You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
242 lines
12 KiB
242 lines
12 KiB
10 years ago
|
/**
|
||
|
|
* Annotation.java
|
||
|
|
* Copyright 2004 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||
|
|
* First released 09.01.2004 at http://yacy.net
|
||
|
|
*
|
||
|
|
* This library is free software; you can redistribute it and/or
|
||
|
|
* modify it under the terms of the GNU Lesser General private
|
||
|
|
* License as published by the Free Software Foundation; either
|
||
|
|
* version 2.1 of the License, or (at your option) any later version.
|
||
|
|
*
|
||
|
|
* This library is distributed in the hope that it will be useful,
|
||
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||
|
|
* Lesser General private License for more details.
|
||
|
|
*
|
||
|
|
* You should have received a copy of the GNU Lesser General private License
|
||
|
|
* along with this program in the file lgpl21.txt
|
||
|
|
* If not, see <http://www.gnu.org/licenses/>.
|
||
|
|
*/
|
||
|
|
|
||
|
|
package net.yacy.document;
|
||
|
|
|
||
|
|
import java.io.IOException;
|
||
|
|
import java.util.ArrayList;
|
||
|
|
import java.util.HashMap;
|
||
|
|
import java.util.HashSet;
|
||
|
|
import java.util.Iterator;
|
||
|
|
import java.util.LinkedHashSet;
|
||
|
|
import java.util.List;
|
||
|
|
import java.util.Locale;
|
||
|
|
import java.util.Map;
|
||
|
|
import java.util.Set;
|
||
|
|
import java.util.TreeMap;
|
||
|
|
|
||
|
|
import net.yacy.cora.document.WordCache;
|
||
|
|
import net.yacy.cora.document.id.DigestURL;
|
||
|
|
import net.yacy.cora.language.synonyms.SynonymLibrary;
|
||
|
|
import net.yacy.cora.lod.vocabulary.Tagging;
|
||
|
|
import net.yacy.cora.order.NaturalOrder;
|
||
|
|
import net.yacy.kelondro.data.word.Word;
|
||
|
|
import net.yacy.kelondro.util.Bitfield;
|
||
|
|
|
||
|
|
public class Tokenizer {
|
||
|
|
|
||
|
|
// this is the page analysis class
|
||
|
|
public final static boolean pseudostemming = false; // switch for removal of words that appear in shortened form
|
||
|
|
public final static int wordminsize = 2;
|
||
|
|
public final static int wordcut = 2;
|
||
|
|
|
||
|
|
// category flags that show how the page can be distinguished in different interest groups
|
||
|
|
public static final int flag_cat_indexof = 0; // a directory listing page (i.e. containing 'index of')
|
||
|
|
public static final int flag_cat_haslocation = 19; // the page has a location metadata attached
|
||
|
|
public static final int flag_cat_hasimage = 20; // the page refers to (at least one) images
|
||
|
|
public static final int flag_cat_hasaudio = 21; // the page refers to (at least one) audio file
|
||
|
|
public static final int flag_cat_hasvideo = 22; // the page refers to (at least one) videos
|
||
|
|
public static final int flag_cat_hasapp = 23; // the page refers to (at least one) application file
|
||
|
|
|
||
|
|
//private Properties analysis;
|
||
|
|
protected final Map<String, Word> words; // a string (the words) to (indexWord) - relation
|
||
|
|
private final Set<String> synonyms; // a set of synonyms to the words
|
||
|
|
protected final Map<String, Set<Tagging.Metatag>> tags = new HashMap<String, Set<Tagging.Metatag>>(); // a set of tags, discovered from Autotagging
|
||
|
|
|
||
|
|
public int RESULT_NUMB_WORDS = -1;
|
||
|
|
public int RESULT_NUMB_SENTENCES = -1;
|
||
|
|
public Bitfield RESULT_FLAGS = new Bitfield(4);
|
||
|
|
|
||
|
|
public Tokenizer(final DigestURL root, final String text, final WordCache meaningLib, boolean doAutotagging, final VocabularyScraper scraper) {
|
||
|
|
this.words = new TreeMap<String, Word>(NaturalOrder.naturalComparator);
|
||
|
|
this.synonyms = new LinkedHashSet<String>();
|
||
|
|
assert text != null;
|
||
|
|
final Set<String> currsentwords = new HashSet<String>();
|
||
|
|
String[] wordcache = new String[LibraryProvider.autotagging.getMaxWordsInTerm() - 1];
|
||
|
|
for (int i = 0; i < wordcache.length; i++) wordcache[i] = "";
|
||
|
|
String k;
|
||
|
|
Tagging.Metatag tag;
|
||
|
|
int wordlen;
|
||
|
|
int wordHandle;
|
||
|
|
int wordHandleCount = 0;
|
||
|
|
//final int sentenceHandleCount = 0;
|
||
|
|
int allwordcounter = 0;
|
||
|
|
final int allsentencecounter = 0;
|
||
|
|
int wordInSentenceCounter = 1;
|
||
|
|
boolean comb_indexof = false, last_last = false, last_index = false;
|
||
|
|
//final Map<StringBuilder, Phrase> sentences = new HashMap<StringBuilder, Phrase>(100);
|
||
|
|
if (LibraryProvider.autotagging.isEmpty()) doAutotagging = false;
|
||
|
|
|
||
|
|
// read source
|
||
|
|
WordTokenizer wordenum = new WordTokenizer(new SentenceReader(text), meaningLib);
|
||
|
|
try {
|
||
|
|
while (wordenum.hasMoreElements()) {
|
||
|
|
String word = wordenum.nextElement().toString().toLowerCase(Locale.ENGLISH);
|
||
|
|
if (word.length() < wordminsize) continue;
|
||
|
|
|
||
|
|
// get tags from autotagging
|
||
|
|
if (doAutotagging) {
|
||
|
|
Set<String> vocabularyNames = LibraryProvider.autotagging.getVocabularyNames();
|
||
|
|
//Collection<Tagging> vocabularies = LibraryProvider.autotagging.getVocabularies();
|
||
|
|
//assert vocabularyNames.size() == vocabularies.size();
|
||
|
|
Map<String, String> vocMap = scraper == null ? null : scraper.removeVocMap(root);
|
||
|
|
if (vocMap != null && vocMap.size() > 0) {
|
||
|
|
for (Map.Entry<String, String> entry: vocMap.entrySet()) {
|
||
|
|
String navigatorName = entry.getKey();
|
||
|
|
String term = entry.getValue();
|
||
|
|
vocabularyNames.remove(navigatorName); // prevent that this is used again for auto-annotation
|
||
|
|
Tagging vocabulary = LibraryProvider.autotagging.getVocabulary(navigatorName);
|
||
|
|
if (vocabulary != null) {
|
||
|
|
// extend the vocabulary
|
||
|
|
String obj = vocabulary.getObjectlink(term);
|
||
|
|
if (obj == null) try {vocabulary.put(term, "", root.toNormalform(true));} catch (IOException e) {} // this makes IO, be careful!
|
||
|
|
// create annotation
|
||
|
|
tag = vocabulary.getMetatagFromTerm(term);
|
||
|
|
Set<Tagging.Metatag> tagset = new HashSet<>();
|
||
|
|
tagset.add(tag);
|
||
|
|
this.tags.put(navigatorName, tagset);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (vocabularyNames.size() > 0) for (int wordc = 1; wordc <= wordcache.length + 1; wordc++) {
|
||
|
|
// wordc is number of words that are tested
|
||
|
|
StringBuilder sb = new StringBuilder();
|
||
|
|
if (wordc == 1) {
|
||
|
|
sb.append(word);
|
||
|
|
} else {
|
||
|
|
for (int w = 0; w < wordc - 1; w++) {
|
||
|
|
sb.append(wordcache[wordcache.length - wordc + w + 1]).append(' ');
|
||
|
|
}
|
||
|
|
sb.append(word);
|
||
|
|
}
|
||
|
|
String testterm = sb.toString().trim();
|
||
|
|
//System.out.println("Testing: " + testterm);
|
||
|
|
tag = LibraryProvider.autotagging.getTagFromTerm(vocabularyNames, testterm);
|
||
|
|
if (tag != null) {
|
||
|
|
String navigatorName = tag.getVocabularyName();
|
||
|
|
Set<Tagging.Metatag> tagset = this.tags.get(navigatorName);
|
||
|
|
if (tagset == null) {
|
||
|
|
tagset = new HashSet<Tagging.Metatag>();
|
||
|
|
this.tags.put(navigatorName, tagset);
|
||
|
|
}
|
||
|
|
tagset.add(tag);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// shift wordcache
|
||
|
|
System.arraycopy(wordcache, 1, wordcache, 0, wordcache.length - 1);
|
||
|
|
wordcache[wordcache.length - 1] = word;
|
||
|
|
|
||
|
|
// distinguish punctuation and words
|
||
|
|
wordlen = word.length();
|
||
|
|
if (wordlen == 1 && SentenceReader.punctuation(word.charAt(0))) { // TODO: wordlen == 1 never true (see earlier if < wordminsize )
|
||
|
|
// store sentence
|
||
|
|
currsentwords.clear();
|
||
|
|
wordInSentenceCounter = 1;
|
||
|
|
} else {
|
||
|
|
// check index.of detection
|
||
|
|
if (last_last && comb_indexof && word.equals("modified")) {
|
||
|
|
this.RESULT_FLAGS.set(flag_cat_indexof, true);
|
||
|
|
wordenum.pre(true); // parse lines as they come with CRLF
|
||
|
|
}
|
||
|
|
if (last_index && (wordminsize > 2 || word.equals("of"))) comb_indexof = true;
|
||
|
|
last_last = word.equals("last");
|
||
|
|
last_index = word.equals("index");
|
||
|
|
|
||
|
|
// store word
|
||
|
|
allwordcounter++;
|
||
|
|
currsentwords.add(word);
|
||
|
|
Word wsp = this.words.get(word);
|
||
|
|
if (wsp != null) {
|
||
|
|
// word already exists
|
||
|
|
wordHandle = wsp.posInText;
|
||
|
|
wsp.inc();
|
||
|
|
} else {
|
||
|
|
// word does not yet exist, create new word entry
|
||
|
|
wordHandle = wordHandleCount++;
|
||
|
|
wsp = new Word(wordHandle, wordInSentenceCounter, /* sentences.size() + */ 100);
|
||
|
|
wsp.flags = this.RESULT_FLAGS.clone();
|
||
|
|
this.words.put(word.toLowerCase(), wsp);
|
||
|
|
}
|
||
|
|
// we now have the unique handle of the word, put it into the sentence:
|
||
|
|
wordInSentenceCounter++;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
} finally {
|
||
|
|
wordenum.close();
|
||
|
|
wordenum = null;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (pseudostemming) {
|
||
|
|
// we search for similar words and reorganize the corresponding sentences
|
||
|
|
// a word is similar, if a shortened version is equal
|
||
|
|
Iterator<Map.Entry<String, Word>> wi = this.words.entrySet().iterator(); // enumerates the keys in descending order?
|
||
|
|
Map.Entry<String, Word> entry;
|
||
|
|
wordsearch: while (wi.hasNext()) {
|
||
|
|
entry = wi.next();
|
||
|
|
String word = entry.getKey();
|
||
|
|
wordlen = word.length();
|
||
|
|
Word wsp = entry.getValue();
|
||
|
|
for (int i = wordcut; i > 0; i--) {
|
||
|
|
if (wordlen > i) {
|
||
|
|
k = word.substring(0, wordlen - i);
|
||
|
|
Word wsp1 = this.words.get(k);
|
||
|
|
if (wsp1 != null) {
|
||
|
|
wsp1.count = wsp1.count + wsp.count; // update word counter
|
||
|
|
wi.remove(); // remove current word
|
||
|
|
continue wordsearch;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// create the synonyms set
|
||
|
|
if (SynonymLibrary.size() > 0) {
|
||
|
|
for (String word: this.words.keySet()) {
|
||
|
|
Set<String> syms = SynonymLibrary.getSynonyms(word);
|
||
|
|
if (syms != null) this.synonyms.addAll(syms);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// store result
|
||
|
|
this.RESULT_NUMB_WORDS = allwordcounter;
|
||
|
|
this.RESULT_NUMB_SENTENCES = allsentencecounter;
|
||
|
|
}
|
||
|
|
|
||
|
|
public Map<String, Word> words() {
|
||
|
|
// returns the words as word/indexWord relation map
|
||
|
|
return this.words;
|
||
|
|
}
|
||
|
|
|
||
|
|
public static Map<String, Word> getWords(final String text, final WordCache meaningLib) {
|
||
|
|
// returns a word/indexWord relation map
|
||
|
|
if (text == null) return null;
|
||
|
|
return new Tokenizer(null, text, meaningLib, false, null).words();
|
||
|
|
}
|
||
|
|
|
||
|
|
public List<String> synonyms() {
|
||
|
|
ArrayList<String> l = new ArrayList<String>(this.synonyms.size());
|
||
|
|
for (String s: this.synonyms) l.add(s);
|
||
|
|
return l;
|
||
|
|
}
|
||
|
|
|
||
|
|
}
|