sof
13 years ago
commit
b09988c9f8
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,429 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package net.yacy.cora.language.phonetic;
|
||||
|
||||
import java.util.Locale;
|
||||
|
||||
import org.apache.commons.codec.EncoderException;
|
||||
import org.apache.commons.codec.StringEncoder;
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* Encodes a string into a Cologne Phonetic value.
|
||||
* </p>
|
||||
* <p>
|
||||
* Implements the <a href="http://de.wikipedia.org/wiki/K%C3%B6lner_Phonetik">Kölner Phonetik</a> (Cologne Phonetic)
|
||||
* algorithm issued by Hans Joachim Postel in 1969.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The <i>Kölner Phonetik</i> is a phonetic algorithm which is optimized for the German language. It is related to the
|
||||
* well-known soundex algorithm.
|
||||
* </p>
|
||||
*
|
||||
* <h2>Algorithm</h2>
|
||||
*
|
||||
* <ul>
|
||||
*
|
||||
* <li>
|
||||
* <h3>Step 1:</h3>
|
||||
* After preprocessing (conversion to upper case, transcription of <a
|
||||
* href="http://en.wikipedia.org/wiki/Germanic_umlaut">germanic umlauts</a>, removal of non alphabetical characters) the
|
||||
* letters of the supplied text are replaced by their phonetic code according to the following table.
|
||||
* <table border="1">
|
||||
* <tbody>
|
||||
* <tr>
|
||||
* <th>Letter</th>
|
||||
* <th>Context</th>
|
||||
* <th align="center">Code</th>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>A, E, I, J, O, U, Y</td>
|
||||
* <td></td>
|
||||
* <td align="center">0</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
*
|
||||
* <td>H</td>
|
||||
* <td></td>
|
||||
* <td align="center">-</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>B</td>
|
||||
* <td></td>
|
||||
* <td rowspan="2" align="center">1</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>P</td>
|
||||
* <td>not before H</td>
|
||||
*
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>D, T</td>
|
||||
* <td>not before C, S, Z</td>
|
||||
* <td align="center">2</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>F, V, W</td>
|
||||
* <td></td>
|
||||
* <td rowspan="2" align="center">3</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
*
|
||||
* <td>P</td>
|
||||
* <td>before H</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>G, K, Q</td>
|
||||
* <td></td>
|
||||
* <td rowspan="3" align="center">4</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td rowspan="2">C</td>
|
||||
* <td>at onset before A, H, K, L, O, Q, R, U, X</td>
|
||||
*
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>before A, H, K, O, Q, U, X except after S, Z</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>X</td>
|
||||
* <td>not after C, K, Q</td>
|
||||
* <td align="center">48</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>L</td>
|
||||
* <td></td>
|
||||
*
|
||||
* <td align="center">5</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>M, N</td>
|
||||
* <td></td>
|
||||
* <td align="center">6</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>R</td>
|
||||
* <td></td>
|
||||
* <td align="center">7</td>
|
||||
* </tr>
|
||||
*
|
||||
* <tr>
|
||||
* <td>S, Z</td>
|
||||
* <td></td>
|
||||
* <td rowspan="6" align="center">8</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td rowspan="3">C</td>
|
||||
* <td>after S, Z</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>at onset except before A, H, K, L, O, Q, R, U, X</td>
|
||||
* </tr>
|
||||
*
|
||||
* <tr>
|
||||
* <td>not before A, H, K, O, Q, U, X</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>D, T</td>
|
||||
* <td>before C, S, Z</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>X</td>
|
||||
* <td>after C, K, Q</td>
|
||||
* </tr>
|
||||
* </tbody>
|
||||
* </table>
|
||||
* <p>
|
||||
* <small><i>(Source: <a href= "http://de.wikipedia.org/wiki/K%C3%B6lner_Phonetik#Buchstabencodes" >Wikipedia (de):
|
||||
* Kölner Phonetik -- Buchstabencodes</a>)</i></small>
|
||||
* </p>
|
||||
*
|
||||
* <h4>Example:</h4>
|
||||
*
|
||||
* {@code "M}ü{@code ller-L}ü{@code denscheidt" => "MULLERLUDENSCHEIDT" => "6005507500206880022"}
|
||||
*
|
||||
* </li>
|
||||
*
|
||||
* <li>
|
||||
* <h3>Step 2:</h3>
|
||||
* Collapse of all multiple consecutive code digits.
|
||||
* <h4>Example:</h4>
|
||||
* {@code "6005507500206880022" => "6050750206802"}</li>
|
||||
*
|
||||
* <li>
|
||||
* <h3>Step 3:</h3>
|
||||
* Removal of all codes "0" except at the beginning. This means that two or more identical consecutive digits can occur
|
||||
* if they occur after removing the "0" digits.
|
||||
*
|
||||
* <h4>Example:</h4>
|
||||
* {@code "6050750206802" => "65752682"}</li>
|
||||
*
|
||||
* </ul>
|
||||
*
|
||||
* @see <a href="http://de.wikipedia.org/wiki/K%C3%B6lner_Phonetik">Wikipedia (de): Kölner Phonetik (in German)</a>
|
||||
* @author Apache Software Foundation
|
||||
* @since 1.5
|
||||
*/
|
||||
public class ColognePhonetic implements StringEncoder {
|
||||
|
||||
private abstract class CologneBuffer {
|
||||
|
||||
protected final char[] data;
|
||||
|
||||
protected int length = 0;
|
||||
|
||||
public CologneBuffer(char[] data) {
|
||||
this.data = data;
|
||||
this.length = data.length;
|
||||
}
|
||||
|
||||
public CologneBuffer(int buffSize) {
|
||||
this.data = new char[buffSize];
|
||||
this.length = 0;
|
||||
}
|
||||
|
||||
protected abstract char[] copyData(int start, final int length);
|
||||
|
||||
public int length() {
|
||||
return length;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return new String(copyData(0, length));
|
||||
}
|
||||
}
|
||||
|
||||
private class CologneOutputBuffer extends CologneBuffer {
|
||||
|
||||
public CologneOutputBuffer(int buffSize) {
|
||||
super(buffSize);
|
||||
}
|
||||
|
||||
public void addRight(char chr) {
|
||||
data[length] = chr;
|
||||
length++;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected char[] copyData(int start, final int length) {
|
||||
char[] newData = new char[length];
|
||||
System.arraycopy(data, start, newData, 0, length);
|
||||
return newData;
|
||||
}
|
||||
}
|
||||
|
||||
private class CologneInputBuffer extends CologneBuffer {
|
||||
|
||||
public CologneInputBuffer(char[] data) {
|
||||
super(data);
|
||||
}
|
||||
|
||||
public void addLeft(char ch) {
|
||||
length++;
|
||||
data[getNextPos()] = ch;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected char[] copyData(int start, final int length) {
|
||||
char[] newData = new char[length];
|
||||
System.arraycopy(data, data.length - this.length + start, newData, 0, length);
|
||||
return newData;
|
||||
}
|
||||
|
||||
public char getNextChar() {
|
||||
return data[getNextPos()];
|
||||
}
|
||||
|
||||
protected int getNextPos() {
|
||||
return data.length - length;
|
||||
}
|
||||
|
||||
public char removeNext() {
|
||||
char ch = getNextChar();
|
||||
length--;
|
||||
return ch;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps some Germanic characters to plain for internal processing. The following characters are mapped:
|
||||
* <ul>
|
||||
* <li>capital a, umlaut mark</li>
|
||||
* <li>capital u, umlaut mark</li>
|
||||
* <li>capital o, umlaut mark</li>
|
||||
* <li>small sharp s, German</li>
|
||||
* </ul>
|
||||
*/
|
||||
private static final char[][] PREPROCESS_MAP = new char[][]{
|
||||
{'\u00C4', 'A'}, // capital a, umlaut mark
|
||||
{'\u00DC', 'U'}, // capital u, umlaut mark
|
||||
{'\u00D6', 'O'}, // capital o, umlaut mark
|
||||
{'\u00DF', 'S'} // small sharp s, German
|
||||
};
|
||||
|
||||
/*
|
||||
* Returns whether the array contains the key, or not.
|
||||
*/
|
||||
private static boolean arrayContains(char[] arr, char key) {
|
||||
for (char element : arr) {
|
||||
if (element == key) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* Implements the <i>Kölner Phonetik</i> algorithm.
|
||||
* </p>
|
||||
* <p>
|
||||
* In contrast to the initial description of the algorithm, this implementation does the encoding in one pass.
|
||||
* </p>
|
||||
*
|
||||
* @param text
|
||||
* @return the corresponding encoding according to the <i>Kölner Phonetik</i> algorithm
|
||||
*/
|
||||
public String colognePhonetic(String text) {
|
||||
if (text == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
text = preprocess(text);
|
||||
|
||||
CologneOutputBuffer output = new CologneOutputBuffer(text.length() * 2);
|
||||
CologneInputBuffer input = new CologneInputBuffer(text.toCharArray());
|
||||
|
||||
char nextChar;
|
||||
|
||||
char lastChar = '-';
|
||||
char lastCode = '/';
|
||||
char code;
|
||||
char chr;
|
||||
|
||||
int rightLength = input.length();
|
||||
|
||||
while (rightLength > 0) {
|
||||
chr = input.removeNext();
|
||||
|
||||
if ((rightLength = input.length()) > 0) {
|
||||
nextChar = input.getNextChar();
|
||||
} else {
|
||||
nextChar = '-';
|
||||
}
|
||||
|
||||
if (arrayContains(new char[]{'A', 'E', 'I', 'J', 'O', 'U', 'Y'}, chr)) {
|
||||
code = '0';
|
||||
} else if (chr == 'H' || chr < 'A' || chr > 'Z') {
|
||||
if (lastCode == '/') {
|
||||
continue;
|
||||
}
|
||||
code = '-';
|
||||
} else if (chr == 'B' || (chr == 'P' && nextChar != 'H')) {
|
||||
code = '1';
|
||||
} else if ((chr == 'D' || chr == 'T') && !arrayContains(new char[]{'S', 'C', 'Z'}, nextChar)) {
|
||||
code = '2';
|
||||
} else if (arrayContains(new char[]{'W', 'F', 'P', 'V'}, chr)) {
|
||||
code = '3';
|
||||
} else if (arrayContains(new char[]{'G', 'K', 'Q'}, chr)) {
|
||||
code = '4';
|
||||
} else if (chr == 'X' && !arrayContains(new char[]{'C', 'K', 'Q'}, lastChar)) {
|
||||
code = '4';
|
||||
input.addLeft('S');
|
||||
rightLength++;
|
||||
} else if (chr == 'S' || chr == 'Z') {
|
||||
code = '8';
|
||||
} else if (chr == 'C') {
|
||||
if (lastCode == '/') {
|
||||
if (arrayContains(new char[]{'A', 'H', 'K', 'L', 'O', 'Q', 'R', 'U', 'X'}, nextChar)) {
|
||||
code = '4';
|
||||
} else {
|
||||
code = '8';
|
||||
}
|
||||
} else {
|
||||
if (arrayContains(new char[]{'S', 'Z'}, lastChar) ||
|
||||
!arrayContains(new char[]{'A', 'H', 'O', 'U', 'K', 'Q', 'X'}, nextChar)) {
|
||||
code = '8';
|
||||
} else {
|
||||
code = '4';
|
||||
}
|
||||
}
|
||||
} else if (arrayContains(new char[]{'T', 'D', 'X'}, chr)) {
|
||||
code = '8';
|
||||
} else if (chr == 'R') {
|
||||
code = '7';
|
||||
} else if (chr == 'L') {
|
||||
code = '5';
|
||||
} else if (chr == 'M' || chr == 'N') {
|
||||
code = '6';
|
||||
} else {
|
||||
code = chr;
|
||||
}
|
||||
|
||||
if (code != '-' && (lastCode != code && (code != '0' || lastCode == '/') || code < '0' || code > '8')) {
|
||||
output.addRight(code);
|
||||
}
|
||||
|
||||
lastChar = chr;
|
||||
lastCode = code;
|
||||
}
|
||||
return output.toString();
|
||||
}
|
||||
|
||||
public Object encode(Object object) throws EncoderException {
|
||||
if (!(object instanceof String)) {
|
||||
throw new EncoderException("This method's parameter was expected to be of the type " +
|
||||
String.class.getName() +
|
||||
". But actually it was of the type " +
|
||||
object.getClass().getName() +
|
||||
".");
|
||||
}
|
||||
return encode((String) object);
|
||||
}
|
||||
|
||||
public String encode(String text) {
|
||||
return colognePhonetic(text);
|
||||
}
|
||||
|
||||
public boolean isEncodeEqual(String text1, String text2) {
|
||||
return colognePhonetic(text1).equals(colognePhonetic(text2));
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts the string to upper case and replaces germanic characters as defined in {@link #PREPROCESS_MAP}.
|
||||
*/
|
||||
private String preprocess(String text) {
|
||||
text = text.toUpperCase(Locale.GERMAN);
|
||||
|
||||
char[] chrs = text.toCharArray();
|
||||
|
||||
for (int index = 0; index < chrs.length; index++) {
|
||||
if (chrs[index] > 'Z') {
|
||||
for (char[] element : PREPROCESS_MAP) {
|
||||
if (chrs[index] == element[0]) {
|
||||
chrs[index] = element[1];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return new String(chrs);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,407 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package net.yacy.cora.language.phonetic;
|
||||
|
||||
import org.apache.commons.codec.EncoderException;
|
||||
import org.apache.commons.codec.StringEncoder;
|
||||
|
||||
/**
|
||||
* Encodes a string into a Metaphone value.
|
||||
* <p>
|
||||
* Initial Java implementation by <CITE>William B. Brogden. December, 1997</CITE>.
|
||||
* Permission given by <CITE>wbrogden</CITE> for code to be used anywhere.
|
||||
* </p>
|
||||
* <p>
|
||||
* <CITE>Hanging on the Metaphone</CITE> by <CITE>Lawrence Philips</CITE> in <CITE>Computer Language of Dec. 1990, p
|
||||
* 39.</CITE>
|
||||
* </p>
|
||||
* <p>
|
||||
* Note, that this does not match the algorithm that ships with PHP, or the algorithm
|
||||
* found in the Perl <a href="http://search.cpan.org/~mschwern/Text-Metaphone-1.96/Metaphone.pm">Text:Metaphone-1.96</a>.
|
||||
* They have had undocumented changes from the originally published algorithm.
|
||||
* For more information, see <a href="https://issues.apache.org/jira/browse/CODEC-57">CODEC-57</a>.
|
||||
* </p>
|
||||
*
|
||||
* @author Apache Software Foundation
|
||||
* @version $Id: Metaphone.java 1157192 2011-08-12 17:27:38Z ggregory $
|
||||
*/
|
||||
public class Metaphone implements StringEncoder {
|
||||
|
||||
/**
|
||||
* Five values in the English language
|
||||
*/
|
||||
private static final String VOWELS = "AEIOU" ;
|
||||
|
||||
/**
|
||||
* Variable used in Metaphone algorithm
|
||||
*/
|
||||
private static final String FRONTV = "EIY" ;
|
||||
|
||||
/**
|
||||
* Variable used in Metaphone algorithm
|
||||
*/
|
||||
private static final String VARSON = "CSPTG" ;
|
||||
|
||||
/**
|
||||
* The max code length for metaphone is 4
|
||||
*/
|
||||
private int maxCodeLen = 4 ;
|
||||
|
||||
/**
|
||||
* Creates an instance of the Metaphone encoder
|
||||
*/
|
||||
public Metaphone() {
|
||||
super();
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the metaphone value of a String. This is similar to the
|
||||
* soundex algorithm, but better at finding similar sounding words.
|
||||
* All input is converted to upper case.
|
||||
* Limitations: Input format is expected to be a single ASCII word
|
||||
* with only characters in the A - Z range, no punctuation or numbers.
|
||||
*
|
||||
* @param txt String to find the metaphone code for
|
||||
* @return A metaphone code corresponding to the String supplied
|
||||
*/
|
||||
public String metaphone(String txt) {
|
||||
boolean hard = false ;
|
||||
if ((txt == null) || (txt.length() == 0)) {
|
||||
return "" ;
|
||||
}
|
||||
// single character is itself
|
||||
if (txt.length() == 1) {
|
||||
return txt.toUpperCase(java.util.Locale.ENGLISH) ;
|
||||
}
|
||||
|
||||
char[] inwd = txt.toUpperCase(java.util.Locale.ENGLISH).toCharArray() ;
|
||||
|
||||
StringBuffer local = new StringBuffer(40); // manipulate
|
||||
StringBuffer code = new StringBuffer(10) ; // output
|
||||
// handle initial 2 characters exceptions
|
||||
switch(inwd[0]) {
|
||||
case 'K' :
|
||||
case 'G' :
|
||||
case 'P' : /* looking for KN, etc*/
|
||||
if (inwd[1] == 'N') {
|
||||
local.append(inwd, 1, inwd.length - 1);
|
||||
} else {
|
||||
local.append(inwd);
|
||||
}
|
||||
break;
|
||||
case 'A': /* looking for AE */
|
||||
if (inwd[1] == 'E') {
|
||||
local.append(inwd, 1, inwd.length - 1);
|
||||
} else {
|
||||
local.append(inwd);
|
||||
}
|
||||
break;
|
||||
case 'W' : /* looking for WR or WH */
|
||||
if (inwd[1] == 'R') { // WR -> R
|
||||
local.append(inwd, 1, inwd.length - 1);
|
||||
break ;
|
||||
}
|
||||
if (inwd[1] == 'H') {
|
||||
local.append(inwd, 1, inwd.length - 1);
|
||||
local.setCharAt(0, 'W'); // WH -> W
|
||||
} else {
|
||||
local.append(inwd);
|
||||
}
|
||||
break;
|
||||
case 'X' : /* initial X becomes S */
|
||||
inwd[0] = 'S';
|
||||
local.append(inwd);
|
||||
break ;
|
||||
default :
|
||||
local.append(inwd);
|
||||
} // now local has working string with initials fixed
|
||||
|
||||
int wdsz = local.length();
|
||||
int n = 0 ;
|
||||
|
||||
while ((code.length() < this.getMaxCodeLen()) &&
|
||||
(n < wdsz) ) { // max code size of 4 works well
|
||||
char symb = local.charAt(n) ;
|
||||
// remove duplicate letters except C
|
||||
if ((symb != 'C') && (isPreviousChar( local, n, symb )) ) {
|
||||
n++ ;
|
||||
} else { // not dup
|
||||
switch(symb) {
|
||||
case 'A' : case 'E' : case 'I' : case 'O' : case 'U' :
|
||||
if (n == 0) {
|
||||
code.append(symb);
|
||||
}
|
||||
break ; // only use vowel if leading char
|
||||
case 'B' :
|
||||
if ( isPreviousChar(local, n, 'M') &&
|
||||
isLastChar(wdsz, n) ) { // B is silent if word ends in MB
|
||||
break;
|
||||
}
|
||||
code.append(symb);
|
||||
break;
|
||||
case 'C' : // lots of C special cases
|
||||
/* discard if SCI, SCE or SCY */
|
||||
if ( isPreviousChar(local, n, 'S') &&
|
||||
!isLastChar(wdsz, n) &&
|
||||
(FRONTV.indexOf(local.charAt(n + 1)) >= 0) ) {
|
||||
break;
|
||||
}
|
||||
if (regionMatch(local, n, "CIA")) { // "CIA" -> X
|
||||
code.append('X');
|
||||
break;
|
||||
}
|
||||
if (!isLastChar(wdsz, n) &&
|
||||
(FRONTV.indexOf(local.charAt(n + 1)) >= 0)) {
|
||||
code.append('S');
|
||||
break; // CI,CE,CY -> S
|
||||
}
|
||||
if (isPreviousChar(local, n, 'S') &&
|
||||
isNextChar(local, n, 'H') ) { // SCH->sk
|
||||
code.append('K') ;
|
||||
break ;
|
||||
}
|
||||
if (isNextChar(local, n, 'H')) { // detect CH
|
||||
if ((n == 0) &&
|
||||
(wdsz >= 3) &&
|
||||
isVowel(local,2) ) { // CH consonant -> K consonant
|
||||
code.append('K');
|
||||
} else {
|
||||
code.append('X'); // CHvowel -> X
|
||||
}
|
||||
} else {
|
||||
code.append('K');
|
||||
}
|
||||
break ;
|
||||
case 'D' :
|
||||
if (!isLastChar(wdsz, n + 1) &&
|
||||
isNextChar(local, n, 'G') &&
|
||||
(FRONTV.indexOf(local.charAt(n + 2)) >= 0)) { // DGE DGI DGY -> J
|
||||
code.append('J'); n += 2 ;
|
||||
} else {
|
||||
code.append('T');
|
||||
}
|
||||
break ;
|
||||
case 'G' : // GH silent at end or before consonant
|
||||
if (isLastChar(wdsz, n + 1) &&
|
||||
isNextChar(local, n, 'H')) {
|
||||
break;
|
||||
}
|
||||
if (!isLastChar(wdsz, n + 1) &&
|
||||
isNextChar(local,n,'H') &&
|
||||
!isVowel(local,n+2)) {
|
||||
break;
|
||||
}
|
||||
if ((n > 0) &&
|
||||
( regionMatch(local, n, "GN") ||
|
||||
regionMatch(local, n, "GNED") ) ) {
|
||||
break; // silent G
|
||||
}
|
||||
if (isPreviousChar(local, n, 'G')) {
|
||||
// NOTE: Given that duplicated chars are removed, I don't see how this can ever be true
|
||||
hard = true ;
|
||||
} else {
|
||||
hard = false ;
|
||||
}
|
||||
if (!isLastChar(wdsz, n) &&
|
||||
(FRONTV.indexOf(local.charAt(n + 1)) >= 0) &&
|
||||
(!hard)) {
|
||||
code.append('J');
|
||||
} else {
|
||||
code.append('K');
|
||||
}
|
||||
break ;
|
||||
case 'H':
|
||||
if (isLastChar(wdsz, n)) {
|
||||
break ; // terminal H
|
||||
}
|
||||
if ((n > 0) &&
|
||||
(VARSON.indexOf(local.charAt(n - 1)) >= 0)) {
|
||||
break;
|
||||
}
|
||||
if (isVowel(local,n+1)) {
|
||||
code.append('H'); // Hvowel
|
||||
}
|
||||
break;
|
||||
case 'F':
|
||||
case 'J' :
|
||||
case 'L' :
|
||||
case 'M':
|
||||
case 'N' :
|
||||
case 'R' :
|
||||
code.append(symb);
|
||||
break;
|
||||
case 'K' :
|
||||
if (n > 0) { // not initial
|
||||
if (!isPreviousChar(local, n, 'C')) {
|
||||
code.append(symb);
|
||||
}
|
||||
} else {
|
||||
code.append(symb); // initial K
|
||||
}
|
||||
break ;
|
||||
case 'P' :
|
||||
if (isNextChar(local,n,'H')) {
|
||||
// PH -> F
|
||||
code.append('F');
|
||||
} else {
|
||||
code.append(symb);
|
||||
}
|
||||
break ;
|
||||
case 'Q' :
|
||||
code.append('K');
|
||||
break;
|
||||
case 'S' :
|
||||
if (regionMatch(local,n,"SH") ||
|
||||
regionMatch(local,n,"SIO") ||
|
||||
regionMatch(local,n,"SIA")) {
|
||||
code.append('X');
|
||||
} else {
|
||||
code.append('S');
|
||||
}
|
||||
break;
|
||||
case 'T' :
|
||||
if (regionMatch(local,n,"TIA") ||
|
||||
regionMatch(local,n,"TIO")) {
|
||||
code.append('X');
|
||||
break;
|
||||
}
|
||||
if (regionMatch(local,n,"TCH")) {
|
||||
// Silent if in "TCH"
|
||||
break;
|
||||
}
|
||||
// substitute numeral 0 for TH (resembles theta after all)
|
||||
if (regionMatch(local,n,"TH")) {
|
||||
code.append('0');
|
||||
} else {
|
||||
code.append('T');
|
||||
}
|
||||
break ;
|
||||
case 'V' :
|
||||
code.append('F'); break ;
|
||||
case 'W' : case 'Y' : // silent if not followed by vowel
|
||||
if (!isLastChar(wdsz,n) &&
|
||||
isVowel(local,n+1)) {
|
||||
code.append(symb);
|
||||
}
|
||||
break ;
|
||||
case 'X' :
|
||||
code.append('K'); code.append('S');
|
||||
break ;
|
||||
case 'Z' :
|
||||
code.append('S'); break ;
|
||||
} // end switch
|
||||
n++ ;
|
||||
} // end else from symb != 'C'
|
||||
if (code.length() > this.getMaxCodeLen()) {
|
||||
code.setLength(this.getMaxCodeLen());
|
||||
}
|
||||
}
|
||||
return code.toString();
|
||||
}
|
||||
|
||||
private boolean isVowel(StringBuffer string, int index) {
|
||||
return VOWELS.indexOf(string.charAt(index)) >= 0;
|
||||
}
|
||||
|
||||
private boolean isPreviousChar(StringBuffer string, int index, char c) {
|
||||
boolean matches = false;
|
||||
if( index > 0 &&
|
||||
index < string.length() ) {
|
||||
matches = string.charAt(index - 1) == c;
|
||||
}
|
||||
return matches;
|
||||
}
|
||||
|
||||
private boolean isNextChar(StringBuffer string, int index, char c) {
|
||||
boolean matches = false;
|
||||
if( index >= 0 &&
|
||||
index < string.length() - 1 ) {
|
||||
matches = string.charAt(index + 1) == c;
|
||||
}
|
||||
return matches;
|
||||
}
|
||||
|
||||
private boolean regionMatch(StringBuffer string, int index, String test) {
|
||||
boolean matches = false;
|
||||
if( index >= 0 &&
|
||||
(index + test.length() - 1) < string.length() ) {
|
||||
String substring = string.substring( index, index + test.length());
|
||||
matches = substring.equals( test );
|
||||
}
|
||||
return matches;
|
||||
}
|
||||
|
||||
private boolean isLastChar(int wdsz, int n) {
|
||||
return n + 1 == wdsz;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Encodes an Object using the metaphone algorithm. This method
|
||||
* is provided in order to satisfy the requirements of the
|
||||
* Encoder interface, and will throw an EncoderException if the
|
||||
* supplied object is not of type java.lang.String.
|
||||
*
|
||||
* @param pObject Object to encode
|
||||
* @return An object (or type java.lang.String) containing the
|
||||
* metaphone code which corresponds to the String supplied.
|
||||
* @throws EncoderException if the parameter supplied is not
|
||||
* of type java.lang.String
|
||||
*/
|
||||
public Object encode(Object pObject) throws EncoderException {
|
||||
if (!(pObject instanceof String)) {
|
||||
throw new EncoderException("Parameter supplied to Metaphone encode is not of type java.lang.String");
|
||||
}
|
||||
return metaphone((String) pObject);
|
||||
}
|
||||
|
||||
/**
|
||||
* Encodes a String using the Metaphone algorithm.
|
||||
*
|
||||
* @param pString String object to encode
|
||||
* @return The metaphone code corresponding to the String supplied
|
||||
*/
|
||||
public String encode(String pString) {
|
||||
return metaphone(pString);
|
||||
}
|
||||
|
||||
/**
|
||||
* Tests is the metaphones of two strings are identical.
|
||||
*
|
||||
* @param str1 First of two strings to compare
|
||||
* @param str2 Second of two strings to compare
|
||||
* @return <code>true</code> if the metaphones of these strings are identical,
|
||||
* <code>false</code> otherwise.
|
||||
*/
|
||||
public boolean isMetaphoneEqual(String str1, String str2) {
|
||||
return metaphone(str1).equals(metaphone(str2));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the maxCodeLen.
|
||||
* @return int
|
||||
*/
|
||||
public int getMaxCodeLen() { return this.maxCodeLen; }
|
||||
|
||||
/**
|
||||
* Sets the maxCodeLen.
|
||||
* @param maxCodeLen The maxCodeLen to set
|
||||
*/
|
||||
public void setMaxCodeLen(int maxCodeLen) { this.maxCodeLen = maxCodeLen; }
|
||||
|
||||
}
|
||||
@ -0,0 +1,73 @@
|
||||
/**
|
||||
* Phonetic
|
||||
* Copyright 201 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany
|
||||
* First released 13.12.2011 at http://yacy.net
|
||||
*
|
||||
* $LastChangedDate$
|
||||
* $LastChangedRevision$
|
||||
* $LastChangedBy$
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.cora.language.phonetic;
|
||||
|
||||
public class Phonetic {
|
||||
|
||||
public enum Encoder {
|
||||
SOUNDEX("Soundex", ""),
|
||||
COLONE_PHONETIC("Koelner Phonetik", "http://de.wikipedia.org/wiki/K%C3%B6lner_Phonetik"),
|
||||
METAPHONE("Metaphone", ""),
|
||||
DOUBLE_METAPHONE("Double Metaphone", ""),
|
||||
NONE("", "");
|
||||
|
||||
final String printName;
|
||||
final String infoUrl;
|
||||
|
||||
private Encoder(final String printName, final String infoUrl) {
|
||||
this.printName = printName;
|
||||
this.infoUrl = infoUrl;
|
||||
}
|
||||
}
|
||||
|
||||
private static final Soundex soundexEncoder = new Soundex();
|
||||
private static final Metaphone metaphoneEncoder = new Metaphone();
|
||||
private static final DoubleMetaphone doubleMetaphoneEncoder = new DoubleMetaphone();
|
||||
private static final ColognePhonetic colognePhonetic = new ColognePhonetic();
|
||||
|
||||
public static String encode(final Encoder encoder, final String s) {
|
||||
try {
|
||||
if (encoder == Encoder.SOUNDEX) return soundexEncoder.encode(s);
|
||||
if (encoder == Encoder.COLONE_PHONETIC) return colognePhonetic.encode(s);
|
||||
if (encoder == Encoder.METAPHONE) return metaphoneEncoder.encode(s);
|
||||
if (encoder == Encoder.DOUBLE_METAPHONE) return doubleMetaphoneEncoder.encode(s);
|
||||
return s;
|
||||
} catch (Throwable e) {
|
||||
// some encoders do not work with non-ASCII charachters and throw an exception
|
||||
return s;
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
for (Encoder encoder: Encoder.values()) {
|
||||
for (String s: args) {
|
||||
System.out.print(Phonetic.encode(encoder, s));
|
||||
System.out.print(" ");
|
||||
}
|
||||
System.out.println();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,340 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package net.yacy.cora.language.phonetic;
|
||||
|
||||
import org.apache.commons.codec.EncoderException;
|
||||
import org.apache.commons.codec.StringEncoder;
|
||||
|
||||
/**
|
||||
* Encodes a string into a Soundex value. Soundex is an encoding used to relate similar names, but can also be used as a
|
||||
* general purpose scheme to find word with similar phonemes.
|
||||
*
|
||||
* @author Apache Software Foundation
|
||||
* @version $Id: Soundex.java 1201529 2011-11-13 21:57:16Z ggregory $
|
||||
*/
|
||||
public class Soundex implements StringEncoder {
|
||||
|
||||
/**
|
||||
* This is a default mapping of the 26 letters used in US English. A value of <code>0</code> for a letter position
|
||||
* means do not encode.
|
||||
* <p>
|
||||
* (This constant is provided as both an implementation convenience and to allow Javadoc to pick
|
||||
* up the value for the constant values page.)
|
||||
* </p>
|
||||
*
|
||||
* @see #US_ENGLISH_MAPPING
|
||||
*/
|
||||
public static final String US_ENGLISH_MAPPING_STRING = "01230120022455012623010202";
|
||||
|
||||
/**
|
||||
* This is a default mapping of the 26 letters used in US English. A value of <code>0</code> for a letter position
|
||||
* means do not encode.
|
||||
*
|
||||
* @see Soundex#Soundex(char[])
|
||||
*/
|
||||
private static final char[] US_ENGLISH_MAPPING = US_ENGLISH_MAPPING_STRING.toCharArray();
|
||||
|
||||
/**
|
||||
* An instance of Soundex using the US_ENGLISH_MAPPING mapping.
|
||||
*
|
||||
* @see #US_ENGLISH_MAPPING
|
||||
*/
|
||||
public static final Soundex US_ENGLISH = new Soundex();
|
||||
|
||||
/**
|
||||
* Every letter of the alphabet is "mapped" to a numerical value. This char array holds the values to which each
|
||||
* letter is mapped. This implementation contains a default map for US_ENGLISH
|
||||
*/
|
||||
private final char[] soundexMapping;
|
||||
|
||||
/**
|
||||
* Creates an instance using US_ENGLISH_MAPPING
|
||||
*
|
||||
* @see Soundex#Soundex(char[])
|
||||
* @see Soundex#US_ENGLISH_MAPPING
|
||||
*/
|
||||
public Soundex() {
|
||||
this.soundexMapping = US_ENGLISH_MAPPING;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a soundex instance using the given mapping. This constructor can be used to provide an internationalized
|
||||
* mapping for a non-Western character set.
|
||||
*
|
||||
* Every letter of the alphabet is "mapped" to a numerical value. This char array holds the values to which each
|
||||
* letter is mapped. This implementation contains a default map for US_ENGLISH
|
||||
*
|
||||
* @param mapping
|
||||
* Mapping array to use when finding the corresponding code for a given character
|
||||
*/
|
||||
public Soundex(char[] mapping) {
|
||||
this.soundexMapping = new char[mapping.length];
|
||||
System.arraycopy(mapping, 0, this.soundexMapping, 0, mapping.length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a refined soundex instance using a custom mapping. This constructor can be used to customize the mapping,
|
||||
* and/or possibly provide an internationalized mapping for a non-Western character set.
|
||||
*
|
||||
* @param mapping
|
||||
* Mapping string to use when finding the corresponding code for a given character
|
||||
* @since 1.4
|
||||
*/
|
||||
public Soundex(String mapping) {
|
||||
this.soundexMapping = mapping.toCharArray();
|
||||
}
|
||||
|
||||
/**
|
||||
* Encodes the Strings and returns the number of characters in the two encoded Strings that are the same. This
|
||||
* return value ranges from 0 through 4: 0 indicates little or no similarity, and 4 indicates strong similarity or
|
||||
* identical values.
|
||||
*
|
||||
* @param s1
|
||||
* A String that will be encoded and compared.
|
||||
* @param s2
|
||||
* A String that will be encoded and compared.
|
||||
* @return The number of characters in the two encoded Strings that are the same from 0 to 4.
|
||||
*
|
||||
* @see SoundexUtils#difference(StringEncoder,String,String)
|
||||
* @see <a href="http://msdn.microsoft.com/library/default.asp?url=/library/en-us/tsqlref/ts_de-dz_8co5.asp"> MS
|
||||
* T-SQL DIFFERENCE </a>
|
||||
*
|
||||
* @throws EncoderException
|
||||
* if an error occurs encoding one of the strings
|
||||
* @since 1.3
|
||||
*/
|
||||
public int difference(String s1, String s2) throws EncoderException {
|
||||
return difference(this, s1, s2);
|
||||
}
|
||||
|
||||
/**
|
||||
* Encodes an Object using the soundex algorithm. This method is provided in order to satisfy the requirements of
|
||||
* the Encoder interface, and will throw an EncoderException if the supplied object is not of type java.lang.String.
|
||||
*
|
||||
* @param pObject
|
||||
* Object to encode
|
||||
* @return An object (or type java.lang.String) containing the soundex code which corresponds to the String
|
||||
* supplied.
|
||||
* @throws EncoderException
|
||||
* if the parameter supplied is not of type java.lang.String
|
||||
* @throws IllegalArgumentException
|
||||
* if a character is not mapped
|
||||
*/
|
||||
public Object encode(Object pObject) throws EncoderException {
|
||||
if (!(pObject instanceof String)) {
|
||||
throw new EncoderException("Parameter supplied to Soundex encode is not of type java.lang.String");
|
||||
}
|
||||
return soundex((String) pObject);
|
||||
}
|
||||
|
||||
/**
|
||||
* Encodes a String using the soundex algorithm.
|
||||
*
|
||||
* @param pString
|
||||
* A String object to encode
|
||||
* @return A Soundex code corresponding to the String supplied
|
||||
* @throws IllegalArgumentException
|
||||
* if a character is not mapped
|
||||
*/
|
||||
public String encode(String pString) {
|
||||
return soundex(pString);
|
||||
}
|
||||
|
||||
/**
|
||||
* Used internally by the SoundEx algorithm.
|
||||
*
|
||||
* Consonants from the same code group separated by W or H are treated as one.
|
||||
*
|
||||
* @param str
|
||||
* the cleaned working string to encode (in upper case).
|
||||
* @param index
|
||||
* the character position to encode
|
||||
* @return Mapping code for a particular character
|
||||
* @throws IllegalArgumentException
|
||||
* if the character is not mapped
|
||||
*/
|
||||
private char getMappingCode(String str, int index) {
|
||||
// map() throws IllegalArgumentException
|
||||
char mappedChar = this.map(str.charAt(index));
|
||||
// HW rule check
|
||||
if (index > 1 && mappedChar != '0') {
|
||||
char hwChar = str.charAt(index - 1);
|
||||
if ('H' == hwChar || 'W' == hwChar) {
|
||||
char preHWChar = str.charAt(index - 2);
|
||||
char firstCode = this.map(preHWChar);
|
||||
if (firstCode == mappedChar || 'H' == preHWChar || 'W' == preHWChar) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
return mappedChar;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the soundex mapping.
|
||||
*
|
||||
* @return soundexMapping.
|
||||
*/
|
||||
private char[] getSoundexMapping() {
|
||||
return this.soundexMapping;
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps the given upper-case character to its Soundex code.
|
||||
*
|
||||
* @param ch
|
||||
* An upper-case character.
|
||||
* @return A Soundex code.
|
||||
* @throws IllegalArgumentException
|
||||
* Thrown if <code>ch</code> is not mapped.
|
||||
*/
|
||||
private char map(char ch) {
|
||||
int index = ch - 'A';
|
||||
if (index < 0 || index >= this.getSoundexMapping().length) {
|
||||
throw new IllegalArgumentException("The character is not mapped: " + ch);
|
||||
}
|
||||
return this.getSoundexMapping()[index];
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves the Soundex code for a given String object.
|
||||
*
|
||||
* @param str
|
||||
* String to encode using the Soundex algorithm
|
||||
* @return A soundex code for the String supplied
|
||||
* @throws IllegalArgumentException
|
||||
* if a character is not mapped
|
||||
*/
|
||||
public String soundex(String str) {
|
||||
if (str == null) {
|
||||
return null;
|
||||
}
|
||||
str = clean(str);
|
||||
if (str.length() == 0) {
|
||||
return str;
|
||||
}
|
||||
char out[] = {'0', '0', '0', '0'};
|
||||
char last, mapped;
|
||||
int incount = 1, count = 1;
|
||||
out[0] = str.charAt(0);
|
||||
// getMappingCode() throws IllegalArgumentException
|
||||
last = getMappingCode(str, 0);
|
||||
while ((incount < str.length()) && (count < out.length)) {
|
||||
mapped = getMappingCode(str, incount++);
|
||||
if (mapped != 0) {
|
||||
if ((mapped != '0') && (mapped != last)) {
|
||||
out[count++] = mapped;
|
||||
}
|
||||
last = mapped;
|
||||
}
|
||||
}
|
||||
return new String(out);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Cleans up the input string before Soundex processing by only returning
|
||||
* upper case letters.
|
||||
*
|
||||
* @param str
|
||||
* The String to clean.
|
||||
* @return A clean String.
|
||||
*/
|
||||
static String clean(String str) {
|
||||
if (str == null || str.length() == 0) {
|
||||
return str;
|
||||
}
|
||||
int len = str.length();
|
||||
char[] chars = new char[len];
|
||||
int count = 0;
|
||||
for (int i = 0; i < len; i++) {
|
||||
if (Character.isLetter(str.charAt(i))) {
|
||||
chars[count++] = str.charAt(i);
|
||||
}
|
||||
}
|
||||
if (count == len) {
|
||||
return str.toUpperCase(java.util.Locale.ENGLISH);
|
||||
}
|
||||
return new String(chars, 0, count).toUpperCase(java.util.Locale.ENGLISH);
|
||||
}
|
||||
|
||||
/**
|
||||
* Encodes the Strings and returns the number of characters in the two
|
||||
* encoded Strings that are the same.
|
||||
* <ul>
|
||||
* <li>For Soundex, this return value ranges from 0 through 4: 0 indicates
|
||||
* little or no similarity, and 4 indicates strong similarity or identical
|
||||
* values.</li>
|
||||
* <li>For refined Soundex, the return value can be greater than 4.</li>
|
||||
* </ul>
|
||||
*
|
||||
* @param encoder
|
||||
* The encoder to use to encode the Strings.
|
||||
* @param s1
|
||||
* A String that will be encoded and compared.
|
||||
* @param s2
|
||||
* A String that will be encoded and compared.
|
||||
* @return The number of characters in the two Soundex encoded Strings that
|
||||
* are the same.
|
||||
*
|
||||
* @see #differenceEncoded(String,String)
|
||||
* @see <a href="http://msdn.microsoft.com/library/default.asp?url=/library/en-us/tsqlref/ts_de-dz_8co5.asp">
|
||||
* MS T-SQL DIFFERENCE</a>
|
||||
*
|
||||
* @throws EncoderException
|
||||
* if an error occurs encoding one of the strings
|
||||
*/
|
||||
static int difference(StringEncoder encoder, String s1, String s2) throws EncoderException {
|
||||
return differenceEncoded(encoder.encode(s1), encoder.encode(s2));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the number of characters in the two Soundex encoded Strings that
|
||||
* are the same.
|
||||
* <ul>
|
||||
* <li>For Soundex, this return value ranges from 0 through 4: 0 indicates
|
||||
* little or no similarity, and 4 indicates strong similarity or identical
|
||||
* values.</li>
|
||||
* <li>For refined Soundex, the return value can be greater than 4.</li>
|
||||
* </ul>
|
||||
*
|
||||
* @param es1
|
||||
* An encoded String.
|
||||
* @param es2
|
||||
* An encoded String.
|
||||
* @return The number of characters in the two Soundex encoded Strings that
|
||||
* are the same.
|
||||
*
|
||||
* @see <a href="http://msdn.microsoft.com/library/default.asp?url=/library/en-us/tsqlref/ts_de-dz_8co5.asp">
|
||||
* MS T-SQL DIFFERENCE</a>
|
||||
*/
|
||||
static int differenceEncoded(String es1, String es2) {
|
||||
|
||||
if (es1 == null || es2 == null) {
|
||||
return 0;
|
||||
}
|
||||
int lengthToMatch = Math.min(es1.length(), es2.length());
|
||||
int diff = 0;
|
||||
for (int i = 0; i < lengthToMatch; i++) {
|
||||
if (es1.charAt(i) == es2.charAt(i)) {
|
||||
diff++;
|
||||
}
|
||||
}
|
||||
return diff;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,74 @@
|
||||
/**
|
||||
* Literal
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 18.12.2011 at http://yacy.net
|
||||
*
|
||||
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
||||
* $LastChangedRevision: 7653 $
|
||||
* $LastChangedBy: orbiter $
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.cora.lod;
|
||||
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import net.yacy.cora.document.MultiProtocolURI;
|
||||
|
||||
/**
|
||||
* A literal is the possible value for a predicate.
|
||||
* A set of literals is the norm of a predicate.
|
||||
* Each literal can have an attached explanation which we express
|
||||
* as a link to the resource that explains the literal.
|
||||
*/
|
||||
public interface Literal {
|
||||
|
||||
/**
|
||||
* the terminal is the actual content of the property and also
|
||||
* the visual representation of the content of a property if the
|
||||
* literal is assigned to that property.
|
||||
* @return a string representing the literal
|
||||
*/
|
||||
public String getTerminal();
|
||||
|
||||
/**
|
||||
* the subject of a literal is a reference to a resource that
|
||||
* explains the literal. If an object has attached properties
|
||||
* from different vocabularies and properties assigned to the
|
||||
* object have actual literal instances assigned, then the set
|
||||
* of subjects of these literals explain the object as a co-notation
|
||||
* to knowledge. Subjects of literals shall therefore be
|
||||
* knowledge authorities for the predicates where the literal is
|
||||
* assigned.
|
||||
* @return an url to a knowledge authority for the literal
|
||||
*/
|
||||
public MultiProtocolURI getSubject();
|
||||
|
||||
/**
|
||||
* if a resource is poorly annotated with metadata an it shall
|
||||
* be automatically annotated, then the terminal of a literal
|
||||
* may be too weak to discover literals in the resource. An additional
|
||||
* discovery pattern may help to reduce the set of literals that can
|
||||
* be discovered automatically. A discovery pattern is then not
|
||||
* a replacement of the terminal itself, it is an additional pattern
|
||||
* that must occur also in the resource where also the terminal of
|
||||
* the literal appears. If the terminal itself is sufficient to discover
|
||||
* the literal, then the discovery pattern may be a catch-all '.*' pattern.
|
||||
* @return the discovery pattern to identify the literal in the resource.
|
||||
*/
|
||||
public Pattern getDiscoveryPattern();
|
||||
|
||||
}
|
||||
@ -0,0 +1,113 @@
|
||||
/**
|
||||
* AbstractScoreMap
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 16.12.2011 at http://yacy.net
|
||||
*
|
||||
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
||||
* $LastChangedRevision: 7653 $
|
||||
* $LastChangedBy: orbiter $
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.cora.lod;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import net.yacy.cora.document.UTF8;
|
||||
import net.yacy.cora.lod.vocabulary.Rdf;
|
||||
|
||||
/**
|
||||
* class for a RDF node element. For a short primer see
|
||||
* http://www.w3.org/TR/REC-rdf-syntax/
|
||||
*/
|
||||
public class Node extends HashMap<String, byte[]> implements Map<String, byte[]> {
|
||||
|
||||
private static final long serialVersionUID = -6715118942251224832L;
|
||||
|
||||
public static final String SUBJECT = "rdf:about";
|
||||
|
||||
private final Rdf type;
|
||||
|
||||
public Node(Rdf type) {
|
||||
super();
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
public Node(Rdf type, byte[] subject) {
|
||||
this(type);
|
||||
this.put(SUBJECT, subject);
|
||||
}
|
||||
|
||||
/**
|
||||
* initialize the triples.
|
||||
* one of the properties must be the resource SUBJECT
|
||||
* for a blank node the SUBJECT can be omitted
|
||||
* @param set
|
||||
*/
|
||||
public Node(Rdf type, Map<String, byte[]> set) {
|
||||
this(type);
|
||||
this.putAll(set);
|
||||
}
|
||||
|
||||
public Rdf getType() {
|
||||
return this.type;
|
||||
}
|
||||
|
||||
public boolean isBlank() {
|
||||
return !this.containsKey(SUBJECT);
|
||||
}
|
||||
|
||||
public byte[] getSubject() {
|
||||
return this.get(SUBJECT);
|
||||
}
|
||||
|
||||
public void setSubject(byte[] subject) {
|
||||
this.put(SUBJECT, subject);
|
||||
}
|
||||
|
||||
public byte[] getObject(Vocabulary predicate) {
|
||||
return this.get(predicate.getPredicate());
|
||||
}
|
||||
|
||||
public byte[] setObject(Vocabulary predicate, byte[] object) {
|
||||
return this.put(predicate.getPredicate(), object);
|
||||
}
|
||||
|
||||
public byte[] removePredicate(Vocabulary predicate) {
|
||||
return this.remove(predicate.getPredicate());
|
||||
}
|
||||
|
||||
public byte[] toObject() {
|
||||
StringBuilder sb = new StringBuilder(this.size() * 50);
|
||||
sb.append("<");
|
||||
sb.append(this.type.getPredicate());
|
||||
byte[] subject = this.get(SUBJECT);
|
||||
if (subject != null) sb.append(" rdf:about=\"").append(UTF8.String(subject)).append('\"');
|
||||
sb.append(">\n");
|
||||
for (Map.Entry<String, byte[]> entry: this.entrySet()) {
|
||||
if (entry.getKey().equals(SUBJECT)) continue;
|
||||
sb.append('<').append(entry.getKey()).append('>');
|
||||
sb.append(UTF8.String(entry.getValue()));
|
||||
sb.append("</").append(entry.getKey()).append(">\n");
|
||||
}
|
||||
sb.append("</");
|
||||
sb.append(this.type.getPredicate());
|
||||
sb.append(">\n");
|
||||
return UTF8.getBytes(sb);
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,78 @@
|
||||
/**
|
||||
* Syntax
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 17.12.2011 at http://yacy.net
|
||||
*
|
||||
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
||||
* $LastChangedRevision: 7653 $
|
||||
* $LastChangedBy: orbiter $
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.cora.lod;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import net.yacy.cora.lod.vocabulary.CreativeCommons;
|
||||
import net.yacy.cora.lod.vocabulary.DublinCore;
|
||||
import net.yacy.cora.lod.vocabulary.Foaf;
|
||||
import net.yacy.cora.lod.vocabulary.Geo;
|
||||
import net.yacy.cora.lod.vocabulary.HttpHeader;
|
||||
import net.yacy.cora.lod.vocabulary.Rdf;
|
||||
import net.yacy.cora.lod.vocabulary.YaCyMetadata;
|
||||
|
||||
/**
|
||||
* helper class to understand xml tags and vocabularies
|
||||
*/
|
||||
public class Syntax {
|
||||
|
||||
private final static Class<?>[] vocabularies = new Class<?>[]{
|
||||
CreativeCommons.class,
|
||||
DublinCore.class,
|
||||
Foaf.class,
|
||||
Geo.class,
|
||||
HttpHeader.class,
|
||||
Rdf.class,
|
||||
YaCyMetadata.class
|
||||
};
|
||||
|
||||
private final static Map<String, Vocabulary> tagMap = new HashMap<String, Vocabulary>();
|
||||
|
||||
static {
|
||||
Vocabulary voc;
|
||||
for (Class<?> v: vocabularies) {
|
||||
Object[] cs = v.getEnumConstants();
|
||||
for (Object c: cs) {
|
||||
voc = (Vocabulary) c;
|
||||
tagMap.put(voc.getPredicate(), voc);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* recognizer for vocabulary tag names
|
||||
* @param tag
|
||||
* @return the vocabulary object for the given tag
|
||||
*/
|
||||
public static Vocabulary getVocabulary(String tag) {
|
||||
return tagMap.get(tag);
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
System.out.println(tagMap);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,124 @@
|
||||
/**
|
||||
* TripleStore
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 16.12.2011 at http://yacy.net
|
||||
*
|
||||
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
||||
* $LastChangedRevision: 7653 $
|
||||
* $LastChangedBy: orbiter $
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
|
||||
package net.yacy.cora.lod;
|
||||
|
||||
import java.util.AbstractMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
|
||||
import net.yacy.cora.lod.vocabulary.Rdf;
|
||||
import net.yacy.cora.order.ByteOrder;
|
||||
import net.yacy.cora.order.CloneableIterator;
|
||||
import net.yacy.cora.storage.MapStore;
|
||||
|
||||
public class TripleStore {
|
||||
|
||||
MapStore store;
|
||||
|
||||
public TripleStore(MapStore store) {
|
||||
this.store = store;
|
||||
}
|
||||
|
||||
public void clear() {
|
||||
this.store.clear();
|
||||
}
|
||||
|
||||
public boolean contains(byte[] id) {
|
||||
return this.store.containsKey(id);
|
||||
}
|
||||
|
||||
public Node get(byte[] id) {
|
||||
Map<String, byte[]> n = this.store.get(id);
|
||||
if (n == null) return null;
|
||||
return new Node(Rdf.Description, n);
|
||||
}
|
||||
|
||||
public boolean isEmpty() {
|
||||
return this.store.isEmpty();
|
||||
}
|
||||
|
||||
public Node put(byte[] id, Node node) {
|
||||
Map<String, byte[]> n = this.store.put(id, node);
|
||||
if (n == null) return null;
|
||||
return new Node(Rdf.Description, n);
|
||||
}
|
||||
|
||||
public void putAll(TripleStore entries) {
|
||||
Iterator<Map.Entry<byte[], Node>> i = entries.iterator();
|
||||
Map.Entry<? extends byte[], ? extends Node> entry;
|
||||
while (i.hasNext()) {
|
||||
entry = i.next();
|
||||
this.put(entry.getKey(), entry.getValue());
|
||||
}
|
||||
}
|
||||
|
||||
public Node remove(byte[] id) {
|
||||
Map<String, byte[]> n = this.store.remove(id);
|
||||
if (n == null) return null;
|
||||
return new Node(Rdf.Description, n);
|
||||
}
|
||||
|
||||
public int size() {
|
||||
return this.store.size();
|
||||
}
|
||||
|
||||
public Iterator<java.util.Map.Entry<byte[], Node>> iterator() {
|
||||
final Iterator<byte[]> id = this.idIterator();
|
||||
return new Iterator<Map.Entry<byte[], Node>>(){
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return id.hasNext();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map.Entry<byte[], Node> next() {
|
||||
byte[] key = id.next();
|
||||
if (key == null) return null;
|
||||
return new AbstractMap.SimpleImmutableEntry<byte[], Node>(key, TripleStore.this.get(key));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void remove() {
|
||||
id.remove();
|
||||
}
|
||||
|
||||
};
|
||||
}
|
||||
|
||||
public ByteOrder getOrdering() {
|
||||
return this.store.getOrdering();
|
||||
}
|
||||
|
||||
public CloneableIterator<byte[]> idIterator() {
|
||||
return this.store.keyIterator();
|
||||
}
|
||||
|
||||
public void close() {
|
||||
this.store.close();
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,72 @@
|
||||
/**
|
||||
* Vocabulary
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 16.12.2011 at http://yacy.net
|
||||
*
|
||||
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
||||
* $LastChangedRevision: 7653 $
|
||||
* $LastChangedBy: orbiter $
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.cora.lod;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
|
||||
/*
|
||||
* A Vocabulary is an interface to an 'extensible enum pattern'.
|
||||
* We want to have an kind of extensible enum for vocabularies.
|
||||
* Since enum classes cannot be extended we use a hack as explained in
|
||||
* http://blogs.oracle.com/darcy/entry/enums_and_mixins .
|
||||
* For an example for 'extensible enum pattern' see
|
||||
* http://stackoverflow.com/questions/1414755/java-extend-enum
|
||||
*/
|
||||
public interface Vocabulary {
|
||||
|
||||
/**
|
||||
* get the RDF identifier
|
||||
* @return
|
||||
*/
|
||||
public String getIdentifier();
|
||||
|
||||
/**
|
||||
* get the prefix for the predicates of this vocabulary
|
||||
* @return
|
||||
*/
|
||||
public String getPrefix();
|
||||
|
||||
/**
|
||||
* get the predicate name which already contains the prefix and the ':'
|
||||
* @return
|
||||
*/
|
||||
public String getPredicate();
|
||||
|
||||
/**
|
||||
* get a set of literals that are allowed for the predicate as values
|
||||
* @return
|
||||
*/
|
||||
public Set<Literal> getLiterals();
|
||||
|
||||
/**
|
||||
* the name method is identical to the java.lang.Enum method.
|
||||
* If an Enum class for vocabularies
|
||||
* implements this interface, the name() method is automatically implemented
|
||||
*
|
||||
* @return Returns the name of the enum constant as declared in the enum declaration.
|
||||
*/
|
||||
public String name();
|
||||
}
|
||||
@ -0,0 +1,207 @@
|
||||
/**
|
||||
* CreativeCommons
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 17.12.2011 at http://yacy.net
|
||||
*
|
||||
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
||||
* $LastChangedRevision: 7653 $
|
||||
* $LastChangedBy: orbiter $
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.cora.lod.vocabulary;
|
||||
|
||||
import java.net.MalformedURLException;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import net.yacy.cora.document.MultiProtocolURI;
|
||||
import net.yacy.cora.lod.Literal;
|
||||
import net.yacy.cora.lod.Vocabulary;
|
||||
|
||||
/**
|
||||
* a vocabulary for creative commons license declarations. see:
|
||||
* http://creativecommons.org/ns#
|
||||
*/
|
||||
public enum CreativeCommons implements Vocabulary {
|
||||
|
||||
// License Properties
|
||||
permits(new Literal[]{
|
||||
PermitLiteral.Reproduction,
|
||||
PermitLiteral.Distribution,
|
||||
PermitLiteral.DerivativeWorks,
|
||||
PermitLiteral.Sharing}),
|
||||
requires(new Literal[]{
|
||||
RequirementLiteral.Notice,
|
||||
RequirementLiteral.Attribution,
|
||||
RequirementLiteral.ShareAlike,
|
||||
RequirementLiteral.SourceCode,
|
||||
RequirementLiteral.Copyleft,
|
||||
RequirementLiteral.LesserCopyleft}),
|
||||
prohibits(new Literal[]{
|
||||
ProhibitionLiteral.CommercialUse,
|
||||
ProhibitionLiteral.HighIncomeNationUse}),
|
||||
jurisdiction,
|
||||
legalcode,
|
||||
deprecatedOn,
|
||||
|
||||
// Work Properties
|
||||
license,
|
||||
morePermissions,
|
||||
attributionName,
|
||||
attributionURL,
|
||||
useGuidelines;
|
||||
|
||||
enum PermitLiteral implements Literal {
|
||||
|
||||
Reproduction("Reproduction", "http://creativecommons.org/ns#Permission", ".*"),
|
||||
Distribution("Distribution", "http://creativecommons.org/ns#Permission", ".*"),
|
||||
DerivativeWorks("Derivative Works", "http://creativecommons.org/ns#Permission", ".*"),
|
||||
Sharing("Sharing", "http://creativecommons.org/ns#Permission", ".*");
|
||||
|
||||
String terminal;
|
||||
MultiProtocolURI subject;
|
||||
Pattern discoveryPattern;
|
||||
|
||||
private PermitLiteral(
|
||||
String terminal,
|
||||
String subject,
|
||||
String discoveryPattern) {
|
||||
this.terminal = terminal;
|
||||
try {
|
||||
this.subject = subject == null ? null : new MultiProtocolURI(subject);
|
||||
} catch (MalformedURLException e) {
|
||||
this.subject = null;
|
||||
}
|
||||
this.discoveryPattern = Pattern.compile(discoveryPattern == null ? ".*" : discoveryPattern);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getTerminal() { return this.terminal; }
|
||||
|
||||
@Override
|
||||
public MultiProtocolURI getSubject() { return this.subject; }
|
||||
|
||||
@Override
|
||||
public Pattern getDiscoveryPattern() { return this.discoveryPattern; }
|
||||
}
|
||||
|
||||
enum RequirementLiteral implements Literal {
|
||||
|
||||
Notice("Notice", "http://creativecommons.org/ns#Requirement", ".*"),
|
||||
Attribution("Attribution", "http://creativecommons.org/ns#Requirement", ".*"),
|
||||
ShareAlike("Share Alike", "http://creativecommons.org/ns#Requirement", ".*"),
|
||||
SourceCode("Source Code", "http://creativecommons.org/ns#Requirement", ".*"),
|
||||
Copyleft("Copyleft", "http://creativecommons.org/ns#Requirement", ".*"),
|
||||
LesserCopyleft("Lesser Copyleft", "http://creativecommons.org/ns#Requirement", ".*");
|
||||
|
||||
String terminal;
|
||||
MultiProtocolURI subject;
|
||||
Pattern discoveryPattern;
|
||||
|
||||
private RequirementLiteral(
|
||||
String terminal,
|
||||
String subject,
|
||||
String discoveryPattern) {
|
||||
this.terminal = terminal;
|
||||
try {
|
||||
this.subject = subject == null ? null : new MultiProtocolURI(subject);
|
||||
} catch (MalformedURLException e) {
|
||||
this.subject = null;
|
||||
}
|
||||
this.discoveryPattern = Pattern.compile(discoveryPattern == null ? ".*" : discoveryPattern);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getTerminal() { return this.terminal; }
|
||||
|
||||
@Override
|
||||
public MultiProtocolURI getSubject() { return this.subject; }
|
||||
|
||||
@Override
|
||||
public Pattern getDiscoveryPattern() { return this.discoveryPattern; }
|
||||
}
|
||||
|
||||
enum ProhibitionLiteral implements Literal {
|
||||
|
||||
CommercialUse("Commercial Use", "http://creativecommons.org/ns#Prohibition", ".*"),
|
||||
HighIncomeNationUse("High Income Nation Use", "http://creativecommons.org/ns#Prohibition", ".*");
|
||||
|
||||
String terminal;
|
||||
MultiProtocolURI subject;
|
||||
Pattern discoveryPattern;
|
||||
|
||||
private ProhibitionLiteral(
|
||||
String terminal,
|
||||
String subject,
|
||||
String discoveryPattern) {
|
||||
this.terminal = terminal;
|
||||
try {
|
||||
this.subject = subject == null ? null : new MultiProtocolURI(subject);
|
||||
} catch (MalformedURLException e) {
|
||||
this.subject = null;
|
||||
}
|
||||
this.discoveryPattern = Pattern.compile(discoveryPattern == null ? ".*" : discoveryPattern);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getTerminal() { return this.terminal; }
|
||||
|
||||
@Override
|
||||
public MultiProtocolURI getSubject() { return this.subject; }
|
||||
|
||||
@Override
|
||||
public Pattern getDiscoveryPattern() { return this.discoveryPattern; }
|
||||
}
|
||||
|
||||
public final static String IDENTIFIER = "http://creativecommons.org/ns#";
|
||||
public final static String PREFIX = "cc";
|
||||
|
||||
private final String predicate;
|
||||
private final Set<Literal> literals;
|
||||
|
||||
private CreativeCommons() {
|
||||
this.predicate = PREFIX + ":" + this.name();
|
||||
this.literals = null;
|
||||
}
|
||||
|
||||
private CreativeCommons(Literal[] literals) {
|
||||
this.predicate = PREFIX + ":" + this.name();
|
||||
this.literals = new HashSet<Literal>();
|
||||
for (Literal l: literals) this.literals.add(l);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getIdentifier() {
|
||||
return IDENTIFIER;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPrefix() {
|
||||
return PREFIX;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<Literal> getLiterals() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPredicate() {
|
||||
return this.predicate;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,79 @@
|
||||
/**
|
||||
* DublinCore
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 16.12.2011 at http://yacy.net
|
||||
*
|
||||
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
||||
* $LastChangedRevision: 7653 $
|
||||
* $LastChangedBy: orbiter $
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
|
||||
package net.yacy.cora.lod.vocabulary;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
import net.yacy.cora.lod.Literal;
|
||||
import net.yacy.cora.lod.Vocabulary;
|
||||
|
||||
public enum DublinCore implements Vocabulary {
|
||||
|
||||
Contributor,
|
||||
Coverage,
|
||||
Creator,
|
||||
Date,
|
||||
Description,
|
||||
Format,
|
||||
Identifier,
|
||||
Language,
|
||||
Publisher,
|
||||
Relation,
|
||||
Rights,
|
||||
Source,
|
||||
Subject,
|
||||
Title,
|
||||
Type;
|
||||
|
||||
public final static String IDENTIFIER = "http://dublincore.org/documents/2010/10/11/dces/";
|
||||
public final static String PREFIX = "dc";
|
||||
|
||||
private final String predicate;
|
||||
|
||||
private DublinCore() {
|
||||
this.predicate = PREFIX + ":" + this.name().toLowerCase();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getIdentifier() {
|
||||
return IDENTIFIER;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPrefix() {
|
||||
return PREFIX;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<Literal> getLiterals() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPredicate() {
|
||||
return this.predicate;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,62 @@
|
||||
/**
|
||||
* Foaf
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 17.12.2011 at http://yacy.net
|
||||
*
|
||||
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
||||
* $LastChangedRevision: 7653 $
|
||||
* $LastChangedBy: orbiter $
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.cora.lod.vocabulary;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
import net.yacy.cora.lod.Literal;
|
||||
import net.yacy.cora.lod.Vocabulary;
|
||||
|
||||
/**
|
||||
* The friend of a friend vocabulary. see:
|
||||
* http://xmlns.com/foaf/spec/
|
||||
*/
|
||||
public enum Foaf implements Vocabulary {
|
||||
;
|
||||
|
||||
@Override
|
||||
public String getIdentifier() {
|
||||
// TODO Auto-generated method stub
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPrefix() {
|
||||
// TODO Auto-generated method stub
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<Literal> getLiterals() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPredicate() {
|
||||
// TODO Auto-generated method stub
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,65 @@
|
||||
/**
|
||||
* Geo
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 16.12.2011 at http://yacy.net
|
||||
*
|
||||
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
||||
* $LastChangedRevision: 7653 $
|
||||
* $LastChangedBy: orbiter $
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.cora.lod.vocabulary;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
import net.yacy.cora.lod.Literal;
|
||||
import net.yacy.cora.lod.Vocabulary;
|
||||
|
||||
public enum Geo implements Vocabulary {
|
||||
|
||||
Long,
|
||||
Lat;
|
||||
|
||||
public final static String IDENTIFIER = "http://www.w3.org/2003/01/geo/wgs84_pos#";
|
||||
public final static String PREFIX = "geo";
|
||||
|
||||
private final String predicate;
|
||||
|
||||
private Geo() {
|
||||
this.predicate = PREFIX + ":" + this.name().toLowerCase();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getIdentifier() {
|
||||
return IDENTIFIER;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPrefix() {
|
||||
return PREFIX;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<Literal> getLiterals() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPredicate() {
|
||||
return this.predicate;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,117 @@
|
||||
/**
|
||||
* HttpHeader
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 16.12.2011 at http://yacy.net
|
||||
*
|
||||
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
||||
* $LastChangedRevision: 7653 $
|
||||
* $LastChangedBy: orbiter $
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
|
||||
package net.yacy.cora.lod.vocabulary;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
import net.yacy.cora.lod.Literal;
|
||||
import net.yacy.cora.lod.Vocabulary;
|
||||
|
||||
public enum HttpHeader implements Vocabulary {
|
||||
|
||||
//The following properties may appear in nodes of type Request:
|
||||
|
||||
accept, // representing an Accept header,
|
||||
acceptCharset, // representing an Accept-Charset header,
|
||||
acceptEncoding, // representing an Accept-Encoding header,
|
||||
acceptLanguage, // representing an Accept-Language header,
|
||||
authorization, // representing an Authorization header,
|
||||
expect, // representing an Expect header,
|
||||
from, // representing a From header,
|
||||
host, // representing a Host header,
|
||||
ifMatch, // representing an If-Match header,
|
||||
ifModifiedSince, // representing an If-Modified-Since header,
|
||||
ifNoneMatch, // representing an If-None-Match header,
|
||||
ifRange, // representing an If-Range header,
|
||||
ifUnmodifiedSince, // representing an If-Unmodified-Since header,
|
||||
maxForwards, // representing a Max-Forwards header,
|
||||
proxyAuthorization, // representing a Proxy-Authorization header,
|
||||
range, // representing a Range header,
|
||||
referer, // representing a Referer header,
|
||||
te, // representing a TE header,
|
||||
userAgent, // representing a User-Agent header.
|
||||
|
||||
//The following properties may appear in nodes of type Response:
|
||||
acceptRanges, // representing a Accept-Ranges header,
|
||||
age, // representing an Age header,
|
||||
etag, // representing an ETag header,
|
||||
location, // representing a Location header,
|
||||
proxyAuthenticate, // representing a Proxy-Authenticate header,
|
||||
retryAfter, // representing a Retry-After header,
|
||||
server, // representing a Server header,
|
||||
vary, // representing a Vary header,
|
||||
wwwAuthenticate, // representing a WWW-Authenticate header.
|
||||
|
||||
//The following properties may appear in nodes of type Request or Response:
|
||||
allow, // representing an Allow header,
|
||||
cacheControl, // representing a Cache-Control header,
|
||||
connection, // representing a Connection header,
|
||||
contentEncoding, // representing a Content-Encoding header,
|
||||
contentLanguage, // representing a Content-Language header,
|
||||
contentLength, // representing a Content-Length header,
|
||||
contentLocation, // representing a Content-Location header,
|
||||
contentMD5, // representing a Content-MD5 header,
|
||||
contentRange, // representing a Content-Range header,
|
||||
contentType, // representing a Content-Type header,
|
||||
date, // representing a Date header,
|
||||
expires, // representing an Expires header,
|
||||
lastModified, // representing a Last-Modified header,
|
||||
pragma, // representing a Pragma header,
|
||||
trailer, // representing a Trailer header,
|
||||
transferEncoding, // representing a Transfer-Encoding header,
|
||||
upgrade, // representing an Upgrade header,
|
||||
via, // representing a Via header,
|
||||
warning; // representing a Warning header.
|
||||
|
||||
public final static String IDENTIFIER = "http://www.w3.org/WAI/ER/HTTP/WD-HTTP-in-RDF-20060131";
|
||||
public final static String PREFIX = "http";
|
||||
|
||||
private final String predicate;
|
||||
|
||||
private HttpHeader() {
|
||||
this.predicate = PREFIX + ":" + this.name();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getIdentifier() {
|
||||
return IDENTIFIER;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPrefix() {
|
||||
return PREFIX;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<Literal> getLiterals() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPredicate() {
|
||||
return this.predicate;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,69 @@
|
||||
/**
|
||||
* Rdf
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 17.12.2011 at http://yacy.net
|
||||
*
|
||||
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
||||
* $LastChangedRevision: 7653 $
|
||||
* $LastChangedBy: orbiter $
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.cora.lod.vocabulary;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
import net.yacy.cora.lod.Literal;
|
||||
import net.yacy.cora.lod.Vocabulary;
|
||||
|
||||
public enum Rdf implements Vocabulary {
|
||||
|
||||
RDF,
|
||||
Description,
|
||||
Bag,
|
||||
Seq,
|
||||
Alt;
|
||||
|
||||
public final static String IDENTIFIER = "http://www.w3.org/1999/02/22-rdf-syntax-ns#";
|
||||
public final static String PREFIX = "rdf";
|
||||
|
||||
private final String predicate;
|
||||
|
||||
private Rdf() {
|
||||
this.predicate = PREFIX + ":" + this.name();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getIdentifier() {
|
||||
return IDENTIFIER;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPrefix() {
|
||||
return PREFIX;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<Literal> getLiterals() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPredicate() {
|
||||
return this.predicate;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,105 @@
|
||||
/**
|
||||
* YaCyMetadata
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 16.12.2011 at http://yacy.net
|
||||
*
|
||||
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
||||
* $LastChangedRevision: 7653 $
|
||||
* $LastChangedBy: orbiter $
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
|
||||
package net.yacy.cora.lod.vocabulary;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
import net.yacy.cora.lod.Literal;
|
||||
import net.yacy.cora.lod.Vocabulary;
|
||||
|
||||
/**
|
||||
* this is the vocabulary of the 'classic' YaCy metadata database
|
||||
*/
|
||||
public enum YaCyMetadata implements Vocabulary {
|
||||
|
||||
hash, // the url's hash
|
||||
mod, // last-modified from the httpd
|
||||
load, // time when the url was loaded
|
||||
fresh, // time until this url is fresh
|
||||
referrer, // (one of) the url's referrer hash(es)
|
||||
md5, // the md5 of the url content (to identify changes)
|
||||
size, // size of file in bytes
|
||||
wc, // size of file by number of words; for video and audio: seconds
|
||||
dt, // doctype, taken from extension or any other heuristic
|
||||
flags, // flags; any stuff (see Word-Entity definition)
|
||||
lang, // language
|
||||
llocal, // # of outlinks to same domain; for video and image: width
|
||||
lother, // # of outlinks to outside domain; for video and image: height
|
||||
limage, // # of embedded image links
|
||||
laudio, // # of embedded audio links; for audio: track number; for video: number of audio tracks
|
||||
lvideo, // # of embedded video links
|
||||
lapp; // # of embedded links to applications
|
||||
|
||||
/*
|
||||
"String hash-12, " + // the url's hash
|
||||
"Cardinal mod-4 {b256}, " + // last-modified from the httpd
|
||||
"Cardinal load-4 {b256}, " + // time when the url was loaded
|
||||
"Cardinal fresh-4 {b256}, " + // time until this url is fresh
|
||||
"String referrer-12, " + // (one of) the url's referrer hash(es)
|
||||
"byte[] md5-8, " + // the md5 of the url content (to identify changes)
|
||||
"Cardinal size-6 {b256}, " + // size of file in bytes
|
||||
"Cardinal wc-3 {b256}, " + // size of file by number of words; for video and audio: seconds
|
||||
"byte[] dt-1, " + // doctype, taken from extension or any other heuristic
|
||||
"Bitfield flags-4, " + // flags; any stuff (see Word-Entity definition)
|
||||
"String lang-2, " + // language
|
||||
"Cardinal llocal-2 {b256}, " + // # of outlinks to same domain; for video and image: width
|
||||
"Cardinal lother-2 {b256}, " + // # of outlinks to outside domain; for video and image: height
|
||||
"Cardinal limage-2 {b256}, " + // # of embedded image links
|
||||
"Cardinal laudio-2 {b256}, " + // # of embedded audio links; for audio: track number; for video: number of audio tracks
|
||||
"Cardinal lvideo-2 {b256}, " + // # of embedded video links
|
||||
"Cardinal lapp-2 {b256}", // # of embedded links to applications
|
||||
*/
|
||||
|
||||
public final static String IDENTIFIER = "http://yacy.net/vocabularies/yacymetadata#";
|
||||
public final static String PREFIX = "yacy";
|
||||
|
||||
private final String predicate;
|
||||
|
||||
private YaCyMetadata() {
|
||||
this.predicate = PREFIX + ":" + this.name();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getIdentifier() {
|
||||
return IDENTIFIER;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPrefix() {
|
||||
return PREFIX;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<Literal> getLiterals() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPredicate() {
|
||||
return this.predicate;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,52 @@
|
||||
/**
|
||||
* ByteOrder
|
||||
* (C) 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
||||
* first published 10.01.2008 on http://yacy.net
|
||||
*
|
||||
* $LastChangedDate$
|
||||
* $LastChangedRevision$
|
||||
* $LastChangedBy$
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.cora.order;
|
||||
|
||||
|
||||
public interface ByteOrder extends Order<byte[]> {
|
||||
|
||||
@Override
|
||||
public boolean wellformed(byte[] a);
|
||||
|
||||
public boolean wellformed(byte[] a, int start, int len);
|
||||
|
||||
@Override
|
||||
public int compare(byte[] a, byte[] b);
|
||||
|
||||
public int compare(byte[] a, byte[] b, int len);
|
||||
|
||||
public int compare(byte[] a, int astart, byte[] b, int bstart, int len);
|
||||
|
||||
@Override
|
||||
public boolean equal(final byte[] a, final byte[] b);
|
||||
|
||||
public boolean equal(final byte[] a, int astart, final byte[] b, int bstart, int length);
|
||||
|
||||
public long cardinal(final byte[] a, int off, int len);
|
||||
|
||||
public byte[] smallest(byte[] a, byte[] b);
|
||||
|
||||
public byte[] largest(byte[] a, byte[] b);
|
||||
}
|
||||
@ -0,0 +1,35 @@
|
||||
/**
|
||||
* ByteOrder
|
||||
* (C) 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
||||
* first published 10.01.2008 on http://yacy.net
|
||||
*
|
||||
* $LastChangedDate$
|
||||
* $LastChangedRevision$
|
||||
* $LastChangedBy$
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.cora.order;
|
||||
|
||||
import java.util.Iterator;
|
||||
|
||||
public interface CloneableIterator<E> extends Iterator<E> {
|
||||
|
||||
// clone the iterator using a modifier
|
||||
// the modifier can be i.e. a re-start position
|
||||
public CloneableIterator<E> clone(Object modifier);
|
||||
|
||||
}
|
||||
@ -0,0 +1,84 @@
|
||||
/**
|
||||
* ByteOrder
|
||||
* (C) 2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
||||
* first published 25.04.2007 on http://yacy.net
|
||||
*
|
||||
* $LastChangedDate$
|
||||
* $LastChangedRevision$
|
||||
* $LastChangedBy$
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.cora.order;
|
||||
|
||||
import java.util.Iterator;
|
||||
import java.util.TreeMap;
|
||||
|
||||
|
||||
|
||||
public class CloneableMapIterator<E> implements CloneableIterator<E> {
|
||||
|
||||
TreeMap<E, ?> map;
|
||||
E next, last;
|
||||
Object start;
|
||||
Iterator<E> iter;
|
||||
|
||||
|
||||
public CloneableMapIterator(final TreeMap<E, ?> map, final E start) {
|
||||
// map must contain eiter a byte[]/Object or a String/Object mapping.
|
||||
// start must be either of type byte[] or String
|
||||
// this iterator iterates then only the key elements of the map
|
||||
this.map = map;
|
||||
this.start = start;
|
||||
this.iter = map.keySet().iterator();
|
||||
if (this.start == null) {
|
||||
if (this.iter.hasNext()) this.next = this.iter.next(); else this.next = null;
|
||||
} else while (this.iter.hasNext()) {
|
||||
this.next = this.iter.next();
|
||||
if (map.comparator().compare(this.next, start) > 1) break;
|
||||
}
|
||||
this.last = null;
|
||||
}
|
||||
|
||||
@Override
|
||||
@SuppressWarnings("unchecked")
|
||||
public CloneableMapIterator<E> clone(final Object modifier) {
|
||||
return new CloneableMapIterator<E>(this.map, (E) modifier);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return this.next != null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public E next() {
|
||||
// returns key-elements, not entry-elements
|
||||
this.last = this.next;
|
||||
if (this.iter.hasNext()) {
|
||||
this.next = this.iter.next();
|
||||
} else {
|
||||
this.next = null;
|
||||
}
|
||||
return this.last;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void remove() {
|
||||
this.map.remove(this.last);
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,69 @@
|
||||
/**
|
||||
* RatingOrder.java
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 25.08.2011 at http://yacy.net
|
||||
*
|
||||
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
||||
* $LastChangedRevision: 7653 $
|
||||
* $LastChangedBy: orbiter $
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
|
||||
package net.yacy.cora.order;
|
||||
|
||||
import net.yacy.cora.sorting.Rating;
|
||||
|
||||
|
||||
|
||||
public class RatingOrder<A> extends AbstractOrder<Rating<A>> implements Order<Rating<A>> {
|
||||
|
||||
Order<A> ordering;
|
||||
|
||||
public RatingOrder(final Order<A> ordering) {
|
||||
this.ordering = ordering;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compare(final Rating<A> a, final Rating<A> b) {
|
||||
return this.ordering.compare(a.getObject(), b.getObject());
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean wellformed(final Rating<A> a) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String signature() {
|
||||
return "RA";
|
||||
}
|
||||
|
||||
@Override
|
||||
public long cardinal(final Rating<A> key) {
|
||||
return key.getScore();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equal(final Rating<A> a, final Rating<A> b) {
|
||||
return this.ordering.compare(a.getObject(), b.getObject()) == 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Order<Rating<A>> clone() {
|
||||
return this;
|
||||
}
|
||||
}
|
||||
@ -1,66 +0,0 @@
|
||||
// RatingOrder.java
|
||||
// -----------------------
|
||||
// (C) by Michael Peter Christen; mc@yacy.net
|
||||
// first published on http://yacy.net
|
||||
// Frankfurt, Germany, 2011
|
||||
// created 25.08.2011
|
||||
//
|
||||
// $LastChangedDate: 2011-03-08 02:51:51 +0100 (Di, 08 Mrz 2011) $
|
||||
// $LastChangedRevision: 7567 $
|
||||
// $LastChangedBy: low012 $
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
|
||||
package net.yacy.cora.ranking;
|
||||
|
||||
|
||||
public class RatingOrder<A> extends AbstractOrder<Rating<A>> implements Order<Rating<A>> {
|
||||
|
||||
Order<A> ordering;
|
||||
|
||||
public RatingOrder(final Order<A> ordering) {
|
||||
this.ordering = ordering;
|
||||
}
|
||||
|
||||
public int compare(final Rating<A> a, final Rating<A> b) {
|
||||
return this.ordering.compare(a.getObject(), b.getObject());
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean wellformed(final Rating<A> a) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String signature() {
|
||||
return "RA";
|
||||
}
|
||||
|
||||
@Override
|
||||
public long cardinal(final Rating<A> key) {
|
||||
return key.getScore();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equal(final Rating<A> a, final Rating<A> b) {
|
||||
return this.ordering.compare(a.getObject(), b.getObject()) == 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Order<Rating<A>> clone() {
|
||||
return this;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,110 @@
|
||||
/**
|
||||
* AbstractMapStore
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 16.12.2011 at http://yacy.net
|
||||
*
|
||||
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
||||
* $LastChangedRevision: 7653 $
|
||||
* $LastChangedBy: orbiter $
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.cora.storage;
|
||||
|
||||
import java.util.AbstractMap;
|
||||
import java.util.Collection;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import net.yacy.cora.document.UTF8;
|
||||
|
||||
public abstract class AbstractMapStore implements MapStore {
|
||||
|
||||
@Override
|
||||
public boolean containsValue(Object arg0) {
|
||||
throw new UnsupportedOperationException("ContainsValue() not appropriate, use outer indexing");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<java.util.Map.Entry<byte[], Map<String, byte[]>>> entrySet() {
|
||||
throw new UnsupportedOperationException("entrySet() not appropriate, use an iterator");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<byte[]> keySet() {
|
||||
throw new UnsupportedOperationException("keySet() not appropriate, use an iterator");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void putAll(Map<? extends byte[], ? extends Map<String, byte[]>> entries) {
|
||||
if (entries instanceof MapStore) {
|
||||
Iterator<Map.Entry<byte[], Map<String, byte[]>>> i = ((MapStore) entries).iterator();
|
||||
Map.Entry<? extends byte[], ? extends Map<String, byte[]>> entry;
|
||||
while (i.hasNext()) {
|
||||
entry = i.next();
|
||||
this.put(entry.getKey(), entry.getValue());
|
||||
}
|
||||
} else {
|
||||
for (Map.Entry<? extends byte[], ? extends Map<String, byte[]>> e: entries.entrySet()) {
|
||||
this.put(e.getKey(), e.getValue());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Collection<Map<String, byte[]>> values() {
|
||||
throw new UnsupportedOperationException("values() not appropriate, use an iterator");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iterator<Map.Entry<byte[], Map<String, byte[]>>> iterator() {
|
||||
final Iterator<byte[]> k = this.keyIterator();
|
||||
return new Iterator<Map.Entry<byte[], Map<String, byte[]>>>(){
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return k.hasNext();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map.Entry<byte[], Map<String, byte[]>> next() {
|
||||
byte[] key = k.next();
|
||||
if (key == null) return null;
|
||||
return new AbstractMap.SimpleImmutableEntry<byte[], Map<String, byte[]>>(key, AbstractMapStore.this.get(key));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void remove() {
|
||||
k.remove();
|
||||
}
|
||||
|
||||
};
|
||||
}
|
||||
|
||||
public static String map2String(Map<String, byte[]> map) {
|
||||
StringBuilder sb = new StringBuilder(map.size() * 50);
|
||||
sb.append("<map>\n");
|
||||
for (Map.Entry<String, byte[]> entry: map.entrySet()) {
|
||||
sb.append('<').append(entry.getKey()).append('>');
|
||||
sb.append(UTF8.String(entry.getValue()));
|
||||
sb.append("</").append(entry.getKey()).append(">\n");
|
||||
}
|
||||
sb.append("</map>\n");
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,56 @@
|
||||
/**
|
||||
* MapStore
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 16.12.2011 at http://yacy.net
|
||||
*
|
||||
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
||||
* $LastChangedRevision: 7653 $
|
||||
* $LastChangedBy: orbiter $
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
|
||||
package net.yacy.cora.storage;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import net.yacy.cora.order.ByteOrder;
|
||||
import net.yacy.cora.order.CloneableIterator;
|
||||
|
||||
/**
|
||||
* this is a placeholder interface
|
||||
* for the complex expressionMap<byte[], Map<String, byte[]>>
|
||||
*
|
||||
*/
|
||||
public interface MapStore extends Map<byte[], Map<String, byte[]>>, Iterable<Map.Entry<byte[], Map<String, byte[]>>> {
|
||||
|
||||
/**
|
||||
* the map should have an ordering on the key elements
|
||||
* @return a byte order on the key elements
|
||||
*/
|
||||
public ByteOrder getOrdering();
|
||||
|
||||
/**
|
||||
* the keys of the map should be iterable
|
||||
* @return an iterator on the map keys
|
||||
*/
|
||||
public CloneableIterator<byte[]> keyIterator();
|
||||
|
||||
/**
|
||||
* most of the MapStore implementations are file-based, so we should consider a close method
|
||||
*/
|
||||
public void close();
|
||||
}
|
||||
@ -0,0 +1,339 @@
|
||||
/**
|
||||
* BEncodedHeapBag
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 16.12.2011 at http://yacy.net
|
||||
*
|
||||
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
||||
* $LastChangedRevision: 7653 $
|
||||
* $LastChangedBy: orbiter $
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.kelondro.blob;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.text.ParseException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import net.yacy.cora.date.GenericFormatter;
|
||||
import net.yacy.cora.document.UTF8;
|
||||
import net.yacy.cora.order.ByteOrder;
|
||||
import net.yacy.cora.order.CloneableIterator;
|
||||
import net.yacy.cora.storage.AbstractMapStore;
|
||||
import net.yacy.cora.storage.MapStore;
|
||||
import net.yacy.kelondro.data.word.Word;
|
||||
import net.yacy.kelondro.logging.Log;
|
||||
import net.yacy.kelondro.order.Base64Order;
|
||||
import net.yacy.kelondro.order.MergeIterator;
|
||||
import net.yacy.kelondro.util.FileUtils;
|
||||
|
||||
public class BEncodedHeapBag extends AbstractMapStore implements MapStore {
|
||||
|
||||
private Map<String, BEncodedHeap> bag; // a map from a date string to a kelondroIndex object
|
||||
private final File baseDir;
|
||||
private final String prefix;
|
||||
private final int keylength, buffermax;
|
||||
private final ByteOrder entryOrder;
|
||||
private String current;
|
||||
private final long fileAgeLimit;
|
||||
private final long fileSizeLimit;
|
||||
|
||||
public BEncodedHeapBag(
|
||||
final File path,
|
||||
final String prefix,
|
||||
final int keylength,
|
||||
final ByteOrder ordering,
|
||||
final int buffermax,
|
||||
final long fileAgeLimit,
|
||||
final long fileSizeLimit) {
|
||||
this.baseDir = path;
|
||||
this.prefix = prefix;
|
||||
this.keylength = keylength;
|
||||
this.buffermax = buffermax;
|
||||
this.entryOrder = ordering;
|
||||
this.fileAgeLimit = fileAgeLimit;
|
||||
this.fileSizeLimit = fileSizeLimit;
|
||||
init();
|
||||
}
|
||||
|
||||
private void init() {
|
||||
this.current = null;
|
||||
|
||||
// initialized tables map
|
||||
this.bag = new HashMap<String, BEncodedHeap>();
|
||||
if (!(this.baseDir.exists())) this.baseDir.mkdirs();
|
||||
String[] tablefile = this.baseDir.list();
|
||||
|
||||
// first pass: find tables
|
||||
final HashMap<String, Long> t = new HashMap<String, Long>();
|
||||
long ram, time, maxtime = 0;
|
||||
Date d;
|
||||
File f;
|
||||
for (final String element : tablefile) {
|
||||
if ((element.startsWith(this.prefix)) &&
|
||||
(element.length() > this.prefix.length()) &&
|
||||
(element.charAt(this.prefix.length()) == '.') &&
|
||||
(element.length() == this.prefix.length() + 23)) {
|
||||
f = new File(this.baseDir, element);
|
||||
try {
|
||||
d = GenericFormatter.SHORT_MILSEC_FORMATTER.parse(element.substring(this.prefix.length() + 1, this.prefix.length() + 18));
|
||||
} catch (final ParseException e) {
|
||||
Log.logSevere("BEncodedHeapBag", "", e);
|
||||
continue;
|
||||
}
|
||||
time = d.getTime();
|
||||
if (time > maxtime) {
|
||||
this.current = element;
|
||||
assert this.current != null;
|
||||
maxtime = time;
|
||||
}
|
||||
|
||||
t.put(element, f.length());
|
||||
}
|
||||
}
|
||||
|
||||
// second pass: open tables
|
||||
Iterator<Map.Entry<String, Long>> i;
|
||||
Map.Entry<String, Long> entry;
|
||||
String maxf;
|
||||
long maxram;
|
||||
while (!t.isEmpty()) {
|
||||
// find maximum table
|
||||
maxram = 0;
|
||||
maxf = null;
|
||||
i = t.entrySet().iterator();
|
||||
while (i.hasNext()) {
|
||||
entry = i.next();
|
||||
ram = entry.getValue().longValue();
|
||||
if (maxf == null || ram > maxram) {
|
||||
maxf = entry.getKey();
|
||||
maxram = ram;
|
||||
}
|
||||
}
|
||||
|
||||
// open next biggest table
|
||||
t.remove(maxf);
|
||||
f = new File(this.baseDir, maxf);
|
||||
try {
|
||||
Log.logInfo("BEncodedHeapBag", "opening partial heap " + f);
|
||||
BEncodedHeap heap = new BEncodedHeap(f, this.keylength, this.entryOrder, this.buffermax);
|
||||
this.bag.put(maxf, heap);
|
||||
} catch (IOException e) {
|
||||
Log.logSevere("BEncodedHeapBag", "error opening partial heap " + f);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void close() {
|
||||
if (this.bag == null) return;
|
||||
|
||||
final Iterator<BEncodedHeap> i = this.bag.values().iterator();
|
||||
while (i.hasNext()) {
|
||||
i.next().close();
|
||||
}
|
||||
this.bag = null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void clear() {
|
||||
close();
|
||||
final String[] l = this.baseDir.list();
|
||||
for (final String element : l) {
|
||||
if (element.startsWith(this.prefix)) {
|
||||
final File f = new File(this.baseDir, element);
|
||||
if (!f.isDirectory()) FileUtils.deletedelete(f);
|
||||
}
|
||||
}
|
||||
init();
|
||||
}
|
||||
|
||||
private MapStore keeperOf(final byte[] key) {
|
||||
if (key == null) return null;
|
||||
if (this.bag == null) return null;
|
||||
for (final MapStore oi: this.bag.values()) {
|
||||
if (oi.containsKey(key)) return oi;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String newFilename() {
|
||||
return this.prefix + "." + GenericFormatter.SHORT_MILSEC_FORMATTER.format() + ".heap";
|
||||
}
|
||||
|
||||
private MapStore newHeap() {
|
||||
this.current = newFilename();
|
||||
final File f = new File(this.baseDir, this.current);
|
||||
BEncodedHeap heap;
|
||||
try {
|
||||
heap = new BEncodedHeap(f, this.keylength, this.entryOrder, this.buffermax);
|
||||
} catch (IOException e) {
|
||||
Log.logSevere("BEncodedHeapBag", "unable to open new heap file: " + e.getMessage(), e);
|
||||
return null;
|
||||
}
|
||||
this.bag.put(this.current, heap);
|
||||
return heap;
|
||||
}
|
||||
|
||||
private MapStore checkHeap(final BEncodedHeap heap) {
|
||||
// check size and age of given table; in case it is too large or too old
|
||||
// create a new table
|
||||
assert heap != null;
|
||||
long t = System.currentTimeMillis();
|
||||
if (((t / 1000) % 10) != 0) return heap; // we check only every 10 seconds because all these file and parser operations are very expensive
|
||||
final String name = heap.getFile().getName();
|
||||
long d;
|
||||
try {
|
||||
d = GenericFormatter.SHORT_MILSEC_FORMATTER.parse(name.substring(this.prefix.length() + 1, this.prefix.length() + 18)).getTime();
|
||||
} catch (final ParseException e) {
|
||||
Log.logSevere("BEncodedHeapBag", "", e);
|
||||
d = 0;
|
||||
}
|
||||
if (d + this.fileAgeLimit < t || new File(this.baseDir, name).length() >= this.fileSizeLimit) {
|
||||
return newHeap();
|
||||
}
|
||||
return heap;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean containsKey(Object key) {
|
||||
if (key == null || !(key instanceof byte[])) return false;
|
||||
synchronized (this.bag) {
|
||||
return keeperOf((byte[]) key) != null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, byte[]> put(byte[] key, Map<String, byte[]> map) {
|
||||
if (this.bag == null) return null;
|
||||
MapStore keeper = null;
|
||||
synchronized (this.bag) {
|
||||
keeper = keeperOf(key);
|
||||
}
|
||||
if (keeper != null) {
|
||||
return keeper.put(key, map);
|
||||
}
|
||||
synchronized (this.bag) {
|
||||
keeper = keeperOf(key); // we must check that again because it could have changed in between
|
||||
if (keeper != null) return keeper.put(key, map);
|
||||
if (this.current == null) {
|
||||
keeper = newHeap();
|
||||
return keeper.put(key, map);
|
||||
}
|
||||
keeper = checkHeap(this.bag.get(this.current));
|
||||
}
|
||||
return keeper.put(key, map);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, byte[]> get(Object key) {
|
||||
if (key == null || !(key instanceof byte[])) return null;
|
||||
synchronized (this.bag) {
|
||||
return keeperOf((byte[]) key).get(key);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isEmpty() {
|
||||
final Iterator<BEncodedHeap> i = this.bag.values().iterator();
|
||||
while (i.hasNext()) if (!i.next().isEmpty()) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int size() {
|
||||
final Iterator<BEncodedHeap> i = this.bag.values().iterator();
|
||||
int s = 0;
|
||||
while (i.hasNext()) s += i.next().size();
|
||||
return s;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, byte[]> remove(Object key) {
|
||||
if (key == null || !(key instanceof byte[])) return null;
|
||||
final MapStore heap;
|
||||
synchronized (this.bag) {
|
||||
heap = keeperOf((byte[]) key);
|
||||
}
|
||||
if (heap == null) return null;
|
||||
return heap.remove(key);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ByteOrder getOrdering() {
|
||||
return this.entryOrder;
|
||||
}
|
||||
|
||||
@Override
|
||||
public CloneableIterator<byte[]> keyIterator() {
|
||||
final List<CloneableIterator<byte[]>> c = new ArrayList<CloneableIterator<byte[]>>(this.bag.size());
|
||||
final Iterator<BEncodedHeap> i = this.bag.values().iterator();
|
||||
CloneableIterator<byte[]> k;
|
||||
while (i.hasNext()) {
|
||||
k = i.next().keyIterator();
|
||||
if (k != null && k.hasNext()) c.add(k);
|
||||
}
|
||||
return MergeIterator.cascade(c, this.entryOrder, MergeIterator.simpleMerge, true);
|
||||
}
|
||||
|
||||
protected static Map<String, byte[]> testMap(int i) {
|
||||
HashMap<String, byte[]> t = new HashMap<String, byte[]>();
|
||||
t.put("rdf:about", UTF8.getBytes("http://abc.de/testmap#" + i));
|
||||
t.put("dc:title", UTF8.getBytes("test nr " + i));
|
||||
return t;
|
||||
}
|
||||
|
||||
private static BEncodedHeapBag testHeapBag(File f) {
|
||||
return new BEncodedHeapBag(
|
||||
f,
|
||||
"testbag",
|
||||
12,
|
||||
Base64Order.enhancedCoder,
|
||||
10,
|
||||
ArrayStack.oneMonth, 100 /*Integer.MAX_VALUE*/);
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
File f = new File("/tmp");
|
||||
BEncodedHeapBag hb = testHeapBag(f);
|
||||
for (int i = 0; i < 10000; i++) {
|
||||
hb.put(Word.word2hash(Integer.toString(i)), testMap(i));
|
||||
}
|
||||
System.out.println("test size after put = " + hb.size());
|
||||
hb.close();
|
||||
hb = testHeapBag(f);
|
||||
Iterator<Map.Entry<byte[], Map<String, byte[]>>> mi = hb.iterator();
|
||||
int c = 1000;
|
||||
Map.Entry<byte[], Map<String, byte[]>> entry;
|
||||
while (mi.hasNext() && c-- > 0) {
|
||||
entry = mi.next();
|
||||
System.out.println(UTF8.String(entry.getKey()) + ": " + AbstractMapStore.map2String(entry.getValue()));
|
||||
}
|
||||
for (int i = 10000; i > 0; i--) {
|
||||
hb.remove(Word.word2hash(Integer.toString(i - 1)));
|
||||
}
|
||||
System.out.println("test size after remove = " + hb.size());
|
||||
hb.close();
|
||||
Log.shutdown();
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,299 @@
|
||||
/**
|
||||
* BEncodedHeapShard
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 16.12.2011 at http://yacy.net
|
||||
*
|
||||
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
||||
* $LastChangedRevision: 7653 $
|
||||
* $LastChangedBy: orbiter $
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.kelondro.blob;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
import net.yacy.cora.document.ASCII;
|
||||
import net.yacy.cora.document.UTF8;
|
||||
import net.yacy.cora.order.ByteOrder;
|
||||
import net.yacy.cora.order.CloneableIterator;
|
||||
import net.yacy.cora.storage.AbstractMapStore;
|
||||
import net.yacy.cora.storage.MapStore;
|
||||
import net.yacy.kelondro.data.word.Word;
|
||||
import net.yacy.kelondro.logging.Log;
|
||||
import net.yacy.kelondro.order.Base64Order;
|
||||
import net.yacy.kelondro.order.MergeIterator;
|
||||
import net.yacy.kelondro.util.FileUtils;
|
||||
|
||||
public class BEncodedHeapShard extends AbstractMapStore implements MapStore {
|
||||
|
||||
public interface Method {
|
||||
/**
|
||||
* a sharding method produces a filename from a given key
|
||||
* @param key
|
||||
* @return
|
||||
*/
|
||||
public String filename(byte[] key);
|
||||
|
||||
/**
|
||||
* get the maximum key length for access keys
|
||||
* @return
|
||||
*/
|
||||
public int getKeylength();
|
||||
|
||||
/**
|
||||
* get the byte order on the keys
|
||||
* @return
|
||||
*/
|
||||
public ByteOrder getOrdering();
|
||||
|
||||
/**
|
||||
* check if the given file name is a part of the shard
|
||||
* @param filename
|
||||
* @return true if the file is part of the shar
|
||||
*/
|
||||
public boolean isShardPart(String filename);
|
||||
|
||||
public String getShardName(String filename);
|
||||
}
|
||||
|
||||
public static class B64ShardMethod implements Method {
|
||||
|
||||
private final int keylength;
|
||||
private final ByteOrder ordering;
|
||||
private final byte[] template;
|
||||
private final int charpos;
|
||||
private final String prefix;
|
||||
|
||||
public B64ShardMethod(
|
||||
final int keylength,
|
||||
final ByteOrder ordering,
|
||||
final String prefix) {
|
||||
this.keylength = keylength;
|
||||
this.ordering = ordering;
|
||||
this.template = ASCII.getBytes(prefix + ".?");
|
||||
this.charpos = ASCII.getBytes(prefix).length + 1;
|
||||
this.prefix = prefix;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String filename(byte[] key) {
|
||||
byte[] s = new byte[this.template.length];
|
||||
System.arraycopy(this.template, 0, s, 0, s.length);
|
||||
s[this.charpos] = key[0];
|
||||
return ASCII.String(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getKeylength() {
|
||||
return this.keylength;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ByteOrder getOrdering() {
|
||||
return this.ordering;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isShardPart(String filename) {
|
||||
// TODO Auto-generated method stub
|
||||
return filename.startsWith(this.prefix) &&
|
||||
filename.charAt(this.prefix.length()) == '.' &&
|
||||
filename.endsWith(".heap");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getShardName(String filename) {
|
||||
return filename.substring(0, this.template.length);
|
||||
}
|
||||
}
|
||||
|
||||
private ConcurrentHashMap<String, MapStore> shard;
|
||||
private final File baseDir;
|
||||
private final Method shardMethod;
|
||||
|
||||
public BEncodedHeapShard(File baseDir, Method shardMethod) {
|
||||
this.shard = new ConcurrentHashMap<String, MapStore>();
|
||||
this.baseDir = baseDir;
|
||||
this.shardMethod = shardMethod;
|
||||
init();
|
||||
}
|
||||
|
||||
private void init() {
|
||||
// initialized tables map
|
||||
this.shard = new ConcurrentHashMap<String, MapStore>();
|
||||
if (!(this.baseDir.exists())) this.baseDir.mkdirs();
|
||||
String[] tablefile = this.baseDir.list();
|
||||
|
||||
// open all tables of this shard
|
||||
for (final String element : tablefile) {
|
||||
if (this.shardMethod.isShardPart(element)) {
|
||||
Log.logInfo("BEncodedHeapShard", "opening partial shard " + element);
|
||||
MapStore bag = openBag(element);
|
||||
this.shard.put(this.shardMethod.getShardName(element), bag);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
if (this.shard == null) return;
|
||||
|
||||
final Iterator<MapStore> i = this.shard.values().iterator();
|
||||
while (i.hasNext()) {
|
||||
i.next().close();
|
||||
}
|
||||
this.shard = null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void clear() {
|
||||
close();
|
||||
final String[] l = this.baseDir.list();
|
||||
for (final String element : l) {
|
||||
if (this.shardMethod.isShardPart(element)) {
|
||||
final File f = new File(this.baseDir, element);
|
||||
if (!f.isDirectory()) FileUtils.deletedelete(f);
|
||||
}
|
||||
}
|
||||
init();
|
||||
}
|
||||
|
||||
private MapStore keeperOf(final byte[] key) {
|
||||
String shardfile = this.shardMethod.filename(key);
|
||||
MapStore bag = this.shard.get(shardfile);
|
||||
if (bag != null) return bag;
|
||||
bag = openBag(shardfile);
|
||||
this.shard.put(shardfile, bag);
|
||||
return bag;
|
||||
}
|
||||
|
||||
public MapStore openBag(String shardfile) {
|
||||
MapStore bag = new BEncodedHeapBag(
|
||||
this.baseDir,
|
||||
shardfile,
|
||||
this.shardMethod.getKeylength(),
|
||||
this.shardMethod.getOrdering(),
|
||||
10,
|
||||
ArrayStack.oneMonth * 12,
|
||||
Integer.MAX_VALUE);
|
||||
return bag;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean containsKey(Object key) {
|
||||
if (key == null || !(key instanceof byte[])) return false;
|
||||
String shardfile = this.shardMethod.filename((byte[]) key);
|
||||
MapStore bag = this.shard.get(shardfile);
|
||||
if (bag == null) return false;
|
||||
return bag.containsKey(key);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, byte[]> put(byte[] key, Map<String, byte[]> map) {
|
||||
if (this.shard == null) return null;
|
||||
MapStore keeper = null;
|
||||
synchronized (this.shard) {
|
||||
keeper = keeperOf(key);
|
||||
}
|
||||
return keeper.put(key, map);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, byte[]> get(Object key) {
|
||||
if (key == null || !(key instanceof byte[])) return null;
|
||||
String shardfile = this.shardMethod.filename((byte[]) key);
|
||||
MapStore bag = this.shard.get(shardfile);
|
||||
if (bag == null) return null;
|
||||
return bag.get(key);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isEmpty() {
|
||||
final Iterator<MapStore> i = this.shard.values().iterator();
|
||||
while (i.hasNext()) if (!i.next().isEmpty()) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int size() {
|
||||
final Iterator<MapStore> i = this.shard.values().iterator();
|
||||
int s = 0;
|
||||
while (i.hasNext()) s += i.next().size();
|
||||
return s;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, byte[]> remove(Object key) {
|
||||
if (key == null || !(key instanceof byte[])) return null;
|
||||
final MapStore bag;
|
||||
synchronized (this.shard) {
|
||||
bag = keeperOf((byte[]) key);
|
||||
}
|
||||
if (bag == null) return null;
|
||||
return bag.remove(key);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ByteOrder getOrdering() {
|
||||
return this.shardMethod.getOrdering();
|
||||
}
|
||||
|
||||
@Override
|
||||
public CloneableIterator<byte[]> keyIterator() {
|
||||
final List<CloneableIterator<byte[]>> c = new ArrayList<CloneableIterator<byte[]>>(this.shard.size());
|
||||
final Iterator<MapStore> i = this.shard.values().iterator();
|
||||
CloneableIterator<byte[]> k;
|
||||
while (i.hasNext()) {
|
||||
k = i.next().keyIterator();
|
||||
if (k != null && k.hasNext()) c.add(k);
|
||||
}
|
||||
return MergeIterator.cascade(c, this.shardMethod.getOrdering(), MergeIterator.simpleMerge, true);
|
||||
}
|
||||
|
||||
private static BEncodedHeapShard testHeapShard(File f) {
|
||||
return new BEncodedHeapShard(f, new B64ShardMethod(12, Base64Order.enhancedCoder, "testshard"));
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
File f = new File("/tmp");
|
||||
BEncodedHeapShard hb = testHeapShard(f);
|
||||
for (int i = 0; i < 10000; i++) {
|
||||
hb.put(Word.word2hash(Integer.toString(i)), BEncodedHeapBag.testMap(i));
|
||||
}
|
||||
System.out.println("test size after put = " + hb.size());
|
||||
hb.close();
|
||||
hb = testHeapShard(f);
|
||||
Iterator<Map.Entry<byte[], Map<String, byte[]>>> mi = hb.iterator();
|
||||
int c = 100;
|
||||
Map.Entry<byte[], Map<String, byte[]>> entry;
|
||||
while (mi.hasNext() && c-- > 0) {
|
||||
entry = mi.next();
|
||||
System.out.println(UTF8.String(entry.getKey()) + ": " + AbstractMapStore.map2String(entry.getValue()));
|
||||
}
|
||||
for (int i = 10000; i > 0; i--) {
|
||||
hb.remove(Word.word2hash(Integer.toString(i - 1)));
|
||||
}
|
||||
System.out.println("test size after remove = " + hb.size());
|
||||
hb.close();
|
||||
Log.shutdown();
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue