boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- accept only location names wit a minimum length

Browse Source 
- remove comma from synonym terms
pull/1/head
Michael Peter Christen 13 years ago
parent cc9ad7198a
commit f1aa4c4390
6 changed files with 26 additions and 18 deletions
Show Stats Download Patch File Download Diff File

8
htroot/DictionaryLoader_p.java
Unescape View File

@ -68,7 +68,7 @@ public class DictionaryLoader_p {
final Response response = sb.loader.load(sb.loader.request(new DigestURI(LibraryProvider.Dictionary.GEON0.url), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, false);
final byte[] b = response.getContent();
FileUtils.copy(b, LibraryProvider.Dictionary.GEON0.file());
LibraryProvider.geoLoc.activateLocalization(LibraryProvider.Dictionary.GEON0.nickname, new GeonamesLocation(LibraryProvider.Dictionary.GEON0.file(), null));
LibraryProvider.geoLoc.activateLocation(LibraryProvider.Dictionary.GEON0.nickname, new GeonamesLocation(LibraryProvider.Dictionary.GEON0.file(), null));
LibraryProvider.autotagging.addPlaces(LibraryProvider.geoLoc);
prop.put("geon0Status", LibraryProvider.Dictionary.GEON0.file().exists() ? 1 : 0);
prop.put("geon0ActionLoaded", 1);
@ -98,7 +98,7 @@ public class DictionaryLoader_p {
if (post.containsKey("geon0Activate")) {
LibraryProvider.Dictionary.GEON0.fileDisabled().renameTo(LibraryProvider.Dictionary.GEON0.file());
LibraryProvider.geoLoc.activateLocalization(LibraryProvider.Dictionary.GEON0.nickname, new GeonamesLocation(LibraryProvider.Dictionary.GEON0.file(), null));
LibraryProvider.geoLoc.activateLocation(LibraryProvider.Dictionary.GEON0.nickname, new GeonamesLocation(LibraryProvider.Dictionary.GEON0.file(), null));
LibraryProvider.autotagging.addPlaces(LibraryProvider.geoLoc);
prop.put("geon0ActionActivated", 1);
}
@ -111,7 +111,7 @@ public class DictionaryLoader_p {
final byte[] b = response.getContent();
FileUtils.copy(b, LibraryProvider.Dictionary.GEODB1.file());
LibraryProvider.geoLoc.deactivateLocalization(LibraryProvider.Dictionary.GEODB0.nickname);
LibraryProvider.geoLoc.activateLocalization(LibraryProvider.Dictionary.GEODB1.nickname, new OpenGeoDBLocation(LibraryProvider.Dictionary.GEODB1.file(), null));
LibraryProvider.geoLoc.activateLocation(LibraryProvider.Dictionary.GEODB1.nickname, new OpenGeoDBLocation(LibraryProvider.Dictionary.GEODB1.file(), null));
LibraryProvider.autotagging.addPlaces(LibraryProvider.geoLoc);
prop.put("geo1Status", LibraryProvider.Dictionary.GEODB1.file().exists() ? 1 : 0);
prop.put("geo1ActionLoaded", 1);
@ -141,7 +141,7 @@ public class DictionaryLoader_p {
if (post.containsKey("geo1Activate")) {
LibraryProvider.Dictionary.GEODB1.fileDisabled().renameTo(LibraryProvider.Dictionary.GEODB1.file());
LibraryProvider.geoLoc.activateLocalization(LibraryProvider.Dictionary.GEODB1.nickname, new OpenGeoDBLocation(LibraryProvider.Dictionary.GEODB1.file(), null));
LibraryProvider.geoLoc.activateLocation(LibraryProvider.Dictionary.GEODB1.nickname, new OpenGeoDBLocation(LibraryProvider.Dictionary.GEODB1.file(), null));
LibraryProvider.autotagging.addPlaces(LibraryProvider.geoLoc);
prop.put("geo1ActionActivated", 1);
}

24
source/net/yacy/cora/lod/vocabulary/Tagging.java
Unescape View File

@ -439,8 +439,9 @@ public class Tagging {
this(name);
Set<String> locNames = localization.locationNames();
for (String loc: locNames) {
this.synonym2term.put(loc.toLowerCase(), loc);
this.term2synonym.put(loc, loc.toLowerCase());
String syn = normalizeTerm(loc);
this.synonym2term.put(syn, loc);
this.term2synonym.put(loc, syn);
}
}
@ -532,13 +533,18 @@ public class Tagging {
private final static Pattern PATTERN_UE = Pattern.compile("\u00FC");
private final static Pattern PATTERN_SZ = Pattern.compile("\u00DF");
public static final String normalizeTerm(String word) {
word = word.trim().toLowerCase();
word = PATTERN_AE.matcher(word).replaceAll("ae");
word = PATTERN_OE.matcher(word).replaceAll("oe");
word = PATTERN_UE.matcher(word).replaceAll("ue");
word = PATTERN_SZ.matcher(word).replaceAll("ss");
return word;
public static final String normalizeTerm(String term) {
term = term.trim().toLowerCase();
term = PATTERN_AE.matcher(term).replaceAll("ae");
term = PATTERN_OE.matcher(term).replaceAll("oe");
term = PATTERN_UE.matcher(term).replaceAll("ue");
term = PATTERN_SZ.matcher(term).replaceAll("ss");
// remove comma
int p;
while ((p = term.indexOf(',')) >= 0) {
term = term.substring(p + 1).trim() + " " + term.substring(0, p);
}
return term;
}
public class Metatag {

6
source/net/yacy/document/LibraryProvider.java
Unescape View File

@ -135,11 +135,11 @@ public class LibraryProvider {
if ( geo0.exists() ) {
geo0.renameTo(Dictionary.GEODB0.fileDisabled());
}
geoLoc.activateLocalization(Dictionary.GEODB1.nickname, new OpenGeoDBLocation(geo1, dymLib));
geoLoc.activateLocation(Dictionary.GEODB1.nickname, new OpenGeoDBLocation(geo1, dymLib));
return;
}
if ( geo0.exists() ) {
geoLoc.activateLocalization(Dictionary.GEODB0.nickname, new OpenGeoDBLocation(geo0, dymLib));
geoLoc.activateLocation(Dictionary.GEODB0.nickname, new OpenGeoDBLocation(geo0, dymLib));
return;
}
}
@ -147,7 +147,7 @@ public class LibraryProvider {
public static void integrateGeonames() {
final File geon = Dictionary.GEON0.file();
if ( geon.exists() ) {
geoLoc.activateLocalization(Dictionary.GEON0.nickname, new GeonamesLocation(geon, dymLib));
geoLoc.activateLocation(Dictionary.GEON0.nickname, new GeonamesLocation(geon, dymLib));
return;
}
}

1
source/net/yacy/document/geolocation/GeonamesLocation.java
Unescape View File

@ -119,6 +119,7 @@ public class GeonamesLocation implements Locations
this.id2loc.put(id, c);
for ( final StringBuilder name : locnames ) {
if (dymLib != null && dymLib.contains(name)) continue;
if (name.length() < OverarchingLocation.MINIMUM_NAME_LENGTH) continue;
List<Integer> locs = this.name2ids.get(name);
if ( locs == null ) {
locs = new ArrayList<Integer>(1);

2
source/net/yacy/document/geolocation/OpenGeoDBLocation.java
Unescape View File

@ -114,7 +114,7 @@ public class OpenGeoDBLocation implements Locations
}
id = Integer.parseInt(v[0]);
h = removeQuotes(v[2]);
if (h.length() < 2) continue;
if (h.length() < OverarchingLocation.MINIMUM_NAME_LENGTH) continue;
if (dymLib != null && dymLib.contains(new StringBuilder(h))) continue;
List<Integer> l = this.name2ids.get(new StringBuilder(h));
if ( l == null ) {

3
source/net/yacy/document/geolocation/OverarchingLocation.java
Unescape View File

@ -30,6 +30,7 @@ import java.util.TreeSet;
public class OverarchingLocation implements Locations {
public static int MINIMUM_NAME_LENGTH = 4;
private final Map<String, Locations> services;
/**
@ -44,7 +45,7 @@ public class OverarchingLocation implements Locations {
* @param nickname the nickname of the service
* @param service the service
*/
public void activateLocalization(final String nickname, final Locations service) {
public void activateLocation(final String nickname, final Locations service) {
this.services.put(nickname, service);
}

Write Preview
Loading…
Cancel
Save