From eaddf2d464926b960539b1a4ede6ddb5cfaa9d4d Mon Sep 17 00:00:00 2001 From: orbiter Date: Fri, 4 Sep 2009 23:33:47 +0000 Subject: [PATCH] - corrected layout of map preview - added caption to maps containing latitude and longitude information - prevented that maps occur on second search page - added location names to did-you-mean - some refactoring of did-you-mean - added equal and compareTo test to Coordinates class to make that work in set - fixed utf-8 support for library files - fixed a bug in images search icon view caption git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6294 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/yacysearch.html | 8 +++- htroot/yacysearch.java | 7 ++-- htroot/yacysearchitem.html | 2 +- source/de/anomic/data/Coordinates.java | 33 ++++++++++++++-- source/de/anomic/data/DidYouMean.java | 40 ++++++-------------- source/de/anomic/data/DidYouMeanLibrary.java | 14 ++++--- source/de/anomic/data/LibraryProvider.java | 20 ++++++++-- source/de/anomic/data/OpenGeoDB.java | 19 +++++++++- source/de/anomic/ymage/ymageOSM.java | 5 ++- 9 files changed, 99 insertions(+), 49 deletions(-) diff --git a/htroot/yacysearch.html b/htroot/yacysearch.html index 7247aaf00..1bd8024c6 100644 --- a/htroot/yacysearch.html +++ b/htroot/yacysearch.html @@ -133,12 +133,16 @@ document.getElementById("Enter").value = "search again";

- Location (click to enlarge)

+ Location -- click on map to enlarge

#{loc}# +

- + +
lat=#[lat]#, lon=#[lon]#
+
lat=#[lat]#,
lon=#[lon]#
+
#{/loc}#

Geographic information provided by OpenGeoDB, Map provided by OpenStreetMap

diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index 1022c72ba..98f9e2cac 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -521,14 +521,15 @@ public class yacysearch { // find geographic info Set coordinates = LibraryProvider.geoDB.find(originalquerystring); - if (coordinates == null || coordinates.size() == 0) { + if (coordinates == null || coordinates.size() == 0 || offset > 0) { prop.put("geoinfo", "0"); } else { int i = 0; for (Coordinates c: coordinates) { - prop.put("geoinfo_loc_" + i + "_lon", c.lon()); - prop.put("geoinfo_loc_" + i + "_lat", c.lat()); + prop.put("geoinfo_loc_" + i + "_lon", Math.round(c.lon() * 10000.0) / 10000.0); + prop.put("geoinfo_loc_" + i + "_lat", Math.round(c.lat() * 10000.0) / 10000.0); i++; + if (i >= 5) break; } prop.put("geoinfo_loc", i); prop.put("geoinfo", "1"); diff --git a/htroot/yacysearchitem.html b/htroot/yacysearchitem.html index 686dcb237..26163ae13 100644 --- a/htroot/yacysearchitem.html +++ b/htroot/yacysearchitem.html @@ -25,7 +25,7 @@ #[name]# -
#[name]#
#[sourcedom]#
+
#[name]#
#[sourcedom]#
#[name]#
#[attr]#
#{/items}# diff --git a/source/de/anomic/data/Coordinates.java b/source/de/anomic/data/Coordinates.java index 9747697a5..88b5b1b0c 100644 --- a/source/de/anomic/data/Coordinates.java +++ b/source/de/anomic/data/Coordinates.java @@ -26,8 +26,10 @@ package de.anomic.data; -public class Coordinates { +public class Coordinates implements Comparable { + private static final double tenmeter = 90.0 / 1.0e6; + private double lon, lat; public Coordinates(double lon, double lat) { @@ -52,12 +54,37 @@ public class Coordinates { /** * compute the hash code of a coordinate - * this produces identical hash codes for locations that are close to each other on longitude + * this produces identical hash codes for locations that are close to each other */ public int hashCode() { int lon1 = coord2int(this.lon) >> 15; int lat1 = coord2int(this.lat) >> 15; - return (lon1 << 15) | lat1; + int h = (lon1 << 15) + lat1; + System.out.println("lon=" + this.lon + ", lat=" + this.lat + ", hash=" + h); + return h; + } + + /** + * comparator that is needed to use the class inside TreeMap/TreeSet + */ + public int compareTo(Coordinates o) { + if (this.equals(o)) return 0; + int s = this.hashCode(); + int t = o.hashCode(); + if (s > t) return 1; + if (s < t) return -1; + return 0; + } + + /** + * equality test that is needed to use the class inside HashMap/HashSet + */ + public boolean equals(Object o) { + if (!(o instanceof Coordinates)) return false; + Coordinates oo = (Coordinates) o; + if (this.lon == oo.lon && this.lat == oo.lat) return true; + // we access fuzzy values that are considered as equal if they are close to each other + return Math.abs(this.lon - oo.lon) < tenmeter && Math.abs(this.lat - oo.lat) < tenmeter; } public String toString() { diff --git a/source/de/anomic/data/DidYouMean.java b/source/de/anomic/data/DidYouMean.java index 46398f0b1..f11ab19cd 100644 --- a/source/de/anomic/data/DidYouMean.java +++ b/source/de/anomic/data/DidYouMean.java @@ -152,6 +152,14 @@ public class DidYouMean { return this.resultSet; } + + public void test(String s) throws InterruptedException { + Set libr = LibraryProvider.dymLib.recommend(s); + libr.addAll(LibraryProvider.geoDB.recommend(s)); + if (libr.size() != 0) createGen = false; + for (String t: libr) guessLib.put(t); + if (createGen) guessGen.put(s); + } /** * DidYouMean's producer thread that changes one letter (e.g. bat/cat) for a given term @@ -161,15 +169,9 @@ public class DidYouMean { public class ChangingOneLetter extends Thread { public void run() { - String s; - Set libr; for (int i = 0; i < wordLen; i++) try { for (char c: alphabet) { - s = word.substring(0, i) + c + word.substring(i + 1); - libr = LibraryProvider.dymLib.recommend(s); - if (libr.size() != 0) createGen = false; - for (String t: libr) guessLib.put(t); - if (createGen) guessGen.put(s); + test(word.substring(0, i) + c + word.substring(i + 1)); if (System.currentTimeMillis() > timeLimit) return; } } catch (InterruptedException e) {} @@ -184,14 +186,8 @@ public class DidYouMean { protected class DeletingOneLetter extends Thread { public void run() { - String s; - Set libr; for (int i = 0; i < wordLen; i++) try { - s = word.substring(0, i) + word.substring(i+1); - libr = LibraryProvider.dymLib.recommend(s); - if (libr.size() != 0) createGen = false; - for (String t: libr) guessLib.put(t); - if (createGen) guessGen.put(s); + test(word.substring(0, i) + word.substring(i+1)); if (System.currentTimeMillis() > timeLimit) return; } catch (InterruptedException e) {} } @@ -205,15 +201,9 @@ public class DidYouMean { protected class AddingOneLetter extends Thread { public void run() { - String s; - Set libr; for (int i = 0; i <= wordLen; i++) try { for (char c: alphabet) { - s = word.substring(0, i) + c + word.substring(i); - libr = LibraryProvider.dymLib.recommend(s); - if (libr.size() != 0) createGen = false; - for (String t: libr) guessLib.put(t); - if (createGen) guessGen.put(s); + test(word.substring(0, i) + c + word.substring(i)); if (System.currentTimeMillis() > timeLimit) return; } } catch (InterruptedException e) {} @@ -228,14 +218,8 @@ public class DidYouMean { protected class ReversingTwoConsecutiveLetters extends Thread { public void run() { - String s; - Set libr; for (int i = 0; i < wordLen - 1; i++) try { - s = word.substring(0, i) + word.charAt(i + 1) + word.charAt(i) + word.substring(i +2); - libr = LibraryProvider.dymLib.recommend(s); - if (libr.size() != 0) createGen = false; - for (String t: libr) guessLib.put(t); - if (createGen) guessGen.put(s); + test(word.substring(0, i) + word.charAt(i + 1) + word.charAt(i) + word.substring(i +2)); if (System.currentTimeMillis() > timeLimit) return; } catch (InterruptedException e) {} } diff --git a/source/de/anomic/data/DidYouMeanLibrary.java b/source/de/anomic/data/DidYouMeanLibrary.java index 70749548e..451797259 100644 --- a/source/de/anomic/data/DidYouMeanLibrary.java +++ b/source/de/anomic/data/DidYouMeanLibrary.java @@ -28,13 +28,15 @@ package de.anomic.data; import java.io.BufferedReader; import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileReader; +import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.util.HashSet; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; +import java.util.zip.GZIPInputStream; /** * provide a completion library for the did-you-mean class @@ -68,14 +70,16 @@ public class DidYouMeanLibrary { for (String f: files) { if (f.endsWith(".words")) try { importFile(new File(dictionaryPath, f)); - } catch (FileNotFoundException e) { + } catch (IOException e) { e.printStackTrace(); } } } - private void importFile(File f) throws FileNotFoundException { - BufferedReader r = new BufferedReader(new FileReader(f)); + private void importFile(File f) throws IOException { + InputStream is = new FileInputStream(f); + if (f.getName().endsWith(".gz")) is = new GZIPInputStream(is); + BufferedReader r = new BufferedReader(new InputStreamReader(is, "UTF-8")); String l; try { while ((l = r.readLine()) != null) { diff --git a/source/de/anomic/data/LibraryProvider.java b/source/de/anomic/data/LibraryProvider.java index a08c54af9..d5b072d3a 100644 --- a/source/de/anomic/data/LibraryProvider.java +++ b/source/de/anomic/data/LibraryProvider.java @@ -70,9 +70,21 @@ public class LibraryProvider { } public static void integrateOpenGeoDB() { - File ogdb = new File(dictSource, "opengeodb-0.2.5a-UTF8-sql.gz"); // may also be named opengeodb-02513_2007-10-02.sql.gz - if (!ogdb.exists()) return; - geoDB = new OpenGeoDB(ogdb); + File ogdb = new File(dictSource, "opengeodb-0.2.5a-UTF8-sql.gz"); + if (ogdb.exists()) { + geoDB = new OpenGeoDB(ogdb); + return; + } + ogdb = new File(dictSource, "opengeodb-02513_2007-10-02.sql.gz"); + if (ogdb.exists()) { + geoDB = new OpenGeoDB(ogdb); + return; + } + ogdb = new File(dictSource, "opengeodb-02513_2007-10-02.sql"); + if (ogdb.exists()) { + geoDB = new OpenGeoDB(ogdb); + return; + } } public static void initDidYouMean() { @@ -140,7 +152,7 @@ public class LibraryProvider { final ArrayList list = new ArrayList(); BufferedReader reader = null; try { - reader = new BufferedReader(new InputStreamReader(new FileInputStream(file))); + reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8")); String line; // read until text starts diff --git a/source/de/anomic/data/OpenGeoDB.java b/source/de/anomic/data/OpenGeoDB.java index 3c7b4aead..42c896bbf 100644 --- a/source/de/anomic/data/OpenGeoDB.java +++ b/source/de/anomic/data/OpenGeoDB.java @@ -38,6 +38,9 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Locale; +import java.util.Set; +import java.util.SortedMap; +import java.util.SortedSet; import java.util.TreeMap; import java.util.zip.GZIPInputStream; @@ -85,7 +88,7 @@ public class OpenGeoDB { try { InputStream is = new FileInputStream(file); if (file.getName().endsWith(".gz")) is = new GZIPInputStream(is); - reader = new BufferedReader(new InputStreamReader(is)); + reader = new BufferedReader(new InputStreamReader(is, "UTF-8")); String line; // read lines @@ -180,4 +183,18 @@ public class OpenGeoDB { return a; } + /** + * read the dictionary and construct a set of recommendations to a given string + * @param s input value that is used to match recommendations + * @return a set that contains all words that start or end with the input value + */ + public Set recommend(String s) { + Set a = new HashSet(); + s = s.trim().toLowerCase(); + SortedMap> t = this.locationName2ids.tailMap(s); + for (String r: t.keySet()) { + if (r.startsWith(s)) a.add(r); else break; + } + return a; + } } diff --git a/source/de/anomic/ymage/ymageOSM.java b/source/de/anomic/ymage/ymageOSM.java index 056db0486..d1425f1ce 100644 --- a/source/de/anomic/ymage/ymageOSM.java +++ b/source/de/anomic/ymage/ymageOSM.java @@ -36,6 +36,7 @@ import java.util.Random; import javax.imageio.ImageIO; +import de.anomic.crawler.CrawlProfile; import de.anomic.crawler.retrieval.Response; import de.anomic.http.client.Cache; import de.anomic.search.Switchboard; @@ -77,13 +78,13 @@ public class ymageOSM { } catch (final MalformedURLException e) { return null; } - System.out.println("*** DEBUG: fetching OSM tile: " + tileURL.toNormalform(true, true)); + //System.out.println("*** DEBUG: fetching OSM tile: " + tileURL.toNormalform(true, true)); InputStream tileStream = Cache.getContentStream(tileURL); if (tileStream == null) { // download resource using the crawler and keep resource in memory if possible Response entry = null; try { - entry = Switchboard.getSwitchboard().loader.load(tileURL, false, false); + entry = Switchboard.getSwitchboard().loader.load(tileURL, false, false, CrawlProfile.CACHE_STRATEGY_IFEXIST); } catch (IOException e) { Log.logWarning("yamyOSM", "cannot load: " + e.getMessage()); return null;