From 1defd580bccb23c043f6c94616cd84d518a8fdce Mon Sep 17 00:00:00 2001 From: orbiter Date: Wed, 19 May 2010 12:53:09 +0000 Subject: [PATCH] - added option to localization search to distinguish between a search for a location according to the search word only or for the relation between a web search results and locations found in the metadata fields - used that to display two layers on map: cities and search result locations - added many marker grafics for the display of the markers on the map - some refactoring of the yacy news code plus bugfixes for latest move from Tree to Table data structure git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6889 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/Blog.java | 4 +- htroot/Bookmarks.java | 4 +- htroot/ConfigProfile_p.java | 4 +- htroot/CrawlMonitorRemoteStart.java | 6 +- htroot/Crawler_p.java | 4 +- htroot/Network.java | 6 +- htroot/News.java | 6 +- htroot/Supporter.java | 14 +- htroot/Surftips.java | 14 +- htroot/ViewProfile.java | 4 +- htroot/Wiki.java | 4 +- htroot/api/bookmarks/posts/add_p.java | 4 +- htroot/env/grafics/dot_blue.png | Bin 0 -> 267 bytes htroot/env/grafics/dot_darkgrey.png | Bin 0 -> 244 bytes htroot/env/grafics/dot_green.png | Bin 0 -> 282 bytes htroot/env/grafics/dot_lightgrey.png | Bin 0 -> 238 bytes htroot/env/grafics/dot_red.png | Bin 0 -> 268 bytes htroot/env/grafics/dot_yellow.png | Bin 0 -> 272 bytes htroot/env/grafics/marker_blue.png | Bin 0 -> 323 bytes htroot/env/grafics/marker_darkgrey.png | Bin 0 -> 314 bytes htroot/env/grafics/marker_green.png | Bin 0 -> 355 bytes htroot/env/grafics/marker_lightgrey.png | Bin 0 -> 295 bytes htroot/env/grafics/marker_red.png | Bin 0 -> 328 bytes htroot/env/grafics/marker_yellow.png | Bin 0 -> 352 bytes htroot/env/grafics/star_blue.png | Bin 0 -> 385 bytes htroot/env/grafics/star_darkgrey.png | Bin 0 -> 393 bytes htroot/env/grafics/star_green.png | Bin 0 -> 460 bytes htroot/env/grafics/star_lightgrey.png | Bin 0 -> 345 bytes htroot/env/grafics/star_red.png | Bin 0 -> 424 bytes htroot/env/grafics/star_yellow.png | Bin 0 -> 446 bytes htroot/yacysearch.java | 6 +- htroot/yacysearch_location.html | 31 ++- htroot/yacysearch_location.java | 66 +++-- htroot/yacysearchtrailer.html | 8 +- htroot/yacysearchtrailer.java | 14 +- source/de/anomic/data/bookmarksDB.java | 4 +- source/de/anomic/search/Switchboard.java | 4 +- source/de/anomic/yacy/yacyCore.java | 2 +- source/de/anomic/yacy/yacyNewsDB.java | 181 ++++++++++++-- source/de/anomic/yacy/yacyNewsPool.java | 52 ++-- source/de/anomic/yacy/yacyNewsQueue.java | 66 +++-- source/de/anomic/yacy/yacyNewsRecord.java | 230 ------------------ source/de/anomic/yacy/yacyPeerActions.java | 2 +- .../geolocalization/GeonamesLocalization.java | 4 + .../geolocalization/Localization.java | 6 + .../OpenGeoDBLocalization.java | 4 + .../OverarchingLocalization.java | 8 + .../yacy/kelondro/table/ChunkIterator.java | 2 +- source/net/yacy/kelondro/table/Table.java | 4 +- 49 files changed, 381 insertions(+), 387 deletions(-) create mode 100644 htroot/env/grafics/dot_blue.png create mode 100644 htroot/env/grafics/dot_darkgrey.png create mode 100644 htroot/env/grafics/dot_green.png create mode 100644 htroot/env/grafics/dot_lightgrey.png create mode 100644 htroot/env/grafics/dot_red.png create mode 100644 htroot/env/grafics/dot_yellow.png create mode 100644 htroot/env/grafics/marker_blue.png create mode 100644 htroot/env/grafics/marker_darkgrey.png create mode 100644 htroot/env/grafics/marker_green.png create mode 100644 htroot/env/grafics/marker_lightgrey.png create mode 100644 htroot/env/grafics/marker_red.png create mode 100644 htroot/env/grafics/marker_yellow.png create mode 100644 htroot/env/grafics/star_blue.png create mode 100644 htroot/env/grafics/star_darkgrey.png create mode 100644 htroot/env/grafics/star_green.png create mode 100644 htroot/env/grafics/star_lightgrey.png create mode 100644 htroot/env/grafics/star_red.png create mode 100644 htroot/env/grafics/star_yellow.png delete mode 100644 source/de/anomic/yacy/yacyNewsRecord.java diff --git a/htroot/Blog.java b/htroot/Blog.java index 5886a5dd5..00ae960be 100644 --- a/htroot/Blog.java +++ b/htroot/Blog.java @@ -46,8 +46,8 @@ import de.anomic.http.server.RequestHeader; import de.anomic.search.Switchboard; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; +import de.anomic.yacy.yacyNewsDB; import de.anomic.yacy.yacyNewsPool; -import de.anomic.yacy.yacyNewsRecord; import java.util.List; import java.util.Map; @@ -175,7 +175,7 @@ public class Blog { map.put("page", pagename); map.put("subject", StrSubject.replace(',', ' ')); map.put("author", StrAuthor.replace(',', ' ')); - sb.peers.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.peers.mySeed(), yacyNewsPool.CATEGORY_BLOG_ADD, map)); + sb.peers.newsPool.publishMyNews(new yacyNewsDB.Record(sb.peers.mySeed(), yacyNewsPool.CATEGORY_BLOG_ADD, map)); } page = sb.blogDB.readBlogEntry(pagename); //maybe "if(page == null)" diff --git a/htroot/Bookmarks.java b/htroot/Bookmarks.java index e3a4a16f2..cdb76ea6c 100644 --- a/htroot/Bookmarks.java +++ b/htroot/Bookmarks.java @@ -52,8 +52,8 @@ import de.anomic.search.Segments; import de.anomic.search.Switchboard; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; +import de.anomic.yacy.yacyNewsDB; import de.anomic.yacy.yacyNewsPool; -import de.anomic.yacy.yacyNewsRecord; public class Bookmarks { @@ -434,7 +434,7 @@ public class Bookmarks { map.put("title", title.replace(',', ' ')); map.put("description", description.replace(',', ' ')); map.put("tags", tagsString.replace(',', ' ')); - sb.peers.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.peers.mySeed(), yacyNewsPool.CATEGORY_BOOKMARK_ADD, map)); + sb.peers.newsPool.publishMyNews(new yacyNewsDB.Record(sb.peers.mySeed(), yacyNewsPool.CATEGORY_BOOKMARK_ADD, map)); } } diff --git a/htroot/ConfigProfile_p.java b/htroot/ConfigProfile_p.java index 2588b477a..862cc77d0 100644 --- a/htroot/ConfigProfile_p.java +++ b/htroot/ConfigProfile_p.java @@ -39,8 +39,8 @@ import de.anomic.http.server.RequestHeader; import de.anomic.search.Switchboard; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; +import de.anomic.yacy.yacyNewsDB; import de.anomic.yacy.yacyNewsPool; -import de.anomic.yacy.yacyNewsRecord; public class ConfigProfile_p { @@ -94,7 +94,7 @@ public class ConfigProfile_p { // generate a news message final Properties news = profile; news.remove("comment"); - sb.peers.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.peers.mySeed(), yacyNewsPool.CATEGORY_PROFILE_UPDATE, news)); + sb.peers.newsPool.publishMyNews(yacyNewsDB.newRecord(sb.peers.mySeed(), yacyNewsPool.CATEGORY_PROFILE_UPDATE, news)); //yacyCore.newsPool.publishMyNews(new yacyNewsRecord(yacyNewsRecord.CATEGORY_PROFILE_UPDATE, profile)); } catch(final IOException e) { } finally { diff --git a/htroot/CrawlMonitorRemoteStart.java b/htroot/CrawlMonitorRemoteStart.java index 05037eff1..aa34ddd4e 100644 --- a/htroot/CrawlMonitorRemoteStart.java +++ b/htroot/CrawlMonitorRemoteStart.java @@ -30,8 +30,8 @@ import de.anomic.http.server.RequestHeader; import de.anomic.search.Switchboard; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; +import de.anomic.yacy.yacyNewsDB; import de.anomic.yacy.yacyNewsPool; -import de.anomic.yacy.yacyNewsRecord; import de.anomic.yacy.yacySeed; public class CrawlMonitorRemoteStart { @@ -44,9 +44,9 @@ public class CrawlMonitorRemoteStart { boolean dark = true; // create other peer crawl table using YaCyNews - Iterator recordIterator = sb.peers.newsPool.recordIterator(yacyNewsPool.INCOMING_DB, true); + Iterator recordIterator = sb.peers.newsPool.recordIterator(yacyNewsPool.INCOMING_DB, true); int showedCrawl = 0; - yacyNewsRecord record; + yacyNewsDB.Record record; yacySeed peer; String peername; while (recordIterator.hasNext()) { diff --git a/htroot/Crawler_p.java b/htroot/Crawler_p.java index 1355e701c..ad48a4df5 100644 --- a/htroot/Crawler_p.java +++ b/htroot/Crawler_p.java @@ -56,8 +56,8 @@ import de.anomic.search.Switchboard; import de.anomic.search.SwitchboardConstants; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; +import de.anomic.yacy.yacyNewsDB; import de.anomic.yacy.yacyNewsPool; -import de.anomic.yacy.yacyNewsRecord; public class Crawler_p { public static final String CRAWLING_MODE_URL = "url"; @@ -288,7 +288,7 @@ public class Crawler_p { m.remove("generalFilter"); m.remove("specificFilter"); m.put("intention", post.get("intention", "").replace(',', '/')); - sb.peers.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.peers.mySeed(), yacyNewsPool.CATEGORY_CRAWL_START, m)); + sb.peers.newsPool.publishMyNews(new yacyNewsDB.Record(sb.peers.mySeed(), yacyNewsPool.CATEGORY_CRAWL_START, m)); } } else { prop.put("info", "5"); //Crawling failed diff --git a/htroot/Network.java b/htroot/Network.java index 705d0316a..c5826dd45 100644 --- a/htroot/Network.java +++ b/htroot/Network.java @@ -48,8 +48,8 @@ import de.anomic.search.SwitchboardConstants; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.yacy.yacyClient; +import de.anomic.yacy.yacyNewsDB; import de.anomic.yacy.yacyNewsPool; -import de.anomic.yacy.yacyNewsRecord; import de.anomic.yacy.yacySeed; import de.anomic.yacy.yacyVersion; @@ -244,8 +244,8 @@ public class Network { final HashMap> updatedWiki = new HashMap>(); final HashMap> updatedBlog = new HashMap>(); final HashMap isCrawling = new HashMap(); - yacyNewsRecord record; - final Iterator recordIterator = sb.peers.newsPool.recordIterator(yacyNewsPool.INCOMING_DB, true); + yacyNewsDB.Record record; + final Iterator recordIterator = sb.peers.newsPool.recordIterator(yacyNewsPool.INCOMING_DB, true); while (recordIterator.hasNext()) { record = recordIterator.next(); if (record == null) { diff --git a/htroot/News.java b/htroot/News.java index 99c169e15..ef20a75cd 100644 --- a/htroot/News.java +++ b/htroot/News.java @@ -35,8 +35,8 @@ import de.anomic.http.server.RequestHeader; import de.anomic.search.Switchboard; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; +import de.anomic.yacy.yacyNewsDB; import de.anomic.yacy.yacyNewsPool; -import de.anomic.yacy.yacyNewsRecord; import de.anomic.yacy.yacySeed; public class News { @@ -103,8 +103,8 @@ public class News { if (sb.peers != null) { final int maxCount = Math.min(1000, sb.peers.newsPool.size(tableID)); - final Iterator recordIterator = sb.peers.newsPool.recordIterator(tableID, false); - yacyNewsRecord record; + final Iterator recordIterator = sb.peers.newsPool.recordIterator(tableID, false); + yacyNewsDB.Record record; yacySeed seed; int i = 0; while ((recordIterator.hasNext()) && (i < maxCount)) { diff --git a/htroot/Supporter.java b/htroot/Supporter.java index 1d8f07dc0..f7a47a1c2 100644 --- a/htroot/Supporter.java +++ b/htroot/Supporter.java @@ -45,8 +45,8 @@ import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.tools.crypt; import de.anomic.tools.nxTools; +import de.anomic.yacy.yacyNewsDB; import de.anomic.yacy.yacyNewsPool; -import de.anomic.yacy.yacyNewsRecord; import de.anomic.yacy.yacySeed; public class Supporter { @@ -79,7 +79,7 @@ public class Supporter { map.put("urlhash", hash); map.put("vote", "negative"); map.put("refid", post.get("refid", "")); - sb.peers.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map)); + sb.peers.newsPool.publishMyNews(new yacyNewsDB.Record(sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map)); } if ((post != null) && ((hash = post.get("votePositive", null)) != null)) { if (!sb.verifyAuthentication(header, false)) { @@ -95,7 +95,7 @@ public class Supporter { map.put("vote", "positive"); map.put("refid", post.get("refid", "")); map.put("comment", post.get("comment", "")); - sb.peers.newsPool.publishMyNews(new yacyNewsRecord(sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map)); + sb.peers.newsPool.publishMyNews(new yacyNewsDB.Record(sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map)); } // create Supporter @@ -170,8 +170,8 @@ public class Supporter { private static void accumulateVotes(final Switchboard sb, final HashMap negativeHashes, final HashMap positiveHashes, final int dbtype) { final int maxCount = Math.min(1000, sb.peers.newsPool.size(dbtype)); - yacyNewsRecord record; - final Iterator recordIterator = sb.peers.newsPool.recordIterator(dbtype, true); + yacyNewsDB.Record record; + final Iterator recordIterator = sb.peers.newsPool.recordIterator(dbtype, true); int j = 0; while ((recordIterator.hasNext()) && (j++ < maxCount)) { record = recordIterator.next(); @@ -200,8 +200,8 @@ public class Supporter { final HashMap Supporter, final ScoreCluster ranking, final Row rowdef, final HashMap negativeHashes, final HashMap positiveHashes, final int dbtype) { final int maxCount = Math.min(1000, sb.peers.newsPool.size(dbtype)); - yacyNewsRecord record; - final Iterator recordIterator = sb.peers.newsPool.recordIterator(dbtype, true); + yacyNewsDB.Record record; + final Iterator recordIterator = sb.peers.newsPool.recordIterator(dbtype, true); int j = 0; String url = "", urlhash; Row.Entry entry; diff --git a/htroot/Surftips.java b/htroot/Surftips.java index 28acccb64..4e5aed3f0 100644 --- a/htroot/Surftips.java +++ b/htroot/Surftips.java @@ -45,8 +45,8 @@ import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.tools.crypt; import de.anomic.tools.nxTools; +import de.anomic.yacy.yacyNewsDB; import de.anomic.yacy.yacyNewsPool; -import de.anomic.yacy.yacyNewsRecord; import de.anomic.yacy.yacySeed; public class Surftips { @@ -87,7 +87,7 @@ public class Surftips { map.put("urlhash", hash); map.put("vote", "negative"); map.put("refid", post.get("refid", "")); - sb.peers.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map)); + sb.peers.newsPool.publishMyNews(new yacyNewsDB.Record(sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map)); } if ((post != null) && ((hash = post.get("votePositive", null)) != null)) { if (!sb.verifyAuthentication(header, false)) { @@ -103,7 +103,7 @@ public class Surftips { map.put("vote", "positive"); map.put("refid", post.get("refid", "")); map.put("comment", post.get("comment", "")); - sb.peers.newsPool.publishMyNews(new yacyNewsRecord(sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map)); + sb.peers.newsPool.publishMyNews(new yacyNewsDB.Record(sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map)); } // create surftips @@ -179,8 +179,8 @@ public class Surftips { private static void accumulateVotes(final Switchboard sb, final HashMap negativeHashes, final HashMap positiveHashes, final int dbtype) { final int maxCount = Math.min(1000, sb.peers.newsPool.size(dbtype)); - yacyNewsRecord record; - final Iterator recordIterator = sb.peers.newsPool.recordIterator(dbtype, true); + yacyNewsDB.Record record; + final Iterator recordIterator = sb.peers.newsPool.recordIterator(dbtype, true); int j = 0; while ((recordIterator.hasNext()) && (j++ < maxCount)) { record = recordIterator.next(); @@ -209,8 +209,8 @@ public class Surftips { final HashMap surftips, final ScoreCluster ranking, final Row rowdef, final HashMap negativeHashes, final HashMap positiveHashes, final int dbtype) { final int maxCount = Math.min(1000, sb.peers.newsPool.size(dbtype)); - yacyNewsRecord record; - final Iterator recordIterator = sb.peers.newsPool.recordIterator(dbtype, true); + yacyNewsDB.Record record; + final Iterator recordIterator = sb.peers.newsPool.recordIterator(dbtype, true); int j = 0; String url = "", urlhash; Row.Entry entry; diff --git a/htroot/ViewProfile.java b/htroot/ViewProfile.java index ba58cbfec..c0437ecae 100644 --- a/htroot/ViewProfile.java +++ b/htroot/ViewProfile.java @@ -50,8 +50,8 @@ import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.yacy.yacyClient; import de.anomic.yacy.yacyCore; +import de.anomic.yacy.yacyNewsDB; import de.anomic.yacy.yacyNewsPool; -import de.anomic.yacy.yacyNewsRecord; import de.anomic.yacy.yacySeed; public class ViewProfile { @@ -99,7 +99,7 @@ public class ViewProfile { } else { // process news if existent try { - final yacyNewsRecord record = sb.peers.newsPool.getByOriginator(yacyNewsPool.INCOMING_DB, yacyNewsPool.CATEGORY_PROFILE_UPDATE, seed.hash); + final yacyNewsDB.Record record = sb.peers.newsPool.getByOriginator(yacyNewsPool.INCOMING_DB, yacyNewsPool.CATEGORY_PROFILE_UPDATE, seed.hash); if (record != null) sb.peers.newsPool.moveOff(yacyNewsPool.INCOMING_DB, record.id()); } catch (final Exception e) { Log.logException(e); diff --git a/htroot/Wiki.java b/htroot/Wiki.java index 18ea19668..636c54ba2 100644 --- a/htroot/Wiki.java +++ b/htroot/Wiki.java @@ -45,8 +45,8 @@ import de.anomic.http.server.RequestHeader; import de.anomic.search.Switchboard; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; +import de.anomic.yacy.yacyNewsDB; import de.anomic.yacy.yacyNewsPool; -import de.anomic.yacy.yacyNewsRecord; public class Wiki { @@ -121,7 +121,7 @@ public class Wiki { map.put("page", pagename); map.put("author", author.replace(',', ' ')); if (post.get("content", "").trim().length() > 0 && !page.page().equals(content)) - sb.peers.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.peers.mySeed(), yacyNewsPool.CATEGORY_WIKI_UPDATE, map)); + sb.peers.newsPool.publishMyNews(new yacyNewsDB.Record(sb.peers.mySeed(), yacyNewsPool.CATEGORY_WIKI_UPDATE, map)); page = newEntry; prop.putHTML("LOCATION", "/Wiki.html?page=" + pagename); } diff --git a/htroot/api/bookmarks/posts/add_p.java b/htroot/api/bookmarks/posts/add_p.java index 7d862764c..2f3321fef 100755 --- a/htroot/api/bookmarks/posts/add_p.java +++ b/htroot/api/bookmarks/posts/add_p.java @@ -10,8 +10,8 @@ import de.anomic.http.server.RequestHeader; import de.anomic.search.Switchboard; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; +import de.anomic.yacy.yacyNewsDB; import de.anomic.yacy.yacyNewsPool; -import de.anomic.yacy.yacyNewsRecord; public class add_p { @@ -83,6 +83,6 @@ public class add_p { map.put("title", title.replace(',', ' ')); map.put("description", description.replace(',', ' ')); map.put("tags", tagsString.replace(',', ' ')); - sb.peers.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.peers.mySeed(), yacyNewsPool.CATEGORY_BOOKMARK_ADD, map)); + sb.peers.newsPool.publishMyNews(new yacyNewsDB.Record(sb.peers.mySeed(), yacyNewsPool.CATEGORY_BOOKMARK_ADD, map)); } } \ No newline at end of file diff --git a/htroot/env/grafics/dot_blue.png b/htroot/env/grafics/dot_blue.png new file mode 100644 index 0000000000000000000000000000000000000000..c9f06ae279aa1ba3c1193bd0ac075e094a76cb89 GIT binary patch literal 267 zcmeAS@N?(olHy`uVBq!ia0vp^AT}2V6Od#IhcWqI$14|0SPL7ww{PyYk`&cJ7D=3I-a7_C5@pb;o zQ@c+T39yK;cp1*w`JXeGuc;x3hdpo4aaGH;Ida}jUJwCyUnRTJw z|F6&QuVFqv;b}w60)<2O9SZ*5ye_I9$0Rs`QJe+np8x-u8H_Yu{&z_B;RSLTJYD@< J);T3K0RZNNV5a~8 literal 0 HcmV?d00001 diff --git a/htroot/env/grafics/dot_darkgrey.png b/htroot/env/grafics/dot_darkgrey.png new file mode 100644 index 0000000000000000000000000000000000000000..8dce96175c04862e5d391caa81a9b84856e502f4 GIT binary patch literal 244 zcmeAS@N?(olHy`uVBq!ia0vp^AT}2V6Od#Ihf#UR>?6d=ta0mlfuF{Erk%9i2F{y8YLJt<#FX-D?-{()`=_Y;w@0sP4w% zb=Q8qlvo@Q=yg`UB{0ZqW20n1OX}{~4uX&7n20ETSnnL9;iQ^;X5Gc7e&^0yyA`gW lxl3oSfBH_K#sB{^Gc=}sv2?n<^*oTv;OXk;vd$@?2>@+yT;Tu! literal 0 HcmV?d00001 diff --git a/htroot/env/grafics/dot_green.png b/htroot/env/grafics/dot_green.png new file mode 100644 index 0000000000000000000000000000000000000000..a85e4756115719b4fd8aa87a0041d61ad507ffdb GIT binary patch literal 282 zcmeAS@N?(olHy`uVBq!ia0vp^AT}2V6Od#Ih2G?Yu9BHWlUT(w1v3i#Z*4+}}Q~*{1B_&2PW%hMg#q{e4%yeh1H2^LYu3K#PwI4`1Vil`sn%5uM-7rzaT=<=x|AyQ@MpBvpWBCgWqo4S8OkR;y%?1T dwDSLdW(J45vNsh1dfb6r22WQ%mvv4FO#puORwMub literal 0 HcmV?d00001 diff --git a/htroot/env/grafics/dot_red.png b/htroot/env/grafics/dot_red.png new file mode 100644 index 0000000000000000000000000000000000000000..b92206bfa8876689766c8e37757d8ca41faafe1f GIT binary patch literal 268 zcmeAS@N?(olHy`uVBq!ia0vp^AT}2V6Od#IhG*Pi_6y3%*GH{EeyNnzN@@v=Yqdi=!attT8O+UN%esQmqY`uXds zO&UcBEFvskhBY^~um$rqH3aeGKREOmhX|Fi~@|31&%=XF#P||%<$yI=EbLq&KZOFp00i_ I>zopr0K8CQwEzGB literal 0 HcmV?d00001 diff --git a/htroot/env/grafics/dot_yellow.png b/htroot/env/grafics/dot_yellow.png new file mode 100644 index 0000000000000000000000000000000000000000..6aa6c554705951faa7a24470dbe16893e3a4f79c GIT binary patch literal 272 zcmeAS@N?(olHy`uVBq!ia0vp^AT}2V6Od#Ihj|eEFU-Hp%d`Kx%i~G^cAlE>@Vc1)oJ-!Jk`8@`6#r(I zs~p~+uV25;?9i&O_g7z5=>JTlMerTYgHHZveW9;s1YThO+&acb;`u Qz`9G$w;ZcMQ^jiXcQJsE>+NliPZl%WZl! zeRFc}$@{#^IY-1lMNz!-0%!l^rPmVb1Jtc`spYDo$4L{@tzT}t8mq#=PZB{WtKF4> zRpCR$c>T!5s-R*pyA&j#Vl=-NB%tDWe-|X6$t+F#kc;uZ)1LjX(pef92S44IH8~jx zoXk#Np{(`NtXtxNy_002ovPDHLkV1g<|ic$ao literal 0 HcmV?d00001 diff --git a/htroot/env/grafics/marker_darkgrey.png b/htroot/env/grafics/marker_darkgrey.png new file mode 100644 index 0000000000000000000000000000000000000000..d5aeaa56c1210c7a5bb9ebdba0c81a9bbfb1bfcd GIT binary patch literal 314 zcmeAS@N?(olHy`uVBq!ia0vp^+(0bA!2~4z?+a-HDYlXzzhH*{AZQ*TT?-UZEOCt} z3C>R|DNig)WpFP_EXc@AcFxZ$OD!r%EmF|%%qvOF(JXKOB?(k~$kW9!q+(9Z#DiQd z3L>ue|JVCZ+`3}w1*Iisf`T`cRH9VeT)LOsdtl<5Dk~uVUE{p=8tbUn``+(fCw;u| zFY}pWmi$czZZ4d{cep^pHPFJGL(z?C>6F*I$`9Z9V))5s`!TjToRd^44GRi9E`K?C z;JV{w6aS6xe@8Gr=vs6lDCA#-c6wM|Nm!ZP}sB6u=eWxD?l!Tr>mdK II;Vst0IFbq%m4rY literal 0 HcmV?d00001 diff --git a/htroot/env/grafics/marker_green.png b/htroot/env/grafics/marker_green.png new file mode 100644 index 0000000000000000000000000000000000000000..44cd224b9328463ba1125f5119025b28be62d3f0 GIT binary patch literal 355 zcmV-p0i6DcP)9?wU-Bg009600{{RhU3gZqwG;pV002ovPDHLkV1hUm BoqYfR literal 0 HcmV?d00001 diff --git a/htroot/env/grafics/marker_lightgrey.png b/htroot/env/grafics/marker_lightgrey.png new file mode 100644 index 0000000000000000000000000000000000000000..a28be425336ad6ac941b58874d02846383f750dd GIT binary patch literal 295 zcmeAS@N?(olHy`uVBq!ia0vp^+(0bA!2~4z?+a-HDYlXzzhH*{AZQ*TT?-UZEOCt} z3C>R|DNig)WpFP_EXc@AcFxZ$OD!r%EmF|%%qvOF(JXKOB?(l#-qXb~q+(7@U^id0 z0?(8G|BH*eJAO#4cZ&7xWlA=iKH-br(<;&EFoBCdUnnMC6Zz9Jqw1~P-o%pTSlyT| zh8v#Mm(-u}%XHP1>^D(%5c=}newM*n#|c}dj~Y}up34_`{D@We+Ln_Bdhh(inSFn) z)%m2?aogMUScLrgBNMfo}c(pPAtq^H0~AGqg4XxeT7JelF{r5}E+&8*!Te literal 0 HcmV?d00001 diff --git a/htroot/env/grafics/marker_red.png b/htroot/env/grafics/marker_red.png new file mode 100644 index 0000000000000000000000000000000000000000..9a6581fdce55d3022b58bc5d204f5ac9d54a993a GIT binary patch literal 328 zcmeAS@N?(olHy`uVBq!ia0vp^+(0bA!2~4z?+a-HDYlXzzhH*{AZQ*TT?-UZEOCt} z3C>R|DNig)WpFP_EXc@AcFxZ$OD!r%EmF|%%qvOF(JXKOB?(k~!PCVtq+(9e4}SLl z|LXQvef+#{u7l8tHg01FjRi+^rSEEQx+B1n!t5%bWtSIQ?>bLhkJt6n9*4pSUHjwh z;;&D3ozAAbz#?GU!^6u=BbF+1Xq^6#6{IdF>JZts{d4+*K*lMlKMJ3#fdnSR%PnJa;#pcyoP8pXR1Q%E+9f{j=b-_x8t_3YG@B7Q=OPn~ov@vG^M;Dhu#qJET zL)_hI3y*VaJo@)%-`fR|DNig)WpFP_EXc@AcFxZ$OD!r%EmF|%%qvOF(JXKOB?(mg!qdeuq+(9e4}SLj zcK>S&|GeLP`2W7Wzxr7u9FEMNUw>}fznFDDi#vrlbVGmEYW%+c-P}O*|G$6trn5|S zcG_=d`Tze<#wCy5-!+|aN#M)#5@wI1?`<#c2>BC}%CV*K;q@x%Ia<>+7-J8A$nIA6 z6ZH-Ssz1l@iIq=uf0=8;osaKdZ=SEcWn=WmHwGp)|6*n}X#TCP=L+ih`|n%!T-C0w z)U5w@_gsTM{;o!vN%6j4c{%t=GH648LzrS2Ut060HPJ>iIE}z?(g9qL-zua$T v_2>U@;pJWR^WXii-SgXC3+P9N|Nog8w&qUL+-7{?42bXP>gTe~DWM4fN$8%c literal 0 HcmV?d00001 diff --git a/htroot/env/grafics/star_blue.png b/htroot/env/grafics/star_blue.png new file mode 100644 index 0000000000000000000000000000000000000000..461393b700a02ade5ec1c231f0f2182a3a3d827a GIT binary patch literal 385 zcmV-{0e=38P)DBhm8dbR>MdkjQb^KT;iFg#wF)4 zbo%;w-}mV|b3UG%_%~jg64xW~;+63~T|2ozMk}?KR7b%Tl1g1C;ZJu9&AELyDm@B1u8@qChuGWWC00000NkvXXu0mjfZQH3` literal 0 HcmV?d00001 diff --git a/htroot/env/grafics/star_darkgrey.png b/htroot/env/grafics/star_darkgrey.png new file mode 100644 index 0000000000000000000000000000000000000000..5e07313b0b6583e56819caecf4803fec16100d70 GIT binary patch literal 393 zcmV;40e1e0P)`J7x-83Tnnp^CCs9K7p}MXwke=tU|I&5+SMog} zQqwdT65_h9E6Xwn0@E~QS^mI~BuO$1!}t9>&*7qR92sbX=lSnYx{@UMtF~?1OK_HD zuRfvRAK{`XJ{{`&UJwL226o`Rg`O!`6h*{W0n;dC9LGcwMw~)gmPI5T$Dxplq7X?{ zRZ$T541=*O3p{`*if9uiWZ!oTc^n7#5DYCa&;4!Nz#C-yFbo)SnkL(}VZGtBu4@>E n@89)BQocVn<_-V=|NjF3<1)3Y?tWyu00000NkvXXu0mjf_Clt- literal 0 HcmV?d00001 diff --git a/htroot/env/grafics/star_green.png b/htroot/env/grafics/star_green.png new file mode 100644 index 0000000000000000000000000000000000000000..62c8c779e0d6644aba9b2c3c2d49966d16109c9b GIT binary patch literal 460 zcmV;-0Wwk literal 0 HcmV?d00001 diff --git a/htroot/env/grafics/star_lightgrey.png b/htroot/env/grafics/star_lightgrey.png new file mode 100644 index 0000000000000000000000000000000000000000..05df339e60e34c9587a231492831bda1184c8513 GIT binary patch literal 345 zcmV-f0jB5`!QNMEU>!thV*82nZ6pQ(HxaOzZ5E;kYCl#gcvo<(R5S72Hhs zN)!E0*8OxVluX;kNf-iy7~446ybV{SY$N1m$lHEMPc%8N%-oOm*_e=os?H3fL)ALJ zeMK~b%}aec$_uwZV8VO4n((T+iPqcx1tYCX rnmt8Ghu6%Hyez1PoCjhJYMPf^|848!O3YQ$`tYQYLvaCf z-mH}mFS1QxU3E}yo4x$K{fP@2?wT>PF3{N*u`_dB-(9ET(wdJypI<(%U1W13R{J>f zl%p#@zkd0+;twZ?b@$TF4v))ID{a^EdmM6Y*kn;^w_2NV)sAI)-!|?3c>XxQ!uR?= z3@Oa64UZfm=UB-fVY6AS|M%}N(WnFlu-F~NFYn&g-(m0-1BUGX|I7>>=X5?e=B^S2 Pav3~b{an^LB{Ts5ZE3lj literal 0 HcmV?d00001 diff --git a/htroot/env/grafics/star_yellow.png b/htroot/env/grafics/star_yellow.png new file mode 100644 index 0000000000000000000000000000000000000000..27e32848f5886eb65d42991c1464ab03b9f99410 GIT binary patch literal 446 zcmV;v0YUzWP)I|JN6Eo&Vlm{&Q>T zpR=vM_ox2aul{@hsjbBQueZ97B%|N4wGY=0ll!>)6w^8cTo z@Wl!+^l;hv@8ey3I)APX{)rBg^+&`Bl;?()~*%7$9-v`qEy}$A2 z@|2&e^r0e$vT^GCbA28t40lKUdwmWe`r|z?$NXHO{cE!;Zk;z5|9y7&KQ{31+sogl o8~^|NjWdLZ2LJ#70RR630R5SgCJT@Tm;e9(07*qoM6N<$f+ybK`v3p{ literal 0 HcmV?d00001 diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index db08cf10d..69235b47b 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -68,8 +68,8 @@ import de.anomic.search.SwitchboardConstants; import de.anomic.server.serverCore; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; +import de.anomic.yacy.yacyNewsDB; import de.anomic.yacy.yacyNewsPool; -import de.anomic.yacy.yacyNewsRecord; import de.anomic.yacy.graphics.ProfilingGraph; public class yacysearch { @@ -413,7 +413,7 @@ public class yacysearch { map.put("urlhash", delHash); map.put("vote", "negative"); map.put("refid", ""); - sb.peers.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map)); + sb.peers.newsPool.publishMyNews(new yacyNewsDB.Record(sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map)); } catch (IOException e) { Log.logException(e); } @@ -438,7 +438,7 @@ public class yacysearch { map.put("description", document.dc_title().replace(',', ' ')); map.put("author", document.dc_creator()); map.put("tags", document.dc_subject(' ')); - sb.peers.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_ADD, map)); + sb.peers.newsPool.publishMyNews(new yacyNewsDB.Record(sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_ADD, map)); document.close(); } } diff --git a/htroot/yacysearch_location.html b/htroot/yacysearch_location.html index 3adf4be60..a55c37b7f 100644 --- a/htroot/yacysearch_location.html +++ b/htroot/yacysearch_location.html @@ -7,7 +7,13 @@ diff --git a/htroot/yacysearch_location.java b/htroot/yacysearch_location.java index 2f0ec95e4..93a362245 100644 --- a/htroot/yacysearch_location.java +++ b/htroot/yacysearch_location.java @@ -33,13 +33,16 @@ import de.anomic.server.serverCore; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.yacy.yacyClient; + import java.util.Date; import net.yacy.kelondro.util.DateFormatter; public class yacysearch_location { + private static final String space = " "; + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { - //final Switchboard sb = (Switchboard) env; + final Switchboard sb = (Switchboard) env; final serverObjects prop = new serverObjects(); prop.put("kml", 0); @@ -52,33 +55,61 @@ public class yacysearch_location { prop.put("kml", 1); if (post == null) return prop; String query = post.get("query", ""); + boolean search_all = !post.containsKey("dom") || post.get("dom", "").equals("all"); + boolean search_query = search_all || post.get("dom", "").indexOf("query") >= 0; + boolean search_mdall = search_all || post.get("dom", "").indexOf("mdall") >= 0; + boolean search_title = search_mdall || post.get("dom", "").indexOf("title") >= 0; + boolean search_publisher = search_mdall || post.get("dom", "").indexOf("publisher") >= 0; + boolean search_creator = search_mdall || post.get("dom", "").indexOf("creator") >= 0; + boolean search_subject = search_mdall || post.get("dom", "").indexOf("subject") >= 0; long maximumTime = post.getLong("maximumTime", 1000); int maximumRecords = post.getInt("maximumRecords", 100); //i.e. http://localhost:8080/yacysearch_location.kml?query=berlin&maximumTime=2000&maximumRecords=100 - // get a queue of search results - BlockingQueue results = yacyClient.search(null, query, false, false, maximumTime, Integer.MAX_VALUE); - - // take the results and compute some locations - RSSMessage message; int placemarkCounter = 0; - try { + if (search_query) { + Set locations = LibraryProvider.geoLoc.find(query, true); + for (Location location: locations) { + // write for all locations a point to this message + prop.put("kml_placemark_" + placemarkCounter + "_location", location.getName()); + prop.put("kml_placemark_" + placemarkCounter + "_name", location.getName()); + prop.put("kml_placemark_" + placemarkCounter + "_author", ""); + prop.put("kml_placemark_" + placemarkCounter + "_copyright", ""); + prop.put("kml_placemark_" + placemarkCounter + "_subject", ""); + prop.put("kml_placemark_" + placemarkCounter + "_description", ""); + prop.put("kml_placemark_" + placemarkCounter + "_date", ""); + prop.putXML("kml_placemark_" + placemarkCounter + "_url", "http://" + sb.peers.mySeed().getPublicAddress() + "/yacysearch.html?query=" + location.getName()); + prop.put("kml_placemark_" + placemarkCounter + "_pointname", location.getName()); + prop.put("kml_placemark_" + placemarkCounter + "_lon", location.lon()); + prop.put("kml_placemark_" + placemarkCounter + "_lat", location.lat()); + placemarkCounter++; + } + } + + if (search_title || search_publisher || search_creator || search_subject) try { + // get a queue of search results + BlockingQueue results = yacyClient.search(null, query, false, false, maximumTime, Integer.MAX_VALUE); + + // take the results and compute some locations + RSSMessage message; loop: while ((message = results.poll(maximumTime, TimeUnit.MILLISECONDS)) != RSSMessage.POISON) { // find all associated locations Set locations = new HashSet(); - String words = message.getTitle() + " " + message.getCopyright() + " " + message.getAuthor(); + StringBuilder words = new StringBuilder(120); + if (search_title) words.append(message.getTitle().trim()).append(space); + if (search_publisher) words.append(message.getCopyright().trim()).append(space); + if (search_creator) words.append(message.getAuthor().trim()).append(space); String subject = ""; - for (String s: message.getSubject()) subject += " " + s; - words += subject; - for (String word: words.split(" ")) if (word.length() >= 3) locations.addAll(LibraryProvider.geoLoc.find(word, true)); - - String locnames = ""; - for (Location location: locations) locnames += ", " + location.getName(); - if (locations.size() > 0) locnames = locnames.substring(2); + for (String s: message.getSubject()) subject += s.trim() + space; + if (search_subject) words.append(subject).append(space); + String[] wordlist = words.toString().trim().split(space); + for (String word: wordlist) if (word.length() >= 3) locations.addAll(LibraryProvider.geoLoc.find(word, true)); + for (int i = 0; i < wordlist.length - 1; i++) locations.addAll(LibraryProvider.geoLoc.find(wordlist[i] + space + wordlist[i + 1], true)); + for (int i = 0; i < wordlist.length - 2; i++) locations.addAll(LibraryProvider.geoLoc.find(wordlist[i] + space + wordlist[i + 1] + space + wordlist[i + 2], true)); for (Location location: locations) { // write for all locations a point to this message - prop.put("kml_placemark_" + placemarkCounter + "_location", locnames); + prop.put("kml_placemark_" + placemarkCounter + "_location", location.getName()); prop.put("kml_placemark_" + placemarkCounter + "_name", message.getTitle()); prop.put("kml_placemark_" + placemarkCounter + "_author", message.getAuthor()); prop.put("kml_placemark_" + placemarkCounter + "_copyright", message.getCopyright()); @@ -93,8 +124,8 @@ public class yacysearch_location { if (placemarkCounter >= maximumRecords) break loop; } } - prop.put("kml_placemark", placemarkCounter); } catch (InterruptedException e) {} + prop.put("kml_placemark", placemarkCounter); } if (header.get(HeaderFramework.CONNECTION_PROP_EXT, "").equals("rss")) { if (post == null) return prop; @@ -117,7 +148,6 @@ public class yacysearch_location { } if (header.get(HeaderFramework.CONNECTION_PROP_EXT, "").equals("html")) { - final Switchboard sb = (Switchboard) env; final boolean authenticated = sb.adminAuthenticated(header) >= 2; int display = (post == null) ? 0 : post.getInt("display", 0); if (!authenticated) display = 2; diff --git a/htroot/yacysearchtrailer.html b/htroot/yacysearchtrailer.html index 726d6cd19..8e3646d28 100644 --- a/htroot/yacysearchtrailer.html +++ b/htroot/yacysearchtrailer.html @@ -44,8 +44,10 @@ #(/nav-about)# +#(cat-location)#:: - +#(/cat-location)# diff --git a/htroot/yacysearchtrailer.java b/htroot/yacysearchtrailer.java index 427267bb2..9eaf862b5 100644 --- a/htroot/yacysearchtrailer.java +++ b/htroot/yacysearchtrailer.java @@ -30,6 +30,7 @@ import java.util.List; import net.yacy.kelondro.util.EventTracker; +import de.anomic.data.LibraryProvider; import de.anomic.http.server.RequestHeader; import de.anomic.search.Navigator; import de.anomic.search.QueryParams; @@ -169,9 +170,16 @@ public class yacysearchtrailer { prop.put("nav-about_body", aboutBody); } - // location search - prop.put("query", theQuery.queryString); - prop.put("display", display); + // category: location search + // show only if there is a location database present and if there had been any search results + if (LibraryProvider.geoLoc.locations() == 0 || + theSearch.getRankingResult().getLocalIndexCount() == 0) { + prop.put("cat-location", 0); + } else { + prop.put("cat-location", 1); + prop.put("cat-location_query", theQuery.queryString); + prop.put("cat-location_display", display); + } EventTracker.update("SEARCH", new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.FINALIZATION + "-" + "bottomline", 0, 0), false, 30000, ProfilingGraph.maxTime); diff --git a/source/de/anomic/data/bookmarksDB.java b/source/de/anomic/data/bookmarksDB.java index a69be7e9b..c2348d7f3 100644 --- a/source/de/anomic/data/bookmarksDB.java +++ b/source/de/anomic/data/bookmarksDB.java @@ -56,8 +56,8 @@ import de.anomic.crawler.CrawlProfile; import de.anomic.crawler.retrieval.Request; import de.anomic.search.Segments; import de.anomic.search.Switchboard; +import de.anomic.yacy.yacyNewsDB; import de.anomic.yacy.yacyNewsPool; -import de.anomic.yacy.yacyNewsRecord; public class bookmarksDB { // ------------------------------------ @@ -291,7 +291,7 @@ public class bookmarksDB { m.remove("generalFilter"); m.remove("specificFilter"); m.put("intention", "Automatic ReCrawl!"); - sb.peers.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.peers.mySeed(), yacyNewsPool.CATEGORY_CRAWL_START, m)); + sb.peers.newsPool.publishMyNews(new yacyNewsDB.Record(sb.peers.mySeed(), yacyNewsPool.CATEGORY_CRAWL_START, m)); } } catch (MalformedURLException e1) {} } // if diff --git a/source/de/anomic/search/Switchboard.java b/source/de/anomic/search/Switchboard.java index 8a44d9184..5b748bda7 100644 --- a/source/de/anomic/search/Switchboard.java +++ b/source/de/anomic/search/Switchboard.java @@ -149,8 +149,8 @@ import de.anomic.tools.CryptoLib; import de.anomic.yacy.yacyBuildProperties; import de.anomic.yacy.yacyClient; import de.anomic.yacy.yacyCore; +import de.anomic.yacy.yacyNewsDB; import de.anomic.yacy.yacyNewsPool; -import de.anomic.yacy.yacyNewsRecord; import de.anomic.yacy.yacySeed; import de.anomic.yacy.Tray; import de.anomic.yacy.yacySeedDB; @@ -1607,7 +1607,7 @@ public final class Switchboard extends serverSwitch { if ((homepage != null) && (homepage.length() > 10)) { final Properties news = new Properties(); news.put("homepage", profile.get("homepage")); - this.peers.newsPool.publishMyNews(yacyNewsRecord.newRecord(peers.mySeed(), yacyNewsPool.CATEGORY_PROFILE_BROADCAST, news)); + this.peers.newsPool.publishMyNews(yacyNewsDB.newRecord(peers.mySeed(), yacyNewsPool.CATEGORY_PROFILE_BROADCAST, news)); } } diff --git a/source/de/anomic/yacy/yacyCore.java b/source/de/anomic/yacy/yacyCore.java index aa007f444..e0f18cf73 100644 --- a/source/de/anomic/yacy/yacyCore.java +++ b/source/de/anomic/yacy/yacyCore.java @@ -346,7 +346,7 @@ public class yacyCore { // include a YaCyNews record to my seed try { - final yacyNewsRecord record = sb.peers.newsPool.myPublication(); + final yacyNewsDB.Record record = sb.peers.newsPool.myPublication(); if (record == null) { sb.peers.mySeed().put("news", ""); } else { diff --git a/source/de/anomic/yacy/yacyNewsDB.java b/source/de/anomic/yacy/yacyNewsDB.java index d685a2c88..12d7b5567 100644 --- a/source/de/anomic/yacy/yacyNewsDB.java +++ b/source/de/anomic/yacy/yacyNewsDB.java @@ -47,13 +47,20 @@ package de.anomic.yacy; import java.io.File; import java.io.IOException; import java.io.UnsupportedEncodingException; +import java.util.Date; +import java.util.HashMap; import java.util.Iterator; +import java.util.Map; +import java.util.Properties; +import java.util.Map.Entry; +import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.index.ObjectIndex; import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; +import net.yacy.kelondro.order.NaturalOrder; import net.yacy.kelondro.table.Table; import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.FileUtils; @@ -66,16 +73,36 @@ public class yacyNewsDB { private final File path; protected ObjectIndex news; + private static final int maxNewsRecordLength = 512; + private static final int categoryStringLength = 8; + public static final int idLength = DateFormatter.PATTERN_SHORT_SECOND.length() + Word.commonHashLength; + + private static final int attributesMaxLength = + maxNewsRecordLength + - idLength + - categoryStringLength + - DateFormatter.PATTERN_SHORT_SECOND.length() + - 2; + + private static final Row rowdef = new Row( + "String idx-" + idLength + " \"id = created + originator\"," + + "String cat-" + categoryStringLength + "," + + "String rec-" + DateFormatter.PATTERN_SHORT_SECOND.length() + "," + + "short dis-2 {b64e}," + + "String att-" + attributesMaxLength, + NaturalOrder.naturalOrder + ); + public yacyNewsDB( final File path, final boolean useTailCache, final boolean exceed134217727) { this.path = path; try { - this.news = new Table(path, yacyNewsRecord.rowdef, 10, 0, useTailCache, exceed134217727); + this.news = new Table(path, rowdef, 10, 0, useTailCache, exceed134217727); } catch (RowSpaceExceededException e) { try { - this.news = new Table(path, yacyNewsRecord.rowdef, 0, 0, false, exceed134217727); + this.news = new Table(path, rowdef, 0, 0, false, exceed134217727); } catch (RowSpaceExceededException e1) { Log.logException(e1); } @@ -86,10 +113,10 @@ public class yacyNewsDB { try {close();} catch (final Exception e) {} if (path.exists()) FileUtils.deletedelete(path); try { - this.news = new Table(path, yacyNewsRecord.rowdef, 10, 0, false, false); + this.news = new Table(path, rowdef, 10, 0, false, false); } catch (RowSpaceExceededException e) { try { - this.news = new Table(path, yacyNewsRecord.rowdef, 0, 0, false, false); + this.news = new Table(path, rowdef, 0, 0, false, false); } catch (RowSpaceExceededException e1) { Log.logException(e1); } @@ -113,7 +140,7 @@ public class yacyNewsDB { news.delete(id.getBytes()); } - public synchronized yacyNewsRecord put(final yacyNewsRecord record) throws IOException, RowSpaceExceededException { + public synchronized Record put(final Record record) throws IOException, RowSpaceExceededException { try { return b2r(news.replace(r2b(record))); } catch (final Exception e) { @@ -122,12 +149,12 @@ public class yacyNewsDB { } } - public synchronized Iterator news() throws IOException { + public synchronized Iterator news() throws IOException { // the iteration iterates yacyNewsRecord - type objects return new recordIterator(); } - public class recordIterator implements Iterator { + public class recordIterator implements Iterator { Iterator rowIterator; @@ -139,17 +166,17 @@ public class yacyNewsDB { return rowIterator.hasNext(); } - public yacyNewsRecord next() { + public Record next() { return b2r(rowIterator.next()); } public void remove() { - throw new UnsupportedOperationException(); + rowIterator.remove(); } } - public synchronized yacyNewsRecord get(final String id) throws IOException { + public synchronized Record get(final String id) throws IOException { try { return b2r(news.get(id.getBytes())); } catch (final kelondroException e) { @@ -158,9 +185,9 @@ public class yacyNewsDB { } } - protected final static yacyNewsRecord b2r(final Row.Entry b) { + protected final static Record b2r(final Row.Entry b) { if (b == null) return null; - return yacyNewsRecord.newRecord( + return new yacyNewsDB.Record( b.getColString(0, null), b.getColString(1, "UTF-8"), (b.empty(2)) ? null : DateFormatter.parseShortSecond(b.getColString(2, null), DateFormatter.UTCDiffString()), @@ -169,11 +196,11 @@ public class yacyNewsDB { ); } - protected final Row.Entry r2b(final yacyNewsRecord r) { + protected final Row.Entry r2b(final Record r) { if (r == null) return null; try { final String attributes = r.attributes().toString(); - if (attributes.length() > yacyNewsRecord.attributesMaxLength) throw new IllegalArgumentException("attribute length=" + attributes.length() + " exceeds maximum size=" + yacyNewsRecord.attributesMaxLength); + if (attributes.length() > attributesMaxLength) throw new IllegalArgumentException("attribute length=" + attributes.length() + " exceeds maximum size=" + attributesMaxLength); final Row.Entry entry = this.news.row().newEntry(); entry.setCol(0, r.id().getBytes()); entry.setCol(1, r.category().getBytes("UTF-8")); @@ -186,5 +213,131 @@ public class yacyNewsDB { return null; } } + + public static Record newRecord(final yacySeed mySeed, final String category, final Properties attributes) { + try { + final HashMap m = new HashMap(); + final Iterator> e = attributes.entrySet().iterator(); + Map.Entry entry; + while (e.hasNext()) { + entry = e.next(); + m.put((String) entry.getKey(), (String) entry.getValue()); + } + return new Record(mySeed, category, m); + } catch (final IllegalArgumentException e) { + yacyCore.log.logWarning("rejected bad yacy news record: " + e.getMessage()); + return null; + } + } + + public static class Record { + + private final String originator; // hash of originating peer + private final Date created; // Date when news was created by originator + private final Date received; // Date when news was received here at this peer + private final String category; // keyword that adresses possible actions + private int distributed; // counter that counts number of distributions of this news record + private final Map attributes; // elemets of the news for a special category + + + public Record(final String newsString) { + this.attributes = MapTools.string2map(newsString, ","); + if (attributes.toString().length() > yacyNewsDB.attributesMaxLength) throw new IllegalArgumentException("attributes length (" + attributes.toString().length() + ") exceeds maximum (" + yacyNewsDB.attributesMaxLength + ")"); + this.category = (attributes.containsKey("cat")) ? attributes.get("cat") : ""; + if (category.length() > yacyNewsDB.categoryStringLength) throw new IllegalArgumentException("category length (" + category.length() + ") exceeds maximum (" + yacyNewsDB.categoryStringLength + ")"); + this.received = (attributes.containsKey("rec")) ? DateFormatter.parseShortSecond(attributes.get("rec"), DateFormatter.UTCDiffString()) : new Date(); + this.created = (attributes.containsKey("cre")) ? DateFormatter.parseShortSecond(attributes.get("cre"), DateFormatter.UTCDiffString()) : new Date(); + this.distributed = (attributes.containsKey("dis")) ? Integer.parseInt(attributes.get("dis")) : 0; + this.originator = (attributes.containsKey("ori")) ? attributes.get("ori") : ""; + removeStandards(); + } + + public Record(final yacySeed mySeed, final String category, final Map attributes) { + if (category.length() > yacyNewsDB.categoryStringLength) throw new IllegalArgumentException("category length (" + category.length() + ") exceeds maximum (" + yacyNewsDB.categoryStringLength + ")"); + if (attributes.toString().length() > yacyNewsDB.attributesMaxLength) throw new IllegalArgumentException("attributes length (" + attributes.toString().length() + ") exceeds maximum (" + yacyNewsDB.attributesMaxLength + ")"); + this.attributes = attributes; + this.received = null; + this.created = new Date(); + this.category = category; + this.distributed = 0; + this.originator = mySeed.hash; + removeStandards(); + } + + protected Record(final String id, final String category, final Date received, final int distributed, final Map attributes) { + if (category.length() > yacyNewsDB.categoryStringLength) throw new IllegalArgumentException("category length (" + category.length() + ") exceeds maximum (" + yacyNewsDB.categoryStringLength + ")"); + if (attributes.toString().length() > yacyNewsDB.attributesMaxLength) throw new IllegalArgumentException("attributes length (" + attributes.toString().length() + ") exceeds maximum (" + yacyNewsDB.attributesMaxLength + ")"); + this.attributes = attributes; + this.received = received; + this.created = DateFormatter.parseShortSecond(id.substring(0, DateFormatter.PATTERN_SHORT_SECOND.length()), DateFormatter.UTCDiffString()); + this.category = category; + this.distributed = distributed; + this.originator = id.substring(DateFormatter.PATTERN_SHORT_SECOND.length()); + removeStandards(); + } + + private void removeStandards() { + attributes.remove("ori"); + attributes.remove("cat"); + attributes.remove("cre"); + attributes.remove("rec"); + attributes.remove("dis"); + } + + public String toString() { + // this creates the string that shall be distributed + // attention: this has no additional encoding + if (this.originator != null) attributes.put("ori", this.originator); + if (this.category != null) attributes.put("cat", this.category); + if (this.created != null) attributes.put("cre", DateFormatter.formatShortSecond(this.created)); + if (this.received != null) attributes.put("rec", DateFormatter.formatShortSecond(this.received)); + attributes.put("dis", Integer.toString(this.distributed)); + final String theString = attributes.toString(); + removeStandards(); + return theString; + } + + public String id() { + return DateFormatter.formatShortSecond(created) + originator; + } + + public String originator() { + return originator; + } + + public Date created() { + return created; + } + + public Date received() { + return received; + } + + public String category() { + return category; + } + + public int distributed() { + return distributed; + } + + public void incDistribution() { + distributed++; + } + + public Map attributes() { + return attributes; + } + + public String attribute(final String key, final String dflt) { + final String s = attributes.get(key); + if ((s == null) || (s.length() == 0)) return dflt; + return s; + } + + public static void main(final String[] args) { + System.out.println((new yacyNewsDB.Record(args[0])).toString()); + } + } } \ No newline at end of file diff --git a/source/de/anomic/yacy/yacyNewsPool.java b/source/de/anomic/yacy/yacyNewsPool.java index 2df801596..c05b0dc11 100644 --- a/source/de/anomic/yacy/yacyNewsPool.java +++ b/source/de/anomic/yacy/yacyNewsPool.java @@ -291,13 +291,13 @@ public class yacyNewsPool { return newsDB.size(); } - public Iterator recordIterator(final int dbKey, final boolean up) { + public Iterator recordIterator(final int dbKey, final boolean up) { // returns an iterator of yacyNewsRecord-type objects final yacyNewsQueue queue = switchQueue(dbKey); return queue.records(up); } - public void publishMyNews(final yacyNewsRecord record) { + public void publishMyNews(final yacyNewsDB.Record record) { // this shall be called if our peer generated a new news record and wants to publish it if (record == null) return; try { @@ -310,23 +310,29 @@ public class yacyNewsPool { } } - public yacyNewsRecord myPublication() throws IOException, RowSpaceExceededException { + public yacyNewsDB.Record myPublication() throws IOException, RowSpaceExceededException { // generate a record for next peer-ping if (outgoingNews.isEmpty()) return null; - final yacyNewsRecord record = outgoingNews.topInc(); - if ((record != null) && (record.distributed() >= maxDistribution)) { + final yacyNewsDB.Record record = outgoingNews.pop(); + if (record == null) return null; + + record.incDistribution(); + + if (record.distributed() >= maxDistribution) { // move record to its final position. This is only for history - publishedNews.push(outgoingNews.pop()); + publishedNews.push(record); + } else { + outgoingNews.push(record); } return record; } - public void enqueueIncomingNews(final yacyNewsRecord record) throws IOException, RowSpaceExceededException { + public void enqueueIncomingNews(final yacyNewsDB.Record record) throws IOException, RowSpaceExceededException { // called if a news is attached to a seed // check consistency if (record.id() == null) return; - if (record.id().length() != yacyNewsRecord.idLength) return; + if (record.id().length() != yacyNewsDB.idLength) return; if (record.category() == null) return; if (!(categories.contains(record.category()))) return; if (record.created().getTime() == 0) return; @@ -359,10 +365,11 @@ public class yacyNewsPool { public int automaticProcess(final yacySeedDB seedDB) throws IOException, InterruptedException, RowSpaceExceededException { // processes news in the incoming-db // returns number of processes - yacyNewsRecord record; + yacyNewsDB.Record record; int pc = 0; synchronized (this.incomingNews) { - final Iterator i = incomingNews.records(true); + final Iterator i = incomingNews.records(true); + HashSet removeIDs = new HashSet(); while (i.hasNext()) { // check for interruption if (Thread.currentThread().isInterrupted()) throw new InterruptedException("Shutdown in progress"); @@ -371,16 +378,17 @@ public class yacyNewsPool { record = i.next(); if (automaticProcessP(seedDB, record)) { this.processedNews.push(record); - i.remove(); + removeIDs.add(record.id()); pc++; } } + for (String id: removeIDs) incomingNews.remove(id); } return pc; } long day = 1000 * 60 * 60 * 24; - private boolean automaticProcessP(final yacySeedDB seedDB, final yacyNewsRecord record) { + private boolean automaticProcessP(final yacySeedDB seedDB, final yacyNewsDB.Record record) { if (record == null) return false; if (record.category() == null) return true; if ((System.currentTimeMillis() - record.created().getTime()) > (14 * day)) { @@ -412,11 +420,11 @@ public class yacyNewsPool { return false; } - public synchronized yacyNewsRecord getSpecific(final int dbKey, final String category, final String key, final String value) { + public synchronized yacyNewsDB.Record getSpecific(final int dbKey, final String category, final String key, final String value) { final yacyNewsQueue queue = switchQueue(dbKey); - yacyNewsRecord record; + yacyNewsDB.Record record; String s; - final Iterator i = queue.records(true); + final Iterator i = queue.records(true); while (i.hasNext()) { record = i.next(); if ((record != null) && (record.category().equals(category))) { @@ -427,10 +435,10 @@ public class yacyNewsPool { return null; } - public synchronized yacyNewsRecord getByOriginator(final int dbKey, final String category, final String originatorHash) { + public synchronized yacyNewsDB.Record getByOriginator(final int dbKey, final String category, final String originatorHash) { final yacyNewsQueue queue = switchQueue(dbKey); - yacyNewsRecord record; - final Iterator i = queue.records(true); + yacyNewsDB.Record record; + final Iterator i = queue.records(true); while (i.hasNext()) { record = i.next(); if ((record != null) && @@ -442,7 +450,7 @@ public class yacyNewsPool { return null; } - public synchronized yacyNewsRecord getByID(final int dbKey, final String id) { + public synchronized yacyNewsDB.Record getByID(final int dbKey, final String id) { switch (dbKey) { case INCOMING_DB: return incomingNews.get(id); case PROCESSED_DB: return processedNews.get(id); @@ -485,7 +493,7 @@ public class yacyNewsPool { private boolean moveOff(final yacyNewsQueue fromqueue, final yacyNewsQueue toqueue, final String id) throws IOException, RowSpaceExceededException { // called if a published news shall be removed - final yacyNewsRecord record = fromqueue.remove(id); + final yacyNewsDB.Record record = fromqueue.remove(id); if (record == null) { return false; } @@ -510,8 +518,8 @@ public class yacyNewsPool { private int moveOffAll(final yacyNewsQueue fromqueue, final yacyNewsQueue toqueue) throws IOException, RowSpaceExceededException { // move off all news from a specific queue to another queue - final Iterator i = fromqueue.records(true); - yacyNewsRecord record; + final Iterator i = fromqueue.records(true); + yacyNewsDB.Record record; if (toqueue == null) return 0; int c = 0; while (i.hasNext()) { diff --git a/source/de/anomic/yacy/yacyNewsQueue.java b/source/de/anomic/yacy/yacyNewsQueue.java index 32258f9af..5348fae04 100644 --- a/source/de/anomic/yacy/yacyNewsQueue.java +++ b/source/de/anomic/yacy/yacyNewsQueue.java @@ -67,7 +67,7 @@ public class yacyNewsQueue { private final yacyNewsDB newsDB; public static final Row rowdef = new Row(new Column[]{ - new Column("newsid", Column.celltype_string, Column.encoder_bytes, yacyNewsRecord.idLength, "id = created + originator"), + new Column("newsid", Column.celltype_string, Column.encoder_bytes, yacyNewsDB.idLength, "id = created + originator"), new Column("last touched", Column.celltype_string, Column.encoder_bytes, DateFormatter.PATTERN_SHORT_SECOND.length(), "") }, NaturalOrder.naturalOrder @@ -83,22 +83,22 @@ public class yacyNewsQueue { this.queueStack = null; } } - - private void resetDB() { - try {close();} catch (final Exception e) {} - if (path.exists()) FileUtils.deletedelete(path); + + public void clear() { try { - this.queueStack = new Table(path, rowdef, 10, 0, false, false); - } catch (RowSpaceExceededException e) { - Log.logException(e); - this.queueStack = null; + this.queueStack.clear(); + } catch (IOException e) { + try {close();} catch (final Exception ee) {} + if (path.exists()) FileUtils.deletedelete(path); + try { + this.queueStack = new Table(path, rowdef, 10, 0, false, false); + } catch (RowSpaceExceededException ee) { + Log.logException(e); + this.queueStack = null; + } } } - public void clear() { - resetDB(); - } - public void close() { if (queueStack != null) queueStack.close(); queueStack = null; @@ -116,28 +116,18 @@ public class yacyNewsQueue { return queueStack.isEmpty(); } - public synchronized void push(final yacyNewsRecord entry) throws IOException, RowSpaceExceededException { + public synchronized void push(final yacyNewsDB.Record entry) throws IOException, RowSpaceExceededException { queueStack.addUnique(r2b(entry, true)); } - public synchronized yacyNewsRecord pop() throws IOException { + public synchronized yacyNewsDB.Record pop() throws IOException { if (queueStack.isEmpty()) return null; return b2r(queueStack.removeOne()); } - public synchronized yacyNewsRecord topInc() throws IOException, RowSpaceExceededException { - if (queueStack.isEmpty()) return null; - final yacyNewsRecord entry = pop(); - if (entry != null) { - entry.incDistribution(); - push(entry); - } - return entry; - } - - public synchronized yacyNewsRecord get(final String id) { - yacyNewsRecord record; - final Iterator i = records(true); + public synchronized yacyNewsDB.Record get(final String id) { + yacyNewsDB.Record record; + final Iterator i = records(true); while (i.hasNext()) { record = i.next(); if ((record != null) && (record.id().equals(id))) return record; @@ -145,9 +135,9 @@ public class yacyNewsQueue { return null; } - public synchronized yacyNewsRecord remove(final String id) { - yacyNewsRecord record; - final Iterator i = records(true); + public synchronized yacyNewsDB.Record remove(final String id) { + yacyNewsDB.Record record; + final Iterator i = records(true); while (i.hasNext()) { record = i.next(); if ((record != null) && (record.id().equals(id))) { @@ -162,19 +152,19 @@ public class yacyNewsQueue { return null; } - yacyNewsRecord b2r(final Row.Entry b) throws IOException { + yacyNewsDB.Record b2r(final Row.Entry b) throws IOException { if (b == null) return null; final String id = b.getColString(0, null); //Date touched = yacyCore.parseUniversalDate(new String(b[1])); return newsDB.get(id); } - private Row.Entry r2b(final yacyNewsRecord r, final boolean updateDB) throws IOException, RowSpaceExceededException { + private Row.Entry r2b(final yacyNewsDB.Record r, final boolean updateDB) throws IOException, RowSpaceExceededException { if (r == null) return null; if (updateDB) { newsDB.put(r); } else { - final yacyNewsRecord r1 = newsDB.get(r.id()); + final yacyNewsDB.Record r1 = newsDB.get(r.id()); if (r1 == null) newsDB.put(r); } final Row.Entry b = queueStack.row().newEntry(new byte[][]{ @@ -183,13 +173,13 @@ public class yacyNewsQueue { return b; } - public Iterator records(final boolean up) { + public Iterator records(final boolean up) { // iterates yacyNewsRecord-type objects - if (queueStack == null) return new HashSet().iterator(); + if (queueStack == null) return new HashSet().iterator(); return new newsIterator(up); } - public class newsIterator implements Iterator { + public class newsIterator implements Iterator { // iterates yacyNewsRecord-type objects Iterator stackNodeIterator; @@ -207,7 +197,7 @@ public class yacyNewsQueue { return stackNodeIterator.hasNext(); } - public yacyNewsRecord next() { + public yacyNewsDB.Record next() { final Row.Entry row = stackNodeIterator.next(); try { return b2r(row); diff --git a/source/de/anomic/yacy/yacyNewsRecord.java b/source/de/anomic/yacy/yacyNewsRecord.java deleted file mode 100644 index 519981b71..000000000 --- a/source/de/anomic/yacy/yacyNewsRecord.java +++ /dev/null @@ -1,230 +0,0 @@ -// yacyNewsRecord.java -// ----------------------- -// part of YaCy -// (C) by Michael Peter Christen; mc@yacy.net -// first published on http://www.anomic.de -// Frankfurt, Germany, 2005 -// -// $LastChangedDate$ -// $LastChangedRevision$ -// $LastChangedBy$ -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -// -// Using this software in any meaning (reading, learning, copying, compiling, -// running) means that you agree that the Author(s) is (are) not responsible -// for cost, loss of data or any harm that may be caused directly or indirectly -// by usage of this softare or this documentation. The usage of this software -// is on your own risk. The installation and usage (starting/running) of this -// software may allow other people or application to access your computer and -// any attached devices and is highly dependent on the configuration of the -// software which must be done by the user of the software; the author(s) is -// (are) also not responsible for proper configuration and usage of the -// software, even if provoked by documentation provided together with -// the software. -// -// Any changes to this file according to the GPL as documented in the file -// gpl.txt aside this file in the shipment you received can be done to the -// lines that follows this copyright notice here, but changes must not be -// done inside the copyright notice above. A re-distribution must contain -// the intact and unchanged copyright notice. -// Contributions and changes to the program code must be marked as such. - -package de.anomic.yacy; - -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.Properties; -import java.util.Map.Entry; - -import net.yacy.kelondro.data.word.Word; -import net.yacy.kelondro.index.Row; -import net.yacy.kelondro.order.NaturalOrder; -import net.yacy.kelondro.util.DateFormatter; -import net.yacy.kelondro.util.MapTools; - - -public class yacyNewsRecord { - - public static final int maxNewsRecordLength = 512; - public static final int categoryStringLength = 8; - public static final int idLength = DateFormatter.PATTERN_SHORT_SECOND.length() + Word.commonHashLength; - - private final String originator; // hash of originating peer - private final Date created; // Date when news was created by originator - private final Date received; // Date when news was received here at this peer - private final String category; // keyword that adresses possible actions - private int distributed; // counter that counts number of distributions of this news record - private final Map attributes; // elemets of the news for a special category - - public static final int attributesMaxLength = maxNewsRecordLength - - idLength - - categoryStringLength - - DateFormatter.PATTERN_SHORT_SECOND.length() - - 2; - - public static final Row rowdef = new Row( - "String idx-" + idLength + " \"id = created + originator\"," + - "String cat-" + categoryStringLength + "," + - "String rec-" + DateFormatter.PATTERN_SHORT_SECOND.length() + "," + - "short dis-2 {b64e}," + - "String att-" + attributesMaxLength, - NaturalOrder.naturalOrder - ); - - public static yacyNewsRecord newRecord(final String newsString) { - try { - return new yacyNewsRecord(newsString); - } catch (final IllegalArgumentException e) { - yacyCore.log.logWarning("rejected bad yacy news record: " + e.getMessage()); - return null; - } - } - - public static yacyNewsRecord newRecord(final yacySeed mySeed, final String category, final Properties attributes) { - try { - final HashMap m = new HashMap(); - final Iterator> e = attributes.entrySet().iterator(); - Map.Entry entry; - while (e.hasNext()) { - entry = e.next(); - m.put((String) entry.getKey(), (String) entry.getValue()); - } - return new yacyNewsRecord(mySeed, category, m); - } catch (final IllegalArgumentException e) { - yacyCore.log.logWarning("rejected bad yacy news record: " + e.getMessage()); - return null; - } - } - - public static yacyNewsRecord newRecord(final yacySeed mySeed, final String category, final Map attributes) { - try { - return new yacyNewsRecord(mySeed, category, attributes); - } catch (final IllegalArgumentException e) { - yacyCore.log.logWarning("rejected bad yacy news record: " + e.getMessage()); - return null; - } - } - - public static yacyNewsRecord newRecord(final String id, final String category, final Date received, final int distributed, final Map attributes) { - try { - return new yacyNewsRecord(id, category, received, distributed, attributes); - } catch (final IllegalArgumentException e) { - yacyCore.log.logWarning("rejected bad yacy news record: " + e.getMessage()); - return null; - } - } - - public yacyNewsRecord(final String newsString) { - this.attributes = MapTools.string2map(newsString, ","); - if (attributes.toString().length() > attributesMaxLength) throw new IllegalArgumentException("attributes length (" + attributes.toString().length() + ") exceeds maximum (" + attributesMaxLength + ")"); - this.category = (attributes.containsKey("cat")) ? attributes.get("cat") : ""; - if (category.length() > categoryStringLength) throw new IllegalArgumentException("category length (" + category.length() + ") exceeds maximum (" + categoryStringLength + ")"); - this.received = (attributes.containsKey("rec")) ? DateFormatter.parseShortSecond(attributes.get("rec"), DateFormatter.UTCDiffString()) : new Date(); - this.created = (attributes.containsKey("cre")) ? DateFormatter.parseShortSecond(attributes.get("cre"), DateFormatter.UTCDiffString()) : new Date(); - this.distributed = (attributes.containsKey("dis")) ? Integer.parseInt(attributes.get("dis")) : 0; - this.originator = (attributes.containsKey("ori")) ? attributes.get("ori") : ""; - removeStandards(); - } - - public yacyNewsRecord(final yacySeed mySeed, final String category, final Map attributes) { - if (category.length() > categoryStringLength) throw new IllegalArgumentException("category length (" + category.length() + ") exceeds maximum (" + categoryStringLength + ")"); - if (attributes.toString().length() > attributesMaxLength) throw new IllegalArgumentException("attributes length (" + attributes.toString().length() + ") exceeds maximum (" + attributesMaxLength + ")"); - this.attributes = attributes; - this.received = null; - this.created = new Date(); - this.category = category; - this.distributed = 0; - this.originator = mySeed.hash; - removeStandards(); - } - - protected yacyNewsRecord(final String id, final String category, final Date received, final int distributed, final Map attributes) { - if (category.length() > categoryStringLength) throw new IllegalArgumentException("category length (" + category.length() + ") exceeds maximum (" + categoryStringLength + ")"); - if (attributes.toString().length() > attributesMaxLength) throw new IllegalArgumentException("attributes length (" + attributes.toString().length() + ") exceeds maximum (" + attributesMaxLength + ")"); - this.attributes = attributes; - this.received = received; - this.created = DateFormatter.parseShortSecond(id.substring(0, DateFormatter.PATTERN_SHORT_SECOND.length()), DateFormatter.UTCDiffString()); - this.category = category; - this.distributed = distributed; - this.originator = id.substring(DateFormatter.PATTERN_SHORT_SECOND.length()); - removeStandards(); - } - - private void removeStandards() { - attributes.remove("ori"); - attributes.remove("cat"); - attributes.remove("cre"); - attributes.remove("rec"); - attributes.remove("dis"); - } - - public String toString() { - // this creates the string that shall be distributed - // attention: this has no additional encoding - if (this.originator != null) attributes.put("ori", this.originator); - if (this.category != null) attributes.put("cat", this.category); - if (this.created != null) attributes.put("cre", DateFormatter.formatShortSecond(this.created)); - if (this.received != null) attributes.put("rec", DateFormatter.formatShortSecond(this.received)); - attributes.put("dis", Integer.toString(this.distributed)); - final String theString = attributes.toString(); - removeStandards(); - return theString; - } - - public String id() { - return DateFormatter.formatShortSecond(created) + originator; - } - - public String originator() { - return originator; - } - - public Date created() { - return created; - } - - public Date received() { - return received; - } - - public String category() { - return category; - } - - public int distributed() { - return distributed; - } - - public void incDistribution() { - distributed++; - } - - public Map attributes() { - return attributes; - } - - public String attribute(final String key, final String dflt) { - final String s = attributes.get(key); - if ((s == null) || (s.length() == 0)) return dflt; - return s; - } - - public static void main(final String[] args) { - System.out.println((newRecord(args[0])).toString()); - } -} diff --git a/source/de/anomic/yacy/yacyPeerActions.java b/source/de/anomic/yacy/yacyPeerActions.java index abf562844..ae7866b38 100644 --- a/source/de/anomic/yacy/yacyPeerActions.java +++ b/source/de/anomic/yacy/yacyPeerActions.java @@ -227,7 +227,7 @@ public class yacyPeerActions { //System.out.println("### triggered news arrival from peer " + peer.getName() + ", news " + ((recordString == null) ? "empty" : "attached")); if ((recordString == null) || (recordString.length() == 0)) return; final String decodedString = de.anomic.tools.crypt.simpleDecode(recordString, ""); - final yacyNewsRecord record = yacyNewsRecord.newRecord(decodedString); + final yacyNewsDB.Record record = new yacyNewsDB.Record(decodedString); if (record != null) { //System.out.println("### news arrival from peer " + peer.getName() + ", decoded=" + decodedString + ", record=" + recordString + ", news=" + record.toString()); final String cre1 = MapTools.string2map(decodedString, ",").get("cre"); diff --git a/source/net/yacy/document/geolocalization/GeonamesLocalization.java b/source/net/yacy/document/geolocalization/GeonamesLocalization.java index 3a9bd98de..887046caf 100644 --- a/source/net/yacy/document/geolocalization/GeonamesLocalization.java +++ b/source/net/yacy/document/geolocalization/GeonamesLocalization.java @@ -124,6 +124,10 @@ public class GeonamesLocalization implements Localization { Log.logException(e); } } + + public int locations() { + return id2loc.size(); + } public Set find(String anyname, boolean locationexact) { HashSet r = new HashSet(); diff --git a/source/net/yacy/document/geolocalization/Localization.java b/source/net/yacy/document/geolocalization/Localization.java index d9076980b..eaccacf81 100644 --- a/source/net/yacy/document/geolocalization/Localization.java +++ b/source/net/yacy/document/geolocalization/Localization.java @@ -32,6 +32,12 @@ import java.util.Set; */ public interface Localization { + /** + * the number of locations that this localization stores + * @return the number of locations + */ + public int locations(); + /** * find a location by name * @param anyname - a name of a location diff --git a/source/net/yacy/document/geolocalization/OpenGeoDBLocalization.java b/source/net/yacy/document/geolocalization/OpenGeoDBLocalization.java index 51fb2ca2d..5568c3a35 100644 --- a/source/net/yacy/document/geolocalization/OpenGeoDBLocalization.java +++ b/source/net/yacy/document/geolocalization/OpenGeoDBLocalization.java @@ -168,6 +168,10 @@ public class OpenGeoDBLocalization implements Localization { s = s.substring(1, s.length() - 1); return s; } + + public int locations() { + return id2loc.size(); + } /** * check database tables against occurrences of this entity diff --git a/source/net/yacy/document/geolocalization/OverarchingLocalization.java b/source/net/yacy/document/geolocalization/OverarchingLocalization.java index ed5566b36..060b09d5a 100644 --- a/source/net/yacy/document/geolocalization/OverarchingLocalization.java +++ b/source/net/yacy/document/geolocalization/OverarchingLocalization.java @@ -54,6 +54,14 @@ public class OverarchingLocalization implements Localization { public void removeLocalization(String nickname) { this.services.remove(nickname); } + + public int locations() { + int locations = 0; + for (Localization service: this.services.values()) { + locations += service.locations(); + } + return locations; + } /** * find (a set of) locations diff --git a/source/net/yacy/kelondro/table/ChunkIterator.java b/source/net/yacy/kelondro/table/ChunkIterator.java index 058b63c4c..d08558c94 100644 --- a/source/net/yacy/kelondro/table/ChunkIterator.java +++ b/source/net/yacy/kelondro/table/ChunkIterator.java @@ -99,6 +99,6 @@ public class ChunkIterator implements Iterator { } public void remove() { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException("no remove in ChunkIterator possible"); } } diff --git a/source/net/yacy/kelondro/table/Table.java b/source/net/yacy/kelondro/table/Table.java index 0036e1fe8..69555c8dd 100644 --- a/source/net/yacy/kelondro/table/Table.java +++ b/source/net/yacy/kelondro/table/Table.java @@ -814,7 +814,7 @@ public class Table implements ObjectIndex, Iterable { } public void remove() { - throw new UnsupportedOperationException("no remove in row iterator"); + ri.remove(); } } @@ -871,7 +871,7 @@ public class Table implements ObjectIndex, Iterable { } public void remove() { - throw new UnsupportedOperationException("no remove in TABLE"); + throw new UnsupportedOperationException("no remove in Table.rowIterator"); } }