boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- more dublin core naming of page metadata

Browse Source 
- better presentation of result counters in search results

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4420 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent fa3b8f0ae1
commit a8a5df4a51
21 changed files with 155 additions and 125 deletions
Show Stats Download Patch File Download Diff File

8
htroot/Bookmarks.java
Unescape View File

@ -203,10 +203,10 @@ public class Bookmarks {
document = plasmaSnippetCache.retrieveDocument(comp.url(), true, 5000, true); document = plasmaSnippetCache.retrieveDocument(comp.url(), true, 5000, true);
prop.put("mode_edit", "0"); // create mode prop.put("mode_edit", "0"); // create mode
prop.put("mode_url", comp.url().toNormalform(false, true)); prop.put("mode_url", comp.url().toNormalform(false, true));
prop.putHTML("mode_title", comp.title()); prop.putHTML("mode_title", comp.dc_title());
prop.putHTML("mode_description", (document == null) ? comp.title(): document.dc_title()); prop.putHTML("mode_description", (document == null) ? comp.dc_title(): document.dc_title());
prop.putHTML("mode_author", comp.author()); prop.putHTML("mode_author", comp.dc_creator());
prop.putHTML("mode_tags", (document == null) ? comp.tags() : document.dc_subject(',')); prop.putHTML("mode_tags", (document == null) ? comp.dc_subject() : document.dc_subject(','));
prop.putHTML("mode_path",""); prop.putHTML("mode_path","");
prop.put("mode_public", "0"); prop.put("mode_public", "0");
prop.put("mode_feed", "0"); //TODO: check if it IS a feed prop.put("mode_feed", "0"); //TODO: check if it IS a feed

4
htroot/CrawlResults.java
Unescape View File

@ -217,11 +217,11 @@ public class CrawlResults {
} else { } else {
prop.put("table_indexed_" + cnt + "_showTitle_available", "1"); prop.put("table_indexed_" + cnt + "_showTitle_available", "1");
if (comp.title() == null || comp.title().trim().length() == 0) if (comp.dc_title() == null || comp.dc_title().trim().length() == 0)
prop.put("table_indexed_" + cnt + "_showTitle_available_nodescr", "0"); prop.put("table_indexed_" + cnt + "_showTitle_available_nodescr", "0");
else else
prop.put("table_indexed_" + cnt + "_showTitle_available_nodescr", "1"); prop.put("table_indexed_" + cnt + "_showTitle_available_nodescr", "1");
prop.putHTML("table_indexed_" + cnt + "_showTitle_available_nodescr_urldescr", comp.title()); prop.putHTML("table_indexed_" + cnt + "_showTitle_available_nodescr_urldescr", comp.dc_title());
prop.put("table_indexed_" + cnt + "_showTitle_available_cachepath", cachepath); prop.put("table_indexed_" + cnt + "_showTitle_available_cachepath", cachepath);
prop.putHTML("table_indexed_" + cnt + "_showTitle_available_urltitle", urlstr); prop.putHTML("table_indexed_" + cnt + "_showTitle_available_urltitle", urlstr);

6
htroot/IndexControlRWIs_p.java
Unescape View File

@ -89,7 +89,7 @@ public class IndexControlRWIs_p {
if (post.containsKey("keystringsearch")) { if (post.containsKey("keystringsearch")) {
keyhash = plasmaCondenser.word2hash(keystring); keyhash = plasmaCondenser.word2hash(keystring);
prop.put("keyhash", keyhash); prop.put("keyhash", keyhash);
final plasmaSearchRankingProcess ranking = plasmaSearchAPI.genSearchresult(prop, sb, keyhash, null, sortorder, false); final plasmaSearchRankingProcess ranking = plasmaSearchAPI.genSearchresult(prop, sb, keyhash, null, sortorder);
if (ranking.filteredCount() == 0) { if (ranking.filteredCount() == 0) {
prop.put("searchresult", 1); prop.put("searchresult", 1);
prop.put("searchresult_word", keystring); prop.put("searchresult_word", keystring);
@ -100,7 +100,7 @@ public class IndexControlRWIs_p {
if (keystring.length() == 0 || !plasmaCondenser.word2hash(keystring).equals(keyhash)) { if (keystring.length() == 0 || !plasmaCondenser.word2hash(keystring).equals(keyhash)) {
prop.put("keystring", "<not possible to compute word from hash>"); prop.put("keystring", "<not possible to compute word from hash>");
} }
final plasmaSearchRankingProcess ranking = plasmaSearchAPI.genSearchresult(prop, sb, keyhash, null, sortorder, false); final plasmaSearchRankingProcess ranking = plasmaSearchAPI.genSearchresult(prop, sb, keyhash, null, sortorder);
if (ranking.filteredCount() == 0) { if (ranking.filteredCount() == 0) {
prop.put("searchresult", 2); prop.put("searchresult", 2);
prop.put("searchresult_wordhash", keyhash); prop.put("searchresult_wordhash", keyhash);
@ -159,7 +159,7 @@ public class IndexControlRWIs_p {
} }
kelondroBitfield flags = plasmaSearchAPI.compileFlags(post); kelondroBitfield flags = plasmaSearchAPI.compileFlags(post);
int count = (post.get("lines", "all").equals("all")) ? -1 : post.getInt("lines", -1); int count = (post.get("lines", "all").equals("all")) ? -1 : post.getInt("lines", -1);
final plasmaSearchRankingProcess ranking = plasmaSearchAPI.genSearchresult(prop, sb, keyhash, flags, sortorder, true); final plasmaSearchRankingProcess ranking = plasmaSearchAPI.genSearchresult(prop, sb, keyhash, flags, sortorder);
plasmaSearchAPI.genURLList(prop, keyhash, keystring, ranking, flags, count, sortorder); plasmaSearchAPI.genURLList(prop, keyhash, keystring, ranking, flags, count, sortorder);
} }

2
htroot/IndexControlURLs_p.java
Unescape View File

@ -241,7 +241,7 @@ public class IndexControlURLs_p {
prop.put("genUrlProfile", "2"); prop.put("genUrlProfile", "2");
prop.putHTML("genUrlProfile_urlNormalform", comp.url().toNormalform(false, true)); prop.putHTML("genUrlProfile_urlNormalform", comp.url().toNormalform(false, true));
prop.put("genUrlProfile_urlhash", urlhash); prop.put("genUrlProfile_urlhash", urlhash);
prop.put("genUrlProfile_urlDescr", comp.title()); prop.put("genUrlProfile_urlDescr", comp.dc_title());
prop.put("genUrlProfile_moddate", entry.moddate().toString()); prop.put("genUrlProfile_moddate", entry.moddate().toString());
prop.put("genUrlProfile_loaddate", entry.loaddate().toString()); prop.put("genUrlProfile_loaddate", entry.loaddate().toString());
prop.put("genUrlProfile_referrer", (le == null) ? 0 : 1); prop.put("genUrlProfile_referrer", (le == null) ? 0 : 1);

2
htroot/ViewFile.java
Unescape View File

@ -124,7 +124,7 @@ public class ViewFile {
return prop; return prop;
} }
url = comp.url(); url = comp.url();
descr = comp.title(); descr = comp.dc_title();
urlEntry.wordCount(); urlEntry.wordCount();
size = urlEntry.size(); size = urlEntry.size();
pre = urlEntry.flags().get(plasmaCondenser.flag_cat_indexof); pre = urlEntry.flags().get(plasmaCondenser.flag_cat_indexof);

11
htroot/js/yacysearch.js
Unescape View File

@ -70,9 +70,12 @@ function addHover() {
} }
} }
function statistics(offset, items, global, total) { function statistics(offset, itemscount, totalcount, localResourceSize, remoteResourceSize, remoteIndexCount, remotePeerCount) {
document.getElementById("resultsOffset").firstChild.nodeValue = offset; document.getElementById("resultsOffset").firstChild.nodeValue = offset;
document.getElementById("itemscount").firstChild.nodeValue = items; document.getElementById("itemscount").firstChild.nodeValue = itemscount;
document.getElementById("globalcount").firstChild.nodeValue = global; document.getElementById("totalcount").firstChild.nodeValue = totalcount;
document.getElementById("totalcount").firstChild.nodeValue = total; document.getElementById("localResourceSize").firstChild.nodeValue = localResourceSize;
document.getElementById("remoteResourceSize").firstChild.nodeValue = remoteResourceSize;
document.getElementById("remoteIndexCount").firstChild.nodeValue = remoteIndexCount;
document.getElementById("remotePeerCount").firstChild.nodeValue = remotePeerCount;
} }

6
htroot/yacy/search.java
Unescape View File

@ -183,7 +183,7 @@ public final class search {
snippetComputationAllTime = theSearch.getSnippetComputationTime(); snippetComputationAllTime = theSearch.getSnippetComputationTime();
// set statistic details of search result and find best result index set // set statistic details of search result and find best result index set
if (theSearch.getLocalCount() == 0) { if (theSearch.getRankingResult().getLocalResourceSize() == 0) {
prop.put("indexcount", ""); prop.put("indexcount", "");
prop.put("joincount", "0"); prop.put("joincount", "0");
} else { } else {
@ -207,11 +207,11 @@ public final class search {
} }
prop.put("indexcount", indexcount.toString()); prop.put("indexcount", indexcount.toString());
if (theSearch.getLocalCount() == 0) { if (theSearch.getRankingResult().getLocalResourceSize() == 0) {
joincount = 0; joincount = 0;
prop.put("joincount", "0"); prop.put("joincount", "0");
} else { } else {
joincount = theSearch.getLocalCount(); joincount = theSearch.getRankingResult().getLocalResourceSize();
prop.put("joincount", Integer.toString(joincount)); prop.put("joincount", Integer.toString(joincount));
accu = theSearch.completeResults(duetime); accu = theSearch.completeResults(duetime);
} }

6
htroot/yacy/urls.java
Unescape View File

@ -104,11 +104,11 @@ public class urls {
referrer = sb.getURL(entry.referrerHash()); referrer = sb.getURL(entry.referrerHash());
// create RSS entry // create RSS entry
comp = entry.comp(); comp = entry.comp();
prop.put("item_" + c + "_title", comp.title()); prop.put("item_" + c + "_title", comp.dc_title());
prop.putHTML("item_" + c + "_link", comp.url().toNormalform(true, false)); prop.putHTML("item_" + c + "_link", comp.url().toNormalform(true, false));
prop.putHTML("item_" + c + "_referrer", (referrer == null) ? "" : referrer.toNormalform(true, false)); prop.putHTML("item_" + c + "_referrer", (referrer == null) ? "" : referrer.toNormalform(true, false));
prop.putHTML("item_" + c + "_description", comp.title()); prop.putHTML("item_" + c + "_description", comp.dc_title());
prop.put("item_" + c + "_author", comp.author()); prop.put("item_" + c + "_author", comp.dc_creator());
prop.put("item_" + c + "_pubDate", serverDate.formatShortSecond(entry.moddate())); prop.put("item_" + c + "_pubDate", serverDate.formatShortSecond(entry.moddate()));
prop.put("item_" + c + "_guid", entry.hash()); prop.put("item_" + c + "_guid", entry.hash());
c++; c++;

2
htroot/yacysearch.html
Unescape View File

@ -99,7 +99,7 @@ document.getElementById("Enter").value = "search again - catch up more links";
<p>No Results. (length of search words must be at least 3 characters)</p> <p>No Results. (length of search words must be at least 3 characters)</p>
:: ::
<div id="results"></div> <div id="results"></div>
<span id="resCounter" style="display: inline;"><strong id="resultsOffset">#[offset]#</strong>-<strong id="itemscount">#[linkcount]#</strong> results from a total number of <strong id="totalcount">#[totalcount]#</strong> known#(globalresults)#.::, <strong id="globalcount">#[globalcount]#</strong> links from other YaCy peers.#(/globalresults)#</span> <span id="resCounter" style="display: inline;"><strong id="resultsOffset">#[offset]#</strong>-<strong id="itemscount">#[itemscount]#</strong> results from a total number of <strong id="totalcount">#[totalcount]#</strong> known#(globalresults)#.:: (<strong id="localResourceSize">#[localResourceSize]#</strong> local, <strong id="remoteResourceSize">#[remoteResourceSize]#</strong> remote), <strong id="remoteIndexCount">#[remoteIndexCount]#</strong> links from <strong id="remotePeerCount">#[remotePeerCount]#</strong> other YaCy peers.#(/globalresults)#</span>
<span id="resNav" style="display: inline;">#[resnav]#</span> <span id="resNav" style="display: inline;">#[resnav]#</span>
:: ::
<p>Searching the web with this peer is disabled for unauthorized users. Please <a href="Status.html?login=">log in</a> as administrator to use the search function</p> <p>Searching the web with this peer is disabled for unauthorized users. Please <a href="Status.html?login=">log in</a> as administrator to use the search function</p>

23
htroot/yacysearch.java
Unescape View File

@ -240,8 +240,8 @@ public class yacysearch {
// create a news message // create a news message
HashMap<String, String> map = new HashMap<String, String>(); HashMap<String, String> map = new HashMap<String, String>();
map.put("url", comp.url().toNormalform(false, true).replace(',', '|')); map.put("url", comp.url().toNormalform(false, true).replace(',', '|'));
map.put("title", comp.title().replace(',', ' ')); map.put("title", comp.dc_title().replace(',', ' '));
map.put("description", ((document == null) ? comp.title() : document.dc_title()).replace(',', ' ')); map.put("description", ((document == null) ? comp.dc_title() : document.dc_title()).replace(',', ' '));
map.put("author", ((document == null) ? "" : document.dc_creator())); map.put("author", ((document == null) ? "" : document.dc_creator()));
map.put("tags", ((document == null) ? "" : document.dc_subject(' '))); map.put("tags", ((document == null) ? "" : document.dc_subject(' ')));
yacyCore.newsPool.publishMyNews(yacyNewsRecord.newRecord(yacyNewsPool.CATEGORY_SURFTIPP_ADD, map)); yacyCore.newsPool.publishMyNews(yacyNewsRecord.newRecord(yacyNewsPool.CATEGORY_SURFTIPP_ADD, map));
@ -306,12 +306,12 @@ public class yacysearch {
// log // log
serverLog.logInfo("LOCAL_SEARCH", "EXIT WORD SEARCH: " + theQuery.queryString + " - " + serverLog.logInfo("LOCAL_SEARCH", "EXIT WORD SEARCH: " + theQuery.queryString + " - " +
(theSearch.getLocalCount() + theSearch.getGlobalCount()) + " links found, " + (theSearch.getRankingResult().getLocalResourceSize() + theSearch.getRankingResult().getRemoteResourceSize()) + " links found, " +
((System.currentTimeMillis() - timestamp) / 1000) + " seconds"); ((System.currentTimeMillis() - timestamp) / 1000) + " seconds");
// prepare search statistics // prepare search statistics
Long trackerHandle = new Long(System.currentTimeMillis()); Long trackerHandle = new Long(System.currentTimeMillis());
HashMap<String, Object> searchProfile = theQuery.resultProfile(theSearch.getLocalCount() + theSearch.getGlobalCount(), System.currentTimeMillis() - timestamp, theSearch.getURLRetrievalTime(), theSearch.getSnippetComputationTime()); HashMap<String, Object> searchProfile = theQuery.resultProfile(theSearch.getRankingResult().getLocalResourceSize() + theSearch.getRankingResult().getRemoteResourceSize(), System.currentTimeMillis() - timestamp, theSearch.getURLRetrievalTime(), theSearch.getSnippetComputationTime());
searchProfile.put("querystring", theQuery.queryString); searchProfile.put("querystring", theQuery.queryString);
searchProfile.put("time", trackerHandle); searchProfile.put("time", trackerHandle);
searchProfile.put("host", client); searchProfile.put("host", client);
@ -323,13 +323,16 @@ public class yacysearch {
sb.localSearchTracker.put(client, handles); sb.localSearchTracker.put(client, handles);
prop = new serverObjects(); prop = new serverObjects();
prop.put("num-results_totalcount", yFormatter.number(theSearch.getLocalCount() + theSearch.getGlobalCount(), !rss));
prop.put("num-results_globalresults", "1");
prop.put("num-results_globalresults_globalcount", yFormatter.number(theSearch.getGlobalCount(), !rss));
prop.put("num-results_offset", offset); prop.put("num-results_offset", offset);
prop.put("num-results_linkcount", "0"); prop.put("num-results_itemscount", "0");
prop.put("num-results_itemsPerPage", itemsPerPage); prop.put("num-results_itemsPerPage", itemsPerPage);
prop.put("num-results_totalcount", yFormatter.number(theSearch.getRankingResult().getLocalResourceSize() + theSearch.getRankingResult().getRemoteResourceSize(), !rss));
prop.put("num-results_globalresults", (globalsearch) ? "1" : "0");
prop.put("num-results_globalresults_localResourceSize", yFormatter.number(theSearch.getRankingResult().getLocalResourceSize(), !rss));
prop.put("num-results_globalresults_remoteResourceSize", yFormatter.number(theSearch.getRankingResult().getRemoteResourceSize(), !rss));
prop.put("num-results_globalresults_remoteIndexCount", yFormatter.number(theSearch.getRankingResult().getRemoteIndexCount(), !rss));
prop.put("num-results_globalresults_remotePeerCount", yFormatter.number(theSearch.getRankingResult().getRemotePeerCount(), !rss));
// compose page navigation // compose page navigation
StringBuffer resnav = new StringBuffer(); StringBuffer resnav = new StringBuffer();
int thispage = offset / theQuery.displayResults(); int thispage = offset / theQuery.displayResults();
@ -337,7 +340,7 @@ public class yacysearch {
resnav.append(navurla(thispage - 1, display, theQuery)); resnav.append(navurla(thispage - 1, display, theQuery));
resnav.append("<strong>&lt;</strong></a>&nbsp;"); resnav.append("<strong>&lt;</strong></a>&nbsp;");
} }
int numberofpages = Math.min(10, Math.min(thispage + 2, (theSearch.getGlobalCount() + theSearch.getLocalCount()) / theQuery.displayResults())); int numberofpages = Math.min(10, Math.min(thispage + 2, (theSearch.getRankingResult().getRemoteResourceSize() + theSearch.getRankingResult().getLocalResourceSize()) / theQuery.displayResults()));
for (int i = 0; i < numberofpages; i++) { for (int i = 0; i < numberofpages; i++) {
if (i == thispage) { if (i == thispage) {
resnav.append("<strong>"); resnav.append("<strong>");

2
htroot/yacysearchitem.html
Unescape View File

@ -62,7 +62,7 @@
#(/rssreferences)# #(/rssreferences)#
#(dynamic)#:: #(dynamic)#::
<script type="text/javascript"> <script type="text/javascript">
statistics("#[offset]#", "#[items]#", "#[global]#", "#[total]#"); statistics("#[offset]#", "#[itemscount]#", "#[totalcount]#", "#[localResourceSize]#", "#[remoteResourceSize]#", "#[remoteIndexCount]#", "#[remotePeerCount]#");
progressbar.step(1); progressbar.step(1);
</script> </script>
#(/dynamic)# #(/dynamic)#

10
htroot/yacysearchitem.java
Unescape View File

@ -44,6 +44,7 @@ import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
import de.anomic.tools.crypt; import de.anomic.tools.crypt;
import de.anomic.tools.nxTools; import de.anomic.tools.nxTools;
import de.anomic.tools.yFormatter;
import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNewsPool; import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacySeed; import de.anomic.yacy.yacySeed;
@ -85,9 +86,12 @@ public class yacysearchitem {
// dynamically update count values // dynamically update count values
if (!rss) { if (!rss) {
prop.put("dynamic_offset", theQuery.neededResults() - theQuery.displayResults() + 1); prop.put("dynamic_offset", theQuery.neededResults() - theQuery.displayResults() + 1);
prop.put("dynamic_global", theSearch.getGlobalCount()); prop.put("dynamic_itemscount", (item < 0) ? theQuery.neededResults() : item + 1);
prop.put("dynamic_total", theSearch.getGlobalCount() + theSearch.getLocalCount()); prop.put("dynamic_totalcount", yFormatter.number(theSearch.getRankingResult().getLocalResourceSize() + theSearch.getRankingResult().getRemoteResourceSize(), !rss));
prop.put("dynamic_items", (item < 0) ? theQuery.neededResults() : item + 1); prop.put("dynamic_localResourceSize", yFormatter.number(theSearch.getRankingResult().getLocalResourceSize(), !rss));
prop.put("dynamic_remoteResourceSize", yFormatter.number(theSearch.getRankingResult().getRemoteResourceSize(), !rss));
prop.put("dynamic_remoteIndexCount", yFormatter.number(theSearch.getRankingResult().getRemoteIndexCount(), !rss));
prop.put("dynamic_remotePeerCount", yFormatter.number(theSearch.getRankingResult().getRemotePeerCount(), !rss));
prop.put("dynamic", "1"); prop.put("dynamic", "1");
} }

48
source/de/anomic/index/indexURLEntry.java
Unescape View File

@ -120,9 +120,9 @@ public class indexURLEntry {
public indexURLEntry( public indexURLEntry(
yacyURL url, yacyURL url,
String descr, String dc_title,
String author, String dc_creator,
String tags, String dc_subject,
String ETag, String ETag,
Date mod, Date mod,
Date load, Date load,
@ -143,7 +143,7 @@ public class indexURLEntry {
// create new entry and store it into database // create new entry and store it into database
this.entry = rowdef.newEntry(); this.entry = rowdef.newEntry();
this.entry.setCol(col_hash, url.hash(), null); this.entry.setCol(col_hash, url.hash(), null);
this.entry.setCol(col_comp, encodeComp(url, descr, author, tags, ETag)); this.entry.setCol(col_comp, encodeComp(url, dc_title, dc_creator, dc_subject, ETag));
encodeDate(col_mod, mod); encodeDate(col_mod, mod);
encodeDate(col_load, load); encodeDate(col_load, load);
encodeDate(col_fresh, fresh); encodeDate(col_fresh, fresh);
@ -175,12 +175,12 @@ public class indexURLEntry {
return new Date(86400000 * this.entry.getColLong(col)); return new Date(86400000 * this.entry.getColLong(col));
} }
public static byte[] encodeComp(yacyURL url, String descr, String author, String tags, String ETag) { public static byte[] encodeComp(yacyURL url, String dc_title, String dc_creator, String dc_subject, String ETag) {
serverCharBuffer s = new serverCharBuffer(200); serverCharBuffer s = new serverCharBuffer(200);
s.append(url.toNormalform(false, true)).append(10); s.append(url.toNormalform(false, true)).append(10);
s.append(descr).append(10); s.append(dc_title).append(10);
s.append(author).append(10); s.append(dc_creator).append(10);
s.append(tags).append(10); s.append(dc_subject).append(10);
s.append(ETag).append(10); s.append(ETag).append(10);
return s.toString().getBytes(); return s.toString().getBytes();
} }
@ -203,13 +203,13 @@ public class indexURLEntry {
url = null; url = null;
} }
String descr = crypt.simpleDecode(prop.getProperty("descr", ""), null); if (descr == null) descr = ""; String descr = crypt.simpleDecode(prop.getProperty("descr", ""), null); if (descr == null) descr = "";
String author = crypt.simpleDecode(prop.getProperty("author", ""), null); if (author == null) author = ""; String dc_creator = crypt.simpleDecode(prop.getProperty("author", ""), null); if (dc_creator == null) dc_creator = "";
String tags = crypt.simpleDecode(prop.getProperty("tags", ""), null); if (tags == null) tags = ""; String tags = crypt.simpleDecode(prop.getProperty("tags", ""), null); if (tags == null) tags = "";
String ETag = crypt.simpleDecode(prop.getProperty("ETag", ""), null); if (ETag == null) ETag = ""; String ETag = crypt.simpleDecode(prop.getProperty("ETag", ""), null); if (ETag == null) ETag = "";
this.entry = rowdef.newEntry(); this.entry = rowdef.newEntry();
this.entry.setCol(col_hash, url.hash(), null); this.entry.setCol(col_hash, url.hash(), null);
this.entry.setCol(col_comp, encodeComp(url, descr, author, tags, ETag)); this.entry.setCol(col_comp, encodeComp(url, descr, dc_creator, tags, ETag));
try { try {
encodeDate(col_mod, serverDate.parseShortDay(prop.getProperty("mod", "20000101"))); encodeDate(col_mod, serverDate.parseShortDay(prop.getProperty("mod", "20000101")));
} catch (ParseException e) { } catch (ParseException e) {
@ -256,9 +256,9 @@ public class indexURLEntry {
try { try {
s.append("hash=").append(hash()); s.append("hash=").append(hash());
s.append(",url=").append(crypt.simpleEncode(comp.url().toNormalform(false, true))); s.append(",url=").append(crypt.simpleEncode(comp.url().toNormalform(false, true)));
s.append(",descr=").append(crypt.simpleEncode(comp.title())); s.append(",descr=").append(crypt.simpleEncode(comp.dc_title()));
s.append(",author=").append(crypt.simpleEncode(comp.author())); s.append(",author=").append(crypt.simpleEncode(comp.dc_creator()));
s.append(",tags=").append(crypt.simpleEncode(comp.tags())); s.append(",tags=").append(crypt.simpleEncode(comp.dc_subject()));
s.append(",ETag=").append(crypt.simpleEncode(comp.ETag())); s.append(",ETag=").append(crypt.simpleEncode(comp.ETag()));
s.append(",mod=").append(serverDate.formatShortDay(moddate())); s.append(",mod=").append(serverDate.formatShortDay(moddate()));
s.append(",load=").append(serverDate.formatShortDay(loaddate())); s.append(",load=").append(serverDate.formatShortDay(loaddate()));
@ -429,7 +429,7 @@ public class indexURLEntry {
null, null,
comp().url(), comp().url(),
referrerHash(), referrerHash(),
comp().title(), comp().dc_title(),
loaddate(), loaddate(),
null, null,
0, 0,
@ -455,7 +455,7 @@ public class indexURLEntry {
public class Components { public class Components {
private yacyURL url; private yacyURL url;
private String title, author, tags, ETag; private String dc_title, dc_creator, dc_subject, ETag;
public Components(String url, String urlhash, String title, String author, String tags, String ETag) { public Components(String url, String urlhash, String title, String author, String tags, String ETag) {
try { try {
@ -463,22 +463,22 @@ public class indexURLEntry {
} catch (MalformedURLException e) { } catch (MalformedURLException e) {
this.url = null; this.url = null;
} }
this.title = title; this.dc_title = title;
this.author = author; this.dc_creator = author;
this.tags = tags; this.dc_subject = tags;
this.ETag = ETag; this.ETag = ETag;
} }
public Components(yacyURL url, String descr, String author, String tags, String ETag) { public Components(yacyURL url, String descr, String author, String tags, String ETag) {
this.url = url; this.url = url;
this.title = descr; this.dc_title = descr;
this.author = author; this.dc_creator = author;
this.tags = tags; this.dc_subject = tags;
this.ETag = ETag; this.ETag = ETag;
} }
public yacyURL url() { return this.url; } public yacyURL url() { return this.url; }
public String title() { return this.title; } public String dc_title() { return this.dc_title; }
public String author() { return this.author; } public String dc_creator() { return this.dc_creator; }
public String tags() { return this.tags; } public String dc_subject() { return this.dc_subject; }
public String ETag() { return this.ETag; } public String ETag() { return this.ETag; }
} }

1
source/de/anomic/kelondro/kelondroEcoTable.java
Unescape View File

@ -343,6 +343,7 @@ public class kelondroEcoTable implements kelondroIndex {
} else { } else {
// read old value // read old value
kelondroRow.Entry v = table.get(i); kelondroRow.Entry v = table.get(i);
assert v != null;
System.arraycopy(row.getPrimaryKeyBytes(), 0, b, 0, rowdef.primaryKeyLength); System.arraycopy(row.getPrimaryKeyBytes(), 0, b, 0, rowdef.primaryKeyLength);
System.arraycopy(v.bytes(), 0, b, rowdef.primaryKeyLength, rowdef.objectsize - rowdef.primaryKeyLength); System.arraycopy(v.bytes(), 0, b, rowdef.primaryKeyLength, rowdef.objectsize - rowdef.primaryKeyLength);
// write new value // write new value

10
source/de/anomic/plasma/plasmaCrawlLURL.java
Unescape View File

@ -72,10 +72,10 @@ import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroCache; import de.anomic.kelondro.kelondroCache;
import de.anomic.kelondro.kelondroCloneableIterator; import de.anomic.kelondro.kelondroCloneableIterator;
import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroSplitTable;
import de.anomic.kelondro.kelondroIndex; import de.anomic.kelondro.kelondroIndex;
import de.anomic.kelondro.kelondroRow; import de.anomic.kelondro.kelondroRow;
import de.anomic.kelondro.kelondroRowSet; import de.anomic.kelondro.kelondroRowSet;
import de.anomic.kelondro.kelondroSplitTable;
import de.anomic.plasma.urlPattern.plasmaURLPattern; import de.anomic.plasma.urlPattern.plasmaURLPattern;
import de.anomic.server.serverCodings; import de.anomic.server.serverCodings;
import de.anomic.server.logging.serverLog; import de.anomic.server.logging.serverLog;
@ -623,14 +623,14 @@ public final class plasmaCrawlLURL {
pw.println(url); pw.println(url);
} }
if (format == 1) { if (format == 1) {
pw.println("<a href=\"" + url + "\">" + htmlTools.encodeUnicode2html(comp.title(), true, true) + "</a><br>"); pw.println("<a href=\"" + url + "\">" + htmlTools.encodeUnicode2html(comp.dc_title(), true, true) + "</a><br>");
} }
if (format == 2) { if (format == 2) {
pw.println("<item>"); pw.println("<item>");
pw.println("<title>" + htmlTools.encodeUnicode2html(comp.title(), true, true) + "</title>"); pw.println("<title>" + htmlTools.encodeUnicode2html(comp.dc_title(), true, true) + "</title>");
pw.println("<link>" + yacyURL.escape(url) + "</link>"); pw.println("<link>" + yacyURL.escape(url) + "</link>");
if (comp.author().length() > 0) pw.println("<author>" + htmlTools.encodeUnicode2html(comp.author(), true, true) + "</author>"); if (comp.dc_creator().length() > 0) pw.println("<author>" + htmlTools.encodeUnicode2html(comp.dc_creator(), true, true) + "</author>");
if (comp.tags().length() > 0) pw.println("<description>" + htmlTools.encodeUnicode2html(comp.tags(), true, true) + "</description>"); if (comp.dc_subject().length() > 0) pw.println("<description>" + htmlTools.encodeUnicode2html(comp.dc_subject(), true, true) + "</description>");
pw.println("<pubDate>" + entry.moddate().toString() + "</pubDate>"); pw.println("<pubDate>" + entry.moddate().toString() + "</pubDate>");
pw.println("<guid isPermaLink=\"false\">" + entry.hash() + "</guid>"); pw.println("<guid isPermaLink=\"false\">" + entry.hash() + "</guid>");
pw.println("</item>"); pw.println("</item>");

4
source/de/anomic/plasma/plasmaSearchAPI.java
Unescape View File

@ -88,10 +88,10 @@ public class plasmaSearchAPI {
} }
} }
public static plasmaSearchRankingProcess genSearchresult(serverObjects prop, plasmaSwitchboard sb, String keyhash, kelondroBitfield filter, int sortorder, boolean fetchURLs) { public static plasmaSearchRankingProcess genSearchresult(serverObjects prop, plasmaSwitchboard sb, String keyhash, kelondroBitfield filter, int sortorder) {
plasmaSearchQuery query = new plasmaSearchQuery(keyhash, -1, sb.getRanking(), filter); plasmaSearchQuery query = new plasmaSearchQuery(keyhash, -1, sb.getRanking(), filter);
plasmaSearchRankingProcess ranked = new plasmaSearchRankingProcess(sb.wordIndex, query, sortorder, Integer.MAX_VALUE); plasmaSearchRankingProcess ranked = new plasmaSearchRankingProcess(sb.wordIndex, query, sortorder, Integer.MAX_VALUE);
ranked.execQuery(fetchURLs); ranked.execQuery();
if (ranked.filteredCount() == 0) { if (ranked.filteredCount() == 0) {
prop.put("searchresult", 2); prop.put("searchresult", 2);

28
source/de/anomic/plasma/plasmaSearchEvent.java
Unescape View File

@ -76,7 +76,6 @@ public final class plasmaSearchEvent {
public TreeMap<String, String> IAResults; public TreeMap<String, String> IAResults;
public TreeMap<String, Integer> IACount; public TreeMap<String, Integer> IACount;
public String IAmaxcounthash, IAneardhthash; public String IAmaxcounthash, IAneardhthash;
private int localcount;
private resultWorker[] workerThreads; private resultWorker[] workerThreads;
private ArrayList<ResultEntry> resultList; private ArrayList<ResultEntry> resultList;
//private int resultListLock; // a pointer that shows that all elements below this pointer are fixed and may not be changed again //private int resultListLock; // a pointer that shows that all elements below this pointer are fixed and may not be changed again
@ -101,7 +100,6 @@ public final class plasmaSearchEvent {
this.IACount = new TreeMap<String, Integer>(); this.IACount = new TreeMap<String, Integer>();
this.IAmaxcounthash = null; this.IAmaxcounthash = null;
this.IAneardhthash = null; this.IAneardhthash = null;
this.localcount = 0;
this.urlRetrievalAllTime = 0; this.urlRetrievalAllTime = 0;
this.snippetComputationAllTime = 0; this.snippetComputationAllTime = 0;
this.workerThreads = null; this.workerThreads = null;
@ -157,8 +155,7 @@ public final class plasmaSearchEvent {
} else { } else {
// do a local search // do a local search
this.rankedCache = new plasmaSearchRankingProcess(wordIndex, query, 2, max_results_preparation); this.rankedCache = new plasmaSearchRankingProcess(wordIndex, query, 2, max_results_preparation);
this.rankedCache.execQuery(true); this.rankedCache.execQuery();
this.localcount = this.rankedCache.filteredCount();
//plasmaWordIndex.Finding finding = wordIndex.retrieveURLs(query, false, 2, ranking, process); //plasmaWordIndex.Finding finding = wordIndex.retrieveURLs(query, false, 2, ranking, process);
if (generateAbstracts) { if (generateAbstracts) {
@ -249,8 +246,7 @@ public final class plasmaSearchEvent {
// sort the local containers and truncate it to a limited count, // sort the local containers and truncate it to a limited count,
// so following sortings together with the global results will be fast // so following sortings together with the global results will be fast
synchronized (rankedCache) { synchronized (rankedCache) {
rankedCache.execQuery(true); rankedCache.execQuery();
localcount = rankedCache.filteredCount();
} }
} }
} }
@ -291,13 +287,13 @@ public final class plasmaSearchEvent {
long startTime = System.currentTimeMillis(); long startTime = System.currentTimeMillis();
indexURLEntry.Components comp = page.comp(); indexURLEntry.Components comp = page.comp();
String pagetitle = comp.title().toLowerCase(); String pagetitle = comp.dc_title().toLowerCase();
if (comp.url() == null) { if (comp.url() == null) {
registerFailure(page.hash(), "url corrupted (null)"); registerFailure(page.hash(), "url corrupted (null)");
return null; // rare case where the url is corrupted return null; // rare case where the url is corrupted
} }
String pageurl = comp.url().toString().toLowerCase(); String pageurl = comp.url().toString().toLowerCase();
String pageauthor = comp.author().toLowerCase(); String pageauthor = comp.dc_creator().toLowerCase();
long dbRetrievalTime = System.currentTimeMillis() - startTime; long dbRetrievalTime = System.currentTimeMillis() - startTime;
// check exclusion // check exclusion
@ -315,7 +311,7 @@ public final class plasmaSearchEvent {
// check constraints // check constraints
if ((query.constraint != null) && if ((query.constraint != null) &&
(query.constraint.get(plasmaCondenser.flag_cat_indexof)) && (query.constraint.get(plasmaCondenser.flag_cat_indexof)) &&
(!(comp.title().startsWith("Index of")))) { (!(comp.dc_title().startsWith("Index of")))) {
final Iterator<String> wi = query.queryHashes.iterator(); final Iterator<String> wi = query.queryHashes.iterator();
while (wi.hasNext()) wordIndex.removeEntry((String) wi.next(), page.hash()); while (wi.hasNext()) wordIndex.removeEntry((String) wi.next(), page.hash());
registerFailure(page.hash(), "index-of constraint not fullfilled"); registerFailure(page.hash(), "index-of constraint not fullfilled");
@ -423,12 +419,8 @@ public final class plasmaSearchEvent {
return secondarySearchThreads; return secondarySearchThreads;
} }
public int getLocalCount() { public plasmaSearchRankingProcess getRankingResult() {
return this.localcount; return this.rankedCache;
}
public int getGlobalCount() {
return this.rankedCache.getGlobalCount();
} }
public long getURLRetrievalTime() { public long getURLRetrievalTime() {
@ -465,7 +457,7 @@ public final class plasmaSearchEvent {
if ((query.onlineSnippetFetch) && if ((query.onlineSnippetFetch) &&
(!event.anyWorkerAlive()) && (!event.anyWorkerAlive()) &&
(event.resultList.size() < query.neededResults() + 10) && (event.resultList.size() < query.neededResults() + 10) &&
((event.getLocalCount() + event.getGlobalCount()) > event.resultList.size())) { ((event.getRankingResult().getLocalResourceSize() + event.getRankingResult().getRemoteResourceSize()) > event.resultList.size())) {
// set new timeout // set new timeout
event.eventTime = System.currentTimeMillis(); event.eventTime = System.currentTimeMillis();
// start worker threads to fetch urls and snippets // start worker threads to fetch urls and snippets
@ -764,7 +756,7 @@ public final class plasmaSearchEvent {
("yacyshare " + ("yacyshare " +
filename.replace('?', ' ') + filename.replace('?', ' ') +
" " + " " +
urlcomps.title()).getBytes(), "UTF-8").keySet(), urlcomps.dc_title()).getBytes(), "UTF-8").keySet(),
urlentry.hash()); urlentry.hash());
wordIndex.loadedURL.remove(urlentry.hash()); // clean up wordIndex.loadedURL.remove(urlentry.hash()); // clean up
throw new RuntimeException("index void"); throw new RuntimeException("index void");
@ -794,7 +786,7 @@ public final class plasmaSearchEvent {
return (alternative_urlname == null) ? urlcomps.url().toNormalform(false, true) : alternative_urlname; return (alternative_urlname == null) ? urlcomps.url().toNormalform(false, true) : alternative_urlname;
} }
public String title() { public String title() {
return urlcomps.title(); return urlcomps.dc_title();
} }
public plasmaSnippetCache.TextSnippet textSnippet() { public plasmaSnippetCache.TextSnippet textSnippet() {
return this.textSnippet; return this.textSnippet;

82
source/de/anomic/plasma/plasmaSearchRankingProcess.java
Unescape View File

@ -57,9 +57,8 @@ public final class plasmaSearchRankingProcess {
private HashMap<String, String> handover; // key = urlhash, value = urlstring; used for double-check of urls that had been handed over to search process private HashMap<String, String> handover; // key = urlhash, value = urlstring; used for double-check of urls that had been handed over to search process
private plasmaSearchQuery query; private plasmaSearchQuery query;
private int sortorder; private int sortorder;
private int filteredCount;
private int maxentries; private int maxentries;
private int globalcount; private int remote_peerCount, remote_indexCount, remote_resourceSize, local_resourceSize;
private indexRWIEntryOrder order; private indexRWIEntryOrder order;
private HashMap<String, Object> urlhashes; // map for double-check; String/Long relation, addresses ranking number (backreference for deletion) private HashMap<String, Object> urlhashes; // map for double-check; String/Long relation, addresses ranking number (backreference for deletion)
private kelondroMScoreCluster<String> ref; // reference score computation for the commonSense heuristic private kelondroMScoreCluster<String> ref; // reference score computation for the commonSense heuristic
@ -76,11 +75,13 @@ public final class plasmaSearchRankingProcess {
this.sortedRWIEntries = new TreeMap<Object, indexRWIRowEntry>(); this.sortedRWIEntries = new TreeMap<Object, indexRWIRowEntry>();
this.doubleDomCache = new HashMap<String, TreeMap<Object, indexRWIRowEntry>>(); this.doubleDomCache = new HashMap<String, TreeMap<Object, indexRWIRowEntry>>();
this.handover = new HashMap<String, String>(); this.handover = new HashMap<String, String>();
this.filteredCount = 0;
this.order = null; this.order = null;
this.query = query; this.query = query;
this.maxentries = maxentries; this.maxentries = maxentries;
this.globalcount = 0; this.remote_peerCount = 0;
this.remote_indexCount = 0;
this.remote_resourceSize = 0;
this.local_resourceSize = 0;
this.urlhashes = new HashMap<String, Object>(); this.urlhashes = new HashMap<String, Object>();
this.ref = new kelondroMScoreCluster<String>(); this.ref = new kelondroMScoreCluster<String>();
this.misses = new TreeSet<String>(); this.misses = new TreeSet<String>();
@ -90,7 +91,7 @@ public final class plasmaSearchRankingProcess {
for (int i = 0; i < 32; i++) {this.flagcount[i] = 0;} for (int i = 0; i < 32; i++) {this.flagcount[i] = 0;}
} }
public void execQuery(boolean fetchURLs) { public void execQuery() {
long timer = System.currentTimeMillis(); long timer = System.currentTimeMillis();
this.localSearchContainerMaps = wordIndex.localSearchContainers(query, null); this.localSearchContainerMaps = wordIndex.localSearchContainers(query, null);
@ -113,16 +114,24 @@ public final class plasmaSearchRankingProcess {
} }
if (sortorder == 2) { if (sortorder == 2) {
insertRanked(index, true); insertRanked(index, true, index.size());
} else { } else {
insertNoOrder(index, fetchURLs); insertNoOrder(index, true, index.size());
} }
} }
private void insertNoOrder(indexContainer index, boolean local) { private void insertNoOrder(indexContainer index, boolean local, int fullResource) {
final Iterator<indexRWIRowEntry> en = index.entries(); final Iterator<indexRWIRowEntry> en = index.entries();
// generate a new map where the urls are sorted (not by hash but by the url text) // generate a new map where the urls are sorted (not by hash but by the url text)
if (local) {
this.local_resourceSize += fullResource;
} else {
this.remote_resourceSize += fullResource;
this.remote_peerCount++;
this.remote_indexCount += index.size();
}
indexRWIRowEntry ientry; indexRWIRowEntry ientry;
indexURLEntry uentry; indexURLEntry uentry;
String u; String u;
@ -141,20 +150,14 @@ public final class plasmaSearchRankingProcess {
if (sortorder == 0) { if (sortorder == 0) {
this.sortedRWIEntries.put(ientry.urlHash(), ientry); this.sortedRWIEntries.put(ientry.urlHash(), ientry);
this.urlhashes.put(ientry.urlHash(), ientry.urlHash()); this.urlhashes.put(ientry.urlHash(), ientry.urlHash());
filteredCount++;
} else { } else {
if (local) { uentry = wordIndex.loadedURL.load(ientry.urlHash(), ientry, 0);
uentry = wordIndex.loadedURL.load(ientry.urlHash(), ientry, 0); if (uentry == null) {
if (uentry == null) { this.misses.add(ientry.urlHash());
this.misses.add(ientry.urlHash());
} else {
u = uentry.comp().url().toNormalform(false, true);
this.sortedRWIEntries.put(u, ientry);
this.urlhashes.put(ientry.urlHash(), u);
filteredCount++;
}
} else { } else {
filteredCount++; u = uentry.comp().url().toNormalform(false, true);
this.sortedRWIEntries.put(u, ientry);
this.urlhashes.put(ientry.urlHash(), u);
} }
} }
@ -163,12 +166,18 @@ public final class plasmaSearchRankingProcess {
} // end loop } // end loop
} }
public void insertRanked(indexContainer index, boolean local) { public void insertRanked(indexContainer index, boolean local, int fullResource) {
// we collect the urlhashes and construct a list with urlEntry objects // we collect the urlhashes and construct a list with urlEntry objects
// attention: if minEntries is too high, this method will not terminate within the maxTime // attention: if minEntries is too high, this method will not terminate within the maxTime
assert (index != null); assert (index != null);
if (index.size() == 0) return; if (index.size() == 0) return;
if (local) {
this.local_resourceSize += fullResource;
} else {
this.remote_resourceSize += fullResource;
this.remote_peerCount++;
}
long timer = System.currentTimeMillis(); long timer = System.currentTimeMillis();
if (this.order == null) { if (this.order == null) {
@ -224,11 +233,8 @@ public final class plasmaSearchRankingProcess {
} }
// increase counter for statistics // increase counter for statistics
if (!local) this.globalcount++; if (!local) this.remote_indexCount++;
} }
this.filteredCount = sortedRWIEntries.size();
//long sc = Math.max(1, System.currentTimeMillis() - s0);
//System.out.println("###DEBUG### time to sort " + container.size() + " entries to " + this.filteredCount + ": " + sc + " milliseconds, " + (container.size() / sc) + " entries/millisecond, ranking = " + tc);
//if ((query.neededResults() > 0) && (container.size() > query.neededResults())) remove(true, true); //if ((query.neededResults() > 0) && (container.size() > query.neededResults())) remove(true, true);
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), plasmaSearchEvent.PRESORT, index.size(), System.currentTimeMillis() - timer)); serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), plasmaSearchEvent.PRESORT, index.size(), System.currentTimeMillis() - timer));
@ -350,14 +356,34 @@ public final class plasmaSearchRankingProcess {
return flagcount; return flagcount;
} }
// "results from a total number of <remote_resourceSize + local_resourceSize> known (<local_resourceSize> local, <remote_resourceSize> remote), <remote_indexCount> links from <remote_peerCount> other YaCy peers."
public int filteredCount() { public int filteredCount() {
return this.filteredCount; // the number of index entries that are considered as result set
return this.sortedRWIEntries.size();
} }
public int getGlobalCount() { public int getRemoteIndexCount() {
return this.globalcount; // the number of result contributions from all the remote peers
return this.remote_indexCount;
} }
public int getRemotePeerCount() {
// the number of remote peers that have contributed
return this.remote_peerCount;
}
public int getRemoteResourceSize() {
// the number of all hits in all the remote peers
return this.remote_resourceSize;
}
public int getLocalResourceSize() {
// the number of hits in the local peer (index size, size of the collection in the own index)
return this.local_resourceSize;
}
public indexRWIEntry remove(String urlHash) { public indexRWIEntry remove(String urlHash) {
Object r = (Long) urlhashes.get(urlHash); Object r = (Long) urlhashes.get(urlHash);
if (r == null) return null; if (r == null) return null;

2
source/de/anomic/plasma/plasmaSnippetCache.java
Unescape View File

@ -874,7 +874,7 @@ public class plasmaSnippetCache {
plasmaSearchEvent event = plasmaSearchEvent.getEvent(eventID); plasmaSearchEvent event = plasmaSearchEvent.getEvent(eventID);
assert plasmaSwitchboard.getSwitchboard() != null; assert plasmaSwitchboard.getSwitchboard() != null;
assert plasmaSwitchboard.getSwitchboard().wordIndex != null; assert plasmaSwitchboard.getSwitchboard().wordIndex != null;
assert event != null; assert event != null : "eventID = " + eventID;
assert event.getQuery() != null; assert event.getQuery() != null;
plasmaSwitchboard.getSwitchboard().wordIndex.removeEntryMultiple(event.getQuery().queryHashes, urlHash); plasmaSwitchboard.getSwitchboard().wordIndex.removeEntryMultiple(event.getQuery().queryHashes, urlHash);
event.remove(urlHash); event.remove(urlHash);

16
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -2205,7 +2205,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
/* ========================================================================= /* =========================================================================
* CREATE INDEX * CREATE INDEX
* ========================================================================= */ * ========================================================================= */
String docDescription = document.dc_title(); String dc_title = document.dc_title();
yacyURL referrerURL = entry.referrerURL(); yacyURL referrerURL = entry.referrerURL();
String noIndexReason = plasmaCrawlEURL.DENIED_UNSPECIFIED_INDEXING_ERROR; String noIndexReason = plasmaCrawlEURL.DENIED_UNSPECIFIED_INDEXING_ERROR;
@ -2236,9 +2236,9 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
long ldate = System.currentTimeMillis(); long ldate = System.currentTimeMillis();
indexURLEntry newEntry = new indexURLEntry( indexURLEntry newEntry = new indexURLEntry(
entry.url(), // URL entry.url(), // URL
docDescription, // document description dc_title, // document description
document.dc_creator(), // author document.dc_creator(), // author
document.dc_subject(' '), // tags document.dc_subject(' '), // tags
"", // ETag "", // ETag
docDate, // modification date docDate, // modification date
new Date(), // loaded date new Date(), // loaded date
@ -2406,7 +2406,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
// of string concatenation // of string concatenation
log.logInfo("*Indexed " + words + " words in URL " + entry.url() + log.logInfo("*Indexed " + words + " words in URL " + entry.url() +
" [" + entry.urlHash() + "]" + " [" + entry.urlHash() + "]" +
"\n\tDescription: " + docDescription + "\n\tDescription: " + dc_title +
"\n\tMimeType: " + document.dc_format() + " | Charset: " + document.getCharset() + " | " + "\n\tMimeType: " + document.dc_format() + " | Charset: " + document.getCharset() + " | " +
"Size: " + document.getTextLength() + " bytes | " + "Size: " + document.getTextLength() + " bytes | " +
"Anchors: " + ((document.getAnchors() == null) ? 0 : document.getAnchors().size()) + "Anchors: " + ((document.getAnchors() == null) ? 0 : document.getAnchors().size()) +
@ -2430,7 +2430,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
} }
} else { } else {
log.logFine("Not Indexed Resource '" + entry.url().toNormalform(false, true) + "': process case=" + processCase); log.logFine("Not Indexed Resource '" + entry.url().toNormalform(false, true) + "': process case=" + processCase);
addURLtoErrorDB(entry.url(), referrerURL.hash(), initiatorPeerHash, docDescription, plasmaCrawlEURL.DENIED_UNKNOWN_INDEXING_PROCESS_CASE, new kelondroBitfield()); addURLtoErrorDB(entry.url(), referrerURL.hash(), initiatorPeerHash, dc_title, plasmaCrawlEURL.DENIED_UNKNOWN_INDEXING_PROCESS_CASE, new kelondroBitfield());
} }
} catch (Exception ee) { } catch (Exception ee) {
if (ee instanceof InterruptedException) throw (InterruptedException)ee; if (ee instanceof InterruptedException) throw (InterruptedException)ee;
@ -2443,7 +2443,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
if (clusterhashes != null) initiatorPeer.setAlternativeAddress((String) clusterhashes.get(initiatorPeer.hash)); if (clusterhashes != null) initiatorPeer.setAlternativeAddress((String) clusterhashes.get(initiatorPeer.hash));
yacyClient.crawlReceipt(initiatorPeer, "crawl", "exception", ee.getMessage(), null, ""); yacyClient.crawlReceipt(initiatorPeer, "crawl", "exception", ee.getMessage(), null, "");
} }
addURLtoErrorDB(entry.url(), (referrerURL == null) ? null : referrerURL.hash(), initiatorPeerHash, docDescription, plasmaCrawlEURL.DENIED_UNSPECIFIED_INDEXING_ERROR, new kelondroBitfield()); addURLtoErrorDB(entry.url(), (referrerURL == null) ? null : referrerURL.hash(), initiatorPeerHash, dc_title, plasmaCrawlEURL.DENIED_UNSPECIFIED_INDEXING_ERROR, new kelondroBitfield());
} }
} else { } else {
@ -2451,7 +2451,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
checkInterruption(); checkInterruption();
log.logInfo("Not indexed any word in URL " + entry.url() + "; cause: " + noIndexReason); log.logInfo("Not indexed any word in URL " + entry.url() + "; cause: " + noIndexReason);
addURLtoErrorDB(entry.url(), (referrerURL == null) ? null : referrerURL.hash(), initiatorPeerHash, docDescription, noIndexReason, new kelondroBitfield()); addURLtoErrorDB(entry.url(), (referrerURL == null) ? null : referrerURL.hash(), initiatorPeerHash, dc_title, noIndexReason, new kelondroBitfield());
if ((processCase == PROCESSCASE_6_GLOBAL_CRAWLING) && (initiatorPeer != null)) { if ((processCase == PROCESSCASE_6_GLOBAL_CRAWLING) && (initiatorPeer != null)) {
if (clusterhashes != null) initiatorPeer.setAlternativeAddress((String) clusterhashes.get(initiatorPeer.hash)); if (clusterhashes != null) initiatorPeer.setAlternativeAddress((String) clusterhashes.get(initiatorPeer.hash));
yacyClient.crawlReceipt(initiatorPeer, "crawl", "rejected", noIndexReason, null, ""); yacyClient.crawlReceipt(initiatorPeer, "crawl", "rejected", noIndexReason, null, "");

7
source/de/anomic/yacy/yacyClient.java
Unescape View File

@ -477,9 +477,10 @@ public final class yacyClient {
// now create a plasmaIndex out of this result // now create a plasmaIndex out of this result
// System.out.println("yacyClient: " + ((urlhashes.length() == 0) ? "primary" : "secondary")+ " search result = " + result.toString()); // debug // System.out.println("yacyClient: " + ((urlhashes.length() == 0) ? "primary" : "secondary")+ " search result = " + result.toString()); // debug
int results = 0; int results = 0, joincount = 0;
try { try {
results = Integer.parseInt((String) result.get("count")); results = Integer.parseInt(result.get("count"));
joincount = Integer.parseInt(result.get("joincount"));
} catch (NumberFormatException e) { } catch (NumberFormatException e) {
yacyCore.log.logFine("SEARCH failed FROM " + target.hash + ":" + target.getName() + ", wrong output format"); yacyCore.log.logFine("SEARCH failed FROM " + target.hash + ":" + target.getName() + ", wrong output format");
yacyCore.peerActions.peerDeparture(target, "search request to peer created number format exception"); yacyCore.peerActions.peerDeparture(target, "search request to peer created number format exception");
@ -557,7 +558,7 @@ public final class yacyClient {
// store remote result to local result container // store remote result to local result container
synchronized (containerCache) { synchronized (containerCache) {
// insert one container into the search result buffer // insert one container into the search result buffer
containerCache.insertRanked(container[0], false); // one is enough containerCache.insertRanked(container[0], false, joincount); // one is enough
// integrate remote topwords // integrate remote topwords
String references = (String) result.get("references"); String references = (String) result.get("references");

Write Preview
Loading…
Cancel
Save