From edc0b33f6d2acb672cf437e23625b861801ac65f Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Thu, 11 Apr 2013 14:46:13 +0200 Subject: [PATCH] - showing references count and clickdepth in host browser - fixed generation and presentation of both values --- htroot/HostBrowser.html | 2 +- htroot/HostBrowser.java | 11 ++++++++--- source/net/yacy/search/index/Segment.java | 2 +- source/net/yacy/search/query/QueryParams.java | 1 - .../yacy/search/schema/CollectionConfiguration.java | 2 +- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/htroot/HostBrowser.html b/htroot/HostBrowser.html index 289f956c6..9f7d1bbca 100644 --- a/htroot/HostBrowser.html +++ b/htroot/HostBrowser.html @@ -128,7 +128,7 @@ function updatepage(str) { #[url]#  #(stored)# #(load)#link, detected from context::load & index#(/load)#:: - indexed#[comment]#:: + indexed#[comment]#:: loading:: #[error]# #(/stored)# diff --git a/htroot/HostBrowser.java b/htroot/HostBrowser.java index 1097255e0..46cb8e42a 100644 --- a/htroot/HostBrowser.java +++ b/htroot/HostBrowser.java @@ -254,14 +254,16 @@ public class HostBrowser { CollectionSchema.inboundlinks_urlstub_txt.getSolrFieldName(), CollectionSchema.outboundlinks_protocol_sxt.getSolrFieldName(), CollectionSchema.outboundlinks_urlstub_txt.getSolrFieldName(), - CollectionSchema.clickdepth_i.getSolrFieldName() + CollectionSchema.clickdepth_i.getSolrFieldName(), + CollectionSchema.references_i.getSolrFieldName() ); SolrDocument doc; Set storedDocs = new HashSet(); Map errorDocs = new HashMap(); Set inboundLinks = new HashSet(); Map> outboundHosts = new HashMap>(); - RowHandleMap clickdepth = new RowHandleMap(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 1, 100, "clickdepth"); + RowHandleMap clickdepth = new RowHandleMap(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 2, 100, "clickdepth"); + RowHandleMap references = new RowHandleMap(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 2, 100, "references"); int hostsize = 0; final List deleteIDs = new ArrayList(); long timeout = System.currentTimeMillis() + TIMEOUT; @@ -271,6 +273,8 @@ public class HostBrowser { FailType error = errortype == null ? null : FailType.valueOf(errortype); Integer cd = (Integer) doc.getFieldValue(CollectionSchema.clickdepth_i.getSolrFieldName()); if (cd != null && cd.intValue() >= 0) clickdepth.add(ASCII.getBytes((String) doc.getFieldValue(CollectionSchema.id.getSolrFieldName())), cd.intValue()); + Integer rc = (Integer) doc.getFieldValue(CollectionSchema.references_i.getSolrFieldName()); + if (rc != null && rc.intValue() >= 0) references.add(ASCII.getBytes((String) doc.getFieldValue(CollectionSchema.id.getSolrFieldName())), rc.intValue()); if (u.startsWith(path)) { if (delete) { deleteIDs.add(ASCII.getBytes((String) doc.getFieldValue(CollectionSchema.id.getSolrFieldName()))); @@ -407,7 +411,8 @@ public class HostBrowser { prop.put("files_list_" + c + "_type_stored", type == StoreType.INDEX ? 1 : error ? 3 : loading ? 2 : 0 /*linked*/); if (type == StoreType.INDEX) { long cd = clickdepth.get(uri.hash()); - prop.put("files_list_" + c + "_type_stored_comment", cd >= 0 ? "clickdepth = " + cd : ""); + long rc = references.get(uri.hash()); + prop.put("files_list_" + c + "_type_stored_comment", (rc >= 0 ? rc + " references" : "") + (rc >= 0 && cd >= 0 ? ", " : "") + (cd >= 0 ? "clickdepth " + cd : "")); } prop.put("files_list_" + c + "_type_stored_load", loadRight ? 1 : 0); if (error) { diff --git a/source/net/yacy/search/index/Segment.java b/source/net/yacy/search/index/Segment.java index 0dac40629..862c75ed1 100644 --- a/source/net/yacy/search/index/Segment.java +++ b/source/net/yacy/search/index/Segment.java @@ -545,7 +545,7 @@ public class Segment { // ENRICH DOCUMENT WITH RANKING INFORMATION if (this.urlCitationIndex != null && this.fulltext.getDefaultConfiguration().contains(CollectionSchema.references_i)) { int references = this.urlCitationIndex.count(url.hash()); - if (references > 0) vector.setField(CollectionSchema.references_i.getSolrFieldName(), references); + vector.setField(CollectionSchema.references_i.getSolrFieldName(), references); } // STORE TO SOLR diff --git a/source/net/yacy/search/query/QueryParams.java b/source/net/yacy/search/query/QueryParams.java index ae85a79d7..746e30c4a 100644 --- a/source/net/yacy/search/query/QueryParams.java +++ b/source/net/yacy/search/query/QueryParams.java @@ -37,7 +37,6 @@ import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrQuery.ORDER; import org.apache.solr.client.solrj.SolrQuery.SortClause; import net.yacy.cora.document.ASCII; diff --git a/source/net/yacy/search/schema/CollectionConfiguration.java b/source/net/yacy/search/schema/CollectionConfiguration.java index 9157a64ef..f3181a884 100644 --- a/source/net/yacy/search/schema/CollectionConfiguration.java +++ b/source/net/yacy/search/schema/CollectionConfiguration.java @@ -354,7 +354,7 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seri clickdepth = 999; processTypes.add(ProcessType.CLICKDEPTH); // postprocessing needed; this is also needed if the depth is positive; there could be a shortcut } - add(doc, CollectionSchema.clickdepth_i, clickdepth); + CollectionSchema.clickdepth_i.add(doc, clickdepth); // no lazy value checking to get a '0' into the index } if (allAttr || contains(CollectionSchema.ip_s)) {