boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'master' of git://gitorious.org/yacy/rc1.git

Browse Source
pull/1/head
reger 12 years ago
parent 31d16f20d7 870aedf3c6
commit 254074b11d
10 changed files with 52 additions and 32 deletions
Show Stats Download Patch File Download Diff File

2
defaults/solr.webgraph.schema
Unescape View File

@ -156,7 +156,7 @@ target_path_folders_sxt
## the values from key-value pairs in the search part of the url (target)
#target_parameter_value_sxt
## "depth of web page according to number of clicks from the 'main' page, which is the page that appears if only the host is entered as url (target)
## depth of web page according to number of clicks from the 'main' page, which is the page that appears if only the host is entered as url (target)
#target_clickdepth_i
## host of the url (target)

2
htroot/portalsearch/yacy-portalsearch.js
Unescape View File

@ -207,7 +207,7 @@ function yrun() {
function yacysearch(clear) {
var url = yconf.url + '/yacysearch.json?callback=?' // JSONP (cross domain) request URL
//var url = yconf.url + '/solr/select?wt=yjson&jsonp=?' // JSONP (cross domain) request URL
//var url = yconf.url + '/solr/select?wt=yjson&callback=?' // JSONP (cross domain) request URL
if(clear) {
$('#ypopup').empty();

11
htroot/solr/select.java
Unescape View File

@ -177,11 +177,12 @@ public class select {
// if this is a call to YaCys special search formats, enhance the query with field assignments
if ((responseWriter instanceof JsonResponseWriter || responseWriter instanceof OpensearchResponseWriter) && "true".equals(post.get("hl", "true"))) {
// add options for snippet generation
post.put("hl", "true");
post.put("hl.fl", "text_t,h1,h2");
post.put("hl.simple.pre", "");
post.put("hl.simple.post", "");
post.put("hl.fragsize", Integer.toString(SearchEvent.SNIPPET_MAX_LENGTH));
if (!post.containsKey("hl.q")) post.put("hl.q", q);
if (!post.containsKey("hl.fl")) post.put("hl.fl", CollectionSchema.h1_txt.getSolrFieldName() + "," + CollectionSchema.h2_txt.getSolrFieldName() + "," + CollectionSchema.text_t.getSolrFieldName());
if (!post.containsKey("hl.alternateField")) post.put("hl.alternateField", CollectionSchema.description.getSolrFieldName());
if (!post.containsKey("hl.simple.pre")) post.put("hl.simple.pre", "<b>");
if (!post.containsKey("hl.simple.post")) post.put("hl.simple.post", "</b>");
if (!post.containsKey("hl.fragsize")) post.put("hl.fragsize", Integer.toString(SearchEvent.SNIPPET_MAX_LENGTH));
}
// get the embedded connector

4
htroot/yacysearch.java
Unescape View File

@ -212,8 +212,8 @@ public class yacysearch {
? 100
: 5000) : (snippetFetchStrategy != null
&& snippetFetchStrategy.isAllowedToFetchOnline() ? 20 : 1000),
post.getInt("maximumRecords", post.getInt("count", 10))); // SRU syntax with old property as alternative
int startRecord = post.getInt("startRecord", post.getInt("offset", 0));
post.getInt("maximumRecords", post.getInt("count", post.getInt("rows", 10)))); // SRU syntax with old property as alternative
int startRecord = post.getInt("startRecord", post.getInt("offset", post.getInt("start", 0)));
boolean global = post.get("resource", "local").equals("global") && sb.peers.sizeConnected() > 0;
final boolean indexof = (post != null && post.get("indexof", "").equals("on"));

8
htroot/yacysearchitem.java
Unescape View File

@ -104,11 +104,13 @@ public class yacysearchitem {
prop.put("navurlBase", QueryParams.navurlBase("html", theSearch.query, null).toString());
final String target_special_pattern = sb.getConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL_PATTERN, "");
long timeout = item == 0 ? 10000 : (theSearch.query.isLocal() ? 1000 : 3000);
if (theSearch.query.contentdom == Classification.ContentDomain.TEXT || theSearch.query.contentdom == Classification.ContentDomain.ALL) {
// text search
// generate result object
final ResultEntry result = theSearch.oneResult(item, theSearch.query.isLocal() ? 1000 : 3000);
final ResultEntry result = theSearch.oneResult(item, timeout);
if (result == null) return prop; // no content
final String resultUrlstring = result.urlstring();
final DigestURI resultURL = result.url();
@ -261,7 +263,7 @@ public class yacysearchitem {
prop.put("content", theSearch.query.contentdom.getCode() + 1); // switch on specific content
//final MediaSnippet ms = theSearch.result().oneImage(item);
final ResultEntry ms = theSearch.oneResult(item, theSearch.query.isLocal() ? 1000 : 5000);
final ResultEntry ms = theSearch.oneResult(item, timeout);
if (ms == null) {
prop.put("content_item", "0");
} else {
@ -297,7 +299,7 @@ public class yacysearchitem {
// any other media content
// generate result object
final ResultEntry ms = theSearch.oneResult(item, theSearch.query.isLocal() ? 1000 : 5000);
final ResultEntry ms = theSearch.oneResult(item, timeout);
prop.put("content", theSearch.query.contentdom.getCode() + 1); // switch on specific content
if (ms == null) {
prop.put("content_item", "0");

2
source/net/yacy/cora/document/RSSFeed.java
Unescape View File

@ -31,7 +31,7 @@ import java.util.Set;
public class RSSFeed implements Iterable<RSSMessage> {
public static final int DEFAULT_MAXSIZE = 1000;
public static final int DEFAULT_MAXSIZE = 10000;
// class variables
private RSSMessage channel;

40
source/net/yacy/cora/federate/solr/responsewriter/GSAResponseWriter.java
Unescape View File

@ -164,21 +164,26 @@ public class GSAResponseWriter implements QueryResponseWriter {
// write header
writer.write(XML_START);
String query = request.getParams().get("originalQuery");
String site = (String) context.get("site");
String site = getContextString(context, "site", "");
String sort = getContextString(context, "sort", "");
String client = getContextString(context, "client", "");
String ip = getContextString(context, "ip", "");
String access = getContextString(context, "access", "");
String entqr = getContextString(context, "entqr", "");
OpensearchResponseWriter.solitaireTag(writer, "TM", Long.toString(System.currentTimeMillis() - start));
OpensearchResponseWriter.solitaireTag(writer, "Q", query);
paramTag(writer, "sort", (String) context.get("sort"));
paramTag(writer, "sort", sort);
paramTag(writer, "output", "xml_no_dtd");
paramTag(writer, "ie", "UTF-8");
paramTag(writer, "oe", "UTF-8");
paramTag(writer, "client", (String) context.get("client"));
paramTag(writer, "client", client);
paramTag(writer, "q", query);
paramTag(writer, "site", site);
paramTag(writer, "start", Integer.toString(resHead.offset));
paramTag(writer, "num", Integer.toString(resHead.rows));
paramTag(writer, "ip", (String) context.get("ip"));
paramTag(writer, "access", (String) context.get("access")); // p - search only public content, s - search only secure content, a - search all content, both public and secure
paramTag(writer, "entqr", (String) context.get("entqr")); // query expansion policy; (entqr=1) -- Uses only the search appliance's synonym file, (entqr=1) -- Uses only the search appliance's synonym file, (entqr=3) -- Uses both standard and local synonym files.
paramTag(writer, "ip", ip);
paramTag(writer, "access", access); // p - search only public content, s - search only secure content, a - search all content, both public and secure
paramTag(writer, "entqr", entqr); // query expansion policy; (entqr=1) -- Uses only the search appliance's synonym file, (entqr=1) -- Uses only the search appliance's synonym file, (entqr=3) -- Uses both standard and local synonym files.
// body introduction
final int responseCount = response.size();
@ -192,16 +197,16 @@ public class GSAResponseWriter implements QueryResponseWriter {
writer.write("<NB>");
if (prevStart >= 0) {
writer.write("<PU>");
XML.escapeCharData("/gsa/search?q=" + request.getParams().get("q") + "&site=" + (String) context.get("site") +
"&lr=&ie=UTF-8&oe=UTF-8&output=xml_no_dtd&client=" + (String) context.get("client") + "&access=" + (String) context.get("access") +
"&sort=" + (String) context.get("sort") + "&start=" + prevStart + "&sa=N", writer); // a relative URL pointing to the NEXT results page.
XML.escapeCharData("/gsa/search?q=" + request.getParams().get("q") + "&site=" + site +
"&lr=&ie=UTF-8&oe=UTF-8&output=xml_no_dtd&client=" + client + "&access=" + access +
"&sort=" + sort + "&start=" + prevStart + "&sa=N", writer); // a relative URL pointing to the NEXT results page.
writer.write("</PU>");
}
if (nextNum > 0) {
writer.write("<NU>");
XML.escapeCharData("/gsa/search?q=" + request.getParams().get("q") + "&site=" + (String) context.get("site") +
"&lr=&ie=UTF-8&oe=UTF-8&output=xml_no_dtd&client=" + (String) context.get("client") + "&access=" + (String) context.get("access") +
"&sort=" + (String) context.get("sort") + "&start=" + nextStart + "&num=" + nextNum + "&sa=N", writer); // a relative URL pointing to the NEXT results page.
XML.escapeCharData("/gsa/search?q=" + request.getParams().get("q") + "&site=" + site +
"&lr=&ie=UTF-8&oe=UTF-8&output=xml_no_dtd&client=" + client + "&access=" + access +
"&sort=" + sort + "&start=" + nextStart + "&num=" + nextNum + "&sa=N", writer); // a relative URL pointing to the NEXT results page.
writer.write("</NU>");
}
writer.write("</NB>");
@ -296,6 +301,17 @@ public class GSAResponseWriter implements QueryResponseWriter {
writer.write(XML_STOP);
}
private static String getContextString(Map<Object,Object> context, String key, String dflt) {
Object v = context.get(key);
if (v == null) return dflt;
if (v instanceof String) return (String) v;
if (v instanceof String[]) {
String[] va = (String[]) v;
return va.length == 0 ? dflt : va[0];
}
return dflt;
}
public static void paramTag(final Writer writer, final String tagname, String value) throws IOException {
if (value == null || value.length() == 0) return;
writer.write("<PARAM name=\"");

6
source/net/yacy/cora/federate/solr/responsewriter/JsonResponseWriter.java
Unescape View File

@ -107,7 +107,7 @@ public class JsonResponseWriter implements QueryResponseWriter {
resHead.offset = response.offset(); // equal to 'start'
resHead.numFound = response.matches();
String jsonp = request.getParams().get("jsonp"); // check for JSONP
String jsonp = request.getParams().get("callback"); // check for JSONP
if (jsonp != null) {
writer.write(jsonp.toCharArray());
writer.write("([".toCharArray());
@ -127,6 +127,7 @@ public class JsonResponseWriter implements QueryResponseWriter {
SolrIndexSearcher searcher = request.getSearcher();
DocIterator iterator = response.iterator();
for (int i = 0; i < responseCount; i++) {
try {
writer.write("{\n".toCharArray());
int id = iterator.nextDoc();
Document doc = searcher.doc(id, OpensearchResponseWriter.SOLR_FIELDS);
@ -216,6 +217,7 @@ public class JsonResponseWriter implements QueryResponseWriter {
if (i < responseCount - 1) {
writer.write(",\n".toCharArray());
}
} catch (Throwable ee) {}
}
writer.write("],\n".toCharArray());
@ -233,7 +235,7 @@ public class JsonResponseWriter implements QueryResponseWriter {
NamedList<Integer> authors = facetFields == null ? null : (NamedList<Integer>) facetFields.get(CollectionSchema.author_sxt.getSolrFieldName());
if (domains != null) {
writer.write("{\"facetname\":\"domains\",\"displayname\":\"Domains\",\"type\":\"String\",\"min\":\"0\",\"max\":\"0\",\"mean\":\"0\",\"elements\":[\n".toCharArray());
writer.write("{\"facetname\":\"domains\",\"displayname\":\"Provider\",\"type\":\"String\",\"min\":\"0\",\"max\":\"0\",\"mean\":\"0\",\"elements\":[\n".toCharArray());
for (int i = 0; i < domains.size(); i++) {
facetEntry(writer, "site", domains.getName(i), Integer.toString(domains.getVal(i)));
if (i < domains.size() - 1) writer.write(',');

2
source/net/yacy/document/parser/htmlParser.java
Unescape View File

@ -51,7 +51,7 @@ import com.ibm.icu.text.CharsetDetector;
public class htmlParser extends AbstractParser implements Parser {
private static final Pattern patternUnderline = Pattern.compile("_");
private static final int maxLinks = 1000;
private static final int maxLinks = 10000;
public htmlParser() {
super("Streaming HTML Parser");

7
source/net/yacy/kelondro/index/RowCollection.java
Unescape View File

@ -54,7 +54,7 @@ public class RowCollection implements Sortable<Row.Entry>, Iterable<Row.Entry>,
private static final byte[] EMPTY_CACHE = new byte[0];
public static final long growfactorLarge100 = 140L;
public static final long growfactorSmall100 = 120L;
public static final long growfactorSmall100 = 110L;
private static final int isortlimit = 20;
private static final int exp_chunkcount = 0;
@ -246,12 +246,11 @@ public class RowCollection implements Sortable<Row.Entry>, Iterable<Row.Entry>,
long allocram = needed * growfactorLarge100 / 100L;
allocram -= allocram % this.rowdef.objectsize;
assert allocram > 0 : "elements = " + elements + ", new = " + allocram;
if (allocram <= Integer.MAX_VALUE && MemoryControl.request(allocram, false)) return allocram;
if (allocram <= Integer.MAX_VALUE && MemoryControl.request(allocram, forcegc)) return allocram;
allocram = needed * growfactorSmall100 / 100L;
allocram -= allocram % this.rowdef.objectsize;
assert allocram >= 0 : "elements = " + elements + ", new = " + allocram;
if (allocram <= Integer.MAX_VALUE && MemoryControl.request(allocram, forcegc)) return allocram;
return needed;
return allocram;
}
private final void ensureSize(final int elements) throws SpaceExceededException {

Write Preview
Loading…
Cancel
Save