boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'master' of https://git.gitorious.org/yacy/rc1.git

Browse Source
pull/1/head
sixcooler 13 years ago
parent 29c2289b5c 84c3fc9d97
commit b51c3fade4
13 changed files with 185 additions and 97 deletions
Show Stats Download Patch File Download Diff File

6
htroot/YMarks.html
Unescape View File

@ -90,8 +90,8 @@
<input type="radio" name="importer" value="surro" /> Surrogate XML<br />
<input type="radio" name="importer" value="dmoz" disabled="disabled" /> DMOZ XML<br />
<input type="radio" name="importer" value="list" disabled="disabled"/> YaCy White/Black List<br />
<input type="radio" name="importer" value="bmks" disabled="disabled"/> YaCy bookmarks.db<br />
<input type="radio" name="importer" value="crawls" disabled="disabled"/> YaCy Crawl-Starts<br />
<input type="radio" name="importer" value="bmks" /> YaCy old bookmarks.db<br />
<input type="radio" name="importer" value="crawls" /> YaCy Crawl Starts<br />
</p>
<p>
<small>Bookmark file</small>
@ -110,7 +110,7 @@
<hr />
<h4>Automatic tagging<img title="help" alt="help" class="help" src="/yacy/ui/img-2/question_blue.png"></h4>
<p>
<input type="radio" name="autotag" value="off" /> Off
<input type="radio" name="autotag" value="off" checked="checked" /> Off
<br />
<input type="radio" name="autotag" value="empty" /> Only for empty tags
<br />

82
htroot/api/ymarks/import_ymark.java
Unescape View File

@ -2,23 +2,35 @@ import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.util.EnumMap;
import java.util.Iterator;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.regex.Pattern;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.document.Document;
import net.yacy.document.Parser.Failure;
import net.yacy.document.content.SurrogateReader;
import net.yacy.kelondro.blob.Tables;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log;
import net.yacy.search.Switchboard;
import org.xml.sax.SAXException;
import de.anomic.data.BookmarksDB;
import de.anomic.data.UserDB;
import de.anomic.data.WorkTables;
import de.anomic.data.ymark.YMarkAutoTagger;
import de.anomic.data.ymark.YMarkEntry;
import de.anomic.data.ymark.YMarkHTMLImporter;
import de.anomic.data.ymark.YMarkJSONImporter;
import de.anomic.data.ymark.YMarkMetadata;
import de.anomic.data.ymark.YMarkTables;
import de.anomic.data.ymark.YMarkUtil;
import de.anomic.data.ymark.YMarkXBELImporter;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
@ -69,7 +81,7 @@ public class import_ymark {
if(post.containsKey("root") && post.get("root").length() > 0) {
root = post.get("root");
}
if(post.containsKey("bmkfile") && post.containsKey("importer")){
if(post.containsKey("bmkfile") && !post.get("bmkfile").isEmpty() && post.containsKey("importer")){
stream = new ByteArrayInputStream(UTF8.getBytes(post.get("bmkfile$file")));
if(post.get("importer").equals("surro") && stream != null) {
SurrogateReader surrogateReader;
@ -133,7 +145,73 @@ public class import_ymark {
prop.put("result", "1");
}
}
}
} else if(post.containsKey("importer") && post.get("importer").equals("crawls")) {
try {
final Pattern pattern = Pattern.compile("^crawl start for.*");
final Iterator<Tables.Row> APIcalls = sb.tables.iterator(WorkTables.TABLE_API_NAME, WorkTables.TABLE_API_COL_COMMENT, pattern);
Tables.Row row = null;
while(APIcalls.hasNext()) {
row = APIcalls.next();
if(row.get(WorkTables.TABLE_API_COL_TYPE, "").equals("crawler")) {
final String url = row.get(WorkTables.TABLE_API_COL_COMMENT, "").substring(16);
final YMarkMetadata meta = new YMarkMetadata(new DigestURI(url), sb.indexSegments);
final Document document = meta.loadDocument(sb.loader);
final EnumMap<YMarkMetadata.METADATA, String> metadata = meta.loadMetadata();
final YMarkEntry bmk_entry = new YMarkEntry(false);
bmk_entry.put(YMarkEntry.BOOKMARK.URL.key(), url);
if(!sb.tables.has(YMarkTables.TABLES.BOOKMARKS.tablename(bmk_user), YMarkUtil.getBookmarkId(url))) {
bmk_entry.put(YMarkEntry.BOOKMARK.PUBLIC.key(), "false");
bmk_entry.put(YMarkEntry.BOOKMARK.TITLE.key(), metadata.get(YMarkMetadata.METADATA.TITLE));
bmk_entry.put(YMarkEntry.BOOKMARK.DESC.key(), metadata.get(YMarkMetadata.METADATA.DESCRIPTION));
}
bmk_entry.put(YMarkEntry.BOOKMARK.FOLDERS.key(), root);
if(autotag) {
bmk_entry.put(YMarkEntry.BOOKMARK.TAGS.key(), YMarkAutoTagger.autoTag(document, 3, sb.tables.bookmarks.getTags(bmk_user)));
}
sb.tables.bookmarks.addBookmark(bmk_user, bmk_entry, merge, true);
}
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (RowSpaceExceededException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (Failure e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
} else if(post.containsKey("importer") && post.get("importer").equals("bmks")) {
final Iterator<String> bit=sb.bookmarksDB.getBookmarksIterator(isAdmin);
BookmarksDB.Bookmark bookmark;
while(bit.hasNext()){
bookmark=sb.bookmarksDB.getBookmark(bit.next());
final YMarkEntry bmk_entry = new YMarkEntry(false);
bmk_entry.put(YMarkEntry.BOOKMARK.URL.key(), bookmark.getUrl());
try {
if(!sb.tables.has(YMarkTables.TABLES.BOOKMARKS.tablename(bmk_user), YMarkUtil.getBookmarkId(bookmark.getUrl()))) {
bmk_entry.put(YMarkEntry.BOOKMARK.PUBLIC.key(), bookmark.getPublic() ? "true" : "false");
bmk_entry.put(YMarkEntry.BOOKMARK.TITLE.key(), bookmark.getTitle());
bmk_entry.put(YMarkEntry.BOOKMARK.DESC.key(), bookmark.getDescription());
bmk_entry.put(YMarkEntry.BOOKMARK.TAGS.key(), bookmark.getTagsString());
bmk_entry.put(YMarkEntry.BOOKMARK.FOLDERS.key(), root+bookmark.getFoldersString().replaceAll(".*"+YMarkUtil.TAGS_SEPARATOR+YMarkUtil.FOLDERS_SEPARATOR, root+YMarkUtil.FOLDERS_SEPARATOR));
}
if(autotag) {
bmk_entry.put(YMarkEntry.BOOKMARK.TAGS.key(), YMarkAutoTagger.autoTag(bookmark.getUrl(), sb.loader, 3, sb.tables.bookmarks.getTags(bmk_user)));
}
sb.tables.bookmarks.addBookmark(bmk_user, bmk_entry, merge, true);
} catch (MalformedURLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (RowSpaceExceededException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
if(post.containsKey("autotag") && !post.get("autotag", "off").equals("off")) {
try {
autoTaggingQueue.put(YMarkAutoTagger.POISON);

31
htroot/index.html
Unescape View File

@ -80,19 +80,19 @@
#(searchaudio)#::<input type="radio" id="audio" name="contentdom" value="audio" #(check)#::checked="checked"#(/check)# /><label for="audio">Audio</label>&nbsp;&nbsp;#(/searchaudio)#
#(searchvideo)#::<input type="radio" id="video" name="contentdom" value="video" #(check)#::checked="checked"#(/check)# /><label for="video">Video</label>&nbsp;&nbsp;#(/searchvideo)#
#(searchapp)#::<input type="radio" id="app" name="contentdom" value="app" #(check)#::checked="checked"#(/check)# /><label for="app">Applications</label>#(/searchapp)#
#(searchoptions)#&nbsp;&nbsp;<a href="/index.html?searchoptions=1" onclick="this.href='/index.html?searchoptions=1&amp;former='+document.getElementById('searchform').search.value+'&amp;contentdom='+radioValue(document.getElementById('searchform').contentdom)">more options...</a>::#(/searchoptions)#
</div>
#(/searchdomswitches)#
<input type="hidden" name="nav" value="#[search.navigation]#" />
<input type="hidden" name="startRecord" value="0" />
<input type="hidden" name="indexof" value="off" />
<input type="hidden" name="meanCount" value="5" />
#(searchoptions)#
<input type="hidden" name="resource" value="global" />
<input type="hidden" name="urlmaskfilter" value=".*" />
<input type="hidden" name="prefermaskfilter" value="" />
<input type="hidden" name="indexof" value="off" />
<input type="hidden" name="meanCount" value="5" />
<input type="hidden" name="maximumRecords" value="#[maximumRecords]#" />
#(searchoptions)#::
</fieldset>
<p><a href="/index.html?searchoptions=2" onclick="this.href='/index.html?searchoptions=2&amp;former='+document.getElementById('searchform').search.value+'&amp;contentdom='+radioValue(document.getElementById('searchform').contentdom)">more options...</a></p>
::
</fieldset>
@ -100,28 +100,17 @@
<tr>
<td><label for="count">Max. number of results</label>:</td>
<td>
<select id="count" name="count">
<option #(count-10)#::selected="selected"#(/count-10)#>10</option>
<option #(count-50)#::selected="selected"#(/count-50)#>50</option>
<option #(count-100)#::selected="selected"#(/count-100)#>100</option>
</select>
<input type="radio" name="maximumRecords" value="10" #(count-10)#::checked="checked"#(/count-10)#/>10
<input type="radio" name="maximumRecords" value="50" #(count-50)#::checked="checked"#(/count-50)#/>50
<input type="radio" name="maximumRecords" value="100" #(count-100)#::checked="checked"#(/count-100)#/>100
</td>
</tr>
<tr>
#(resource-select)#::
<td><label for="resource">Resource</label>:</td>
<td>
<select id="resource" name="resource">
<option value="global" #(global)#::selected="selected"#(/global)#
#(global-disabled)#::disabled="disabled"#(/global-disabled)#>global</option>
<option value="local" #(local)#::selected="selected"#(/local)#>local</option>
</select>
#(global-disabled)#::
Global search is disabled because<br />#(reason)#
<a class="lock" href="ConfigNetwork_p.html">DHT Distribution</a> is::
<a class="lock" href="ConfigNetwork_p.html">Index Receive</a> is::
<a class="lock" href="ConfigNetwork_p.html">DHT Distribution and Index Receive</a> are#(/reason)#
disabled.#(/global-disabled)#
<input type="radio" name="resource" value="global" #(global)#::checked="checked"#(/global)# #(global-disabled)#::disabled="disabled"#(/global-disabled)#/>global
<input type="radio" name="resource" value="local" #(local)#::checked="checked"#(/local)#/>local
</td>
#(/resource-select)#
</tr>
@ -159,7 +148,7 @@
<tr><td></td><td></td></tr>
<tr>
<td>
<a href="http://www.yacy-websuche.de/wiki/index.php/En:SearchParameters">advanced parameters</a>
<a href="http://www.yacy-websuche.de/wiki/index.php/En:SearchParameters">help: advanced parameters</a>
</td>
<td>
</td>

32
htroot/index.java
Unescape View File

@ -9,7 +9,7 @@
// $LastChangedBy$
//
// LICENSE
//
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
@ -41,15 +41,15 @@ public class index {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
String forwardTarget = sb.getConfig(SwitchboardConstants.INDEX_FORWARD, "");
final String forwardTarget = sb.getConfig(SwitchboardConstants.INDEX_FORWARD, "");
if (forwardTarget.length() > 0) {
// forward the page
prop.put("forward", 1);
prop.put("forward_target", forwardTarget);
return prop;
}
// access control
final boolean authorizedAccess = sb.verifyAuthentication(header, false);
if ((post != null) && (post.containsKey("publicPage"))) {
@ -58,10 +58,10 @@ public class index {
return prop;
}
}
final boolean global = (post == null) ? true : post.get("resource", "global").equals("global");
int searchoptions = (post == null) ? 1 : post.getInt("searchoptions", 1);
boolean global = (post == null) ? true : post.get("resource", "global").equals("global");
int searchoptions = (post == null) ? 0 : Math.min(1, post.getInt("searchoptions", 0));
if (!sb.getConfigBool("search.options", true)) searchoptions = 0;
final String former = (post == null) ? "" : post.get("former", "");
final int count = Math.min(100, (post == null) ? 10 : post.getInt("count", 10));
@ -71,12 +71,12 @@ public class index {
final String constraint = (post == null) ? "" : post.get("constraint", "");
final String cat = (post == null) ? "href" : post.get("cat", "href");
final int type = (post == null) ? 0 : post.getInt("type", 0);
final boolean indexDistributeGranted = sb.getConfigBool(SwitchboardConstants.INDEX_DIST_ALLOW, true);
//final boolean indexDistributeGranted = sb.getConfigBool(SwitchboardConstants.INDEX_DIST_ALLOW, true);
final boolean indexReceiveGranted = sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW, true) ||
sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_AUTODISABLED, true);
//global = global && indexDistributeGranted && indexReceiveGranted;
global = global && indexReceiveGranted;
// search domain
ContentDomain contentdom = ContentDomain.TEXT;
final String cds = (post == null) ? "text" : post.get("contentdom", "text");
@ -85,7 +85,7 @@ public class index {
if (cds.equals("video")) contentdom = ContentDomain.VIDEO;
if (cds.equals("image")) contentdom = ContentDomain.IMAGE;
if (cds.equals("app")) contentdom = ContentDomain.APP;
// we create empty entries for template strings
String promoteSearchPageGreeting = env.getConfig(SwitchboardConstants.GREETING, "");
if (env.getConfigBool(SwitchboardConstants.GREETING_NETWORK_NAME, false)) promoteSearchPageGreeting = env.getConfig("network.unit.description", "");
@ -102,10 +102,9 @@ public class index {
prop.put("searchoptions_count-10", (count == 10) ? "1" : "0");
prop.put("searchoptions_count-50", (count == 50) ? "1" : "0");
prop.put("searchoptions_count-100", (count == 100) ? "1" : "0");
prop.put("searchoptions_resource-select", (sb.peers == null) ? 0 : sb.peers.sizeConnected() > 0 ? 1 : 0);
prop.put("searchoptions_resource-select", (sb.peers == null || sb.peers.sizeConnected() == 0 || !global) ? 0 : 1);
prop.put("searchoptions_resource-select_global", global ? "1" : "0");
prop.put("searchoptions_resource-select_global-disabled", (indexReceiveGranted && indexDistributeGranted) ? "0" : "1");
prop.put("searchoptions_resource-select_global-disabled_reason", (indexReceiveGranted) ? "0" : (indexDistributeGranted ? "1" : "2"));
prop.put("searchoptions_resource-select_global-disabled", indexReceiveGranted ? "0" : "1");
prop.put("searchoptions_resource-select_local", global ? "0" : "1");
prop.put("searchoptions_urlmaskoptions", "0");
prop.putHTML("searchoptions_urlmaskoptions_urlmaskfilter", urlmaskfilter);
@ -120,6 +119,7 @@ public class index {
prop.put("pi", sb.getConfigBool("publicAdministratorPi", false) ? 1 : 0);
prop.putHTML("constraint", constraint);
prop.put("searchdomswitches", sb.getConfigBool("search.text", true) || sb.getConfigBool("search.audio", true) || sb.getConfigBool("search.video", true) || sb.getConfigBool("search.image", true) || sb.getConfigBool("search.app", true) ? 1 : 0);
prop.put("searchdomswitches_searchoptions", searchoptions);
prop.put("searchdomswitches_searchtext", sb.getConfigBool("search.text", true) ? 1 : 0);
prop.put("searchdomswitches_searchaudio", sb.getConfigBool("search.audio", true) ? 1 : 0);
prop.put("searchdomswitches_searchvideo", sb.getConfigBool("search.video", true) ? 1 : 0);
@ -134,7 +134,7 @@ public class index {
prop.put("search.verify", sb.getConfig("search.verify", "iffresh") );
// online caution timing
sb.localSearchLastAccess = System.currentTimeMillis();
return prop;
}
}

2
htroot/js/yacy-ymarks-bookmark-actions.js
Unescape View File

@ -18,6 +18,7 @@ function bm_action(com,grid) {
}
else if (com=='Add') {
$('#bmaddform').resetForm();
$("input[name='bm_url']").removeAttr("disabled");
$("#bm_url").blur(function() {
var url = $("input[name='bm_url']").getValue();
$.ajax({
@ -47,6 +48,7 @@ function bm_action(com,grid) {
alert("Editing of more than one selected bookmark is currently not supportet!");
return false;
}
$("input[name='bm_url']").attr("disabled","disabled");
$("input[name='bm_url']").setValue($('.trSelected',grid).find('.url').text());
$("input[name='bm_title']").setValue($('.trSelected',grid).find('h3.linktitle').text().trim());
$("textarea[name='bm_desc']").setValue($('.trSelected',grid).find('p.desc').text().trim());

1
htroot/js/yacy-ymarks-tag-actions.js
Unescape View File

@ -1,5 +1,6 @@
/* Initialize Tag Actions */
function tag_action(com,grid) {
alert("Sorry, the function you have requested is not yet available!");
if (com=='Add') {
flex = grid;
$('#tagaddform').resetForm();

39
htroot/js/yacy-ymarks.js
Unescape View File

@ -20,7 +20,7 @@ $(document).ready(function() {
{display: 'Title', name : 'title', width : 400, sortable : true, align: 'left'},
{display: 'Tags', name : 'tags', width : 160, sortable : false, align: 'left'},
{display: 'Folders', name : 'folders', width : 160, sortable : true, align: 'left', hide: true},
{display: 'Date added', name : 'date_added', width : 100, sortable : true, align: 'left', hide: true},
{display: 'Date added', name : 'date_added', width : 100, sortable : true, align: 'left'},
{display: 'Date modified', name : 'date_modified', width : 100, sortable : true, align: 'left'},
{display: 'Date visited', name : 'date_visited', width : 100, sortable : true, align: 'left', hide: true},
{display: 'API PK', name : 'apicall_pk', width : 85, sortable : true, align: 'left', hide: true},
@ -98,17 +98,34 @@ $(document).ready(function() {
});
$("#ymarks_treeview").bind("click", function(event) {
if ($(event.target).is("li") || $(event.target).parents("li").length) {
var folder = $(event.target).parents("li").filter(":first").attr("id");
$('#ymarks_flexigrid').flexOptions({
query: folder,
qtype: "_folder",
newp: 1
});
$('#ymarks_flexigrid').flexReload();
return false;
}
if ($(event.target).is("li") || $(event.target).parents("li").length) {
var folder = $(event.target).parents("li").filter(":first").attr("id");
$('#ymarks_flexigrid').flexOptions({
query: folder,
qtype: "_folder",
newp: 1
});
$('#ymarks_flexigrid').flexReload();
return false;
}
});
$('input[name=importer]').change(function() {
if ($("input[name=importer]:checked").val() == 'crawls') {
$("input[name='bmkfile']").attr("disabled","disabled");
$("input[name='root']").setValue("/Crawl Start");
}
else {
$("input[name='bmkfile']").removeAttr("disabled");
$("input[name='root']").setValue("/Imported Bookmarks");
}
if ($("input[name=importer]:checked").val() == 'bmks') {
$("input[name='bmkfile']").attr("disabled","disabled");
}
else {
$("input[name='bmkfile']").removeAttr("disabled");
}
});
$("#example").multiselect();

8
htroot/yacy/search.java
Unescape View File

@ -56,10 +56,10 @@ import net.yacy.kelondro.util.ByteBuffer;
import net.yacy.kelondro.util.EventTracker;
import net.yacy.kelondro.util.ISO639;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.peers.Seed;
import net.yacy.peers.EventChannel;
import net.yacy.peers.Protocol;
import net.yacy.peers.Network;
import net.yacy.peers.Protocol;
import net.yacy.peers.Seed;
import net.yacy.peers.graphics.ProfilingGraph;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
@ -230,7 +230,7 @@ public final class search {
count,
0,
filter,
QueryParams.SEARCHDOM_LOCAL,
QueryParams.Searchdom.LOCAL,
-1,
null,
false,
@ -290,7 +290,7 @@ public final class search {
count,
0,
filter,
QueryParams.SEARCHDOM_LOCAL,
QueryParams.Searchdom.LOCAL,
-1,
constraint,
false,

4
htroot/yacysearch.java
Unescape View File

@ -584,8 +584,8 @@ public class yacysearch {
itemsPerPage,
offset,
urlmask,
(clustersearch && globalsearch) ? QueryParams.SEARCHDOM_CLUSTERALL :
((globalsearch) ? QueryParams.SEARCHDOM_GLOBALDHT : QueryParams.SEARCHDOM_LOCAL),
(clustersearch && globalsearch) ? QueryParams.Searchdom.CLUSTER :
((globalsearch) ? QueryParams.Searchdom.GLOBAL : QueryParams.Searchdom.LOCAL),
20,
constraint,
true,

2
source/de/anomic/data/ymark/YMarkEntry.java
Unescape View File

@ -139,7 +139,7 @@ public class YMarkEntry extends TreeMap<String, String> {
}
}
private void setDefaults() {
public void setDefaults() {
for (BOOKMARK b : BOOKMARK.values()) {
if(!b.deflt().isEmpty() && !this.containsKey(b.key())) {
this.put(b.key(), b.deflt());

6
source/de/anomic/data/ymark/YMarkTables.java
Unescape View File

@ -267,7 +267,11 @@ public class YMarkTables {
bmk_row = this.worktables.select(bmk_table, urlHash);
if (bmk_row == null) {
// create and insert new entry
this.worktables.insert(bmk_table, urlHash, bmk.getData());
if(!bmk.containsKey(YMarkEntry.BOOKMARK.DATE_ADDED.key())) {
bmk.put(YMarkEntry.BOOKMARK.DATE_ADDED.key(), date);
bmk.put(YMarkEntry.BOOKMARK.DATE_MODIFIED.key(), date);
}
this.worktables.insert(bmk_table, urlHash, bmk.getData());
} else {
// modify and update existing entry
HashSet<String> oldSet;

18
source/net/yacy/search/query/QueryParams.java
Unescape View File

@ -61,11 +61,9 @@ import net.yacy.search.snippet.ContentDomain;
public final class QueryParams {
public static final int SEARCHDOM_LOCAL = 0;
public static final int SEARCHDOM_CLUSTERDHT = 1;
public static final int SEARCHDOM_CLUSTERALL = 2;
public static final int SEARCHDOM_GLOBALDHT = 3;
public static final int SEARCHDOM_GLOBALALL = 4;
public enum Searchdom {
LOCAL, CLUSTER, GLOBAL;
}
private static final String ampersand = "&amp;";
@ -89,7 +87,7 @@ public final class QueryParams {
public final ContentDomain contentdom;
public final String targetlang;
public final String navigators;
public final int domType;
public final Searchdom domType;
public final int zonecode;
public final int domMaxTargets;
public final int maxDistance;
@ -148,7 +146,7 @@ public final class QueryParams {
this.itemsPerPage = itemsPerPage;
this.offset = 0;
this.targetlang = "en";
this.domType = SEARCHDOM_LOCAL;
this.domType = Searchdom.LOCAL;
this.zonecode = DigestURI.TLD_any_zone_filter;
this.domMaxTargets = 0;
this.constraint = constraint;
@ -178,7 +176,7 @@ public final class QueryParams {
final String navigators,
final CacheStrategy snippetCacheStrategy,
final int itemsPerPage, final int offset, final String urlMask,
final int domType, final int domMaxTargets,
final Searchdom domType, final int domMaxTargets,
final Bitfield constraint, final boolean allofconstraint,
final String site,
final String authorhash,
@ -257,7 +255,7 @@ public final class QueryParams {
}
public boolean isLocal() {
return this.domType == SEARCHDOM_LOCAL;
return this.domType == Searchdom.LOCAL;
}
public static HandleSet hashes2Set(final String query) {
@ -471,7 +469,7 @@ public final class QueryParams {
context.append(asterisk);
context.append(this.maxDistance);
context.append(asterisk);
context.append(this.snippetCacheStrategy == null ? "null" : this.snippetCacheStrategy.name());
context.append(this.snippetCacheStrategy == null ? "null" : this.snippetCacheStrategy.name());
if (anonymized) {
this.idCacheAnon = context.toString();
} else {

51
source/net/yacy/search/query/SearchEvent.java
Unescape View File

@ -103,7 +103,7 @@ public final class SearchEvent {
this.peers = peers;
this.workTables = workTables;
this.query = query;
this.secondarySearchSuperviser = (query.queryHashes.size() > 1) ? new SecondarySearchSuperviser() : null; // generate abstracts only for combined searches
this.secondarySearchSuperviser = (this.query.queryHashes.size() > 1) ? new SecondarySearchSuperviser() : null; // generate abstracts only for combined searches
if (this.secondarySearchSuperviser != null) this.secondarySearchSuperviser.start();
this.primarySearchThreads = null;
this.secondarySearchThreads = null;
@ -114,9 +114,8 @@ public final class SearchEvent {
this.IAmaxcounthash = null;
this.IAneardhthash = null;
this.localSearchThread = null;
this.order = new ReferenceOrder(query.ranking, UTF8.getBytes(query.targetlang));
boolean remote = (query.domType == QueryParams.SEARCHDOM_GLOBALDHT) || (query.domType == QueryParams.SEARCHDOM_CLUSTERALL);
if (remote && peers.sizeConnected() == 0) remote = false;
this.order = new ReferenceOrder(this.query.ranking, UTF8.getBytes(this.query.targetlang));
final boolean remote = (this.query.domType == QueryParams.Searchdom.GLOBAL || this.query.domType == QueryParams.Searchdom.CLUSTER) && peers.sizeConnected() > 0 && peers.mySeed().getFlagAcceptRemoteIndex();
final long start = System.currentTimeMillis();
if (remote) {
// initialize a ranking process that is the target for data
@ -128,32 +127,32 @@ public final class SearchEvent {
// start global searches
final long timer = System.currentTimeMillis();
this.primarySearchThreads = (query.queryHashes.isEmpty()) ? null : RemoteSearch.primaryRemoteSearches(
QueryParams.hashSet2hashString(query.queryHashes),
QueryParams.hashSet2hashString(query.excludeHashes),
query.prefer,
query.urlMask,
query.snippetMatcher,
query.targetlang == null ? "" : query.targetlang,
query.sitehash == null ? "" : query.sitehash,
query.authorhash == null ? "" : query.authorhash,
this.primarySearchThreads = (this.query.queryHashes.isEmpty()) ? null : RemoteSearch.primaryRemoteSearches(
QueryParams.hashSet2hashString(this.query.queryHashes),
QueryParams.hashSet2hashString(this.query.excludeHashes),
this.query.prefer,
this.query.urlMask,
this.query.snippetMatcher,
this.query.targetlang == null ? "" : this.query.targetlang,
this.query.sitehash == null ? "" : this.query.sitehash,
this.query.authorhash == null ? "" : this.query.authorhash,
remote_maxcount,
remote_maxtime,
query.maxDistance,
query.getSegment(),
this.query.maxDistance,
this.query.getSegment(),
peers,
this.rankingProcess,
this.secondarySearchSuperviser,
Switchboard.urlBlacklist,
query.ranking,
query.constraint,
(query.domType == QueryParams.SEARCHDOM_GLOBALDHT) ? null : preselectedPeerHashes,
this.query.ranking,
this.query.constraint,
(this.query.domType == QueryParams.Searchdom.GLOBAL) ? null : preselectedPeerHashes,
burstRobinsonPercent,
burstMultiwordPercent);
if (this.primarySearchThreads != null) {
Log.logFine("SEARCH_EVENT", "STARTING " + this.primarySearchThreads.length + " THREADS TO CATCH EACH " + remote_maxcount + " URLs");
this.rankingProcess.moreFeeders(this.primarySearchThreads.length);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(query.id(true), Type.REMOTESEARCH_START, "", this.primarySearchThreads.length, System.currentTimeMillis() - timer), false);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), Type.REMOTESEARCH_START, "", this.primarySearchThreads.length, System.currentTimeMillis() - timer), false);
// finished searching
Log.logFine("SEARCH_EVENT", "SEARCH TIME AFTER GLOBAL-TRIGGER TO " + this.primarySearchThreads.length + " PEERS: " + ((System.currentTimeMillis() - start) / 1000) + " seconds");
} else {
@ -162,7 +161,7 @@ public final class SearchEvent {
}
// start worker threads to fetch urls and snippets
this.resultFetcher = new SnippetProcess(loader, this.rankingProcess, query, this.peers, this.workTables, 3000, deleteIfSnippetFail);
this.resultFetcher = new SnippetProcess(loader, this.rankingProcess, this.query, this.peers, this.workTables, 3000, deleteIfSnippetFail);
} else {
// do a local search
this.rankingProcess = new RWIProcess(this.query, this.order, max_results_preparation);
@ -192,7 +191,7 @@ public final class SearchEvent {
this.IACount.put(wordhash, LargeNumberCache.valueOf(container.size()));
this.IAResults.put(wordhash, WordReferenceFactory.compressIndex(container, null, 1000).toString());
}
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(query.id(true), Type.ABSTRACTS, "", this.rankingProcess.searchContainerMap().size(), System.currentTimeMillis() - timer), false);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), Type.ABSTRACTS, "", this.rankingProcess.searchContainerMap().size(), System.currentTimeMillis() - timer), false);
} else {
this.rankingProcess.start(); // start concurrently
// but give process time to accumulate a certain amount of data
@ -206,16 +205,16 @@ public final class SearchEvent {
}
// start worker threads to fetch urls and snippets
this.resultFetcher = new SnippetProcess(loader, this.rankingProcess, query, this.peers, this.workTables, 500, deleteIfSnippetFail);
this.resultFetcher = new SnippetProcess(loader, this.rankingProcess, this.query, this.peers, this.workTables, 500, deleteIfSnippetFail);
}
// clean up events
SearchEventCache.cleanupEvents(false);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(query.id(true), Type.CLEANUP, "", 0, 0), false);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), Type.CLEANUP, "", 0, 0), false);
// store this search to a cache so it can be re-used
if (MemoryControl.available() < 1024 * 1024 * 100) SearchEventCache.cleanupEvents(true);
SearchEventCache.put(query.id(false), this);
SearchEventCache.put(this.query.id(false), this);
}
@ -372,8 +371,8 @@ public final class SearchEvent {
}
public ResultEntry oneResult(final int item, final long timeout) {
if ((this.query.domType == QueryParams.SEARCHDOM_GLOBALDHT) ||
(this.query.domType == QueryParams.SEARCHDOM_CLUSTERALL)) {
if ((this.query.domType == QueryParams.Searchdom.GLOBAL) ||
(this.query.domType == QueryParams.Searchdom.CLUSTER)) {
// this is a search using remote search threads. Also the local
// search thread is started as background process
if ((this.localSearchThread != null) && (this.localSearchThread.isAlive())) {

Write Preview
Loading…
Cancel
Save