boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added show from cache in search results (not yet finished)

Browse Source
pull/1/head
Michael Peter Christen 13 years ago
parent 5e562dcdb7
commit 5aee19daa4
9 changed files with 99 additions and 16 deletions
Show Stats Download Patch File Download Diff File

5
defaults/yacy.init
Unescape View File

@ -744,9 +744,10 @@ search.target.special.pattern =
# these information pieces may be switched on or off
search.result.show.date = true
search.result.show.size = false
search.result.show.metadata = true
search.result.show.parser = true
search.result.show.metadata = false
search.result.show.parser = false
search.result.show.pictures = false
search.result.show.cache = true
# search navigators: comma-separated list of default values for search navigation.
# can be temporary different if search string is given with differen navigation values

5
htroot/ConfigPortal.html
Unescape View File

@ -62,7 +62,7 @@
<img src="env/grafics/idea.png" width="32" height="32" alt="idea" align="center"/>Speed up search results with this option! (use CACHEONLY or FALSE to switch off verification)<br/>
<input type="radio" name="search.verify" value="nocache" #(search.verify.nocache)#::checked="checked"#(/search.verify.nocache)# onclick="document.getElementById('search_verify_delete').disabled=false;document.getElementById('search_verify_delete').checked=true;"/>NOCACHE: no use of web cache, load all snippets online<br/>
<input type="radio" name="search.verify" value="iffresh" #(search.verify.iffresh)#::checked="checked"#(/search.verify.iffresh)# onclick="document.getElementById('search_verify_delete').disabled=false;document.getElementById('search_verify_delete').checked=true;"/>IFFRESH: use the cache if the cache exists and is fresh otherwise load online<br/>
<input type="radio" name="search.verify" value="ifexist" #(search.verify.ifexist)#::checked="checked"#(/search.verify.ifexist)# onclick="document.getElementById('search_verify_delete').disabled=false;document.getElementById('search_verify_delete').checked=true;"/>IFEXIST: use the cache if the cache exist or load online<br/>
<input type="radio" name="search.verify" value="ifexist" #(search.verify.ifexist)#::checked="checked"#(/search.verify.ifexist)# />IFEXIST: use the cache if the cache exist or load online<br/>
<input type="checkbox" name="search.verify.delete" id="search_verify_delete" value="true" #(search.verify.delete)#::checked="checked"#(/search.verify.delete)# />If verification fails, delete index reference<br/><br/>
<input type="radio" name="search.verify" value="cacheonly" #(search.verify.cacheonly)#::checked="checked"#(/search.verify.cacheonly)# onclick="document.getElementById('search_verify_delete').disabled=true;document.getElementById('search_verify_delete').checked=false;"/>CACHEONLY: never go online, use all content from cache. If no cache entry exist, consider content nevertheless as available and show result without snippet<br/>
<input type="radio" name="search.verify" value="false" #(search.verify.false)#::checked="checked"#(/search.verify.false)# onclick="document.getElementById('search_verify_delete').disabled=true;document.getElementById('search_verify_delete').checked=false;"/>FALSE: no link verification and not snippet generation: all search results are valid without verification
@ -74,7 +74,8 @@
<input type="checkbox" name="search.result.show.size" value="true" #(search.result.show.size)#::checked="checked"#(/search.result.show.size)# />Size&nbsp;
<input type="checkbox" name="search.result.show.metadata" value="true" #(search.result.show.metadata)#::checked="checked"#(/search.result.show.metadata)# />Metadata&nbsp;
<input type="checkbox" name="search.result.show.parser" value="true" #(search.result.show.parser)#::checked="checked"#(/search.result.show.parser)# />Parser&nbsp;
<input type="checkbox" name="search.result.show.pictures" value="true" #(search.result.show.pictures)#::checked="checked"#(/search.result.show.pictures)# />Pictures
<input type="checkbox" name="search.result.show.pictures" value="true" #(search.result.show.pictures)#::checked="checked"#(/search.result.show.pictures)# />Pictures&nbsp;
<input type="checkbox" name="search.result.show.cache" value="true" #(search.result.show.cache)#::checked="checked"#(/search.result.show.cache)# />Cache
</dd>
<dt>Show Navigation on Side-Bar</dt>

3
htroot/ConfigPortal.java
Unescape View File

@ -92,6 +92,7 @@ public class ConfigPortal {
sb.setConfig("search.result.show.metadata", post.getBoolean("search.result.show.metadata", false));
sb.setConfig("search.result.show.parser", post.getBoolean("search.result.show.parser", false));
sb.setConfig("search.result.show.pictures", post.getBoolean("search.result.show.pictures", false));
sb.setConfig("search.result.show.cache", post.getBoolean("search.result.show.cache", false));
sb.setConfig(SwitchboardConstants.SEARCH_VERIFY, post.get("search.verify", "ifexist"));
sb.setConfig(SwitchboardConstants.SEARCH_VERIFY_DELETE, post.getBoolean("search.verify.delete", false));
@ -136,6 +137,7 @@ public class ConfigPortal {
sb.setConfig("search.result.show.metadata", true);
sb.setConfig("search.result.show.parser", true);
sb.setConfig("search.result.show.pictures", false);
sb.setConfig("search.result.show.cache", false);
sb.setConfig(SwitchboardConstants.SEARCH_VERIFY, "iffresh");
sb.setConfig(SwitchboardConstants.SEARCH_VERIFY_DELETE, "true");
sb.setConfig("about.headline", "");
@ -165,6 +167,7 @@ public class ConfigPortal {
prop.put("search.result.show.metadata", sb.getConfigBool("search.result.show.metadata", false) ? 1 : 0);
prop.put("search.result.show.parser", sb.getConfigBool("search.result.show.parser", false) ? 1 : 0);
prop.put("search.result.show.pictures", sb.getConfigBool("search.result.show.pictures", false) ? 1 : 0);
prop.put("search.result.show.cache", sb.getConfigBool("search.result.show.cache", false) ? 1 : 0);
prop.put("search.navigation.hosts", sb.getConfig("search.navigation", "").indexOf("hosts",0) >= 0 ? 1 : 0);
prop.put("search.navigation.authors", sb.getConfig("search.navigation", "").indexOf("authors",0) >= 0 ? 1 : 0);

6
htroot/yacy/search.java
Unescape View File

@ -41,12 +41,12 @@ import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.Classification;
import net.yacy.cora.document.Classification.ContentDomain;
import net.yacy.cora.document.RSSMessage;
import net.yacy.cora.lod.SimpleVocabulary;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.sorting.ScoreMap;
import net.yacy.cora.sorting.WeakPriorityBlockingQueue;
import net.yacy.document.Autotagging.Metatag;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.data.word.WordReference;
import net.yacy.kelondro.data.word.WordReferenceFactory;
@ -236,7 +236,7 @@ public final class search {
prefer,
ContentDomain.contentdomParser(contentdom),
language,
new HashSet<Metatag>(),
new HashSet<SimpleVocabulary.Metatag>(),
"", // no navigation
null, // no snippet computation
count,
@ -299,7 +299,7 @@ public final class search {
prefer,
ContentDomain.contentdomParser(contentdom),
language,
new HashSet<Metatag>(),
new HashSet<SimpleVocabulary.Metatag>(),
"", // no navigation
null, // no snippet computation
count,

9
htroot/yacysearch.java
Unescape View File

@ -45,13 +45,12 @@ import net.yacy.cora.document.Classification;
import net.yacy.cora.document.Classification.ContentDomain;
import net.yacy.cora.document.RSSMessage;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.lod.SimpleVocabulary;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.ResponseHeader;
import net.yacy.cora.services.federated.yacy.CacheStrategy;
import net.yacy.document.Autotagging.Metatag;
import net.yacy.document.Autotagging.Vocabulary;
import net.yacy.document.Condenser;
import net.yacy.document.Document;
import net.yacy.document.LibraryProvider;
@ -125,9 +124,9 @@ public class yacysearch {
prop.put("focus", ((post == null) ? true : post.get("focus", "1").equals("1")) ? 1 : 0);
// produce vocabulary navigation sidebars
Collection<Vocabulary> vocabularies = LibraryProvider.autotagging.getVocabularies();
Collection<SimpleVocabulary> vocabularies = LibraryProvider.autotagging.getVocabularies();
int j = 0;
for (Vocabulary v: vocabularies) {
for (SimpleVocabulary v: vocabularies) {
prop.put("sidebarVocabulary_" + j + "_vocabulary", v.getName());
j++;
}
@ -464,7 +463,7 @@ public class yacysearch {
}
int voc = 0;
Collection<Metatag> metatags = new ArrayList<Metatag>(1);
Collection<SimpleVocabulary.Metatag> metatags = new ArrayList<SimpleVocabulary.Metatag>(1);
while ((voc = querystring.indexOf("/vocabulary/", 0)) >= 0) {
String vocabulary = "";
int ve = querystring.indexOf(' ', voc + 12);

1
htroot/yacysearchitem.html
Unescape View File

@ -29,6 +29,7 @@
#(showMetadata)#::&nbsp;|&nbsp;<a href="api/yacydoc.html?urlhash=#[urlhash]#" target="_blank" onclick="return hs.htmlExpand(this, { objectType: 'ajax'} )">Metadata</a>#(/showMetadata)#
#(showParser)#::&nbsp;|&nbsp;<a href="ViewFile.html?urlHash=#[urlhash]#&amp;words=#[words]#&amp" target="_blank">Parser</a>#(/showParser)#
#(showPictures)#::&nbsp;|&nbsp;<a href="yacysearch.html?cat=image&amp;url=#[link]#&amp;query=#[former]#">Pictures</a></p>#(/showPictures)#
#(showCache)#::&nbsp;|&nbsp;<a href="yacysearch.html?cat=image&amp;url=#[link]#&amp;query=#[former]#">Cache</a></p>#(/showCache)#
</div>
::
#(item)#::<div class="thumbcontainer">

3
htroot/yacysearchitem.java
Unescape View File

@ -128,6 +128,7 @@ public class yacysearchitem {
prop.put("content_showMetadata", sb.getConfigBool("search.result.show.metadata", true) ? 1 : 0);
prop.put("content_showParser", sb.getConfigBool("search.result.show.parser", true) ? 1 : 0);
prop.put("content_showPictures", sb.getConfigBool("search.result.show.pictures", true) ? 1 : 0);
prop.put("content_showCache", sb.getConfigBool("search.result.show.cache", true) ? 1 : 0);
prop.put("content_authorized", authenticated ? "1" : "0");
final String urlhash = ASCII.String(result.hash());
prop.put("content_authorized_bookmark", sb.tables.bookmarks.hasBookmark("admin", urlhash) ? "0" : "1");
@ -198,7 +199,7 @@ public class yacysearchitem {
} else {
prop.put("content_code", "");
}
if (result.lat() == 0.0f || result.lon() == 0.0f) {
if (result.lat() == 0.0d || result.lon() == 0.0d) {
prop.put("content_loc", 0);
} else {
prop.put("content_loc", 1);

4
htroot/yacysearchtrailer.java
Unescape View File

@ -28,9 +28,9 @@ import java.util.Iterator;
import java.util.Map;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.lod.SimpleVocabulary;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.sorting.ScoreMap;
import net.yacy.document.Autotagging;
import net.yacy.document.LibraryProvider;
import net.yacy.kelondro.util.Formatter;
import net.yacy.peers.graphics.ProfilingGraph;
@ -307,7 +307,7 @@ public class yacysearchtrailer {
while (i < 20 && navigatorIterator.hasNext()) {
name = navigatorIterator.next();
count = ve.getValue().get(name);
nav = "%2Fvocabulary%2F" + navname + "%2F" + MultiProtocolURI.escape(Autotagging.encodePrintname(name)).toString();
nav = "%2Fvocabulary%2F" + navname + "%2F" + MultiProtocolURI.escape(SimpleVocabulary.Metatag.encodePrintname(name)).toString();
queryStringForUrl = theQuery.queryStringForUrl();
p = queryStringForUrl.indexOf(nav);
if (p < 0) {

79
source/net/yacy/cora/storage/Files.java
Unescape View File

@ -1,5 +1,5 @@
/**
* ConfigurationSet
* Files
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany
* First released 29.06.2011 at http://yacy.net
*
@ -26,15 +26,92 @@ package net.yacy.cora.storage;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.zip.GZIPInputStream;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
public class Files {
/**
* open text files for reading. If the files are compressed, choose the
* appropriate decompression method automatically
* @param f
* @return the input stream for the file
* @throws IOException
*/
public static InputStream read(File f) throws IOException {
// make input stream
InputStream is = new BufferedInputStream(new FileInputStream(f));
if (f.toString().endsWith(".bz2")) is = new BZip2CompressorInputStream(is);
if (f.toString().endsWith(".gz")) is = new GZIPInputStream(is);
return is;
}
/**
* reading a file line by line should be done with two concurrent processes
* - one reading the file and doing IO operations
* - one processing the result
* This method makes is easy to create concurrent file readers by providing
* a process that fills a blocking queue with lines from a file.
* After the method is called, it returns immediately a blocking queue which is
* filled concurrently with the lines of the file. When the reading is finished,
* this is signalled with a poison entry, the POISON_LINE String which can be
* compared with an "==" operation.
* @param f the file to read
* @param maxQueueSize
* @return a blocking queue which is filled with the lines, terminated by POISON_LINE
* @throws IOException
*/
public final static String POISON_LINE = "__@POISON__";
public static BlockingQueue<String> concurentLineReader(final File f, final int maxQueueSize) throws IOException {
final BlockingQueue<String> q = new ArrayBlockingQueue<String>(maxQueueSize);
final InputStream is = read(f);
final BufferedReader br = new BufferedReader(new InputStreamReader(is, "UTF-8"));
Thread t = new Thread() {
public void run() {
String line;
try {
while ((line = br.readLine()) != null) {
q.put(line);
}
} catch (IOException e) {
} catch (InterruptedException e) {
} finally {
try {
q.put(POISON_LINE);
try {
br.close();
is.close();
} catch (IOException ee) {
}
} catch (InterruptedException e) {
// last try
q.add(POISON_LINE);
try {
br.close();
is.close();
} catch (IOException ee) {
}
}
}
}
};
t.start();
return q;
}
/**
* copy a file or a complete directory
* @param from the source file or directory

Write Preview
Loading…
Cancel
Save