boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'master' of ssh://git@gitorious.org/yacy/rc1.git

Browse Source
pull/1/head
Michael Peter Christen 13 years ago
parent 89142d1e8d d10627d591
commit 3e1bc9477f
20 changed files with 63 additions and 73 deletions
Show Stats Download Patch File Download Diff File

0
yacy.logging → defaults/yacy.logging
Unescape View File

9
htroot/Load_RSS_p.java
Unescape View File

@ -1,5 +1,5 @@
/** /**
* oad_RSS_p * Load_RSS_p
* Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany * Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
* First released 20.08.2010 at http://yacy.net * First released 20.08.2010 at http://yacy.net
* *
@ -26,6 +26,7 @@ import java.util.Date;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.regex.Pattern;
import net.yacy.cora.document.Hit; import net.yacy.cora.document.Hit;
import net.yacy.cora.document.RSSFeed; import net.yacy.cora.document.RSSFeed;
@ -83,7 +84,7 @@ public class Load_RSS_p {
if (messageurl.length() == 0) continue; if (messageurl.length() == 0) continue;
final byte[] api_pk = row.get("api_pk"); final byte[] api_pk = row.get("api_pk");
final Row r = api_pk == null ? null : sb.tables.select("api", api_pk); final Row r = api_pk == null ? null : sb.tables.select("api", api_pk);
if (r == null || !r.get("comment", "").matches(".*\\Q" + messageurl + "\\E.*")) { if (r == null || !r.get("comment", "").matches(".*" + Pattern.quote(messageurl) + ".*")) {
d.add(row.getPK()); d.add(row.getPK());
} }
} }
@ -125,7 +126,7 @@ public class Load_RSS_p {
if (messageurl.length() == 0) continue; if (messageurl.length() == 0) continue;
final byte[] api_pk = row.get("api_pk"); final byte[] api_pk = row.get("api_pk");
final Row r = api_pk == null ? null : sb.tables.select("api", api_pk); final Row r = api_pk == null ? null : sb.tables.select("api", api_pk);
if (r != null && r.get("comment", "").matches(".*\\Q" + messageurl + "\\E.*")) { if (r != null && r.get("comment", "").matches(".*" + Pattern.quote(messageurl) + ".*")) {
d.add(row.getPK()); d.add(row.getPK());
} }
} }
@ -194,7 +195,7 @@ public class Load_RSS_p {
// check if feed is registered in scheduler // check if feed is registered in scheduler
final byte[] api_pk = row.get("api_pk"); final byte[] api_pk = row.get("api_pk");
final Row r = api_pk == null ? null : sb.tables.select("api", api_pk); final Row r = api_pk == null ? null : sb.tables.select("api", api_pk);
if (r != null && r.get("comment", "").matches(".*\\Q" + messageurl + "\\E.*")) { if (r != null && r.get("comment", "").matches(".*" + Pattern.quote(messageurl) + ".*")) {
// this is a recorded entry // this is a recorded entry
final Date date_next_exec = r.get(WorkTables.TABLE_API_COL_DATE_NEXT_EXEC, (Date) null); final Date date_next_exec = r.get(WorkTables.TABLE_API_COL_DATE_NEXT_EXEC, (Date) null);
prop.put("showscheduledfeeds_list_" + apic + "_pk", UTF8.String(row.getPK())); prop.put("showscheduledfeeds_list_" + apic + "_pk", UTF8.String(row.getPK()));

1
htroot/index.html
Unescape View File

@ -1,4 +1,3 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml"> <html xmlns="http://www.w3.org/1999/xhtml">
<head> <head>
#(forward)#::<meta http-equiv="REFRESH" content="0; url=#[target]#" />#(/forward)# #(forward)#::<meta http-equiv="REFRESH" content="0; url=#[target]#" />#(/forward)#

2
htroot/yacyinteractive.java
Unescape View File

@ -54,7 +54,7 @@ public class yacyinteractive {
prop.put("startRecord", startRecord); prop.put("startRecord", startRecord);
prop.put("maximumRecords", maximumRecords); prop.put("maximumRecords", maximumRecords);
prop.putHTML("querys", query.replaceAll(" ", "+")); prop.putHTML("querys", query.replaceAll(" ", "+"));
prop.put("serverlist", query.length() == 0 ? 1 : 0); prop.put("serverlist", query.isEmpty() ? 1 : 0);
prop.put("focus", focus ? 1 : 0); prop.put("focus", focus ? 1 : 0);
prop.put("allowrealtime", sb.indexSegments.URLCount() < 100000 ? 1 : 0); prop.put("allowrealtime", sb.indexSegments.URLCount() < 100000 ? 1 : 0);
return prop; return prop;

9
htroot/yacysearch.java
Unescape View File

@ -28,6 +28,7 @@
// if the shell's current path is HTROOT // if the shell's current path is HTROOT
import java.io.IOException; import java.io.IOException;
import java.net.InetAddress;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
@ -120,8 +121,7 @@ public class yacysearch {
prop.put("topmenu", sb.getConfigBool("publicTopmenu", true) ? 1 : 0); prop.put("topmenu", sb.getConfigBool("publicTopmenu", true) ? 1 : 0);
//get focus option //get focus option
final boolean focus = (post == null) ? true : post.get("focus", "1").equals("1"); prop.put("focus", ((post == null) ? true : post.get("focus", "1").equals("1")) ? 1 : 0);
prop.put("focus", focus ? 1 : 0);
// produce vocabulary navigation sidebars // produce vocabulary navigation sidebars
Collection<Vocabulary> vocabularies = LibraryProvider.autotagging.getVocabularies(); Collection<Vocabulary> vocabularies = LibraryProvider.autotagging.getVocabularies();
@ -1106,6 +1106,11 @@ public class yacysearch {
sb.localSearchLastAccess = System.currentTimeMillis(); sb.localSearchLastAccess = System.currentTimeMillis();
// hostname and port (assume locahost if nothing helps)
final InetAddress hostIP = Domains.myPublicLocalIP();
prop.put("myhost", hostIP != null ? hostIP.getHostAddress() : "localhost");
prop.put("myport", serverCore.getPortNr(sb.getConfig("port", "8090")));
// return rewrite properties // return rewrite properties
return prop; return prop;
} }

7
source/de/anomic/data/ymark/YMarkCrawlStart.java
Unescape View File

@ -119,10 +119,9 @@ public class YMarkCrawlStart extends HashMap<String,String>{
private void load() { private void load() {
try { try {
final StringBuilder buffer = new StringBuilder(500); final StringBuilder buffer = new StringBuilder(500);
//buffer.append("^.*crawlingURL=\\Q"); buffer.append("^crawl start for ");
buffer.append("^crawl start for \\Q"); buffer.append(Pattern.quote(url));
buffer.append(url); buffer.append("?.*");
buffer.append("\\E?.*");
final Pattern pattern = Pattern.compile(buffer.toString()); final Pattern pattern = Pattern.compile(buffer.toString());
//final Iterator<Tables.Row> APIcalls = this.worktables.iterator(WorkTables.TABLE_API_NAME, WorkTables.TABLE_API_COL_URL, pattern); //final Iterator<Tables.Row> APIcalls = this.worktables.iterator(WorkTables.TABLE_API_NAME, WorkTables.TABLE_API_COL_URL, pattern);
final Iterator<Tables.Row> APIcalls = this.worktables.iterator(WorkTables.TABLE_API_NAME, WorkTables.TABLE_API_COL_COMMENT, pattern); final Iterator<Tables.Row> APIcalls = this.worktables.iterator(WorkTables.TABLE_API_NAME, WorkTables.TABLE_API_COL_COMMENT, pattern);

10
source/de/anomic/data/ymark/YMarkTables.java
Unescape View File

@ -92,8 +92,6 @@ public class YMarkTables {
public final static String ADMIN_AUTHENTICATE_MSG = "Admin authentication required!"; public final static String ADMIN_AUTHENTICATE_MSG = "Admin authentication required!";
public final static String p1 = "(?:^|.*,)"; public final static String p1 = "(?:^|.*,)";
public final static String p2 = "\\Q";
public final static String p3 = "\\E";
public final static String p4 = "(?:,.*|$)"; public final static String p4 = "(?:,.*|$)";
public final static String p5 = "((?:"; public final static String p5 = "((?:";
public final static String p6 = ")(?:,.*|$)){"; public final static String p6 = ")(?:,.*|$)){";
@ -194,9 +192,7 @@ public class YMarkTables {
patternBuilder.setLength(0); patternBuilder.setLength(0);
patternBuilder.append(p1); patternBuilder.append(p1);
patternBuilder.append('('); patternBuilder.append('(');
patternBuilder.append(p2); patternBuilder.append(Pattern.quote(folder));
patternBuilder.append(folder);
patternBuilder.append(p3);
patternBuilder.append(')'); patternBuilder.append(')');
patternBuilder.append(p4); patternBuilder.append(p4);
final Pattern p = Pattern.compile(patternBuilder.toString()); final Pattern p = Pattern.compile(patternBuilder.toString());
@ -210,9 +206,7 @@ public class YMarkTables {
patternBuilder.append(p1); patternBuilder.append(p1);
patternBuilder.append(p5); patternBuilder.append(p5);
for (final String tag : tagArray) { for (final String tag : tagArray) {
patternBuilder.append(p2); patternBuilder.append(Pattern.quote(tag));
patternBuilder.append(tag);
patternBuilder.append(p3);
patternBuilder.append('|'); patternBuilder.append('|');
} }
patternBuilder.deleteCharAt(patternBuilder.length()-1); patternBuilder.deleteCharAt(patternBuilder.length()-1);

2
source/net/yacy/cora/document/Classification.java
Unescape View File

@ -129,7 +129,7 @@ public class Classification {
return appsExtSet.contains(appsExt.trim().toLowerCase()); return appsExtSet.contains(appsExt.trim().toLowerCase());
} }
public static ContentDomain getContentDomain(String ext) { public static ContentDomain getContentDomain(final String ext) {
if (isTextExtension(ext)) return ContentDomain.TEXT; if (isTextExtension(ext)) return ContentDomain.TEXT;
if (isImageExtension(ext)) return ContentDomain.IMAGE; if (isImageExtension(ext)) return ContentDomain.IMAGE;
if (isAudioExtension(ext)) return ContentDomain.AUDIO; if (isAudioExtension(ext)) return ContentDomain.AUDIO;

4
source/net/yacy/kelondro/logging/GuiHandler.java
Unescape View File

@ -208,8 +208,8 @@ public class GuiHandler extends Handler {
} }
public void close() throws SecurityException { public synchronized void close() throws SecurityException {
// Nothing implement here
} }
} }

6
source/net/yacy/kelondro/table/Table.java
Unescape View File

@ -690,7 +690,11 @@ public class Table implements Index, Iterable<Row.Entry> {
} }
this.file.put(i, p, 0); this.file.put(i, p, 0);
byte[] pk = lr.getPrimaryKeyBytes(); byte[] pk = lr.getPrimaryKeyBytes();
if (pk == null) continue; if (pk == null) {
// Table file might be corrupt
Log.logWarning("TABLE", "Possible corruption found in table " + this.filename() + " detected. i=" + i + ",p=" + p);
continue;
}
this.index.put(pk, i); this.index.put(pk, i);
break; break;
} }

10
source/net/yacy/kelondro/workflow/InstantBusyThread.java
Unescape View File

@ -182,4 +182,14 @@ public final class InstantBusyThread extends AbstractBusyThread implements BusyT
return oneTimeJob( thread, startupDelay); return oneTimeJob( thread, startupDelay);
} }
@Override
public void open() {
// Not implemented in this thread
}
@Override
public synchronized void close() {
// Not implemented in this thread
}
} }

18
source/net/yacy/search/index/DocumentMetadata.java
Unescape View File

@ -50,7 +50,7 @@ public class DocumentMetadata implements Metadata {
} }
@Override @Override
public void close() { public synchronized void close() {
// TODO Auto-generated method stub // TODO Auto-generated method stub
} }
@ -62,37 +62,37 @@ public class DocumentMetadata implements Metadata {
} }
@Override @Override
public URIMetadata load(Element<WordReference> obrwi) { public URIMetadata load(final Element<WordReference> obrwi) {
// TODO Auto-generated method stub // TODO Auto-generated method stub
return null; return null;
} }
@Override @Override
public URIMetadata load(byte[] urlHash) { public URIMetadata load(final byte[] urlHash) {
// TODO Auto-generated method stub // TODO Auto-generated method stub
return null; return null;
} }
@Override @Override
public void store(URIMetadata entry) throws IOException { public void store(final URIMetadata entry) throws IOException {
// TODO Auto-generated method stub // TODO Auto-generated method stub
} }
@Override @Override
public boolean remove(byte[] urlHashBytes) { public boolean remove(final byte[] urlHashBytes) {
// TODO Auto-generated method stub // TODO Auto-generated method stub
return false; return false;
} }
@Override @Override
public boolean exists(byte[] urlHash) { public boolean exists(final byte[] urlHash) {
// TODO Auto-generated method stub // TODO Auto-generated method stub
return false; return false;
} }
@Override @Override
public CloneableIterator<byte[]> keys(boolean up, byte[] firstKey) { public CloneableIterator<byte[]> keys(final boolean up, final byte[] firstKey) {
// TODO Auto-generated method stub // TODO Auto-generated method stub
return null; return null;
} }
@ -104,14 +104,14 @@ public class DocumentMetadata implements Metadata {
} }
@Override @Override
public CloneableIterator<URIMetadata> entries(boolean up, String firstHash) public CloneableIterator<URIMetadata> entries(final boolean up, final String firstHash)
throws IOException { throws IOException {
// TODO Auto-generated method stub // TODO Auto-generated method stub
return null; return null;
} }
@Override @Override
public int deleteDomain(String hosthash) throws IOException { public int deleteDomain(final String hosthash) throws IOException {
// TODO Auto-generated method stub // TODO Auto-generated method stub
return 0; return 0;
} }

2
source/net/yacy/search/index/DocumentReference.java
Unescape View File

@ -42,7 +42,7 @@ public class DocumentReference {
return data.size(); return data.size();
} }
public void close() { public synchronized void close() {
if (data != null) { if (data != null) {
data.close(); data.close();
} }

4
source/net/yacy/search/index/Segment.java
Unescape View File

@ -72,8 +72,8 @@ import de.anomic.crawler.retrieval.Response;
public class Segment { public class Segment {
// catchall word // catchall word
final static String catchallString = "yacyall"; // a word that is always in all indexes; can be used for zero-word searches to find ALL documents public final static String catchallString = "yacyall"; // a word that is always in all indexes; can be used for zero-word searches to find ALL documents
final static byte[] catchallHash; public final static byte[] catchallHash;
final static Word catchallWord = new Word(0, 0, 0); final static Word catchallWord = new Word(0, 0, 0);
static { static {
catchallHash = Word.word2hash(catchallString); // "KZzU-Vf6h5k-" catchallHash = Word.word2hash(catchallString); // "KZzU-Vf6h5k-"

2
source/net/yacy/search/index/SolrConfiguration.java
Unescape View File

@ -466,7 +466,7 @@ public class SolrConfiguration extends ConfigurationSet implements Serializable
* @param rel * @param rel
* @return binary encoded information about rel * @return binary encoded information about rel
*/ */
private int relEval(final String[] rel) { private static int relEval(final String[] rel) {
int i = 0; int i = 0;
for (final String s: rel) { for (final String s: rel) {
final String s0 = s.toLowerCase().trim(); final String s0 = s.toLowerCase().trim();

5
source/net/yacy/search/query/SnippetProcess.java
Unescape View File

@ -62,6 +62,7 @@ import org.apache.solr.common.SolrDocumentList;
import de.anomic.crawler.Cache; import de.anomic.crawler.Cache;
import de.anomic.data.WorkTables; import de.anomic.data.WorkTables;
import net.yacy.search.index.Segment;
import net.yacy.search.index.SolrField; import net.yacy.search.index.SolrField;
public class SnippetProcess { public class SnippetProcess {
@ -606,6 +607,10 @@ public class SnippetProcess {
return new ResultEntry(page, this.query.getSegment(), this.peers, null, null, dbRetrievalTime, snippetComputationTime); // result without snippet return new ResultEntry(page, this.query.getSegment(), this.peers, null, null, dbRetrievalTime, snippetComputationTime); // result without snippet
} else { } else {
// problems with snippet fetch // problems with snippet fetch
if (this.snippetFetchWordHashes.has(Segment.catchallHash)) {
// we accept that because the word cannot be on the page
return new ResultEntry(page, this.query.getSegment(), this.peers, null, null, dbRetrievalTime, 0);
}
final String reason = "no text snippet; errorCode = " + snippet.getErrorCode(); final String reason = "no text snippet; errorCode = " + snippet.getErrorCode();
if (this.deleteIfSnippetFail) { if (this.deleteIfSnippetFail) {
this.workTables.failURLsRegisterMissingWord(this.query.getSegment().termIndex(), page.url(), this.query.queryHashes, reason); this.workTables.failURLsRegisterMissingWord(this.query.getSegment().termIndex(), page.url(), this.query.queryHashes, reason);

2
source/net/yacy/yacy.java
Unescape View File

@ -173,7 +173,7 @@ public final class yacy {
f = new File(dataHome, "DATA/LOG/"); f = new File(dataHome, "DATA/LOG/");
mkdirsIfNeseccary(f); mkdirsIfNeseccary(f);
f = new File(dataHome, "DATA/LOG/yacy.logging"); f = new File(dataHome, "DATA/LOG/yacy.logging");
final File f0 = new File(appHome, "yacy.logging"); final File f0 = new File(appHome, "defaults/yacy.logging");
if (!f.exists() || f0.lastModified() > f.lastModified()) try { if (!f.exists() || f0.lastModified() > f.lastModified()) try {
FileUtils.copy(f0, f); FileUtils.copy(f0, f);
} catch (final IOException e){ } catch (final IOException e){

27
vocabularies/de/Freizeitaktivitäten.vocabulary
Unescape View File

@ -1,27 +0,0 @@
Eislaufen
Drachensteigen
Saunabaden=Sauna
Spazieren und Wandern=Spazieren,Wandern,Park,Ufer
Skaten=Inline,Inliner
Skateboarden=Skateboard
Museumsbesuch=Museum
Altstadtbesuch=Altstadt
Parkbesuch=Park,Zoo,Freilichtmuseum,Burg
Ausflug=Schifstouren,Schifffahrt,Aussicht,Aussichtspunkt,Geocaching
Schwimmen=Schwimmbad,Freibad,Hallenbad,Schwimmbäder,Freibäder,Hallenbäder,Therme
Fastnacht
Eislaufen
Saunabaden=Sauna
Theater=Bühnen
Oper=Operette,Opernhaus
Spielplätze=Spielplatz
Museum
Camping
Picknick=Picknicken
Essen und Trinken=Essen,Trinken,Bar,Restaurant,Kneipe
Kochen
Singen=Gesang,Chor
Kegeln
Bowling
Kartbahn=Kart
Kino=Kinoprogramm,Cine
Write Preview
Loading…
Cancel
Save