boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'master' of ssh://git@gitorious.org/yacy/rc1.git

Browse Source
pull/1/head
Michael Peter Christen 11 years ago
parent bfaf42b0b8 37f2a82a5d
commit 36594d0348
14 changed files with 127 additions and 13 deletions
Show Stats Download Patch File Download Diff File

2
.classpath
Unescape View File

@ -84,6 +84,8 @@
<classpathentry kind="lib" path="lib/jetty-servlet-8.1.14.v20131031.jar"/>
<classpathentry kind="lib" path="lib/jetty-servlets-8.1.14.v20131031.jar"/>
<classpathentry kind="lib" path="lib/jetty-util-8.1.14.v20131031.jar"/>
<classpathentry kind="lib" path="lib/jetty-webapp-8.1.14.v20131031.jar"/>
<classpathentry kind="lib" path="lib/jetty-xml-8.1.14.v20131031.jar"/>
<classpathentry kind="lib" path="lib/servlet-api-3.0.jar"/>
<classpathentry kind="output" path="gen"/>
</classpath>

2
addon/YaCy.app/Contents/Info.plist
Unescape View File

@ -74,6 +74,8 @@
<string>$JAVAROOT/lib/jetty-servlet-8.1.14.v20131031.jar</string>
<string>$JAVAROOT/lib/jetty-servlets-8.1.14.v20131031.jar</string>
<string>$JAVAROOT/lib/jetty-util-8.1.14.v20131031.jar</string>
<string>$JAVAROOT/lib/jetty-webapp-8.1.14.v20131031.jar</string>
<string>$JAVAROOT/lib/jetty-xml-8.1.14.v20131031.jar</string>
<string>$JAVAROOT/lib/jsch-0.1.50.jar</string>
<string>$JAVAROOT/lib/json-simple-1.1.1.jar</string>
<string>$JAVAROOT/lib/log4j-over-slf4j-1.7.2.jar</string>

4
build.xml
Unescape View File

@ -198,6 +198,8 @@
<pathelement location="${lib}/jetty-servlet-8.1.14.v20131031.jar" />
<pathelement location="${lib}/jetty-servlets-8.1.14.v20131031.jar" />
<pathelement location="${lib}/jetty-util-8.1.14.v20131031.jar" />
<pathelement location="${lib}/jetty-webapp-8.1.14.v20131031.jar" />
<pathelement location="${lib}/jetty-xml-8.1.14.v20131031.jar" />
<pathelement location="${lib}/jsch-0.1.50.jar" />
<pathelement location="${lib}/json-simple-1.1.1.jar" />
<pathelement location="${lib}/jsoup-1.6.3.jar" />
@ -653,7 +655,7 @@
<!--<arg value="-Xrunhprof"/>-->
<arg line="-start"/>
<jvmarg line="-Xms180m"/>
<jvmarg line="-Xmx600m"/>
<jvmarg line="-Xmx800m"/>
<!-- <arg line="-migratewords"/>-->
<!-- <arg line="-start ${user.dir}"/>-->
</java>

6
defaults/solr.collection.schema
Unescape View File

@ -273,7 +273,7 @@ images_height_val
images_width_val
## size of images as number of pixels (easier for ranking than using with and height)
images_pixel_val
#images_pixel_val
## number of image links with alt tag
#images_withalt_i
@ -360,7 +360,7 @@ url_protocol_s
url_file_name_s
## tokens generated from url_file_name_s which can be used for better matching and result boosting
url_file_name_tokens_t
#url_file_name_tokens_t
## the file name extension
url_file_ext_s
@ -446,7 +446,7 @@ host_extent_i
#cr_host_norm_i
## custom rating; to be set with external rating information
rating_i
#rating_i
## names of cms attributes; if several are recognized then they are listen in decreasing order of number of matching criterias
#ext_cms_txt

69
defaults/web.xml
Unescape View File

@ -0,0 +1,69 @@
<?xml version="1.0" encoding="UTF-8"?>
<web-app version="3.0" xmlns="http://java.sun.com/xml/ns/javaee" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_3_0.xsd">
<display-name>YaCy</display-name>
<description>Decentralized Web Search</description>
<!-- Standard YaCy Servlets -->
<!-- Default servlet for all YaCy output
-as this is mandatory the servlet activated automatically by the application
-no need to include this in this web.xml (provided only for completness)
<servlet>
<servlet-name>YaCyDefaultServlet</servlet-name>
<servlet-class>net.yacy.http.servlets.YaCyDefaultServlet</servlet-class>
</servlet>
-->
<!-- servlet to provide direct access to the embedded solr, also used for P2P searches -->
<servlet>
<servlet-name>SolrServlet</servlet-name>
<servlet-class>net.yacy.http.servlets.SolrServlet</servlet-class>
</servlet>
<!-- servlet to provide Google Search Appliance (GSA) formatted search results -->
<servlet>
<servlet-name>GSAServlet</servlet-name>
<servlet-class>net.yacy.http.servlets.GSAsearchServlet</servlet-class>
</servlet>
<!-- servlet to provide searchresults via proxy and needed for Augmented Browsing -->
<servlet>
<servlet-name>URLProxyServlet</servlet-name>
<servlet-class>net.yacy.http.servlets.YaCyProxyServlet</servlet-class>
</servlet>
<!-- mapping activated by the application
<servlet-mapping>
<servlet-name>SolrServlet</servlet-name>
<url-pattern>/</url-pattern>
</servlet-mapping>
-->
<servlet-mapping>
<servlet-name>SolrServlet</servlet-name>
<url-pattern>/solr/select</url-pattern>
</servlet-mapping>
<servlet-mapping>
<servlet-name>GSAServlet</servlet-name>
<url-pattern>/gsa/search</url-pattern>
</servlet-mapping>
<servlet-mapping>
<servlet-name>URLProxyServlet</servlet-name>
<url-pattern>/proxy.html</url-pattern>
</servlet-mapping>
<session-config>
<session-timeout>
30
</session-timeout>
</session-config>
<welcome-file-list>
<welcome-file>index.html</welcome-file>
<welcome-file>Index.html</welcome-file>
<welcome-file>Welcome.html</welcome-file>
</welcome-file-list>
</web-app>

BIN
lib/jetty-webapp-8.1.14.v20131031.jar
View File

Binary file not shown.

BIN
lib/jetty-xml-8.1.14.v20131031.jar
View File

Binary file not shown.

7
nbproject/project.xml
Unescape View File

@ -22,6 +22,11 @@
<location>htroot</location>
<encoding>UTF-8</encoding>
</source-folder>
<source-folder>
<label>YaCy</label>
<location>.</location>
<encoding>UTF-8</encoding>
</source-folder>
</folders>
<ide-actions>
<action name="build">
@ -78,7 +83,7 @@
<compilation-unit>
<package-root>source</package-root>
<package-root>htroot</package-root>
<classpath mode="compile">lib/J7Zip-modified.jar;lib/activation.jar;lib/apache-mime4j-0.6.jar;lib/arq-2.8.7.jar;lib/bcmail-jdk15-145.jar;lib/bcprov-jdk15-145.jar;lib/commons-codec-1.7.jar;lib/commons-compress-1.4.1.jar;lib/commons-fileupload-1.2.2.jar;lib/commons-io-2.1.jar;lib/commons-jxpath-1.3.jar;lib/commons-lang-2.6.jar;lib/commons-logging-1.1.3.jar;lib/fontbox-1.8.3.jar;lib/geronimo-stax-api_1.0_spec-1.0.1.jar;lib/guava-15.0.jar;lib/htmllexer.jar;lib/httpclient-4.3.1.jar;lib/httpcore-4.3.jar;lib/httpmime-4.3.1.jar;lib/icu4j-core.jar;lib/iri-0.8.jar;lib/jakarta-oro-2.0.8.jar;lib/jaudiotagger-2.0.4-20111207.115108-15.jar;lib/jcifs-1.3.17.jar;lib/jcl-over-slf4j-1.7.2.jar;lib/jempbox-1.8.3.jar;lib/jena-2.6.4.jar;lib/jetty-client-8.1.14.v20131031.jar;lib/jetty-continuation-8.1.14.v20131031.jar;lib/jetty-http-8.1.14.v20131031.jar;lib/jetty-io-8.1.14.v20131031.jar;lib/jetty-security-8.1.14.v20131031.jar;lib/jetty-server-8.1.14.v20131031.jar;lib/jetty-servlet-8.1.14.v20131031.jar;lib/jetty-servlets-8.1.14.v20131031.jar;lib/jetty-util-8.1.14.v20131031.jar;lib/jsch-0.1.50.jar;lib/json-simple-1.1.1.jar;lib/jsoup-1.6.3.jar;lib/log4j-over-slf4j-1.7.2.jar;lib/lucene-analyzers-common-4.6.0.jar;lib/lucene-analyzers-phonetic-4.6.0.jar;lib/lucene-classification-4.6.0.jar;lib/lucene-codecs-4.6.0.jar;lib/lucene-core-4.6.0.jar;lib/lucene-facet-4.6.0.jar;lib/lucene-grouping-4.6.0.jar;lib/lucene-highlighter-4.6.0.jar;lib/lucene-join-4.6.0.jar;lib/lucene-memory-4.6.0.jar;lib/lucene-misc-4.6.0.jar;lib/lucene-queries-4.6.0.jar;lib/lucene-queryparser-4.6.0.jar;lib/lucene-spatial-4.6.0.jar;lib/lucene-suggest-4.6.0.jar;lib/metadata-extractor-2.6.2.jar;lib/noggit-0.5.jar;lib/pdfbox-1.8.3.jar;lib/poi-3.9-20121203.jar;lib/poi-scratchpad-3.9-20121203.jar;lib/sax-2.0.1.jar;lib/servlet-api-3.0.jar;lib/slf4j-api-1.7.2.jar;lib/slf4j-jdk14-1.7.2.jar;lib/solr-core-4.6.0.jar;lib/solr-solrj-4.6.0.jar;lib/spatial4j-0.3.jar;lib/webcat-0.1-swf.jar;lib/wstx-asl-3.2.9.jar;lib/xercesImpl.jar;lib/xml-apis.jar;lib/zookeeper-3.4.5.jar</classpath>
<classpath mode="compile">lib/J7Zip-modified.jar;lib/activation.jar;lib/apache-mime4j-0.6.jar;lib/arq-2.8.7.jar;lib/bcmail-jdk15-145.jar;lib/bcprov-jdk15-145.jar;lib/commons-codec-1.7.jar;lib/commons-compress-1.4.1.jar;lib/commons-fileupload-1.2.2.jar;lib/commons-io-2.1.jar;lib/commons-jxpath-1.3.jar;lib/commons-lang-2.6.jar;lib/commons-logging-1.1.3.jar;lib/fontbox-1.8.3.jar;lib/geronimo-stax-api_1.0_spec-1.0.1.jar;lib/guava-15.0.jar;lib/htmllexer.jar;lib/httpclient-4.3.1.jar;lib/httpcore-4.3.jar;lib/httpmime-4.3.1.jar;lib/icu4j-core.jar;lib/iri-0.8.jar;lib/jakarta-oro-2.0.8.jar;lib/jaudiotagger-2.0.4-20111207.115108-15.jar;lib/jcifs-1.3.17.jar;lib/jcl-over-slf4j-1.7.2.jar;lib/jempbox-1.8.3.jar;lib/jena-2.6.4.jar;lib/jetty-client-8.1.14.v20131031.jar;lib/jetty-continuation-8.1.14.v20131031.jar;lib/jetty-http-8.1.14.v20131031.jar;lib/jetty-io-8.1.14.v20131031.jar;lib/jetty-security-8.1.14.v20131031.jar;lib/jetty-server-8.1.14.v20131031.jar;lib/jetty-servlet-8.1.14.v20131031.jar;lib/jetty-servlets-8.1.14.v20131031.jar;lib/jetty-util-8.1.14.v20131031.jar;lib/jetty-webapp-8.1.14.v20131031.jar;lib/jetty-xml-8.1.14.v20131031.jar;lib/jsch-0.1.50.jar;lib/json-simple-1.1.1.jar;lib/jsoup-1.6.3.jar;lib/log4j-over-slf4j-1.7.2.jar;lib/lucene-analyzers-common-4.6.0.jar;lib/lucene-analyzers-phonetic-4.6.0.jar;lib/lucene-classification-4.6.0.jar;lib/lucene-codecs-4.6.0.jar;lib/lucene-core-4.6.0.jar;lib/lucene-facet-4.6.0.jar;lib/lucene-grouping-4.6.0.jar;lib/lucene-highlighter-4.6.0.jar;lib/lucene-join-4.6.0.jar;lib/lucene-memory-4.6.0.jar;lib/lucene-misc-4.6.0.jar;lib/lucene-queries-4.6.0.jar;lib/lucene-queryparser-4.6.0.jar;lib/lucene-spatial-4.6.0.jar;lib/lucene-suggest-4.6.0.jar;lib/metadata-extractor-2.6.2.jar;lib/noggit-0.5.jar;lib/pdfbox-1.8.3.jar;lib/poi-3.9-20121203.jar;lib/poi-scratchpad-3.9-20121203.jar;lib/sax-2.0.1.jar;lib/servlet-api-3.0.jar;lib/slf4j-api-1.7.2.jar;lib/slf4j-jdk14-1.7.2.jar;lib/solr-core-4.6.0.jar;lib/solr-solrj-4.6.0.jar;lib/spatial4j-0.3.jar;lib/webcat-0.1-swf.jar;lib/wstx-asl-3.2.9.jar;lib/xercesImpl.jar;lib/xml-apis.jar;lib/zookeeper-3.4.5.jar</classpath>
<built-to>lib/yacycore.jar</built-to>
<source-level>1.6</source-level>
</compilation-unit>

9
source/net/yacy/crawler/data/CrawlQueues.java
Unescape View File

@ -186,6 +186,15 @@ public class CrawlQueues {
}
}
public void freemem() {
if ((this.errorURL.stackSize() > 1)) {
log.warn("freemem: Cleaning Error-URLs report stack, "
+ this.errorURL.stackSize()
+ " entries on stack");
this.errorURL.clearStack();
}
}
public Request[] activeWorkerEntries() {
synchronized (this.workers) {
final Request[] e = new Request[this.workers.size()];

31
source/net/yacy/http/Jetty8HttpServerImpl.java
Unescape View File

@ -26,6 +26,7 @@ package net.yacy.http;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.Inet4Address;
import java.net.InetAddress;
import java.net.InetSocketAddress;
@ -57,9 +58,10 @@ import org.eclipse.jetty.server.handler.HandlerList;
import org.eclipse.jetty.server.handler.IPAccessHandler;
import org.eclipse.jetty.server.nio.SelectChannelConnector;
import org.eclipse.jetty.server.ssl.SslSelectChannelConnector;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.eclipse.jetty.util.resource.Resource;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.eclipse.jetty.webapp.WebAppContext;
/**
* class to embedded Jetty 8 http server into YaCy
@ -102,8 +104,31 @@ public class Jetty8HttpServerImpl implements YaCyHttpServer {
domainHandler.setAlternativeResolver(sb.peers);
// configure root context
ServletContextHandler htrootContext = new ServletContextHandler(ServletContextHandler.SESSIONS);
htrootContext.setContextPath("/");
// ServletContextHandler htrootContext = new ServletContextHandler(ServletContextHandler.SESSIONS);
WebAppContext htrootContext = new WebAppContext();
htrootContext.setContextPath("/");
try {
htrootContext.setBaseResource(Resource.newResource("htroot"));
// set web.xml to use
// make use of Jetty feature to define web.xml other as default WEB-INF/web.xml
// look in DATA/SETTINGS or use the one in DEFAULTS
Resource webxml = Resource.newResource(sb.dataPath + "/DATA/SETTINGS/web.xml");
if (webxml.exists()) {
htrootContext.setDescriptor(webxml.getName());
} else {
htrootContext.setDescriptor(sb.appPath + "/defaults/web.xml");
}
} catch (IOException ex) {
if (htrootContext.getBaseResource() == null) {
ConcurrentLog.severe("SERVER", "could not find directory: htroot ");
} else {
ConcurrentLog.warn("SERVER", "could not find: defaults/web.xml or DATA/SETTINGS/web.xml");
}
}
// as fundamental component leave this hardcoded, other servlets may be defined in web.xml only
ServletHolder sholder = new ServletHolder(YaCyDefaultServlet.class);
sholder.setInitParameter("resourceBase", "htroot");
//sholder.setInitParameter("welcomeFile", "index.html"); // default is index.html, welcome.html

4
source/net/yacy/http/servlets/YaCyProxyServlet.java
Unescape View File

@ -82,7 +82,7 @@ public class YaCyProxyServlet extends ProxyServlet implements Servlet {
}
final String remoteHost = req.getRemoteHost();
if (!Domains.isThisHostIP(remoteHost)) {
if (!Domains.isThisHostIP(remoteHost)) {
response.sendError(HttpServletResponse.SC_FORBIDDEN,
"proxy use not granted for IP " + remoteHost);
return;
@ -333,7 +333,7 @@ public class YaCyProxyServlet extends ProxyServlet implements Servlet {
private boolean proxyippatternmatch(final String key) {
// the cfgippattern is a comma-separated list of patterns
// each pattern may contain one wildcard-character '*' which matches anything
final String cfgippattern = Switchboard.getSwitchboard().getConfig("proxyClient", "*");
final String cfgippattern = Switchboard.getSwitchboard().getConfig("proxyURL.access", "*");
if (cfgippattern.equals("*")) {
return true;
}

2
source/net/yacy/search/SwitchboardConstants.java
Unescape View File

@ -95,7 +95,7 @@ public final class SwitchboardConstants {
public static final String CRAWLJOB_LOCAL_CRAWL = "50_localcrawl";
public static final String CRAWLJOB_LOCAL_CRAWL_METHOD_START = "coreCrawlJob";
public static final String CRAWLJOB_LOCAL_CRAWL_METHOD_JOBCOUNT = "coreCrawlJobSize";
public static final String CRAWLJOB_LOCAL_CRAWL_METHOD_FREEMEM = null;
public static final String CRAWLJOB_LOCAL_CRAWL_METHOD_FREEMEM = "freemem";
public static final String CRAWLJOB_LOCAL_CRAWL_IDLESLEEP = "50_localcrawl_idlesleep";
public static final String CRAWLJOB_LOCAL_CRAWL_BUSYSLEEP = "50_localcrawl_busysleep";
// 60_remotecrawlloader

2
source/net/yacy/search/index/Segment.java
Unescape View File

@ -455,7 +455,7 @@ public class Segment {
*/
public int getWordCountGuess(String word) {
if (this.fulltext.getDefaultConnector() == null) return 0;
if (word == null || word.indexOf(':') >= 0 || word.indexOf(' ') >= 0 || word.indexOf('/') >= 0) return 0;
if (word == null || word.indexOf(':') >= 0 || word.indexOf(' ') >= 0 || word.indexOf('/') >= 0 || word.indexOf('\"') >= 0) return 0;
if (this.termIndex != null) {
int count = this.termIndex.count(Word.word2hash(word));
if (count > 0) return count;

2
source/net/yacy/yacy.java
Unescape View File

@ -523,7 +523,7 @@ public final class yacy {
final HTTPClient con = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);
con.setHeader(requestHeader.entrySet());
try {
con.GETbytes("http://localhost:"+ port +"/" + path, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME,"admin"), encodedPassword);
con.GETbytes("http://localhost:"+ port +"/" + path, config.getProperty(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME,"admin"), encodedPassword);
if (con.getStatusCode() > 199 && con.getStatusCode() < 300) {
ConcurrentLog.config("COMMAND-STEERING", "YACY accepted steering command: " + processdescription);

Write Preview
Loading…
Cancel
Save