boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'master' of git@github.com:yacy/yacy_search_server.git

Browse Source
pull/8/head
Michael Peter Christen 10 years ago
parent fb9e1dd3f5 5183ad718d
commit e0a23c56c7
11 changed files with 472 additions and 559 deletions
Show Stats Download Patch File Download Diff File

4
.classpath
Unescape View File

@ -48,8 +48,8 @@
<classpathentry kind="lib" path="lib/pdfbox-1.8.9.jar"/>
<classpathentry kind="lib" path="lib/bcmail-jdk15-1.46.jar"/>
<classpathentry kind="lib" path="lib/bcprov-jdk15-1.46.jar"/>
<classpathentry kind="lib" path="lib/poi-3.11-20141221.jar"/>
<classpathentry kind="lib" path="lib/poi-scratchpad-3.11-20141221.jar"/>
<classpathentry kind="lib" path="lib/poi-3.12-20150511.jar"/>
<classpathentry kind="lib" path="lib/poi-scratchpad-3.12-20150511.jar"/>
<classpathentry kind="lib" path="lib/commons-io-2.4.jar"/>
<classpathentry kind="lib" path="lib/slf4j-api-1.7.9.jar"/>
<classpathentry kind="lib" path="lib/lucene-analyzers-common-4.10.3.jar"/>

4
build.xml
Unescape View File

@ -224,8 +224,8 @@
<pathelement location="${lib}/noggit-0.5.jar" />
<pathelement location="${lib}/org.restlet.jar" />
<pathelement location="${lib}/pdfbox-1.8.9.jar" />
<pathelement location="${lib}/poi-3.11-20141221.jar" />
<pathelement location="${lib}/poi-scratchpad-3.11-20141221.jar" />
<pathelement location="${lib}/poi-3.12-20150511.jar" />
<pathelement location="${lib}/poi-scratchpad-3.12-20150511.jar" />
<pathelement location="${lib}/slf4j-api-1.7.9.jar" />
<pathelement location="${lib}/slf4j-jdk14-1.7.9.jar" />
<pathelement location="${lib}/solr-core-4.10.3.jar" />

45
htroot/AugmentedParsing_p.html
Unescape View File

@ -1,45 +0,0 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>YaCy '#[clientname]#': Augmented Parsing</title>
#%env/templates/metas.template%#
</head>
<body id="Settings">
#%env/templates/header.template%#
#%env/templates/submenuSemantic.template%#
<h2>Augmented Parsing</h2>
<form id="parsersettings" action="AugmentedParsing_p.html" method="post" enctype="multipart/form-data">
<fieldset><legend id="urlproxy">Global Status</legend>
<p>
With this settings you can activate or deactivate augmented parsing which combines the documents with information from external sources (tags etc.).
</p>
<dl>
<dt><label for="content">Augmented Parser:</label></dt>
<dd>
<input type="checkbox" name="augmentedparserenabled" id="augmentedparserenabled" #(augmentedparserenabled_checked)#:: checked="checked"#(/augmentedparserenabled_checked)# />Enabled<br/>
<p class="help">
Globally enables or disables the augmented parser. This setting requires a restart.
</p>
</dd>
<dt><label for="content">Augmented Parser - RDFa:</label></dt>
<dd>
<input type="checkbox" name="augmentedparserRDFenabled" id="augmentedparserRDFenabled" #(augmentedparserRDFenabled_checked)#:: checked="checked"#(/augmentedparserRDFenabled_checked)# />Enabled<br/>
<p class="help">
Globally enables or disables the RDFa parser. This setting requires a restart.
</p>
</dd>
</dl>
<input type="submit" name="augmentedparserSettings" value="Submit" class="btn btn-primary"/>
</fieldset>
</form>
#%env/templates/footer.template%#
</body>
</html>

38
htroot/AugmentedParsing_p.java
Unescape View File

@ -1,38 +0,0 @@
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
public final class AugmentedParsing_p {
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {
// return variable that accumulates replacements
final serverObjects prop = new serverObjects();
if (post != null) {
if (post.containsKey("augmentedparserSettings")) {
env.setConfig("parserAugmentation",
"on".equals(post.get("augmentedparserenabled")) ? true : false);
env.setConfig("parserAugmentation.RDFa",
"on".equals(post.get("augmentedparserRDFenabled")) ? true : false);
}
}
prop.put("augmentedparserenabled_checked",
env.getConfigBool("parserAugmentation", false) ? "1" : "0");
prop.put("augmentedparserRDFenabled_checked",
env.getConfigBool("parserAugmentation.RDFa", false) ? "1" : "0");
// return rewrite properties
return prop;
}
}

1
htroot/env/templates/submenuSemantic.template vendored
Unescape View File

@ -14,7 +14,6 @@
<h3>Augmented Content</h3>
<ul class="SubMenu">
<li><a href="AugmentedBrowsing_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Augmented Browsing</a></li>
<li><a href="AugmentedParsing_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Augmented Parsing</a></li>
</ul>
</div>
</div>

BIN
lib/poi-3.11-20141221.jar → lib/poi-3.12-20150511.jar
View File

Binary file not shown.

0
lib/poi-3.11.License → lib/poi-3.12.License
Unescape View File

BIN
lib/poi-scratchpad-3.11-20141221.jar → lib/poi-scratchpad-3.12-20150511.jar
View File

Binary file not shown.

2
nbproject/project.xml
Unescape View File

@ -83,7 +83,7 @@
<compilation-unit>
<package-root>source</package-root>
<package-root>htroot</package-root>
<classpath mode="compile">lib/J7Zip-modified.jar;lib/apache-mime4j-0.6.jar;lib/bcmail-jdk15-1.46.jar;lib/bcprov-jdk15-1.46.jar;lib/chardet.jar;lib/commons-codec-1.10.jar;lib/commons-compress-1.9.jar;lib/commons-fileupload-1.3.1.jar;lib/commons-io-2.4.jar;lib/commons-jxpath-1.3.jar;lib/commons-lang-2.6.jar;lib/commons-logging-1.2.jar;lib/fontbox-1.8.9.jar;lib/geronimo-stax-api_1.0_spec-1.0.1.jar;lib/guava-18.0.jar;lib/htmllexer.jar;lib/httpclient-4.4.jar;lib/httpcore-4.4.1.jar;lib/httpmime-4.4.jar;lib/icu4j-core.jar;lib/jakarta-oro-2.0.8.jar;lib/jaudiotagger-2.0.4-20111207.115108-15.jar;lib/javax.servlet-api-3.1.0.jar;lib/jcifs-1.3.17.jar;lib/jcl-over-slf4j-1.7.9.jar;lib/jempbox-1.8.9.jar;lib/jetty-client-9.2.10.v20150310.jar;lib/jetty-continuation-9.2.10.v20150310.jar;lib/jetty-deploy-9.2.10.v20150310.jar;lib/jetty-http-9.2.10.v20150310.jar;lib/jetty-io-9.2.10.v20150310.jar;lib/jetty-jmx-9.2.10.v20150310.jar;lib/jetty-proxy-9.2.10.v20150310.jar;lib/jetty-security-9.2.10.v20150310.jar;lib/jetty-server-9.2.10.v20150310.jar;lib/jetty-servlet-9.2.10.v20150310.jar;lib/jetty-servlets-9.2.10.v20150310.jar;lib/jetty-util-9.2.10.v20150310.jar;lib/jetty-webapp-9.2.10.v20150310.jar;lib/jetty-xml-9.2.10.v20150310.jar;lib/jsch-0.1.52.jar;lib/json-simple-1.1.1.jar;lib/jsoup-1.8.2.jar;lib/log4j-over-slf4j-1.7.9.jar;lib/lucene-analyzers-common-4.10.3.jar;lib/lucene-analyzers-phonetic-4.10.3.jar;lib/lucene-classification-4.10.3.jar;lib/lucene-codecs-4.10.3.jar;lib/lucene-core-4.10.3.jar;lib/lucene-facet-4.10.3.jar;lib/lucene-grouping-4.10.3.jar;lib/lucene-highlighter-4.10.3.jar;lib/lucene-join-4.10.3.jar;lib/lucene-memory-4.10.3.jar;lib/lucene-misc-4.10.3.jar;lib/lucene-queries-4.10.3.jar;lib/lucene-queryparser-4.10.3.jar;lib/lucene-spatial-4.10.3.jar;lib/lucene-suggest-4.10.3.jar;lib/metadata-extractor-2.8.1.jar;lib/noggit-0.5.jar;lib/org.restlet.jar;lib/pdfbox-1.8.9.jar;lib/poi-3.11-20141221.jar;lib/poi-scratchpad-3.11-20141221.jar;lib/slf4j-api-1.7.9.jar;lib/slf4j-jdk14-1.7.9.jar;lib/solr-core-4.10.3.jar;lib/solr-solrj-4.10.3.jar;lib/spatial4j-0.4.1.jar;lib/webcat-0.1-swf.jar;lib/weupnp-0.1.2.jar;lib/wstx-asl-3.2.9.jar;lib/xercesImpl.jar;lib/xml-apis.jar;lib/xmpcore-5.1.2.jar;lib/zookeeper-3.4.6.jar</classpath>
<classpath mode="compile">lib/J7Zip-modified.jar;lib/apache-mime4j-0.6.jar;lib/bcmail-jdk15-1.46.jar;lib/bcprov-jdk15-1.46.jar;lib/chardet.jar;lib/commons-codec-1.10.jar;lib/commons-compress-1.9.jar;lib/commons-fileupload-1.3.1.jar;lib/commons-io-2.4.jar;lib/commons-jxpath-1.3.jar;lib/commons-lang-2.6.jar;lib/commons-logging-1.2.jar;lib/fontbox-1.8.9.jar;lib/geronimo-stax-api_1.0_spec-1.0.1.jar;lib/guava-18.0.jar;lib/htmllexer.jar;lib/httpclient-4.4.jar;lib/httpcore-4.4.1.jar;lib/httpmime-4.4.jar;lib/icu4j-core.jar;lib/jakarta-oro-2.0.8.jar;lib/jaudiotagger-2.0.4-20111207.115108-15.jar;lib/javax.servlet-api-3.1.0.jar;lib/jcifs-1.3.17.jar;lib/jcl-over-slf4j-1.7.9.jar;lib/jempbox-1.8.9.jar;lib/jetty-client-9.2.10.v20150310.jar;lib/jetty-continuation-9.2.10.v20150310.jar;lib/jetty-deploy-9.2.10.v20150310.jar;lib/jetty-http-9.2.10.v20150310.jar;lib/jetty-io-9.2.10.v20150310.jar;lib/jetty-jmx-9.2.10.v20150310.jar;lib/jetty-proxy-9.2.10.v20150310.jar;lib/jetty-security-9.2.10.v20150310.jar;lib/jetty-server-9.2.10.v20150310.jar;lib/jetty-servlet-9.2.10.v20150310.jar;lib/jetty-servlets-9.2.10.v20150310.jar;lib/jetty-util-9.2.10.v20150310.jar;lib/jetty-webapp-9.2.10.v20150310.jar;lib/jetty-xml-9.2.10.v20150310.jar;lib/jsch-0.1.52.jar;lib/json-simple-1.1.1.jar;lib/jsoup-1.8.2.jar;lib/log4j-over-slf4j-1.7.9.jar;lib/lucene-analyzers-common-4.10.3.jar;lib/lucene-analyzers-phonetic-4.10.3.jar;lib/lucene-classification-4.10.3.jar;lib/lucene-codecs-4.10.3.jar;lib/lucene-core-4.10.3.jar;lib/lucene-facet-4.10.3.jar;lib/lucene-grouping-4.10.3.jar;lib/lucene-highlighter-4.10.3.jar;lib/lucene-join-4.10.3.jar;lib/lucene-memory-4.10.3.jar;lib/lucene-misc-4.10.3.jar;lib/lucene-queries-4.10.3.jar;lib/lucene-queryparser-4.10.3.jar;lib/lucene-spatial-4.10.3.jar;lib/lucene-suggest-4.10.3.jar;lib/metadata-extractor-2.8.1.jar;lib/noggit-0.5.jar;lib/org.restlet.jar;lib/pdfbox-1.8.9.jar;lib/poi-3.12-20150511.jar;lib/poi-scratchpad-3.12-20150511.jar;lib/slf4j-api-1.7.9.jar;lib/slf4j-jdk14-1.7.9.jar;lib/solr-core-4.10.3.jar;lib/solr-solrj-4.10.3.jar;lib/spatial4j-0.4.1.jar;lib/webcat-0.1-swf.jar;lib/weupnp-0.1.2.jar;lib/wstx-asl-3.2.9.jar;lib/xercesImpl.jar;lib/xml-apis.jar;lib/xmpcore-5.1.2.jar;lib/zookeeper-3.4.6.jar</classpath>
<built-to>lib/yacycore.jar</built-to>
<source-level>1.7</source-level>
</compilation-unit>

4
pom.xml
Unescape View File

@ -521,12 +521,12 @@
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi</artifactId>
<version>3.11</version>
<version>3.12</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-scratchpad</artifactId>
<version>3.11</version>
<version>3.12</version>
</dependency>
<dependency>
<groupId>sax</groupId>

7
source/net/yacy/document/TextParser.java
Unescape View File

@ -64,13 +64,10 @@ import net.yacy.document.parser.vcfParser;
import net.yacy.document.parser.vsdParser;
import net.yacy.document.parser.xlsParser;
import net.yacy.document.parser.zipParser;
import net.yacy.document.parser.augment.AugmentParser;
import net.yacy.document.parser.images.genericImageParser;
import net.yacy.document.parser.images.metadataImageParser;
import net.yacy.document.parser.rdfa.impl.RDFaParser;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.search.Switchboard;
public final class TextParser {
@ -91,9 +88,9 @@ public final class TextParser {
initParser(new docParser());
initParser(new gzipParser());
// AugmentParser calls internally RDFaParser (therefore add before RDFa)
if (Switchboard.getSwitchboard().getConfigBool("parserAugmentation", true)) initParser(new AugmentParser());
// if (Switchboard.getSwitchboard().getConfigBool("parserAugmentation", true)) initParser(new AugmentParser()); // experimental implementation, not working yet (2015-06-05)
// RDFaParser calls internally htmlParser (therefore add before html)
if (Switchboard.getSwitchboard().getConfigBool("parserAugmentation.RDFa", true)) initParser(new RDFaParser());
// if (Switchboard.getSwitchboard().getConfigBool("parserAugmentation.RDFa", true)) initParser(new RDFaParser()); // experimental implementation, not working yet (2015-06-04)
initParser(new htmlParser()); // called within rdfa parser
initParser(new genericImageParser());
initParser(new metadataImageParser());

Write Preview
Loading…
Cancel
Save