boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'master' of ssh://git@gitorious.org/yacy/rc1.git

Browse Source
pull/1/head
Michael Peter Christen 12 years ago
parent d988ba50cf e74d66e28c
commit ea49a8aa8c
6 changed files with 29 additions and 250 deletions
Show Stats Download Patch File Download Diff File

2
.classpath
Unescape View File

@ -64,11 +64,11 @@
<classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/> <classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/>
<classpathentry kind="lib" path="lib/icu4j-core.jar"/> <classpathentry kind="lib" path="lib/icu4j-core.jar"/>
<classpathentry kind="lib" path="lib/htmllexer.jar"/> <classpathentry kind="lib" path="lib/htmllexer.jar"/>
<classpathentry kind="lib" path="lib/htmlparser.jar"/>
<classpathentry kind="lib" path="lib/jena-2.6.4.jar" sourcepath="/Users/admin/.m2/repository/com/hp/hpl/jena/jena/2.6.4/jena-2.6.4-sources.jar"/> <classpathentry kind="lib" path="lib/jena-2.6.4.jar" sourcepath="/Users/admin/.m2/repository/com/hp/hpl/jena/jena/2.6.4/jena-2.6.4-sources.jar"/>
<classpathentry kind="lib" path="lib/sax-2.0.1.jar"/> <classpathentry kind="lib" path="lib/sax-2.0.1.jar"/>
<classpathentry kind="lib" path="lib/arq-2.8.7.jar"/> <classpathentry kind="lib" path="lib/arq-2.8.7.jar"/>
<classpathentry kind="lib" path="lib/iri-0.8.jar"/> <classpathentry kind="lib" path="lib/iri-0.8.jar"/>
<classpathentry kind="lib" path="lib/apache-solr-core-3.6.0.jar" sourcepath="/Volumes/Raptor/Data/workspace/apache-solr-3.6.0/src/java"/> <classpathentry kind="lib" path="lib/apache-solr-core-3.6.0.jar" sourcepath="/Volumes/Raptor/Data/workspace/apache-solr-3.6.0/src/java"/>
<classpathentry kind="lib" path="lib/jsoup-1.6.3.jar"/>
<classpathentry kind="output" path="gen"/> <classpathentry kind="output" path="gen"/>
</classpath> </classpath>

2
build.xml
Unescape View File

@ -176,7 +176,6 @@
<pathelement location="${lib}/geronimo-stax-api_1.0_spec-1.0.1.jar" /> <pathelement location="${lib}/geronimo-stax-api_1.0_spec-1.0.1.jar" />
<pathelement location="${lib}/guava-r05.jar" /> <pathelement location="${lib}/guava-r05.jar" />
<pathelement location="${lib}/htmllexer.jar" /> <pathelement location="${lib}/htmllexer.jar" />
<pathelement location="${lib}/htmlparser.jar" />
<pathelement location="${lib}/httpclient-4.2.1.jar" /> <pathelement location="${lib}/httpclient-4.2.1.jar" />
<pathelement location="${lib}/httpcore-4.2.1.jar" /> <pathelement location="${lib}/httpcore-4.2.1.jar" />
<pathelement location="${lib}/httpmime-4.2.1.jar" /> <pathelement location="${lib}/httpmime-4.2.1.jar" />
@ -191,6 +190,7 @@
<pathelement location="${lib}/jetty-util-6.1.26-patched-JETTY-1340.jar" /> <pathelement location="${lib}/jetty-util-6.1.26-patched-JETTY-1340.jar" />
<pathelement location="${lib}/jsch-0.1.42.jar" /> <pathelement location="${lib}/jsch-0.1.42.jar" />
<pathelement location="${lib}/json-simple-1.1.jar" /> <pathelement location="${lib}/json-simple-1.1.jar" />
<pathelement location="${lib}/jsoup-1.6.3.jar" />
<pathelement location="${lib}/log4j-1.2.16.jar" /> <pathelement location="${lib}/log4j-1.2.16.jar" />
<pathelement location="${lib}/log4j-over-slf4j-1.6.1.jar" /> <pathelement location="${lib}/log4j-over-slf4j-1.6.1.jar" />
<pathelement location="${lib}/lucene-analyzers-3.6.0.jar" /> <pathelement location="${lib}/lucene-analyzers-3.6.0.jar" />

1
defaults/yacy.init
Unescape View File

@ -508,6 +508,7 @@ proxyURL=false
proxyURL.access=127.0.0.1,0:0:0:0:0:0:0:1 proxyURL.access=127.0.0.1,0:0:0:0:0:0:0:1
# which urls to rewrite to /proxy.html?url=x (values: all, domainlist) # which urls to rewrite to /proxy.html?url=x (values: all, domainlist)
proxyURL.rewriteURLs=domainlist proxyURL.rewriteURLs=domainlist
proxyURL.useforresults=false
# From the 'IndexCreate' menu point you can also define a crawling start point. # From the 'IndexCreate' menu point you can also define a crawling start point.
# The crawling works the same way as the prefetch, but it is possible to # The crawling works the same way as the prefetch, but it is possible to

BIN
lib/htmlparser.jar
View File

Binary file not shown.

BIN
lib/jsoup-1.6.3.jar
View File

Binary file not shown.

246
source/net/yacy/interaction/AugmentHtmlStream.java
Unescape View File

@ -21,10 +21,7 @@ import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.logging.Log;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import org.htmlparser.Tag; import org.jsoup.Jsoup;
import org.htmlparser.Text;
import org.htmlparser.util.NodeList;
import org.htmlparser.visitors.NodeVisitor;
import de.anomic.http.server.ServerSideIncludes; import de.anomic.http.server.ServerSideIncludes;
@ -33,101 +30,6 @@ public class AugmentHtmlStream {
static RequestHeader globalrequestHeader; static RequestHeader globalrequestHeader;
/**
* creates a NodeVisitor which assigns a unique ID to every node
*
* @return customized NodeVisitor
*/
private static class VisitorAddUniqueID extends NodeVisitor {
private int counter;
public VisitorAddUniqueID() {
this.setCounter(0);
}
@Override
public void visitTag(Tag tag) {
if (tag.getAttribute("id") == null) {
this.setCounter(this.getCounter() + 1);
tag.setAttribute("id", "\"sci" + this.getCounter() + "\"");
}
if (tag instanceof org.htmlparser.tags.LinkTag) {
// Link
Log.logInfo("AUGMENTATION", tag.getAttribute("href"));
//LinkTag lt = (LinkTag)tag;
}
}
@Override
public void visitStringNode(Text string) {
}
public void setCounter(int counter) {
this.counter = counter;
}
public int getCounter() {
return this.counter;
}
}
/**
* creates a NodeVisitor which inspects the element if it contains useful
* text
*
* @return customized NodeVisitor
*/
private static class VisitorText extends NodeVisitor {
private int counter;
public VisitorText() {
this.counter = 0;
}
@Override
public void visitTag(Tag tag) {
// tag.setText(tag.getText()+" <span>augmented</span>");
// Node node = new org.htmlparser.nodes.TextNode(loadInternal("interactionparts/scibutton.html", globalrequestHeader));
// NodeList nl = tag.getChildren();
// nl.add (node);
// tag.setChildren(nl);
}
@Override
public void visitStringNode(Text string) {
// if (string.getParent() != null) {
//
// string.setText(string
// .getText()
// .replaceAll("und",
// "<a href=\"http://www.kit.edu/\" target=\"_blank\">KIT</a>"));
//
//
// }
}
public void setCounter(int counter) {
this.counter = counter;
}
public int getCounter() {
return this.counter;
}
}
/** /**
* send web page to external REFLECT web service * send web page to external REFLECT web service
* *
@ -244,159 +146,35 @@ public class AugmentHtmlStream {
// Add DOCTYPE if not present. // Add DOCTYPE if not present.
// This is required for IE to render position:absolute correctly. // This is required for IE to render position:absolute correctly.
if (sb.getConfigBool("augmentation.addDoctype", true) == true) { if (sb.getConfigBool("augmentation.addDoctype", false) == true) {
Doc = processAddDoctype(Doc); Doc = processAddDoctype(Doc);
augmented = true; augmented = true;
} }
if (sb.getConfigBool("augmentation.reparse", true) == true) { if (sb.getConfigBool("augmentation.reparse", false) == true) {
NodeList list = new NodeList();
// Fill NodeList with parsed Document
try {
org.htmlparser.Parser par = new org.htmlparser.Parser();
par.setInputHTML(Doc);
list = par.parse(null);
Log.logInfo ("AUGMENTATION", url.toString());
} catch (Exception e) {
}
// Add Unique ID to every node element which has no id yet.
// This allows consistent interaction between client (browser) and
// back-end (data store) by providing "position awareness" in the
// document.
if (sb.getConfigBool("augmentation.reparse.adduniqueid", true) == true) {
try {
NodeVisitor visitorAddUniqueID = new AugmentHtmlStream.VisitorAddUniqueID();
list.visitAllNodesWith(visitorAddUniqueID);
} catch (Exception e) {
}
}
// Inspect on text tags
try {
NodeVisitor visitorText = new AugmentHtmlStream.VisitorText();
list.visitAllNodesWith(visitorText);
} catch (Exception e) {
}
// System.out.println("Starting augmentation for " + url);
// System.out.println("Content: " + Doc);
if (!(list == null)) {
// DOCUMENT IS MANIPULABLE BY HTML REWRITER
// SO SEND IT TO YACY PARSER
Document document = null;
try {
final StringReader stringReader = new StringReader(Doc);
InputStream inputStream = new InputStream() {
@Override
public int read() throws IOException {
return stringReader.read();
}
};
document = Document.mergeDocuments(
url,
"text/html",
TextParser.parseSource(url, "text/html", null,
data.length(), inputStream));
} catch (Exception e) {
}
if (document != null) {
if (document.dc_format() == "text/html") {
// SCI_TITLE = document.dc_title();
// SCI_CREATOR = document.dc_creator();
// SCI_DESCRIPTION = document.dc_description();
// SCI_IDENTIFIER = document.dc_identifier();
}
}
// ADD AUGMENTED HEADER INFORMATION org.jsoup.nodes.Document d = Jsoup.parse(Doc);
NodeList header = list.extractAllNodesThatMatch( d.title ("yacy - "+d.title());
new org.htmlparser.filters.NodeClassFilter(
org.htmlparser.tags.HeadTag.class), true);
org.htmlparser.util.SimpleNodeIterator iterHeader = header if (sb.getConfigBool("interaction.overlayinteraction.enabled", false) == true) {
.elements();
while (iterHeader.hasMoreNodes()) { d.head().append (loadInternal("env/templates/jqueryheader.template", requestHeader));
org.htmlparser.tags.HeadTag ht = ((org.htmlparser.tags.HeadTag) iterHeader d.head().append ("<script type='text/javascript'>"+loadInternal("interaction_elements/interaction.js", requestHeader)+"</script>");
.nextNode()); d.head().append ("<script type='text/javascript'>"+loadInternal("interaction_elements/interaction_metadata.js", requestHeader)+"</script>");
NodeList headchildren = ht.getChildren();
headchildren.add(new org.htmlparser.nodes.TextNode(loadInternal("env/templates/jqueryheader.template", requestHeader))); d.body().append (loadInternal("interaction_elements/OverlayInteraction.html?action="+action+"&urlhash="+ ASCII.String(url.hash()) +"&url="+url.toNormalform(false, true), requestHeader));
headchildren.add(new org.htmlparser.nodes.TextNode("<script type='text/javascript'>"+loadInternal("interaction_elements/interaction.js", requestHeader)+"</script>")); d.body().append (loadInternal("interaction_elements/Footer.html?action="+action+"&urlhash="+ ASCII.String(url.hash()) +"&url="+url.toNormalform(false, true), requestHeader));
headchildren.add(new org.htmlparser.nodes.TextNode("<script type='text/javascript'>"+loadInternal("interaction_elements/interaction_metadata.js", requestHeader)+"</script>"));
augmented = true;
ht.setChildren(headchildren);
} }
// ADD AUGMENTED BODY INFORMATION Doc = d.html();
NodeList body = list.extractAllNodesThatMatch(
new org.htmlparser.filters.NodeClassFilter(
org.htmlparser.tags.BodyTag.class), true);
org.htmlparser.util.SimpleNodeIterator iterBody = body
.elements();
while (iterBody.hasMoreNodes()) {
org.htmlparser.tags.BodyTag bt = ((org.htmlparser.tags.BodyTag) iterBody
.nextNode());
NodeList bodychildren = bt.getChildren();
bodychildren.add(new org.htmlparser.nodes.TextNode(loadInternal("interaction_elements/OverlayInteraction.html?action="+action+"&urlhash="+ ASCII.String(url.hash()) +"&url="+url.toNormalform(false, true), requestHeader)));
bodychildren.add(new org.htmlparser.nodes.TextNode(loadInternal("interaction_elements/Footer.html?action="+action+"&urlhash="+ ASCII.String(url.hash()) +"&url="+url.toNormalform(false, true), requestHeader)));
bt.setChildren(bodychildren);
augmented = true; augmented = true;
} }
Doc = list.toHtml(true);
augmented = true;
} // not list = null
} // reparse
if (augmented) { if (augmented) {
return (new StringBuffer (Doc)); return (new StringBuffer (Doc));

Write Preview
Loading…
Cancel
Save