|
|
|
@ -1,4 +1,4 @@
|
|
|
|
// wikiParser.java
|
|
|
|
// knwikiParser.java
|
|
|
|
// ---------
|
|
|
|
// ---------
|
|
|
|
// part of YaCy
|
|
|
|
// part of YaCy
|
|
|
|
// (C) by Michael Peter Christen; mc@yacy.net
|
|
|
|
// (C) by Michael Peter Christen; mc@yacy.net
|
|
|
|
@ -8,9 +8,9 @@
|
|
|
|
//
|
|
|
|
//
|
|
|
|
// This file is contributed by Franz Brausze
|
|
|
|
// This file is contributed by Franz Brausze
|
|
|
|
//
|
|
|
|
//
|
|
|
|
// $LastChangedDate: $
|
|
|
|
// $LastChangedDate$
|
|
|
|
// $LastChangedRevision: $
|
|
|
|
// $LastChangedRevision$
|
|
|
|
// $LastChangedBy: $
|
|
|
|
// $LastChangedBy$
|
|
|
|
//
|
|
|
|
//
|
|
|
|
// This program is free software; you can redistribute it and/or modify
|
|
|
|
// This program is free software; you can redistribute it and/or modify
|
|
|
|
// it under the terms of the GNU General Public License as published by
|
|
|
|
// it under the terms of the GNU General Public License as published by
|
|
|
|
@ -42,55 +42,55 @@ import de.anomic.search.Switchboard;
|
|
|
|
|
|
|
|
|
|
|
|
public class knwikiParser implements wikiParser {
|
|
|
|
public class knwikiParser implements wikiParser {
|
|
|
|
|
|
|
|
|
|
|
|
public Token[] tokens;
|
|
|
|
public Token[] tokens;
|
|
|
|
private String[] BEs;
|
|
|
|
private String[] BEs;
|
|
|
|
private final Switchboard sb;
|
|
|
|
private final Switchboard sb;
|
|
|
|
|
|
|
|
|
|
|
|
private knwikiParser(final Switchboard sb) {
|
|
|
|
private knwikiParser(final Switchboard sb) {
|
|
|
|
this.sb = sb;
|
|
|
|
this.sb = sb;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
public static void main(final String[] args) {
|
|
|
|
public static void main(final String[] args) {
|
|
|
|
final String text = "===T<pre>itle===\n" +
|
|
|
|
final String text = "===T<pre>itle===\n" +
|
|
|
|
"==blubb== was ==ein '''shice'''==...och.bla\n" +
|
|
|
|
"==blubb== was ==ein '''shice'''==...och.bla\n" +
|
|
|
|
"* ein \n" +
|
|
|
|
"* ein \n" +
|
|
|
|
"*==test=</pre>=\n" +
|
|
|
|
"*==test=</pre>=\n" +
|
|
|
|
"** doppelt\n" +
|
|
|
|
"** doppelt\n" +
|
|
|
|
"* ''tess*sst''\n" +
|
|
|
|
"* ''tess*sst''\n" +
|
|
|
|
"*** xyz\n" +
|
|
|
|
"*** xyz\n" +
|
|
|
|
"=]*** huch\n" +
|
|
|
|
"=]*** huch\n" +
|
|
|
|
"* ehehe***\n" +
|
|
|
|
"* ehehe***\n" +
|
|
|
|
"* blubb\n" +
|
|
|
|
"* blubb\n" +
|
|
|
|
"bliblablo\n\n\n" +
|
|
|
|
"bliblablo\n\n\n" +
|
|
|
|
"* blubb\n" +
|
|
|
|
"* blubb\n" +
|
|
|
|
"{|border=-1\n" +
|
|
|
|
"{|border=-1\n" +
|
|
|
|
"|-\n" +
|
|
|
|
"|-\n" +
|
|
|
|
"||bla|| blubb\n" +
|
|
|
|
"||bla|| blubb\n" +
|
|
|
|
"|-\n" +
|
|
|
|
"|-\n" +
|
|
|
|
"||align center|och||huch||\n" +
|
|
|
|
"||align center|och||huch||\n" +
|
|
|
|
"|}\n" +
|
|
|
|
"|}\n" +
|
|
|
|
"\n" +
|
|
|
|
"\n" +
|
|
|
|
"# bla\n" +
|
|
|
|
"# bla\n" +
|
|
|
|
"# blubb\n" +
|
|
|
|
"# blubb\n" +
|
|
|
|
"'''''ehehehe''''', ne?!\n" +
|
|
|
|
"'''''ehehehe''''', ne?!\n" +
|
|
|
|
"[http://www/index.html,ne?!] -\n" +
|
|
|
|
"[http://www/index.html,ne?!] -\n" +
|
|
|
|
"[[Image:blubb|BLA]] ---- och\n" +
|
|
|
|
"[[Image:blubb|BLA]] ---- och\n" +
|
|
|
|
" blubb1\n" +
|
|
|
|
" blubb1\n" +
|
|
|
|
" blubb2\n" +
|
|
|
|
" blubb2\n" +
|
|
|
|
":doppel-blubb[= huch =]\n" +
|
|
|
|
":doppel-blubb[= huch =]\n" +
|
|
|
|
";hier:da\n" +
|
|
|
|
";hier:da\n" +
|
|
|
|
";dort:und so\n" +
|
|
|
|
";dort:und so\n" +
|
|
|
|
";;und:doppelt\n\n\n\n" +
|
|
|
|
";;und:doppelt\n\n\n\n" +
|
|
|
|
"[[Image:blubb|BLA]]";
|
|
|
|
"[[Image:blubb|BLA]]";
|
|
|
|
// text = "[=\n=]* bla";
|
|
|
|
// text = "[=\n=]* bla";
|
|
|
|
String t = "[=] ein fucking [= test =]-text[=,ne?!=] joa, [=alles=]wunderbar," +
|
|
|
|
String t = "[=] ein fucking [= test =]-text[=,ne?!=] joa, [=alles=]wunderbar," +
|
|
|
|
"[=denk ich=] mal =]";
|
|
|
|
"[=denk ich=] mal =]";
|
|
|
|
final long l = System.currentTimeMillis();
|
|
|
|
final long l = System.currentTimeMillis();
|
|
|
|
t = new knwikiParser(null).parse((args.length > 0) ? args[0] : text, "localhost:8080");
|
|
|
|
t = new knwikiParser(null).parse((args.length > 0) ? args[0] : text, "localhost:8080");
|
|
|
|
System.out.println("parsing time: " + (System.currentTimeMillis() - l) + " ms");
|
|
|
|
System.out.println("parsing time: " + (System.currentTimeMillis() - l) + " ms");
|
|
|
|
System.out.println("--- --- ---");
|
|
|
|
System.out.println("--- --- ---");
|
|
|
|
System.out.println(t);
|
|
|
|
System.out.println(t);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
public String transform(final String content) {
|
|
|
|
public String transform(final String content) {
|
|
|
|
return parse(content, null);
|
|
|
|
return parse(content, null);
|
|
|
|
@ -105,7 +105,7 @@ public class knwikiParser implements wikiParser {
|
|
|
|
return parse(new String(content, encoding), null);
|
|
|
|
return parse(new String(content, encoding), null);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private String parse(String text, final String publicAddress) {
|
|
|
|
private String parse(String text, final String publicAddress) {
|
|
|
|
tokens = new Token[] {
|
|
|
|
tokens = new Token[] {
|
|
|
|
new SimpleToken('=', '=', new String[][] { null, { "h2" }, { "h3" }, { "h4" } }, true),
|
|
|
|
new SimpleToken('=', '=', new String[][] { null, { "h2" }, { "h3" }, { "h4" } }, true),
|
|
|
|
new SimpleToken('\'', '\'', new String[][] { null, { "i" }, { "b" }, null, { "b", "i" } }, false),
|
|
|
|
new SimpleToken('\'', '\'', new String[][] { null, { "i" }, { "b" }, null, { "b", "i" } }, false),
|
|
|
|
@ -130,124 +130,130 @@ public class knwikiParser implements wikiParser {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
r.add("hr");
|
|
|
|
r.add("hr");
|
|
|
|
BEs = r.toArray(new String[r.size()]);
|
|
|
|
BEs = r.toArray(new String[r.size()]);
|
|
|
|
|
|
|
|
|
|
|
|
Text[] tt = Text.split2Texts(text, "[=", "=]");
|
|
|
|
Text[] tt = Text.split2Texts(text, "[=", "=]");
|
|
|
|
for (int i=0; i<tt.length; i+=2)
|
|
|
|
for (int i=0; i<tt.length; i+=2)
|
|
|
|
tt[i].setText(parseUnescaped(tt[i].getText()));
|
|
|
|
tt[i].setText(parseUnescaped(tt[i].getText()));
|
|
|
|
text = Text.mergeTexts(tt);
|
|
|
|
text = Text.mergeTexts(tt);
|
|
|
|
|
|
|
|
|
|
|
|
tt = Text.split2Texts(text, "<pre>", "</pre>");
|
|
|
|
tt = Text.split2Texts(text, "<pre>", "</pre>");
|
|
|
|
for (int i=0; i<tt.length; i+=2)
|
|
|
|
for (int i=0; i<tt.length; i+=2)
|
|
|
|
tt[i].setText(replaceBRs(tt[i].getText()));
|
|
|
|
tt[i].setText(replaceBRs(tt[i].getText()));
|
|
|
|
return Text.mergeTexts(tt);
|
|
|
|
return Text.mergeTexts(tt);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private String parseUnescaped(String text) {
|
|
|
|
private String parseUnescaped(String text) {
|
|
|
|
Token st;
|
|
|
|
Token st;
|
|
|
|
Matcher m;
|
|
|
|
Matcher m;
|
|
|
|
StringBuffer sb;
|
|
|
|
StringBuffer stringBuffer;
|
|
|
|
for (int i=0; i<tokens.length; i++) {
|
|
|
|
for (int i=0; i<tokens.length; i++) {
|
|
|
|
st = tokens[i];
|
|
|
|
st = tokens[i];
|
|
|
|
for (int j=0; j<st.getRegex().length; j++) {
|
|
|
|
for (int j=0; j<st.getRegex().length; j++) {
|
|
|
|
m = st.getRegex()[j].matcher(text);
|
|
|
|
m = st.getRegex()[j].matcher(text);
|
|
|
|
sb = new StringBuffer();
|
|
|
|
stringBuffer = new StringBuffer();
|
|
|
|
while (m.find()) try {
|
|
|
|
while (m.find()) try {
|
|
|
|
//System.out.print("found " + st.getClass().getSimpleName() + ": " +
|
|
|
|
if (!st.setText(m.group(), j)) {
|
|
|
|
// m.group().replaceAll("\n", "\\\\n").replaceAll("\t", " ") + ", ");
|
|
|
|
continue;
|
|
|
|
if (!st.setText(m.group(), j)) {
|
|
|
|
}
|
|
|
|
// System.out.println("not usable");
|
|
|
|
m.appendReplacement(stringBuffer, (st.getMarkup() == null) ? m.group() : st.getMarkup());
|
|
|
|
continue;
|
|
|
|
} catch (final wikiParserException e) {
|
|
|
|
//} else {
|
|
|
|
m.appendReplacement(stringBuffer, st.getText());
|
|
|
|
// System.out.println("usable");
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
m.appendReplacement(sb, (st.getMarkup() == null) ? m.group() : st.getMarkup());
|
|
|
|
|
|
|
|
} catch (final wikiParserException e) {
|
|
|
|
|
|
|
|
m.appendReplacement(sb, st.getText());
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
text = new String(m.appendTail(sb));
|
|
|
|
text = new String(m.appendTail(stringBuffer));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return text.replaceAll("----", "<hr />");
|
|
|
|
return text.replaceAll("----", "<hr />");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private String replaceBRs(final String text) {
|
|
|
|
private String replaceBRs(final String text) {
|
|
|
|
final StringBuilder sb = new StringBuilder(text.length());
|
|
|
|
final StringBuilder stringBuffer = new StringBuilder(text.length());
|
|
|
|
final String[] tt = text.split("\n");
|
|
|
|
final String[] tt = text.split("\n");
|
|
|
|
boolean replace;
|
|
|
|
boolean replace;
|
|
|
|
for (int i=0, j; i<tt.length; i++) {
|
|
|
|
for (int i=0, j; i<tt.length; i++) {
|
|
|
|
replace = true;
|
|
|
|
replace = true;
|
|
|
|
for (j=0; j<BEs.length; j++)
|
|
|
|
for (j=0; j<BEs.length; j++)
|
|
|
|
if (tt[i].endsWith(BEs[j] + ">")) { replace = false; break; }
|
|
|
|
if (tt[i].endsWith(BEs[j] + ">")) { replace = false; break; }
|
|
|
|
sb.append(tt[i]);
|
|
|
|
stringBuffer.append(tt[i]);
|
|
|
|
if (i < tt.length - 1) {
|
|
|
|
if (i < tt.length - 1) {
|
|
|
|
if (replace) sb.append("<br />");
|
|
|
|
if (replace) stringBuffer.append("<br />");
|
|
|
|
sb.append("\n");
|
|
|
|
stringBuffer.append("\n");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return new String(sb);
|
|
|
|
return new String(stringBuffer);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static class Text {
|
|
|
|
private static class Text {
|
|
|
|
|
|
|
|
|
|
|
|
public static final String escapeNewLine = "@";
|
|
|
|
public static final String escapeNewLine = "@";
|
|
|
|
|
|
|
|
|
|
|
|
private String text;
|
|
|
|
private String text;
|
|
|
|
private final boolean nl;
|
|
|
|
private final boolean nl;
|
|
|
|
|
|
|
|
|
|
|
|
public Text(final String text, final boolean escaped, final boolean newLineBefore) {
|
|
|
|
public Text(final String text, final boolean escaped, final boolean newLineBefore) {
|
|
|
|
this.text = text;
|
|
|
|
this.text = text;
|
|
|
|
this.nl = newLineBefore;
|
|
|
|
this.nl = newLineBefore;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
public String setText(final String text) {
|
|
|
|
public String setText(final String text) {
|
|
|
|
if (this.nl)
|
|
|
|
if (this.nl) {
|
|
|
|
this.text = text.substring(escapeNewLine.length());
|
|
|
|
this.text = text.substring(escapeNewLine.length());
|
|
|
|
else
|
|
|
|
} else {
|
|
|
|
this.text = text;
|
|
|
|
this.text = text;
|
|
|
|
return this.text;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return this.text;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
public String getTextPlain() { return this.text; }
|
|
|
|
public String getTextPlain() {
|
|
|
|
public String getText() {
|
|
|
|
return this.text;
|
|
|
|
if (this.nl)
|
|
|
|
}
|
|
|
|
return escapeNewLine + this.text;
|
|
|
|
|
|
|
|
return this.text;
|
|
|
|
public String getText() {
|
|
|
|
}
|
|
|
|
if (this.nl) {
|
|
|
|
|
|
|
|
return escapeNewLine + this.text;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
return this.text;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
public String toString() { return this.text; }
|
|
|
|
@Override
|
|
|
|
|
|
|
|
public String toString() {
|
|
|
|
|
|
|
|
return this.text;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
static Text[] split2Texts(final String text, final String escapeBegin, final String escapeEnd) {
|
|
|
|
static Text[] split2Texts(final String text, final String escapeBegin, final String escapeEnd) {
|
|
|
|
if (text == null) return null;
|
|
|
|
|
|
|
|
if (text.length() < 2) return new Text[] { new Text(text, false, true) };
|
|
|
|
if (text == null) return null;
|
|
|
|
|
|
|
|
|
|
|
|
final int startLen = escapeBegin.length();
|
|
|
|
if (text.length() < 2) return new Text[] {new Text(text, false, true) };
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
final int startLen = escapeBegin.length();
|
|
|
|
final int endLen = escapeEnd.length();
|
|
|
|
final int endLen = escapeEnd.length();
|
|
|
|
final ArrayList<Text> r = new ArrayList<Text>();
|
|
|
|
final ArrayList<Text> r = new ArrayList<Text>();
|
|
|
|
boolean escaped = text.startsWith(escapeBegin);
|
|
|
|
boolean escaped = text.startsWith(escapeBegin);
|
|
|
|
if (escaped) r.add(new Text("", false, true));
|
|
|
|
if (escaped) r.add(new Text("", false, true));
|
|
|
|
int i, j = 0;
|
|
|
|
int i, j = 0;
|
|
|
|
while ((i = text.indexOf((escaped) ? escapeEnd : escapeBegin, j)) > -1) {
|
|
|
|
while ((i = text.indexOf((escaped) ? escapeEnd : escapeBegin, j)) > -1) {
|
|
|
|
r.add(resolve2Text(text, escaped, (j > 0) ? j + ((escaped) ? startLen : endLen) : 0, i, escapeEnd));
|
|
|
|
r.add(resolve2Text(text, escaped, (j > 0) ? j + ((escaped) ? startLen : endLen) : 0, i, escapeEnd));
|
|
|
|
j = i;
|
|
|
|
j = i;
|
|
|
|
escaped = !escaped;
|
|
|
|
escaped = !escaped;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
r.add(resolve2Text(text, escaped, (escaped) ? j : (j > 0) ? j + endLen : 0, -1, escapeEnd));
|
|
|
|
r.add(resolve2Text(text, escaped, (escaped) ? j : (j > 0) ? j + endLen : 0, -1, escapeEnd));
|
|
|
|
return r.toArray(new Text[r.size()]);
|
|
|
|
return r.toArray(new Text[r.size()]);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static Text resolve2Text(final String text, final boolean escaped, final int from, int to, final String escapeEnd) {
|
|
|
|
private static Text resolve2Text(final String text, final boolean escaped, final int from, int to, final String escapeEnd) {
|
|
|
|
if (to == -1) to = text.length();
|
|
|
|
if (to == -1) to = text.length();
|
|
|
|
return new Text(
|
|
|
|
return new Text(
|
|
|
|
text.substring(from, to),
|
|
|
|
text.substring(from, to),
|
|
|
|
escaped,
|
|
|
|
escaped,
|
|
|
|
from < escapeEnd.length() + 2 || (!escaped && text.charAt(from - escapeEnd.length() - 1) == '\n'));
|
|
|
|
from < escapeEnd.length() + 2 || (!escaped && text.charAt(from - escapeEnd.length() - 1) == '\n'));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
static String mergeTexts(final Text[] texts) {
|
|
|
|
static String mergeTexts(final Text[] texts) {
|
|
|
|
final StringBuilder sb = new StringBuilder(2000);
|
|
|
|
final StringBuilder sb = new StringBuilder(2000);
|
|
|
|
for (int n=0; n < texts.length; n++)
|
|
|
|
for (int n=0; n < texts.length; n++) {
|
|
|
|
sb.append(texts[n].getTextPlain());
|
|
|
|
sb.append(texts[n].getTextPlain());
|
|
|
|
return new String(sb);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return new String(sb);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
low012
16 years ago