boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

*) cleaned up code for better readability

Browse Source 
*) added a few copyright notices
*) removed redundancy in constructors of ListToken

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6295 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
low012 16 years ago
parent eaddf2d464
commit 248f3fd9b5
14 changed files with 681 additions and 557 deletions
Show Stats Download Patch File Download Diff File

6
source/de/anomic/data/list/ListAccumulator.java
Unescape View File

@ -4,9 +4,9 @@
//
// (C) 2009 by Marc Nause
//
// $LastChangedDate: $
// $LastChangedRevision: $
// $LastChangedBy: $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by

6
source/de/anomic/data/list/XMLBlacklistImporter.java
Unescape View File

@ -4,9 +4,9 @@
//
// (C) 2009 by Marc Nause
//
// $LastChangedDate: $
// $LastChangedRevision: $
// $LastChangedBy: $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by

25
source/de/anomic/data/wiki/abstractWikiParser.java
Unescape View File

@ -1,3 +1,28 @@
// abstractWikiParser.java
// ---------
// part of YaCy
// (C) by Michael Peter Christen; mc@yacy.net
// first published on http://www.anomic.de
// Frankfurt, Germany, 2007
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.data.wiki;
import java.io.BufferedReader;

56
source/de/anomic/data/wiki/knwikiParser.java
Unescape View File

@ -1,4 +1,4 @@
// wikiParser.java
// knwikiParser.java
// ---------
// part of YaCy
// (C) by Michael Peter Christen; mc@yacy.net
@ -8,9 +8,9 @@
//
// This file is contributed by Franz Brausze
//
// $LastChangedDate: $
// $LastChangedRevision: $
// $LastChangedBy: $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@ -145,46 +145,41 @@ public class knwikiParser implements wikiParser {
private String parseUnescaped(String text) {
Token st;
Matcher m;
StringBuffer sb;
StringBuffer stringBuffer;
for (int i=0; i<tokens.length; i++) {
st = tokens[i];
for (int j=0; j<st.getRegex().length; j++) {
m = st.getRegex()[j].matcher(text);
sb = new StringBuffer();
stringBuffer = new StringBuffer();
while (m.find()) try {
//System.out.print("found " + st.getClass().getSimpleName() + ": " +
// m.group().replaceAll("\n", "\\\\n").replaceAll("\t", " ") + ", ");
if (!st.setText(m.group(), j)) {
// System.out.println("not usable");
continue;
//} else {
// System.out.println("usable");
}
m.appendReplacement(sb, (st.getMarkup() == null) ? m.group() : st.getMarkup());
m.appendReplacement(stringBuffer, (st.getMarkup() == null) ? m.group() : st.getMarkup());
} catch (final wikiParserException e) {
m.appendReplacement(sb, st.getText());
m.appendReplacement(stringBuffer, st.getText());
}
text = new String(m.appendTail(sb));
text = new String(m.appendTail(stringBuffer));
}
}
return text.replaceAll("----", "<hr />");
}
private String replaceBRs(final String text) {
final StringBuilder sb = new StringBuilder(text.length());
final StringBuilder stringBuffer = new StringBuilder(text.length());
final String[] tt = text.split("\n");
boolean replace;
for (int i=0, j; i<tt.length; i++) {
replace = true;
for (j=0; j<BEs.length; j++)
if (tt[i].endsWith(BEs[j] + ">")) { replace = false; break; }
sb.append(tt[i]);
stringBuffer.append(tt[i]);
if (i < tt.length - 1) {
if (replace) sb.append("<br />");
sb.append("\n");
if (replace) stringBuffer.append("<br />");
stringBuffer.append("\n");
}
}
return new String(sb);
return new String(stringBuffer);
}
private static class Text {
@ -200,24 +195,34 @@ public class knwikiParser implements wikiParser {
}
public String setText(final String text) {
if (this.nl)
if (this.nl) {
this.text = text.substring(escapeNewLine.length());
else
} else {
this.text = text;
}
return this.text;
}
public String getTextPlain() {
return this.text;
}
public String getTextPlain() { return this.text; }
public String getText() {
if (this.nl)
if (this.nl) {
return escapeNewLine + this.text;
}
return this.text;
}
public String toString() { return this.text; }
@Override
public String toString() {
return this.text;
}
static Text[] split2Texts(final String text, final String escapeBegin, final String escapeEnd) {
if (text == null) return null;
if (text.length() < 2) return new Text[] {new Text(text, false, true) };
final int startLen = escapeBegin.length();
@ -245,8 +250,9 @@ public class knwikiParser implements wikiParser {
static String mergeTexts(final Text[] texts) {
final StringBuilder sb = new StringBuilder(2000);
for (int n=0; n < texts.length; n++)
for (int n=0; n < texts.length; n++) {
sb.append(texts[n].getTextPlain());
}
return new String(sb);
}
}

26
source/de/anomic/data/wiki/tokens/AbstractToken.java
Unescape View File

@ -8,9 +8,9 @@
//
// This file is contributed by Franz Brausze
//
// $LastChangedDate: $
// $LastChangedRevision: $
// $LastChangedBy: $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@ -39,13 +39,25 @@ public abstract class AbstractToken implements Token {
protected abstract void parse() throws wikiParserException;
public String getMarkup() throws wikiParserException {
if (this.text == null)
if (this.text == null) {
throw new IllegalArgumentException();
if (!this.parsed) parse();
}
if (!this.parsed) {
parse();
}
return this.markup;
}
public String getText() { return this.text; }
public String getText() {
return this.text;
}
public String toString() { try { return getMarkup(); } catch (final wikiParserException e) { return null; } }
@Override
public String toString() {
try {
return getMarkup();
} catch (final wikiParserException e) {
return null;
}
}
}

18
source/de/anomic/data/wiki/tokens/DefinitionListToken.java
Unescape View File

@ -8,9 +8,9 @@
//
// This file is contributed by Franz Brausze
//
// $LastChangedDate: $
// $LastChangedRevision: $
// $LastChangedBy: $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@ -36,6 +36,7 @@ public class DefinitionListToken extends ListToken {
super(';', null, null);
}
@Override
protected StringBuilder parse(final String[] t, final int depth, final StringBuilder sb) {
sb.append("<dl>\n");
while (super.aktline < t.length && getGrade(t[super.aktline]) >= depth) {
@ -49,20 +50,25 @@ public class DefinitionListToken extends ListToken {
}
sb.append("</");
if (t[super.aktline].indexOf(':') == -1 || getGrade(t[super.aktline]) > depth)
if (t[super.aktline].indexOf(':') == -1 || getGrade(t[super.aktline]) > depth) {
sb.append("dt");
else
} else {
sb.append("dd");
}
sb.append(">\n");
super.aktline++;
}
for (int j=0; j<depth; j++) sb.append("\t");
for (int j=0; j<depth; j++) {
sb.append("\t");
}
sb.append("</dl>");
super.aktline--;
return sb;
}
@Override
public String[] getBlockElementNames() {
return blockElements;
}
}

70
source/de/anomic/data/wiki/tokens/LinkToken.java
Unescape View File

@ -8,9 +8,9 @@
//
// This file is contributed by Franz Brausze
//
// $LastChangedDate: $
// $LastChangedRevision: $
// $LastChangedBy: $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@ -73,8 +73,7 @@ public class LinkToken extends AbstractToken {
"( ([^\\]]*))?" + // optional description
"\\]"); // end
private static final Pattern[] patterns = new Pattern[] {
imgPattern, bkmPattern, intPattern, extPattern };
private static final Pattern[] patterns = new Pattern[] { imgPattern, bkmPattern, intPattern, extPattern };
private final String localhost;
private final String wikiPath;
@ -88,32 +87,39 @@ public class LinkToken extends AbstractToken {
}
protected void parse() throws wikiParserException {
final StringBuilder sb = new StringBuilder();
if (this.patternNr < 0 || this.patternNr >= patterns.length)
final StringBuilder stringBuilder = new StringBuilder();
if (this.patternNr < 0 || this.patternNr >= patterns.length) {
throw new wikiParserException("patternNr was not set correctly: " + this.patternNr);
}
final Matcher m = patterns[this.patternNr].matcher(this.text);
if (!m.find())
if (!m.find()) {
throw new wikiParserException("Didn't find match for: (" + this.patternNr + ") " + this.text);
}
switch (this.patternNr) {
case IMG:
sb.append("<img src=\"").append(formatHref(m.group(1).substring(6))).append("\"");
if (m.group(5) != null) sb.append(" align=\"").append(m.group(5)).append("\"");
sb.append(" alt=\"").append((m.group(7) == null) ? formatHref(m.group(1).substring(6)) : m.group(7)).append("\"");
sb.append(" />");
stringBuilder.append("<img src=\"").append(formatHref(m.group(1).substring(6))).append("\"");
if (m.group(5) != null) {
stringBuilder.append(" align=\"").append(m.group(5)).append("\"");
}
stringBuilder.append(" alt=\"").append((m.group(7) == null) ? formatHref(m.group(1).substring(6)) : m.group(7)).append("\"");
stringBuilder.append(" />");
break;
case BKM:
final Link[] links = getLinksFromBookmarkTag(m.group(2));
if (links == null) {
sb.append("<span class=\"error\">Couldn't find Bookmark-Tag '").append(m.group(2)).append("'.</span>");
stringBuilder.append("<span class=\"error\">Couldn't find Bookmark-Tag '").append(m.group(2)).append("'.</span>");
} else {
appendLinks(links, sb);
appendLinks(links, stringBuilder);
}
break;
case INT:
sb.append(new Link(
stringBuilder.append(new Link(
"http://" + this.localhost + "/" + this.wikiPath + m.group(1),
m.group(4),
(m.group(4) == null) ? m.group(1) : m.group(4)
@ -121,7 +127,7 @@ public class LinkToken extends AbstractToken {
break;
case EXT:
sb.append(new Link(
stringBuilder.append(new Link(
m.group(1),
m.group(3),
(m.group(3) == null) ? m.group(1) : m.group(3)
@ -129,7 +135,7 @@ public class LinkToken extends AbstractToken {
break;
}
this.parsed = true;
this.markup = new String(sb);
this.markup = new String(stringBuilder);
}
private String formatHref(final String link) {
@ -171,25 +177,35 @@ public class LinkToken extends AbstractToken {
this.desc = desc;
}
@Override
public String toString() {
final StringBuilder sb = new StringBuilder();
sb.append("<a href=\"").append(this.href).append("\"");
if (this.title != null) sb.append(" title=\"").append(this.title).append("\"");
sb.append(">");
if (this.desc == null) sb.append(this.href); else sb.append(this.desc);
sb.append("</a>");
return new String(sb);
final StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append("<a href=\"").append(this.href).append("\"");
if (this.title != null) stringBuilder.append(" title=\"").append(this.title).append("\"");
stringBuilder.append(">");
if (this.desc == null) stringBuilder.append(this.href); else stringBuilder.append(this.desc);
stringBuilder.append("</a>");
return new String(stringBuilder);
}
}
public String[] getBlockElementNames() { return null; }
public Pattern[] getRegex() { return patterns; }
public String[] getBlockElementNames() {
return null;
}
public Pattern[] getRegex() {
return patterns;
}
public boolean setText(final String text, final int patternNr) {
this.text = text;
this.patternNr = patternNr;
this.parsed = false;
if (text == null) { this.markup = null; this.patternNr = -1; }
if (text == null) {
this.markup = null;
this.patternNr = -1;
}
return true;
}
}

42
source/de/anomic/data/wiki/tokens/ListToken.java
Unescape View File

@ -8,9 +8,9 @@
//
// This file is contributed by Franz Brausze
//
// $LastChangedDate: $
// $LastChangedRevision: $
// $LastChangedBy: $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@ -44,31 +44,11 @@ public class ListToken extends AbstractToken {
protected int aktline = 0;
public ListToken(final char firstChar, final String listBlockElement) {
this.firstChar = firstChar;
this.listBlockElement = listBlockElement;
this.listElement = "li";
this.recursion = true;
this.pattern = new Pattern[] { Pattern.compile("^[" + firstChar + "]([^\n]|\n[" + firstChar + "])*", Pattern.MULTILINE) };
final ArrayList<String> r = new ArrayList<String>();
if (this.listBlockElement != null) {
if (this.recursion) r.add(this.listBlockElement);
if (this.listElement != null) r.add(this.listElement);
}
blockElements = r.toArray(new String[r.size()]);
this(firstChar, listBlockElement, "li");
}
public ListToken(final char firstChar, final String listBlockElement, final String listElement) {
this.firstChar = firstChar;
this.listBlockElement = listBlockElement;
this.listElement = listElement;
this.recursion = true;
this.pattern = new Pattern[] { Pattern.compile("^[" + firstChar + "]([^\n]|\n[" + firstChar + "])*", Pattern.MULTILINE) };
final ArrayList<String> r = new ArrayList<String>();
if (this.listBlockElement != null) {
if (this.recursion) r.add(this.listBlockElement);
if (this.listElement != null) r.add(this.listElement);
}
blockElements = r.toArray(new String[r.size()]);
this(firstChar, listBlockElement, listElement, true);
}
public ListToken(final char firstChar, final String listBlockElement, final String listElement, final boolean recursion) {
@ -79,8 +59,12 @@ public class ListToken extends AbstractToken {
this.pattern = new Pattern[] { Pattern.compile("^[" + firstChar + "]([^\n]|\n[" + firstChar + "])*", Pattern.MULTILINE) };
final ArrayList<String> r = new ArrayList<String>();
if (this.listBlockElement != null) {
if (this.recursion) r.add(this.listBlockElement);
if (this.listElement != null) r.add(this.listElement);
if (this.recursion) {
r.add(this.listBlockElement);
}
if (this.listElement != null) {
r.add(this.listElement);
}
}
blockElements = r.toArray(new String[r.size()]);
}
@ -116,8 +100,9 @@ public class ListToken extends AbstractToken {
protected int getGrade(final String t) {
int i = 0;
for (i=0; i<t.length(); i++)
for (i=0; i<t.length(); i++) {
if (t.charAt(i) != this.firstChar) break;
}
return i - 1;
}
@ -140,4 +125,5 @@ public class ListToken extends AbstractToken {
this.aktline = 0;
return true;
}
}

27
source/de/anomic/data/wiki/tokens/SimpleToken.java
Unescape View File

@ -8,9 +8,9 @@
//
// This file is contributed by Franz Brausze
//
// $LastChangedDate: $
// $LastChangedRevision: $
// $LastChangedBy: $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@ -58,17 +58,22 @@ public class SimpleToken extends AbstractToken {
this.blockElements = null;
}
for (i=0; i<definitionList.length; i++)
for (i=0; i<definitionList.length; i++) {
if (definitionList[i] != null) {
i++;
break;
}
this.pattern = new Pattern[] { Pattern.compile(
}
this.pattern = new Pattern[] {
Pattern.compile(
"([\\" + firstChar + "]{" + i + "," + definitionList.length + "})" +
"(.*?)" +
"([\\" + lastChar + "]{" + i + "," + definitionList.length + "})")};
"([\\" + lastChar + "]{" + i + "," + definitionList.length + "})")
};
}
@Override
public String getMarkup() throws wikiParserException {
if (this.content == null) {
if (this.text == null) {
@ -136,6 +141,12 @@ public class SimpleToken extends AbstractToken {
return false;
}
public Pattern[] getRegex() { return this.pattern; }
public String[] getBlockElementNames() { return this.blockElements; }
public Pattern[] getRegex() {
return this.pattern;
}
public String[] getBlockElementNames() {
return this.blockElements;
}
}

19
source/de/anomic/data/wiki/tokens/TableToken.java
Unescape View File

@ -8,9 +8,9 @@
//
// This file is contributed by Franz Brausze
//
// $LastChangedDate: $
// $LastChangedRevision: $
// $LastChangedBy: $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@ -40,6 +40,7 @@ public class TableToken extends AbstractToken {
"([^\n]|\n\\|[|-])*\n" + // new line must start with "||" or "|-"
"\\|\\}") // "|}"
};
private static final String[] blockElementNames = new String[] { "table", "tr", "td" };
protected void parse() {
@ -73,7 +74,9 @@ public class TableToken extends AbstractToken {
// from de.anomic.data.wikiCode.java.parseTableProperties, modified by [FB]
private static final String[] tps = { "rowspan", "colspan", "vspace", "hspace", "cellspacing", "cellpadding", "border" };
private static final HashMap<String, String[]> ps = new HashMap<String, String[]>();
static {
Arrays.sort(tps);
String[] array;
@ -125,8 +128,13 @@ public class TableToken extends AbstractToken {
return sb.append(" ").append(key).append("=\"").append(value).append("\"");
}
public Pattern[] getRegex() { return pattern; }
public String[] getBlockElementNames() { return blockElementNames; }
public Pattern[] getRegex() {
return pattern;
}
public String[] getBlockElementNames() {
return blockElementNames;
}
public boolean setText(final String text, final int patternNr) {
this.text = text;
@ -134,4 +142,5 @@ public class TableToken extends AbstractToken {
this.markup = null;
return true;
}
}

6
source/de/anomic/data/wiki/tokens/Token.java
Unescape View File

@ -8,9 +8,9 @@
//
// This file is contributed by Franz Brausze
//
// $LastChangedDate: $
// $LastChangedRevision: $
// $LastChangedBy: $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by

13
source/de/anomic/data/wiki/wikiBoard.java
Unescape View File

@ -3,18 +3,21 @@
//(C) by Michael Peter Christen; mc@yacy.net
//first published on http://www.anomic.de
//Frankfurt, Germany, 2004
//last major change: 20.07.2004
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
//This program is free software; you can redistribute it and/or modify
//it under the terms of the GNU General Public License as published by
//the Free Software Foundation; either version 2 of the License, or
//(at your option) any later version.
//
//This program is distributed in the hope that it will be useful,
//but WITHOUT ANY WARRANTY; without even the implied warranty of
//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
//GNU General Public License for more details.
//
//You should have received a copy of the GNU General Public License
//along with this program; if not, write to the Free Software
//Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
@ -41,7 +44,7 @@ public class wikiBoard {
public static final int keyLength = 64;
private static final String dateFormat = "yyyyMMddHHmmss";
static SimpleDateFormat SimpleFormatter = new SimpleDateFormat(dateFormat);
private static final SimpleDateFormat SimpleFormatter = new SimpleDateFormat(dateFormat);
static {
SimpleFormatter.setTimeZone(TimeZone.getTimeZone("GMT"));

25
source/de/anomic/data/wiki/wikiParser.java
Unescape View File

@ -1,3 +1,28 @@
// wikiParser.java
// ---------
// part of YaCy
// (C) by Michael Peter Christen; mc@yacy.net
// first published on http://www.anomic.de
// Frankfurt, Germany, 2007
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.data.wiki;
import java.io.UnsupportedEncodingException;

25
source/de/anomic/data/wiki/wikiParserException.java
Unescape View File

@ -1,3 +1,28 @@
// wikiParserException.java
// ---------
// part of YaCy
// (C) by Michael Peter Christen; mc@yacy.net
// first published on http://www.anomic.de
// Frankfurt, Germany, 2007
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.data.wiki;
public class wikiParserException extends Exception {

Write Preview
Loading…
Cancel
Save