boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

*) Bugfix. re-enabling inheritance of serverCharBuffer from writer class

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2618 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
theli 19 years ago
parent 97d2a08ef1
commit ad7f600f25
6 changed files with 128 additions and 63 deletions
Show Stats Download Patch File Download Diff File

2
source/de/anomic/htmlFilter/htmlFilterAbstractScraper.java
View File

File diff suppressed because one or more lines are too long

28
source/de/anomic/htmlFilter/htmlFilterContentScraper.java
Unescape View File

@ -43,6 +43,9 @@
package de.anomic.htmlFilter;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.nio.charset.Charset;
@ -57,7 +60,9 @@ import java.util.Properties;
import java.util.TreeSet;
import de.anomic.net.URL;
import de.anomic.server.serverByteBuffer;
import de.anomic.server.serverCharBuffer;
import de.anomic.server.serverFileUtils;
public class htmlFilterContentScraper extends htmlFilterAbstractScraper implements htmlFilterScraper {
@ -113,6 +118,23 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
this.content = new serverCharBuffer(1024);
}
public void transformCharset(String oldCharset, String newCharset) throws UnsupportedEncodingException {
// // convert the content back to the old bytearray
// ByteArrayInputStream temp = new ByteArrayInputStream(new String(this.content.getChars()).getBytes(oldCharset));
//
// // create a reader with the new charset
// serverCharBuffer newContent = new serverCharBuffer(this.content.length());
// try {
// InputStreamReader reader = new InputStreamReader(temp,newCharset);
// serverFileUtils.copy(reader, newContent);
// reader.close();
// } catch (IOException e) {
// // ignore this
// }
//
// this.content = newContent;
}
public void scrapeText(char[] newtext) {
// System.out.println("SCRAPE: " + new String(newtext));
if ((content.length() != 0) && (content.charAt(content.length() - 1) != 32)) content.append(32);
@ -248,8 +270,12 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
}
public byte[] getText() {
return this.getText("UTF-8");
}
public byte[] getText(String charSet) {
try {
return content.toString().getBytes("UTF-8");
return content.toString().getBytes(charSet);
} catch (UnsupportedEncodingException e) {
return content.toString().getBytes();
}

20
source/de/anomic/htmlFilter/htmlFilterContentTransformer.java
Unescape View File

@ -46,6 +46,7 @@ package de.anomic.htmlFilter;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.text.Collator;
import java.util.ArrayList;
import java.util.Locale;
@ -112,14 +113,19 @@ public class htmlFilterContentTransformer extends htmlFilterAbstractTransformer
}
private static char[] genBlueLetters(int length) {
serverCharBuffer bb = new serverCharBuffer(" <FONT COLOR=#0000FF>".toCharArray());
length = length / 2;
if (length > 10) length = 7;
while (length-- > 0) {
bb.append('X');
try {
serverCharBuffer bb = new serverCharBuffer(" <FONT COLOR=#0000FF>".toCharArray());
length = length / 2;
if (length > 10) length = 7;
while (length-- > 0) {
bb.append('X');
}
bb.append("</FONT> ");
return bb.getChars();
} catch (IOException e) {
// ignore this.
return null;
}
bb.append("</FONT> ");
return bb.getChars();
}
private boolean bluelistHit(char[] text) {

113
source/de/anomic/htmlFilter/htmlFilterWriter.java
Unescape View File

@ -117,67 +117,92 @@ public final class htmlFilterWriter extends Writer {
}
public static char[] genTag0raw(String tagname, boolean opening, char[] tagopts) {
serverCharBuffer bb = new serverCharBuffer(tagname.length() + tagopts.length + 3);
bb.append('<');
if (!opening) {
bb.append('/');
}
bb.append(tagname);
if (tagopts.length > 0) {
// if (tagopts[0] == (byte) 32)
bb.append(tagopts);
// else bb.append((byte) 32).append(tagopts);
try {
serverCharBuffer bb = new serverCharBuffer(tagname.length() + tagopts.length + 3);
bb.append('<');
if (!opening) {
bb.append('/');
}
bb.append(tagname);
if (tagopts.length > 0) {
// if (tagopts[0] == (byte) 32)
bb.append(tagopts);
// else bb.append((byte) 32).append(tagopts);
}
bb.append('>');
return bb.getChars();
} catch (IOException e) {
// ignore this
return null;
}
bb.append('>');
return bb.getChars();
}
public static char[] genTag1raw(String tagname, char[] tagopts, char[] text) {
serverCharBuffer bb = new serverCharBuffer(2 * tagname.length() + tagopts.length + text.length + 5);
bb.append('<').append(tagname);
if (tagopts.length > 0) {
// if (tagopts[0] == (byte) 32)
bb.append(tagopts);
// else bb.append((byte) 32).append(tagopts);
try {
serverCharBuffer bb = new serverCharBuffer(2 * tagname.length() + tagopts.length + text.length + 5);
bb.append('<').append(tagname);
if (tagopts.length > 0) {
// if (tagopts[0] == (byte) 32)
bb.append(tagopts);
// else bb.append((byte) 32).append(tagopts);
}
bb.append('>');
bb.append(text);
bb.append('<').append('/').append(tagname).append('>');
return bb.getChars();
} catch (IOException e) {
// ignore this
return null;
}
bb.append('>');
bb.append(text);
bb.append('<').append('/').append(tagname).append('>');
return bb.getChars();
}
public static char[] genTag0(String tagname, Properties tagopts, char quotechar) {
char[] tagoptsx = (tagopts.size() == 0) ? null : genOpts(tagopts, quotechar);
serverCharBuffer bb = new serverCharBuffer(tagname.length() + ((tagoptsx == null) ? 0 : (tagoptsx.length + 1)) + tagname.length() + 2);
bb.append('<').append(tagname);
if (tagoptsx != null) {
bb.append(32);
bb.append(tagoptsx);
try {
char[] tagoptsx = (tagopts.size() == 0) ? null : genOpts(tagopts, quotechar);
serverCharBuffer bb = new serverCharBuffer(tagname.length() + ((tagoptsx == null) ? 0 : (tagoptsx.length + 1)) + tagname.length() + 2);
bb.append('<').append(tagname);
if (tagoptsx != null) {
bb.append(32);
bb.append(tagoptsx);
}
bb.append('>');
return bb.getChars();
} catch (IOException e) {
// ignore this
return null;
}
bb.append('>');
return bb.getChars();
}
public static char[] genTag1(String tagname, Properties tagopts, char[] text, char quotechar) {
char[] gt0 = genTag0(tagname, tagopts, quotechar);
serverCharBuffer cb = new serverCharBuffer(gt0, gt0.length + text.length + tagname.length() + 3);
cb.append(text).append('<').append('/').append(tagname).append('>');
return cb.getChars();
try {
char[] gt0 = genTag0(tagname, tagopts, quotechar);
serverCharBuffer cb = new serverCharBuffer(gt0, gt0.length + text.length + tagname.length() + 3);
cb.append(text).append('<').append('/').append(tagname).append('>');
return cb.getChars();
} catch (IOException e) {
// ignore this
return null;
}
}
// a helper method for pretty-printing of properties for html tags
public static char[] genOpts(Properties prop, char quotechar) {
Enumeration e = prop.propertyNames();
serverCharBuffer bb = new serverCharBuffer(prop.size() * 40);
String key;
while (e.hasMoreElements()) {
key = (String) e.nextElement();
bb.append(32).append(key).append('=').append(quotechar);
bb.append(prop.getProperty(key));
bb.append(quotechar);
try {
Enumeration e = prop.propertyNames();
serverCharBuffer bb = new serverCharBuffer(prop.size() * 40);
String key;
while (e.hasMoreElements()) {
key = (String) e.nextElement();
bb.append(32).append(key).append('=').append(quotechar);
bb.append(prop.getProperty(key));
bb.append(quotechar);
}
if (bb.length() > 0) return bb.getChars(1);
return bb.getChars();
}catch (IOException e) {
// ignore this
return null;
}
if (bb.length() > 0) return bb.getChars(1);
return bb.getChars();
}
private char[] filterTag(String tag, boolean opening, char[] content, char quotechar) {

8
source/de/anomic/plasma/plasmaParser.java
Unescape View File

@ -598,6 +598,14 @@ public final class plasmaParser {
public plasmaParserDocument transformScraper(URL location, String mimeType, String charSet, htmlFilterContentScraper scraper) {
try {
if (scraper.getMetas().containsKey("content-type")) {
String newCharset = (String) scraper.getMetas().get("content-type");
if (!charSet.equals(newCharset)) {
// TODO: transformation of content needed
this.theLogger.logFine("Charset transformation needed from '" + charSet + "' to '" + newCharset + "'");
}
}
String[] sections = new String[scraper.getHeadlines(1).length + scraper.getHeadlines(2).length + scraper.getHeadlines(3).length + scraper.getHeadlines(4).length];
int p = 0;
for (int i = 1; i <= 4; i++) for (int j = 0; j < scraper.getHeadlines(i).length; j++) sections[p++] = scraper.getHeadlines(i)[j];

10
source/de/anomic/server/serverCharBuffer.java
Unescape View File

@ -50,7 +50,7 @@ import java.io.IOException;
import java.io.Writer;
import java.util.Properties;
public final class serverCharBuffer /* extends Writer */ {
public final class serverCharBuffer extends Writer {
public static final char singlequote = '\'';
public static final char doublequote = '"';
@ -164,10 +164,10 @@ public final class serverCharBuffer /* extends Writer */ {
length += le;
}
public serverCharBuffer append(char b) {
write(b);
return this;
}
// public serverCharBuffer append(char b) {
// write(b);
// return this;
// }
public serverCharBuffer append(int i) {
write((char) (i));

Write Preview
Loading…
Cancel
Save