boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

code cleanup

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1173 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent 8f1f2daa5e
commit 76618442e0
7 changed files with 96 additions and 95 deletions
Show Stats Download Patch File Download Diff File

18
source/de/anomic/htmlFilter/htmlFilterContentScraper.java
Unescape View File

@ -86,7 +86,7 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
private HashMap images;
private String title;
private String headline;
private serverByteBuffer text;
private serverByteBuffer content;
private URL root;
public htmlFilterContentScraper(URL root) {
@ -98,13 +98,13 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
this.images = new HashMap();
this.title = "";
this.headline = "";
this.text = new serverByteBuffer(1024);
this.content = new serverByteBuffer(1024);
}
public void scrapeText(byte[] newtext) {
// System.out.println("SCRAPE: " + new String(newtext));
if ((text.length() != 0) && (text.byteAt(text.length() - 1) != 32)) text.append(32);
text.append(super.stripAll(new serverByteBuffer(newtext, newtext.length + 1)).trim()).append(32);
if ((content.length() != 0) && (content.byteAt(content.length() - 1) != 32)) content.append(32);
content.append(super.stripAll(new serverByteBuffer(newtext, newtext.length + 1)).trim()).append(32);
}
@ -222,8 +222,8 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
// extract headline from content
if (title.length() > 0) hl = title.trim();
else if (headline.length() > 0) hl = headline.trim();
else if (text.length() > 80) hl = new String(text.getBytes(), 0, 80).trim();
else hl = text.trim().toString();
else if (content.length() > 80) hl = new String(content.getBytes(), 0, 80).trim();
else hl = content.trim().toString();
// clean the line: may contain too many funny symbols
for (int i = 0; i < hl.length(); i++)
@ -238,7 +238,7 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
}
public byte[] getText() {
return text.getBytes();
return content.getBytes();
}
public Map getAnchors() {
@ -256,7 +256,7 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
images = null;
title = null;
headline = null;
text = null;
content = null;
root = null;
}
@ -265,7 +265,7 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
System.out.println("HEADLINE:" + headline);
System.out.println("ANCHORS :" + anchors.toString());
System.out.println("IMAGES :" + images.toString());
System.out.println("TEXT :" + new String(text.getBytes()));
System.out.println("TEXT :" + new String(content.getBytes()));
}
public static void main(String[] args) {

140
source/de/anomic/http/httpc.java
Unescape View File

@ -131,7 +131,7 @@ public final class httpc {
// class variables
private Socket socket = null; // client socket for commands
private Thread socketOwner = null;
String host = null;
private String host = null;
//private long timeout;
// output and input streams for client control connection
@ -872,15 +872,17 @@ public final class httpc {
/**
* Call the server with the CONNECT-method.
* This is used to establish https-connections through a https-proxy
*
* @param host To which host should a connection be made?
* @param port Which port should be connected?
* @param requestHeader prefilled httpHeader.
* @return Instance of response with the content.
*/
public response CONNECT(String host, int port, httpHeader requestHeader) throws IOException {
public response CONNECT(String remotehost, int remoteport, httpHeader requestHeader) throws IOException {
try {
send(httpHeader.METHOD_CONNECT, host + ":" + port, requestHeader, false);
send(httpHeader.METHOD_CONNECT, remotehost + ":" + remoteport, requestHeader, false);
return new response(false);
} catch (SocketException e) {
throw new IOException(e.getMessage());
@ -1520,6 +1522,67 @@ do upload
return openSockets;
}
}
/**
* This method outputs the input stream to either an output socket or an
* file or both. If the length of the input stream is given in the
* header, exactly that lenght is written. Otherwise the stream is
* written, till it is closed. If this instance is zipped, stream the
* input stream through gzip to unzip it on the fly.
*
* @param procOS OutputStream where the stream is to be written. If null
* no write happens.
* @param bufferOS OutputStream where the stream is to be written too.
* If null no write happens.
* @param clientInput InputStream where the content is to be read from.
* @throws IOException
*/
public static void writeContentX(InputStream clientInput, boolean usegzip, long length, OutputStream procOS, OutputStream bufferOS) throws IOException {
// we write length bytes, but if length == -1 (or < 0) then we
// write until the input stream closes
// procOS == null -> no write to procOS
// file == null -> no write to file
// If the Content-Encoding is gzip, we gunzip on-the-fly
// and change the Content-Encoding and Content-Length attributes in the header
byte[] buffer = new byte[2048];
int l;
long len = 0;
// using the proper intput stream
InputStream dis = (usegzip) ? (InputStream) new GZIPInputStream(clientInput) : (InputStream) clientInput;
// we have three methods of reading: length-based, length-based gzip and connection-close-based
try {
if (length > 0) {
// we read exactly 'length' bytes
while ((len < length) && ((l = dis.read(buffer)) >= 0)) {
if (procOS != null) procOS.write(buffer, 0, l);
if (bufferOS != null) bufferOS.write(buffer, 0, l);
len += l;
}
} else {
// no content-length was given, thus we read until the connection closes
while ((l = dis.read(buffer, 0, buffer.length)) >= 0) {
if (procOS != null) procOS.write(buffer, 0, l);
if (bufferOS != null) bufferOS.write(buffer, 0, l);
}
}
} catch (java.net.SocketException e) {
throw new IOException("Socket exception: " + e.getMessage());
} catch (java.net.SocketTimeoutException e) {
throw new IOException("Socket time-out: " + e.getMessage());
} finally {
// close the streams
if (procOS != null) {
if (procOS instanceof httpChunkedOutputStream)
((httpChunkedOutputStream)procOS).finish();
procOS.flush();
}
if (bufferOS != null) bufferOS.flush();
buffer = null;
}
}
/**
* Inner Class to get the response of an http-request and parse it.
@ -1669,7 +1732,7 @@ do upload
public byte[] writeContent() throws IOException {
int contentLength = (int) this.responseHeader.contentLength();
serverByteBuffer sbb = new serverByteBuffer((contentLength==-1)?8192:contentLength);
writeContentX(null, sbb, httpc.this.clientInput);
writeContentX(httpc.this.clientInput, this.gzip, this.responseHeader.contentLength(), null, sbb);
return sbb.getBytes();
}
@ -1684,7 +1747,7 @@ do upload
public byte[] writeContent(OutputStream procOS) throws IOException {
int contentLength = (int) this.responseHeader.contentLength();
serverByteBuffer sbb = new serverByteBuffer((contentLength==-1)?8192:contentLength);
writeContentX(procOS, sbb, httpc.this.clientInput);
writeContentX(httpc.this.clientInput, this.gzip, this.responseHeader.contentLength(), procOS, sbb);
return sbb.getBytes();
}
@ -1702,7 +1765,7 @@ do upload
FileOutputStream bufferOS = null;
try {
if (file != null) bufferOS = new FileOutputStream(file);
writeContentX(procOS, bufferOS, httpc.this.clientInput);
writeContentX(httpc.this.clientInput, this.gzip, this.responseHeader.contentLength(), procOS, bufferOS);
} finally {
if (bufferOS != null) {
bufferOS.close();
@ -1710,70 +1773,7 @@ do upload
}
}
}
/**
* This method outputs the input stream to either an output socket or an
* file or both. If the length of the input stream is given in the
* header, exactly that lenght is written. Otherwise the stream is
* written, till it is closed. If this instance is zipped, stream the
* input stream through gzip to unzip it on the fly.
*
* @param procOS OutputStream where the stream is to be written. If null
* no write happens.
* @param bufferOS OutputStream where the stream is to be written too.
* If null no write happens.
* @param clientInput InputStream where the content is to be read from.
* @throws IOException
*/
public void writeContentX(OutputStream procOS, OutputStream bufferOS, InputStream clientInput) throws IOException {
// we write length bytes, but if length == -1 (or < 0) then we
// write until the input stream closes
// procOS == null -> no write to procOS
// file == null -> no write to file
// If the Content-Encoding is gzip, we gunzip on-the-fly
// and change the Content-Encoding and Content-Length attributes in the header
byte[] buffer = new byte[2048];
int l;
long len = 0;
// find out length
long length = this.responseHeader.contentLength();
// using the proper intput stream
InputStream dis = (this.gzip) ? (InputStream) new GZIPInputStream(clientInput) : (InputStream) clientInput;
// we have three methods of reading: length-based, length-based gzip and connection-close-based
try {
if (length > 0) {
// we read exactly 'length' bytes
while ((len < length) && ((l = dis.read(buffer)) >= 0)) {
if (procOS != null) procOS.write(buffer, 0, l);
if (bufferOS != null) bufferOS.write(buffer, 0, l);
len += l;
}
} else {
// no content-length was given, thus we read until the connection closes
while ((l = dis.read(buffer, 0, buffer.length)) >= 0) {
if (procOS != null) procOS.write(buffer, 0, l);
if (bufferOS != null) bufferOS.write(buffer, 0, l);
}
}
} catch (java.net.SocketException e) {
throw new IOException("Socket exception: " + e.getMessage());
} catch (java.net.SocketTimeoutException e) {
throw new IOException("Socket time-out: " + e.getMessage());
} finally {
// close the streams
if (procOS != null) {
if (procOS instanceof httpChunkedOutputStream)
((httpChunkedOutputStream)procOS).finish();
procOS.flush();
}
if (bufferOS != null) bufferOS.flush();
buffer = null;
}
}
/**
* This method outputs a logline to the serverlog with the current
* status of this instance.

4
source/de/anomic/http/httpd.java
Unescape View File

@ -172,8 +172,8 @@ public final class httpd implements serverHandler {
* Must be called at least once, but can be called again to re-use the object.
* @see de.anomic.server.serverHandler#initSession(de.anomic.server.serverCore.Session)
*/
public void initSession(serverCore.Session session) throws IOException {
this.session = session;
public void initSession(serverCore.Session newsession) throws IOException {
this.session = newsession;
this.userAddress = session.userAddress; // client InetAddress
this.clientIP = this.userAddress.getHostAddress();
if (this.userAddress.isAnyLocalAddress()) this.clientIP = "localhost";

1
source/de/anomic/http/httpdFileHandler.java
Unescape View File

@ -211,6 +211,7 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
serverLog.logWarning("HTTPDFileHandler", "Content-MD5 support not availabel ...");
}
}
/*
* Returns the path of a (existing) localized File, or the english one, if not availible.
* @param path is relative from htroot

14
source/de/anomic/kelondro/kelondroAttrSeq.java
Unescape View File

@ -109,13 +109,13 @@ public class kelondroAttrSeq {
this.theLogger.warning("ATTRSEQ WARNING for file " + this.file + ": " + message);
}
private void readAttrFile(File file) throws IOException {
private void readAttrFile(File loadfile) throws IOException {
BufferedReader br = null;
int p;
if (file.toString().endsWith(".gz")) {
br = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(file))));
if (loadfile.toString().endsWith(".gz")) {
br = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(loadfile))));
} else {
br = new BufferedReader(new InputStreamReader(new FileInputStream(file)));
br = new BufferedReader(new InputStreamReader(new FileInputStream(loadfile)));
}
String line;
String key;
@ -137,7 +137,7 @@ public class kelondroAttrSeq {
if ((p = line.indexOf('=')) > 0) {
key = line.substring(0, p).trim();
if (entries.containsKey(key)) {
logWarning("read PropFile " + file.toString() + ", key " + key + ": double occurrence");
logWarning("read PropFile " + loadfile.toString() + ", key " + key + ": double occurrence");
} else {
entries.put(key, line.substring(p + 1).trim());
}
@ -309,10 +309,10 @@ public class kelondroAttrSeq {
} else return null;
int p = a.indexOf('-');
if (p < 0) return null;
String name = a.substring(0, p);
String atomname = a.substring(0, p);
try {
int x = Integer.parseInt(a.substring(p + 1));
return new Object[]{name, new Integer(x)};
return new Object[]{atomname, new Integer(x)};
} catch (NumberFormatException e) {
return null;
}

6
source/de/anomic/kelondro/kelondroTree.java
Unescape View File

@ -941,19 +941,19 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr
nextNode = null;
} else {
Object[] stacktop;
Node parent = null;
Node parentNode = null;
int parentpointer = (up) ? rightchild : leftchild;
while ((nodeStack.size() != 0) && (parentpointer == ((up) ? rightchild : leftchild))) {
//System.out.println("step up");
// go on, walk up further
stacktop = (Object[]) nodeStack.removeLast(); // top of stack: Node/parentpointer pair
parent = (Node) stacktop[0];
parentNode = (Node) stacktop[0];
parentpointer = ((Integer) stacktop[1]).intValue();
}
if ((nodeStack.size() == 0) && (parentpointer == ((up) ? rightchild : leftchild))) {
nextNode = null;
} else {
nextNode = parent;
nextNode = parentNode;
}
}
}

8
source/yacy.java
Unescape View File

@ -142,8 +142,8 @@ public final class yacy {
public static String combinedVersionString2PrettyString(String s) {
long svn;
try {svn = (long) (100000000.0 * Double.parseDouble(s));} catch (NumberFormatException ee) {svn = 0;}
double version = (Math.floor((double) svn / (double) 100000) / (double) 1000);
String vStr = (version < 0.11) ? "dev" : Double.toString(version);
double v = (Math.floor((double) svn / (double) 100000) / (double) 1000);
String vStr = (v < 0.11) ? "dev" : Double.toString(v);
//while (vStr.length() < 5) vStr = vStr + "0";
svn = svn % 100000;
if (svn > 4000) svn=svn / 10; // fix a previous bug online
@ -160,8 +160,8 @@ public final class yacy {
* @param svn Current version given from svn.
* @return String with the combined version
*/
public static float versvn2combinedVersion(float version, int svn) {
return (float) (((double) version * 100000000.0 + ((double) svn)) / 100000000.0);
public static float versvn2combinedVersion(float v, int svn) {
return (float) (((double) v * 100000000.0 + ((double) svn)) / 100000000.0);
}
/**

Write Preview
Loading…
Cancel
Save