boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fixed utf-8 decoding in htmlFilterAbstractScraper and removed httpd timing

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@323 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 20 years ago
parent 3addf58046
commit abba8fe61b
3 changed files with 117 additions and 103 deletions
Show Stats Download Patch File Download Diff File

25
source/de/anomic/htmlFilter/htmlFilterAbstractScraper.java
Unescape View File

@ -357,12 +357,27 @@ public abstract class htmlFilterAbstractScraper implements htmlFilterScraper {
public static serverByteBuffer convertUmlaute(serverByteBuffer bb) { public static serverByteBuffer convertUmlaute(serverByteBuffer bb) {
serverByteBuffer t = new serverByteBuffer(bb.length() + 20); serverByteBuffer t = new serverByteBuffer(bb.length() + 20);
byte b; int b0, b1, b2;
String z; String z;
for (int i = 0; i < bb.length(); i++) { int i = 0;
b = bb.byteAt(i); while (i < bb.length()) {
z = code_iso8859s(b & 0xff); b0 = bb.byteAt(i) & 0xff;
if (z == null) t.append(b); else t.append(z); // check utf-8 encoding
if (b0 < 128) {
t.append(b0);
i++;
} else {
b1 = bb.byteAt(i + 1) & 0xff;
if ((b0 > 0xbf) && (b0 < 0xe0)) {
z = code_iso8859s(((b0 & 0x1f) << 0x6) | (b1 & 0x3f));
i += 2;
} else {
b2 = bb.byteAt(i + 2) & 0xff;
z = code_iso8859s(((b0 & 0xf) << 0xc) | ((b1 & 0x3f) << 0x6) | (b2 & 0x3f));
i += 3;
}
if (z == null) t.append(b0); else t.append(z);
}
} }
return t; return t;
} }

2
source/de/anomic/server/serverAbstractThread.java
Unescape View File

@ -81,7 +81,7 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
protected final void announceMoreExecTime(long millis) { protected final void announceMoreExecTime(long millis) {
this.busytime += millis; this.busytime += millis;
} }
protected final void announceMoreSleepTime(long millis) { protected final void announceMoreSleepTime(long millis) {
this.idletime += millis; this.idletime += millis;
} }

193
source/de/anomic/server/serverCore.java
Unescape View File

@ -637,6 +637,9 @@ public final class serverCore extends serverAbstractThread implements serverThre
public Session(ThreadGroup theThreadGroup) { public Session(ThreadGroup theThreadGroup) {
super(theThreadGroup,"Session"); super(theThreadGroup,"Session");
// setting the session startup time
this.start = System.currentTimeMillis();
} }
public void setStopped(boolean stopped) { public void setStopped(boolean stopped) {
@ -724,56 +727,50 @@ public final class serverCore extends serverAbstractThread implements serverThre
this.running = true; this.running = true;
// The thread keeps running. // The thread keeps running.
while (!this.stopped && !Thread.interrupted()) { while (!this.stopped && !Thread.interrupted()) {
if (this.done) { if (this.done) {
// We are waiting for a task now. // We are waiting for a task now.
synchronized (this) { synchronized (this) {
try { try {
this.wait(); //Wait until we get a request to process. this.wait(); //Wait until we get a request to process.
} } catch (InterruptedException e) {
catch (InterruptedException e) { this.stopped = true;
this.stopped = true; // log.error("", e);
// log.error("", e); }
}
} }
} } else {
else
{
//There is a task....let us execute it. //There is a task....let us execute it.
try { try {
execute(); execute();
if (this.syncObject != null) { if (this.syncObject != null) {
synchronized (this.syncObject) { synchronized (this.syncObject) {
//Notify the completion. //Notify the completion.
this.syncObject.notifyAll(); this.syncObject.notifyAll();
} }
} }
} catch (Exception e) { } catch (Exception e) {
// log.error("", e); // log.error("", e);
} } finally {
finally {
reset(); reset();
if (!this.stopped && !this.isInterrupted()) { if (!this.stopped && !this.isInterrupted()) {
try { try {
this.setName("Session_inPool"); this.setName("Session_inPool");
serverCore.this.theSessionPool.returnObject(this); serverCore.this.theSessionPool.returnObject(this);
} } catch (Exception e1) {
catch (Exception e1) {
// e1.printStackTrace(); // e1.printStackTrace();
this.stopped = true; this.stopped = true;
} }
} }
} }
} }
} }
} }
private void execute() throws InterruptedException { private void execute() throws InterruptedException {
try { try {
// setting the session startup time
this.start = System.currentTimeMillis();
// settin the session identity // settin the session identity
this.identity = "-"; this.identity = "-";
@ -820,53 +817,52 @@ public final class serverCore extends serverAbstractThread implements serverThre
} }
//log.logDebug("* session " + handle + " completed. time = " + (System.currentTimeMillis() - handle)); //log.logDebug("* session " + handle + " completed. time = " + (System.currentTimeMillis() - handle));
announceMoreExecTime(System.currentTimeMillis() - this.start);
} }
private void listen() { private void listen() {
try { try {
// set up some reflection // set up some reflection
Class[] stringType = {"".getClass()}; Class[] stringType = {"".getClass()};
Class[] exceptionType = {Class.forName("java.lang.Throwable")}; Class[] exceptionType = {Class.forName("java.lang.Throwable")};
// send greeting // send greeting
Object result = commandObj.greeting(); Object result = commandObj.greeting();
if (result != null) { if (result != null) {
if ((result instanceof String) && (((String) result).length() > 0)) writeLine((String) result); if ((result instanceof String) && (((String) result).length() > 0)) writeLine((String) result);
} }
// start dialog // start dialog
byte[] requestBytes = null; byte[] requestBytes = null;
boolean terminate = false; boolean terminate = false;
int pos; int pos;
String cmd; String cmd;
String tmp; String tmp;
Object[] stringParameter = new String[1]; Object[] stringParameter = new String[1];
while ((this.in != null) && ((requestBytes = readLine()) != null)) { while ((this.in != null) && ((requestBytes = readLine()) != null)) {
this.commandCounter++; this.commandCounter++;
this.setName("Session_" + this.userAddress.getHostAddress() + ":" + this.controlSocket.getPort() + "#" + commandCounter); this.setName("Session_" + this.userAddress.getHostAddress() + ":" + this.controlSocket.getPort() + "#" + commandCounter);
this.request = new String(requestBytes); this.request = new String(requestBytes);
//log.logDebug("* session " + handle + " received command '" + request + "'. time = " + (System.currentTimeMillis() - handle)); //log.logDebug("* session " + handle + " received command '" + request + "'. time = " + (System.currentTimeMillis() - handle));
log(false, this.request); log(false, this.request);
try { try {
// if we can not determine the proper command string we try to call function emptyRequest // if we can not determine the proper command string we try to call function emptyRequest
// of the commandObject // of the commandObject
if (this.request.trim().length() == 0) this.request = "EMPTY"; if (this.request.trim().length() == 0) this.request = "EMPTY";
pos = this.request.indexOf(' '); pos = this.request.indexOf(' ');
if (pos < 0) { if (pos < 0) {
cmd = this.request.trim().toUpperCase(); cmd = this.request.trim().toUpperCase();
stringParameter[0] = ""; stringParameter[0] = "";
} else { } else {
cmd = this.request.substring(0, pos).trim().toUpperCase(); cmd = this.request.substring(0, pos).trim().toUpperCase();
stringParameter[0] = this.request.substring(pos).trim(); stringParameter[0] = this.request.substring(pos).trim();
} }
// setting the socket timeout for reading of the request content // setting the socket timeout for reading of the request content
this.controlSocket.setSoTimeout(this.socketTimeout); this.controlSocket.setSoTimeout(this.socketTimeout);
// exec command and return value // exec command and return value
Object commandMethod = this.commandObjMethodCache.get(cmd); Object commandMethod = this.commandObjMethodCache.get(cmd);
if (commandMethod == null) { if (commandMethod == null) {
try { try {
@ -877,38 +873,40 @@ public final class serverCore extends serverAbstractThread implements serverThre
stringParameter[0] = this.request.trim(); stringParameter[0] = this.request.trim();
} }
} }
result = ((Method)commandMethod).invoke(this.commandObj, stringParameter); //long commandStart = System.currentTimeMillis();
//log.logDebug("* session " + handle + " completed command '" + request + "'. time = " + (System.currentTimeMillis() - handle)); result = ((Method)commandMethod).invoke(this.commandObj, stringParameter);
//announceMoreExecTime(commandStart - System.currentTimeMillis()); // shall be negative!
//log.logDebug("* session " + handle + " completed command '" + request + "'. time = " + (System.currentTimeMillis() - handle));
this.out.flush(); this.out.flush();
if (result == null) { if (result == null) {
/* /*
log(2, true, "(NULL RETURNED/STREAM PASSED)"); log(2, true, "(NULL RETURNED/STREAM PASSED)");
*/ */
} else if (result instanceof Boolean) { } else if (result instanceof Boolean) {
if (((Boolean) result).equals(TERMINATE_CONNECTION)) break; if (((Boolean) result).equals(TERMINATE_CONNECTION)) break;
// deactivating timeout. this is needed because of persistent connections // deactivating timeout. this is needed because of persistent connections
this.controlSocket.setSoTimeout(0); this.controlSocket.setSoTimeout(0);
} else if (result instanceof String) { } else if (result instanceof String) {
if (((String) result).startsWith("!")) { if (((String) result).startsWith("!")) {
result = ((String) result).substring(1); result = ((String) result).substring(1);
terminate = true; terminate = true;
} }
writeLine((String) result); writeLine((String) result);
} else if (result instanceof InputStream) { } else if (result instanceof InputStream) {
tmp = send(out, (InputStream) result); tmp = send(out, (InputStream) result);
if ((tmp.length() > 4) && (tmp.toUpperCase().startsWith("PASS"))) { if ((tmp.length() > 4) && (tmp.toUpperCase().startsWith("PASS"))) {
log(true, "PASS ********"); log(true, "PASS ********");
} else { } else {
log(true, tmp); log(true, tmp);
} }
tmp = null; tmp = null;
} }
if (terminate) break; if (terminate) break;
} catch (InvocationTargetException ite) { } catch (InvocationTargetException ite) {
System.out.println("ERROR A " + userAddress.getHostAddress()); System.out.println("ERROR A " + userAddress.getHostAddress());
// we extract a target exception and let the thread survive // we extract a target exception and let the thread survive
writeLine((String) commandObj.error(ite.getTargetException())); writeLine((String) commandObj.error(ite.getTargetException()));
} catch (NoSuchMethodException nsme) { } catch (NoSuchMethodException nsme) {
System.out.println("ERROR B " + userAddress.getHostAddress()); System.out.println("ERROR B " + userAddress.getHostAddress());
if (isNotLocal(userAddress.getHostAddress().toString())) { if (isNotLocal(userAddress.getHostAddress().toString())) {
@ -933,14 +931,15 @@ public final class serverCore extends serverAbstractThread implements serverThre
writeLine("UNKNOWN REASON:" + (String) commandObj.error(e)); writeLine("UNKNOWN REASON:" + (String) commandObj.error(e));
} }
} // end of while } // end of while
} catch (java.lang.ClassNotFoundException e) { } catch (java.lang.ClassNotFoundException e) {
System.out.println("Internal Error: wrapper class not found: " + e.getMessage()); System.out.println("Internal Error: wrapper class not found: " + e.getMessage());
System.exit(0); System.exit(0);
} catch (java.io.IOException e) { } catch (java.io.IOException e) {
// connection interruption: more or less normal // connection interruption: more or less normal
} }
} //announceMoreExecTime(System.currentTimeMillis() - this.start);
}
} }
public static byte[] receive(PushbackInputStream pbis, serverByteBuffer readLineBuffer, int maxSize, boolean logerr) { public static byte[] receive(PushbackInputStream pbis, serverByteBuffer readLineBuffer, int maxSize, boolean logerr) {

Write Preview
Loading…
Cancel
Save