boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

replaced some more .getBytes() with UTF8/ASCII.getBytes()

Browse Source
pull/1/head
Michael Peter Christen 13 years ago
parent bd769de604
commit a06930662c
9 changed files with 27 additions and 39 deletions
Show Stats Download Patch File Download Diff File

2
htroot/yacy/transferRWI.java
Unescape View File

@ -193,7 +193,7 @@ public final class transferRWI {
// learn entry
try {
sb.index.storeRWI(wordHash.getBytes(), iEntry);
sb.index.storeRWI(ASCII.getBytes(wordHash), iEntry);
} catch (final Exception e) {
Log.logException(e);
}

12
source/net/yacy/crawler/retrieval/FTPLoader.java
Unescape View File

@ -32,7 +32,9 @@ import java.io.IOException;
import java.io.PrintStream;
import java.util.Date;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.ResponseHeader;
@ -126,14 +128,14 @@ public class FTPLoader {
final ResponseHeader responseHeader = new ResponseHeader(200);
responseHeader.put(HeaderFramework.LAST_MODIFIED, HeaderFramework.formatRFC1123(new Date()));
responseHeader.put(HeaderFramework.CONTENT_TYPE, "text/html");
final CrawlProfile profile = this.sb.crawler.getActive(request.profileHandle().getBytes());
final CrawlProfile profile = this.sb.crawler.getActive(ASCII.getBytes(request.profileHandle()));
response = new Response(
request,
requestHeader,
responseHeader,
profile,
false,
dirList.toString().getBytes());
UTF8.getBytes(dirList.toString()));
}
} else {
// file -> download
@ -152,7 +154,7 @@ public class FTPLoader {
if (berr.size() > 0 || response == null) {
// some error logging
final String detail = (berr.size() > 0) ? "Errorlog: " + berr.toString() : "";
this.sb.crawlQueues.errorURL.push(request, this.sb.peers.mySeed().hash.getBytes(), new Date(), 1, FailCategory.TEMPORARY_NETWORK_FAILURE, " ftp server download, " + detail, -1);
this.sb.crawlQueues.errorURL.push(request, ASCII.getBytes(this.sb.peers.mySeed().hash), new Date(), 1, FailCategory.TEMPORARY_NETWORK_FAILURE, " ftp server download, " + detail, -1);
throw new IOException("FTPLoader: Unable to download URL '" + request.url().toString() + "': " + detail);
}
@ -245,7 +247,7 @@ public class FTPLoader {
// create response with metadata only
responseHeader.put(HeaderFramework.CONTENT_TYPE, "text/plain");
final CrawlProfile profile = this.sb.crawler.getActive(request.profileHandle().getBytes());
final CrawlProfile profile = this.sb.crawler.getActive(ASCII.getBytes(request.profileHandle()));
final Response response = new Response(
request,
requestHeader,
@ -260,7 +262,7 @@ public class FTPLoader {
final byte[] b = ftpClient.get(path);
// create a response
final CrawlProfile profile = this.sb.crawler.getActive(request.profileHandle().getBytes());
final CrawlProfile profile = this.sb.crawler.getActive(ASCII.getBytes(request.profileHandle()));
final Response response = new Response(
request,
requestHeader,

20
source/net/yacy/crawler/retrieval/FileLoader.java
Unescape View File

@ -30,7 +30,9 @@ import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.Classification;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.ResponseHeader;
@ -68,14 +70,6 @@ public class FileLoader {
String[] l = null;
try {l = url.list();} catch (IOException e) {}
if (l != null) {
/*
if (l == null) {
// this can only happen if there is no connection or the directory does not exist
//log.logInfo("directory listing not available. URL = " + request.url().toString());
sb.crawlQueues.errorURL.push(request, this.sb.peers.mySeed().hash.getBytes(), new Date(), 1, "directory listing not available. URL = " + request.url().toString());
throw new IOException("directory listing not available. URL = " + request.url().toString());
}
*/
String u = url.toNormalform(true, true);
List<String> list = new ArrayList<String>();
for (String s: l) {
@ -87,14 +81,14 @@ public class FileLoader {
ResponseHeader responseHeader = new ResponseHeader(200);
responseHeader.put(HeaderFramework.LAST_MODIFIED, HeaderFramework.formatRFC1123(new Date()));
responseHeader.put(HeaderFramework.CONTENT_TYPE, "text/html");
final CrawlProfile profile = this.sb.crawler.getActive(request.profileHandle().getBytes());
final CrawlProfile profile = this.sb.crawler.getActive(ASCII.getBytes(request.profileHandle()));
Response response = new Response(
request,
requestHeader,
responseHeader,
profile,
false,
content.toString().getBytes());
UTF8.getBytes(content.toString()));
return response;
}
@ -127,14 +121,14 @@ public class FileLoader {
// create response with metadata only
responseHeader.put(HeaderFramework.CONTENT_TYPE, "text/plain");
final CrawlProfile profile = this.sb.crawler.getActive(request.profileHandle().getBytes());
final CrawlProfile profile = this.sb.crawler.getActive(ASCII.getBytes(request.profileHandle()));
Response response = new Response(
request,
requestHeader,
responseHeader,
profile,
false,
url.toTokens().getBytes());
UTF8.getBytes(url.toTokens()));
return response;
}
@ -144,7 +138,7 @@ public class FileLoader {
is.close();
// create response with loaded content
final CrawlProfile profile = this.sb.crawler.getActive(request.profileHandle().getBytes());
final CrawlProfile profile = this.sb.crawler.getActive(ASCII.getBytes(request.profileHandle()));
Response response = new Response(
request,
requestHeader,

5
source/net/yacy/crawler/retrieval/HTTPLoader.java
Unescape View File

@ -27,6 +27,7 @@ package net.yacy.crawler.retrieval;
import java.io.IOException;
import java.util.Date;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.protocol.ClientIdentification;
import net.yacy.cora.protocol.HeaderFramework;
@ -78,7 +79,7 @@ public final class HTTPLoader {
private Response load(final Request request, final int retryCount, final int maxFileSize, final BlacklistType blacklistType) throws IOException {
byte[] myHash = this.sb.peers.mySeed().hash.getBytes();
byte[] myHash = ASCII.getBytes(this.sb.peers.mySeed().hash);
if (retryCount < 0) {
this.sb.crawlQueues.errorURL.push(request, myHash, new Date(), 1, FailCategory.TEMPORARY_NETWORK_FAILURE, "retry counter exceeded", -1);
@ -198,7 +199,7 @@ public final class HTTPLoader {
}
// create a new cache entry
final CrawlProfile profile = this.sb.crawler.getActive(request.profileHandle().getBytes());
final CrawlProfile profile = this.sb.crawler.getActive(ASCII.getBytes(request.profileHandle()));
response = new Response(
request,
requestHeader,

5
source/net/yacy/crawler/retrieval/Request.java
Unescape View File

@ -29,6 +29,7 @@ package net.yacy.crawler.retrieval;
import java.io.IOException;
import java.util.Date;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.order.Base64Order;
import net.yacy.cora.order.NaturalOrder;
@ -212,11 +213,11 @@ public class Request extends WorkflowJob
new byte[][] {
this.url.hash(),
this.initiator,
this.url.toString().getBytes(),
UTF8.getBytes(this.url.toString()),
this.refhash,
namebytes,
appdatestr,
(this.profileHandle == null) ? null : this.profileHandle.getBytes(),
(this.profileHandle == null) ? null : ASCII.getBytes(this.profileHandle),
NaturalOrder.encodeLong(this.depth, rowdef.width(7)),
NaturalOrder.encodeLong(this.anchors, rowdef.width(8)),
NaturalOrder.encodeLong(this.forkfactor, rowdef.width(9)),

2
source/net/yacy/crawler/retrieval/Response.java
Unescape View File

@ -173,7 +173,7 @@ public class Response {
if (!request.isEmpty()) this.responseHeader.put(HeaderFramework.CONTENT_LENGTH, Long.toString(request.size()));
this.profile = profile;
this.status = QUEUE_STATE_FRESH;
this.content = request.name().length() > 0 ? request.name().getBytes() : request.url().toTokens().getBytes();
this.content = request.name().length() > 0 ? UTF8.getBytes(request.name()) : UTF8.getBytes(request.url().toTokens());
this.fromCache = true;
}

14
source/net/yacy/crawler/retrieval/SMBLoader.java
Unescape View File

@ -38,8 +38,10 @@ import java.util.List;
import jcifs.smb.SmbException;
import jcifs.smb.SmbFile;
import jcifs.smb.SmbFileInputStream;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.Classification;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.ResponseHeader;
@ -80,14 +82,6 @@ public class SMBLoader {
String[] l = null;
try {l = url.list();} catch (IOException e) {}
if (l != null) {
/*
if (l == null) {
// this can only happen if there is no connection or the directory does not exist
//log.logInfo("directory listing not available. URL = " + request.url().toString());
sb.crawlQueues.errorURL.push(request, this.sb.peers.mySeed().hash.getBytes(), new Date(), 1, "directory listing not available. URL = " + request.url().toString());
throw new IOException("directory listing not available. URL = " + request.url().toString());
}
*/
String u = url.toNormalform(true, true);
List<String> list = new ArrayList<String>();
for (String s: l) {
@ -106,14 +100,14 @@ public class SMBLoader {
ResponseHeader responseHeader = new ResponseHeader(200);
responseHeader.put(HeaderFramework.LAST_MODIFIED, HeaderFramework.formatRFC1123(new Date()));
responseHeader.put(HeaderFramework.CONTENT_TYPE, "text/html");
final CrawlProfile profile = this.sb.crawler.getActive(request.profileHandle().getBytes());
final CrawlProfile profile = this.sb.crawler.getActive(ASCII.getBytes(request.profileHandle()));
Response response = new Response(
request,
requestHeader,
responseHeader,
profile,
false,
content.toString().getBytes());
UTF8.getBytes(content.toString()));
return response;
}

4
source/net/yacy/kelondro/util/ByteBuffer.java
Unescape View File

@ -35,10 +35,6 @@ import net.yacy.cora.document.UTF8;
public final class ByteBuffer extends OutputStream {
private static final byte singlequote = (byte) 39;
private static final byte doublequote = (byte) 34;
private static final byte equal = (byte) '=';
private byte[] buffer;
private int offset;
private int length;

2
source/net/yacy/search/snippet/MediaSnippet.java
Unescape View File

@ -259,7 +259,7 @@ public class MediaSnippet implements Comparable<MediaSnippet>, Comparator<MediaS
// check if url is in blacklist
if (Switchboard.urlBlacklist.isListed(blacklistType, url.getHost().toLowerCase(), url.getFile())) {
Switchboard.getSwitchboard().crawlQueues.errorURL.push(new Request(url, null), Switchboard.getSwitchboard().peers.mySeed().hash.getBytes(), new Date(), 1, FailCategory.FINAL_LOAD_CONTEXT, "url in blacklist", -1);
Switchboard.getSwitchboard().crawlQueues.errorURL.push(new Request(url, null), ASCII.getBytes(Switchboard.getSwitchboard().peers.mySeed().hash), new Date(), 1, FailCategory.FINAL_LOAD_CONTEXT, "url in blacklist", -1);
Log.logFine("snippet fetch", "MEDIA-SNIPPET Rejecting URL '" + url.toString() + "'. URL is in blacklist.");
isBlacklisted = true;
}

Write Preview
Loading…
Cancel
Save