boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fix for http://bugs.yacy.net/view.php?id=126

Browse Source
pull/1/head
Michael Peter Christen 13 years ago
parent 1473e2258e
commit acf8d521a2
7 changed files with 37 additions and 40 deletions
Show Stats Download Patch File Download Diff File

17
source/net/yacy/cora/document/MultiProtocolURI.java
Unescape View File

@ -828,19 +828,20 @@ public class MultiProtocolURI implements Serializable, Comparable<MultiProtocolU
}
public String toNormalform(final boolean excludeReference, final boolean stripAmp) {
return toNormalform(excludeReference, stripAmp, false, false);
return toNormalform(excludeReference, stripAmp, false);
}
private static final Pattern ampPattern = Pattern.compile("&amp;");
public String toNormalform(final boolean excludeReference, final boolean stripAmp, final boolean resolveHost, final boolean removeSessionID) {
String result = toNormalform0(excludeReference, resolveHost, removeSessionID);
public String toNormalform(final boolean excludeReference, final boolean stripAmp, final boolean removeSessionID) {
String result = toNormalform0(excludeReference, removeSessionID);
if (stripAmp) {
result = ampPattern.matcher(result).replaceAll("&");
}
return result;
}
private String toNormalform0(final boolean excludeReference, final boolean resolveHost, final boolean removeSessionID) {
private String toNormalform0(final boolean excludeReference, final boolean removeSessionID) {
// generates a normal form of the URL
boolean defaultPort = false;
if (this.protocol.equals("mailto")) {
@ -866,13 +867,7 @@ public class MultiProtocolURI implements Serializable, Comparable<MultiProtocolU
u.append("@");
}
final String hl = getHost().toLowerCase();
if (resolveHost) {
final InetAddress r = getInetAddress();
u.append(r == null ? hl : r.getHostAddress());
} else {
u.append(hl);
}
u.append(hl);
}
if (!defaultPort) {
u.append(":");

12
source/net/yacy/cora/protocol/http/HTTPClient.java
Unescape View File

@ -335,7 +335,7 @@ public class HTTPClient {
*/
public byte[] GETbytes(final MultiProtocolURI url, final int maxBytes) throws IOException {
final boolean localhost = url.getHost().equals("localhost");
final String urix = url.toNormalform(true, false, !localhost, false);
final String urix = url.toNormalform(true, false);
final HttpGet httpGet = new HttpGet(urix);
if (!localhost) setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
return getContentBytes(httpGet, maxBytes);
@ -352,7 +352,7 @@ public class HTTPClient {
public void GET(final String uri) throws IOException {
if (this.currentRequest != null) throw new IOException("Client is in use!");
final MultiProtocolURI url = new MultiProtocolURI(uri);
final HttpGet httpGet = new HttpGet(url.toNormalform(true, false, true, false));
final HttpGet httpGet = new HttpGet(url.toNormalform(true, false));
setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
this.currentRequest = httpGet;
execute(httpGet);
@ -367,7 +367,7 @@ public class HTTPClient {
*/
public HttpResponse HEADResponse(final String uri) throws IOException {
final MultiProtocolURI url = new MultiProtocolURI(uri);
final HttpHead httpHead = new HttpHead(url.toNormalform(true, false, true, false));
final HttpHead httpHead = new HttpHead(url.toNormalform(true, false));
setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
execute(httpHead);
finish();
@ -388,7 +388,7 @@ public class HTTPClient {
public void POST(final String uri, final InputStream instream, final long length) throws IOException {
if (this.currentRequest != null) throw new IOException("Client is in use!");
final MultiProtocolURI url = new MultiProtocolURI(uri);
final HttpPost httpPost = new HttpPost(url.toNormalform(true, false, true, false));
final HttpPost httpPost = new HttpPost(url.toNormalform(true, false));
String host = url.getHost();
if (host == null) host = "127.0.0.1";
setHost(host); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
@ -424,7 +424,7 @@ public class HTTPClient {
* @throws IOException
*/
public byte[] POSTbytes(final MultiProtocolURI url, final String vhost, final Map<String, ContentBody> post, final boolean usegzip) throws IOException {
final HttpPost httpPost = new HttpPost(url.toNormalform(true, false, true, false));
final HttpPost httpPost = new HttpPost(url.toNormalform(true, false));
setHost(vhost); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
if (vhost == null) setHost("127.0.0.1");
@ -455,7 +455,7 @@ public class HTTPClient {
*/
public byte[] POSTbytes(final String uri, final InputStream instream, final long length) throws IOException {
final MultiProtocolURI url = new MultiProtocolURI(uri);
final HttpPost httpPost = new HttpPost(url.toNormalform(true, false, true, false));
final HttpPost httpPost = new HttpPost(url.toNormalform(true, false));
String host = url.getHost();
if (host == null) host = "127.0.0.1";
setHost(host); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service

2
source/net/yacy/document/parser/html/EmbedEntry.java
Unescape View File

@ -58,7 +58,7 @@ public class EmbedEntry {
@Override
public String toString() {
return "<embed url=\"" + this.url.toNormalform(false, false, false, false) + "\"" +
return "<embed url=\"" + this.url.toNormalform(false, false) + "\"" +
(this.type != null && this.type.length() > 0 ? " type=\"" + this.type + "\"" : "") +
(this.pluginspage != null && this.pluginspage.length() > 0 ? " pluginspage=\"" + this.pluginspage + "\"" : "") +
(this.width >= 0 ? " width=\"" + this.width + "\"" : "") +

28
source/net/yacy/document/parser/html/ImageEntry.java
Unescape View File

@ -34,7 +34,7 @@ public class ImageEntry implements Comparable<ImageEntry>, Comparator<ImageEntry
private final String alt;
private final int width, height;
private final long fileSize;
public ImageEntry(final MultiProtocolURI url, final String alt, final int width, final int height, long fileSize) {
assert url != null;
this.url = url;
@ -47,7 +47,7 @@ public class ImageEntry implements Comparable<ImageEntry>, Comparator<ImageEntry
public MultiProtocolURI url() {
return this.url;
}
public String alt() {
return this.alt;
}
@ -59,17 +59,17 @@ public class ImageEntry implements Comparable<ImageEntry>, Comparator<ImageEntry
public int height() {
return this.height;
}
public long fileSize() {
return this.fileSize;
}
@Override
public String toString() {
return "<img url=\"" + url.toNormalform(false, false, false, false) + "\"" +
(alt != null && alt.length() > 0 ? " alt=\"" + alt + "\"" : "") +
(width >= 0 ? " width=\"" + width + "\"" : "") +
(height >= 0 ? " height=\"" + height + "\"" : "") +
return "<img url=\"" + this.url.toNormalform(false, false) + "\"" +
(this.alt != null && this.alt.length() > 0 ? " alt=\"" + this.alt + "\"" : "") +
(this.width >= 0 ? " width=\"" + this.width + "\"" : "") +
(this.height >= 0 ? " height=\"" + this.height + "\"" : "") +
">";
}
@ -79,17 +79,18 @@ public class ImageEntry implements Comparable<ImageEntry>, Comparator<ImageEntry
// this hash method therefore tries to compute a 'perfect hash' based on the size of the images
// unfortunately it can not be ensured that all images get different hashes, but this should appear
// only in very rare cases
if (width < 0 || height < 0)
return /*0x7FFF0000 |*/ (url.hashCode() & 0xFFFF);
return ((0x7FFF - (((width * height) >> 9) & 0x7FFF)) << 16) | (url.hashCode() & 0xFFFF);
if (this.width < 0 || this.height < 0)
return /*0x7FFF0000 |*/ (this.url.hashCode() & 0xFFFF);
return ((0x7FFF - (((this.width * this.height) >> 9) & 0x7FFF)) << 16) | (this.url.hashCode() & 0xFFFF);
}
@Override
public int compareTo(final ImageEntry h) {
// this is needed if this object is stored in a TreeSet
// this method uses the image-size ordering from the hashCode method
// assuming that hashCode would return a 'perfect hash' this method would
// create a total ordering on images with respect on the image size
assert (url != null);
assert (this.url != null);
if (this.url.toNormalform(true, true).equals((h).url.toNormalform(true, true))) return 0;
final int thc = this.hashCode();
final int ohc = (h).hashCode();
@ -98,10 +99,11 @@ public class ImageEntry implements Comparable<ImageEntry>, Comparator<ImageEntry
return this.url.toString().compareTo((h).url.toString());
}
@Override
public int compare(ImageEntry o1, ImageEntry o2) {
return o1.compareTo(o2);
}
@Override
public boolean equals(final Object o) {
if(o != null && o instanceof ImageEntry) {

8
source/net/yacy/kelondro/data/meta/DigestURI.java
Unescape View File

@ -73,7 +73,7 @@ public class DigestURI extends MultiProtocolURI implements Serializable {
}
return (url == null) ? null : ASCII.String(url.hash(), 6, 6);
}
/**
* from a given list of hosts make a list of host hashes
* the list is separated by comma
@ -93,7 +93,7 @@ public class DigestURI extends MultiProtocolURI implements Serializable {
}
return sb.toString();
}
public static Set<String> hosthashess(String hosthashes) {
if (hosthashes == null || hosthashes.length() == 0) return null;
HashSet<String> h = new HashSet<String>();
@ -103,7 +103,7 @@ public class DigestURI extends MultiProtocolURI implements Serializable {
}
return h;
}
/**
* DigestURI from File
@ -244,7 +244,7 @@ public class DigestURI extends MultiProtocolURI implements Serializable {
final StringBuilder hashs = new StringBuilder(12);
assert hashs.length() == 0;
// form the 'local' part of the hash
final String normalform = toNormalform(true, true, false, true);
final String normalform = toNormalform(true, true, true);
final String b64l = Base64Order.enhancedCoder.encode(Digest.encodeMD5Raw(normalform));
if (b64l.length() < 5) return null;
hashs.append(b64l.substring(0, 5)); // 5 chars

2
source/net/yacy/search/Switchboard.java
Unescape View File

@ -2376,7 +2376,7 @@ public final class Switchboard extends serverSwitch
// process the next hyperlink
nextUrl = nextEntry.getKey();
final String u = nextUrl.toNormalform(true, true, false, true);
final String u = nextUrl.toNormalform(true, true, true);
if ( !(u.startsWith("http://")
|| u.startsWith("https://")
|| u.startsWith("ftp://")

8
source/net/yacy/search/index/SolrConfiguration.java
Unescape View File

@ -283,7 +283,7 @@ public class SolrConfiguration extends ConfigurationSet implements Serializable
final String[] css_url = new String[csss.size()];
c = 0;
for (final Map.Entry<MultiProtocolURI, String> entry: csss.entrySet()) {
final String url = entry.getKey().toNormalform(false, false, false, false);
final String url = entry.getKey().toNormalform(false, false);
inboundLinks.remove(url);
ouboundLinks.remove(url);
css_tag[c] =
@ -305,7 +305,7 @@ public class SolrConfiguration extends ConfigurationSet implements Serializable
for (final MultiProtocolURI url: scriptss) {
inboundLinks.remove(url);
ouboundLinks.remove(url);
scripts[c++] = url.toNormalform(false, false, false, false);
scripts[c++] = url.toNormalform(false, false);
}
addSolr(solrdoc, SolrField.scriptscount_i, scripts.length);
if (scripts.length > 0) addSolr(solrdoc, SolrField.scripts_txt, scripts);
@ -319,7 +319,7 @@ public class SolrConfiguration extends ConfigurationSet implements Serializable
for (final MultiProtocolURI url: framess) {
inboundLinks.remove(url);
ouboundLinks.remove(url);
frames[c++] = url.toNormalform(false, false, false, false);
frames[c++] = url.toNormalform(false, false);
}
addSolr(solrdoc, SolrField.framesscount_i, frames.length);
if (frames.length > 0) addSolr(solrdoc, SolrField.frames_txt, frames);
@ -333,7 +333,7 @@ public class SolrConfiguration extends ConfigurationSet implements Serializable
for (final MultiProtocolURI url: iframess) {
inboundLinks.remove(url);
ouboundLinks.remove(url);
iframes[c++] = url.toNormalform(false, false, false, false);
iframes[c++] = url.toNormalform(false, false);
}
addSolr(solrdoc, SolrField.iframesscount_i, iframes.length);
if (iframes.length > 0) addSolr(solrdoc, SolrField.iframes_txt, iframes);

Write Preview
Loading…
Cancel
Save