boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fix for http://bugs.yacy.net/view.php?id=59

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7997 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 14 years ago
parent d3df03838a
commit a9838f8b99
4 changed files with 101 additions and 72 deletions
Show Stats Download Patch File Download Diff File

9
htroot/yacysearchitem.java
Unescape View File

@ -32,6 +32,7 @@ import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.RequestHeader.FileType;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.EventTracker;
@ -71,9 +72,7 @@ public class yacysearchitem {
final boolean authenticated = sb.adminAuthenticated(header) >= 2;
final int item = post.getInt("item", -1);
final boolean auth = (header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "")).equals("localhost") || sb.verifyAuthentication(header, true);
final String path = header.get(HeaderFramework.CONNECTION_PROP_PATH);
final boolean isHtml = path.endsWith(".html");
final boolean isJson = path.endsWith(".json");
final RequestHeader.FileType fileType = header.fileType();
// default settings for blank item
prop.put("content", "0");
@ -114,7 +113,7 @@ public class yacysearchitem {
final DigestURI resultURL = result.url();
final int port = resultURL.getPort();
DigestURI faviconURL = null;
if ((isHtml || isJson) && !sb.isIntranetMode() && !resultURL.isLocal()) try {
if ((fileType == FileType.HTML || fileType == FileType.JSON) && !sb.isIntranetMode() && !resultURL.isLocal()) try {
faviconURL = new DigestURI(resultURL.getProtocol() + "://" + resultURL.getHost() + ((port != -1) ? (":" + port) : "") + "/favicon.ico");
} catch (final MalformedURLException e1) {
Log.logException(e1);
@ -139,7 +138,7 @@ public class yacysearchitem {
prop.putHTML("content_link", result.urlstring());
prop.putHTML("content_showPictures_link", result.urlstring());
prop.putHTML("content_target", target);
if (faviconURL != null && isHtml) sb.loader.loadIfNotExistBackground(faviconURL.toNormalform(true, false), 1024 * 1024 * 10);
if (faviconURL != null && fileType == FileType.HTML) sb.loader.loadIfNotExistBackground(faviconURL.toNormalform(true, false), 1024 * 1024 * 10);
prop.putHTML("content_faviconCode", sb.licensedURLs.aquireLicense(faviconURL)); // acquire license for favicon url loading
prop.put("content_urlhash", resulthashString);
prop.put("content_ranking", result.ranking);

31
htroot/yacysearchtrailer.java
Unescape View File

@ -55,6 +55,7 @@ public class yacysearchtrailer {
return prop;
}
final QueryParams theQuery = theSearch.getQuery();
final RequestHeader.FileType fileType = header.fileType();
// compose search navigation
@ -72,11 +73,11 @@ public class yacysearchtrailer {
while (i < 10 && navigatorIterator.hasNext()) {
name = navigatorIterator.next();
count = namespaceNavigator.get(name);
prop.putJSON("nav-namespace_element_" + i + "_name", name);
prop.put(fileType, "nav-namespace_element_" + i + "_name", name);
prop.put("nav-namespace_element_" + i + "_url", "<a href=\"" + QueryParams.navurl("html", 0, theQuery, theQuery.queryStringForUrl() + "+" + "inurl:" + name, theQuery.urlMask.toString(), theQuery.navigators).toString() + "\">" + name + " (" + count + ")</a>");
prop.putJSON("nav-namespace_element_" + i + "_url-json", QueryParams.navurl("json", 0, theQuery, theQuery.queryStringForUrl() + "+" + "inurl:" + name, theQuery.urlMask.toString(), theQuery.navigators).toString());
prop.put("nav-namespace_element_" + i + "_count", count);
prop.put("nav-namespace_element_" + i + "_modifier", "inurl:" + name);
prop.put(fileType, "nav-namespace_element_" + i + "_modifier", "inurl:" + name);
prop.put("nav-namespace_element_" + i + "_nl", 1);
i++;
}
@ -98,11 +99,11 @@ public class yacysearchtrailer {
name = navigatorIterator.next();
count = hostNavigator.get(name);
dnav = "site:" + name;
prop.putJSON("nav-domains_element_" + i + "_name", name);
prop.put(fileType, "nav-domains_element_" + i + "_name", name);
prop.put("nav-domains_element_" + i + "_url", "<a href=\"" + QueryParams.navurl("html", 0, theQuery, theQuery.queryStringForUrl() + "+" + dnav, theQuery.urlMask.toString(), theQuery.navigators).toString() + "\">" + name + " (" + count + ")</a>");
prop.putJSON("nav-domains_element_" + i + "_url-json", QueryParams.navurl("json", 0, theQuery, theQuery.queryStringForUrl() + "+" + dnav, theQuery.urlMask.toString(), theQuery.navigators).toString());
prop.put("nav-domains_element_" + i + "_count", count);
prop.put("nav-domains_element_" + i + "_modifier", dnav);
prop.put(fileType, "nav-domains_element_" + i + "_modifier", dnav);
prop.put("nav-domains_element_" + i + "_nl", 1);
i++;
}
@ -124,11 +125,11 @@ public class yacysearchtrailer {
name = navigatorIterator.next().trim();
count = authorNavigator.get(name);
anav = (name.indexOf(' ') < 0) ? "author:" + name : "author:'" + name.replace(" ", "+") + "'";
prop.putJSON("nav-authors_element_" + i + "_name", name);
prop.put(fileType, "nav-authors_element_" + i + "_name", name);
prop.put("nav-authors_element_" + i + "_url", "<a href=\"" + QueryParams.navurl("html", 0, theQuery, theQuery.queryStringForUrl() + "+" + anav, theQuery.urlMask.toString(), theQuery.navigators).toString() + "\">" + name + " (" + count + ")</a>");
prop.putJSON("nav-authors_element_" + i + "_url-json", QueryParams.navurl("json", 0, theQuery, theQuery.queryStringForUrl() + "+" + anav, theQuery.urlMask.toString(), theQuery.navigators).toString());
prop.put("nav-authors_element_" + i + "_count", count);
prop.put("nav-authors_element_" + i + "_modifier", anav);
prop.put(fileType, "nav-authors_element_" + i + "_modifier", anav);
prop.put("nav-authors_element_" + i + "_nl", 1);
i++;
}
@ -150,13 +151,13 @@ public class yacysearchtrailer {
count = topicNavigator.get(name);
if (/*(theQuery == null) ||*/ (theQuery.queryString == null)) break;
if (name != null) {
prop.putJSON("nav-topics_element_" + i + "_name", name);
prop.put(fileType, "nav-topics_element_" + i + "_name", name);
prop.put("nav-topics_element_" + i + "_url",
"<a href=\"" + QueryParams.navurl("html", 0, theQuery, theQuery.queryStringForUrl() + "+" + name, theQuery.urlMask.toString(), theQuery.navigators).toString() + "\">" + name + "</a>");
//+"<a href=\"" + QueryParams.navurl("html", 0, display, theQuery, theQuery.queryStringForUrl() + "+-" + name, theQuery.urlMask.toString(), theQuery.navigators) + "\">-</a>")*/;
prop.putJSON("nav-topics_element_" + i + "_url-json", QueryParams.navurl("json", 0, theQuery, theQuery.queryStringForUrl() + "+" + name, theQuery.urlMask.toString(), theQuery.navigators).toString());
prop.put("nav-topics_element_" + i + "_count", count);
prop.put("nav-topics_element_" + i + "_modifier", name);
prop.put(fileType, "nav-topics_element_" + i + "_modifier", name);
prop.put("nav-topics_element_" + i + "_nl", 1);
i++;
}
@ -179,11 +180,11 @@ public class yacysearchtrailer {
name = navigatorIterator.next().trim();
count = protocolNavigator.get(name);
pnav = "/" + name;
prop.putJSON("nav-protocols_element_" + i + "_name", name);
prop.put(fileType, "nav-protocols_element_" + i + "_name", name);
prop.put("nav-protocols_element_" + i + "_url", "<a href=\"" + QueryParams.navurl("html", 0, theQuery, theQuery.queryStringForUrl() + "+" + pnav, theQuery.urlMask.toString(), theQuery.navigators).toString() + "\">" + name + " (" + count + ")</a>");
prop.putJSON("nav-protocols_element_" + i + "_url-json", QueryParams.navurl("json", 0, theQuery, theQuery.queryStringForUrl() + "+" + pnav, theQuery.urlMask.toString(), theQuery.navigators).toString());
prop.put("nav-protocols_element_" + i + "_count", count);
prop.put("nav-protocols_element_" + i + "_modifier", pnav);
prop.put(fileType, "nav-protocols_element_" + i + "_modifier", pnav);
prop.put("nav-protocols_element_" + i + "_nl", 1);
i++;
}
@ -205,11 +206,11 @@ public class yacysearchtrailer {
name = navigatorIterator.next().trim();
count = filetypeNavigator.get(name);
tnav = "filetype:" + name;
prop.putJSON("nav-filetypes_element_" + i + "_name", name);
prop.put(fileType, "nav-filetypes_element_" + i + "_name", name);
prop.put("nav-filetypes_element_" + i + "_url", "<a href=\"" + QueryParams.navurl("html", 0, theQuery, theQuery.queryStringForUrl() + "+" + tnav, theQuery.urlMask.toString(), theQuery.navigators).toString() + "\">" + name + " (" + count + ")</a>");
prop.putJSON("nav-filetypes_element_" + i + "_url-json", QueryParams.navurl("json", 0, theQuery, theQuery.queryStringForUrl() + "+" + tnav, theQuery.urlMask.toString(), theQuery.navigators).toString());
prop.put("nav-filetypes_element_" + i + "_count", count);
prop.put("nav-filetypes_element_" + i + "_modifier", tnav);
prop.put(fileType, "nav-filetypes_element_" + i + "_modifier", tnav);
prop.put("nav-filetypes_element_" + i + "_nl", 1);
i++;
}
@ -237,10 +238,10 @@ public class yacysearchtrailer {
prop.put("cat-location", 0);
} else {
prop.put("cat-location", 1);
prop.put("cat-location_query", theQuery.queryString(true));
prop.put("cat-location_queryenc", theQuery.queryString(true).replace(' ', '+'));
prop.put(fileType, "cat-location_query", theQuery.queryString(true));
prop.put(fileType, "cat-location_queryenc", theQuery.queryString(true).replace(' ', '+'));
}
final int indexcount = theSearch.getRankingResult().getLocalIndexCount() - theSearch.getRankingResult().getMissCount() - theSearch.getRankingResult().getSortOutCount() + theSearch.getRankingResult().getRemoteIndexCount();
prop.put("num-results_totalcount", Formatter.number(indexcount, true));

71
source/de/anomic/server/serverObjects.java
Unescape View File

@ -1,4 +1,4 @@
//serverObjects.java
//serverObjects.java
//-----------------------
//(C) by Michael Peter Christen; mc@yacy.net
//first published on http://www.anomic.de
@ -56,6 +56,8 @@ import java.util.regex.Pattern;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.RequestHeader.FileType;
import net.yacy.document.parser.html.CharacterCoding;
import net.yacy.kelondro.util.Formatter;
import net.yacy.search.Switchboard;
@ -70,10 +72,10 @@ public class serverObjects extends HashMap<String, String> implements Cloneable
private final static Pattern patternF = Pattern.compile("\f");
private final static Pattern patternR = Pattern.compile("\r");
private final static Pattern patternT = Pattern.compile("\t");
private static final long serialVersionUID = 1L;
private boolean localized = true;
private boolean localized = true;
private final static char BOM = '\uFEFF'; // ByteOrderMark character that may appear at beginnings of Strings (Browser may append that)
public serverObjects() {
@ -87,7 +89,7 @@ public class serverObjects extends HashMap<String, String> implements Cloneable
public serverObjects(final Map<String, String> input) {
super(input);
}
private static final String removeByteOrderMark(final String s) {
if (s == null || s.length() == 0) return s;
if (s.charAt(0) == BOM) return s.substring(1);
@ -100,7 +102,7 @@ public class serverObjects extends HashMap<String, String> implements Cloneable
* @param value The value that should be mapped to the key.
* If value is <code>null</code>, then the element at <code>key</code>
* is removed from the map.
* @return The value that was added to the map.
* @return The value that was added to the map.
* @see java.util.Hashtable#insert(K, V)
*/
@Override
@ -148,21 +150,21 @@ public class serverObjects extends HashMap<String, String> implements Cloneable
public String put(final String key, final java.util.Date value) {
return this.put(key, value.toString());
}
public String put(final String key, final InetAddress value) {
return this.put(key, value.toString());
}
/**
* Add a String to the map. The content of the String is escaped to be usable in JSON output.
* @param key key name as String.
* @param value a String that will be reencoded for JSON output.
* @return the modified String that was added to the map.
*/
public String putJSON(final String key, String value) {
public String putJSON(final String key, final String value) {
return put(key, toJSON(value));
}
private static String toJSON(String value) {
// value = value.replaceAll("\\", "\\\\");
value = patternDoublequote.matcher(value).replaceAll("'");
@ -174,11 +176,11 @@ public class serverObjects extends HashMap<String, String> implements Cloneable
value = patternT.matcher(value).replaceAll("\\t");
return value;
}
public String putJSON(final String key, final byte[] value) {
return putJSON(key, UTF8.String(value));
}
/**
* Add a String to the map. The content of the String is escaped to be usable in HTML output.
* @param key key name as String.
@ -189,7 +191,7 @@ public class serverObjects extends HashMap<String, String> implements Cloneable
public String putHTML(final String key, final String value) {
return put(key, CharacterCoding.unicode2html(value, true));
}
public String putHTML(final String key, final byte[] value) {
return putHTML(key, UTF8.String(value));
}
@ -204,11 +206,24 @@ public class serverObjects extends HashMap<String, String> implements Cloneable
return put(key, CharacterCoding.unicode2xml(value, true));
}
/**
* put the key/value pair with a special method according to the given file type
* @param fileType
* @param key
* @param value
* @return
*/
public String put(final RequestHeader.FileType fileType, final String key, final String value) {
if (fileType == FileType.JSON) return putJSON(key, value);
if (fileType == FileType.XML) return putXML(key, value);
return putHTML(key, value);
}
/**
* Add a byte/long/integer to the map. The number will be encoded into a String using
* a localized format specified by {@link Formatter} and {@link #setLocalized(boolean)}.
* @param key key name as String.
* @param value integer type value to be added to the map in its formatted String
* @param value integer type value to be added to the map in its formatted String
* representation.
* @return the String value added to the map.
*/
@ -232,12 +247,12 @@ public class serverObjects extends HashMap<String, String> implements Cloneable
return this.put(key, value == null ? "" : Formatter.number(value));
}
public String putWiki(String hostport, final String key, final String wikiCode){
public String putWiki(final String hostport, final String key, final String wikiCode){
return this.put(key, Switchboard.wikiParser.transform(hostport, wikiCode));
}
public String putWiki(String hostport, final String key, final byte[] wikiCode) {
public String putWiki(final String hostport, final String key, final byte[] wikiCode) {
try {
return this.put(key, Switchboard.wikiParser.transform(hostport, wikiCode));
} catch (final UnsupportedEncodingException e) {
@ -302,7 +317,7 @@ public class serverObjects extends HashMap<String, String> implements Cloneable
s = s.toLowerCase();
return s.equals("true") || s.equals("on") || s.equals("1");
}
public boolean hasValue(final String key) {
final String s = super.get(key);
return (s != null && !s.isEmpty());
@ -324,7 +339,7 @@ public class serverObjects extends HashMap<String, String> implements Cloneable
// put all elements of another hashtable into the own table
public void putAll(final serverObjects add) {
for (Map.Entry<String, String> entry: add.entrySet()) {
for (final Map.Entry<String, String> entry: add.entrySet()) {
put(entry.getKey(), entry.getValue());
}
}
@ -359,7 +374,7 @@ public class serverObjects extends HashMap<String, String> implements Cloneable
* Currently it is used for numbers added with the putNum() methods only.
* @param loc if <code>true</code> store numbers in a localized format, otherwise
* use a default english locale without grouping.
* @see Formatter#setLocale(String)
* @see Formatter#setLocale(String)
*/
public void setLocalized(final boolean loc) {
this.localized = loc;
@ -375,9 +390,9 @@ public class serverObjects extends HashMap<String, String> implements Cloneable
*/
@Override
public String toString() {
if (this.isEmpty()) return "";
final StringBuilder param = new StringBuilder(this.size() * 40);
for (final Map.Entry<String, String> entry: this.entrySet()) {
if (isEmpty()) return "";
final StringBuilder param = new StringBuilder(size() * 40);
for (final Map.Entry<String, String> entry: entrySet()) {
param.append(MultiProtocolURI.escape(entry.getKey()));
param.append('=');
param.append(MultiProtocolURI.escape(entry.getValue()));
@ -386,10 +401,10 @@ public class serverObjects extends HashMap<String, String> implements Cloneable
param.setLength(param.length() - 1);
return param.toString();
}
public static void main(String[] args) {
String v = "ein \"zitat\"";
public static void main(final String[] args) {
final String v = "ein \"zitat\"";
System.out.println(toJSON(v));
}
}

62
source/net/yacy/cora/protocol/RequestHeader.java
Unescape View File

@ -7,12 +7,12 @@
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
@ -28,81 +28,95 @@ import net.yacy.cora.document.MultiProtocolURI;
public class RequestHeader extends HeaderFramework {
// request header properties
// request header properties
public static final String CONNECTION = "Connection";
public static final String PROXY_CONNECTION = "Proxy-Connection";
public static final String KEEP_ALIVE = "Keep-Alive";
public static final String USER_AGENT = "User-Agent";
public static final String AUTHORIZATION = "Authorization";
public static final String WWW_AUTHENTICATE = "WWW-Authenticate";
public static final String PROXY_AUTHORIZATION = "Proxy-Authorization";
public static final String PROXY_AUTHENTICATE = "Proxy-Authenticate";
public static final String UPGRADE = "Upgrade";
public static final String UPGRADE = "Upgrade";
public static final String TE = "TE";
public static final String X_CACHE = "X-Cache";
public static final String X_CACHE_LOOKUP = "X-Cache-Lookup";
public static final String COOKIE = "Cookie";
public static final String IF_MODIFIED_SINCE = "If-Modified-Since";
public static final String IF_RANGE = "If-Range";
public static final String REFERER = "Referer";
private static final long serialVersionUID = 0L;
public enum FileType {
HTML, JSON, XML
}
public RequestHeader() {
super();
}
public RequestHeader(final Map<String, String> reverseMappingCache) {
super(reverseMappingCache);
}
public RequestHeader(final Map<String, String> reverseMappingCache, final Map<String, String> othermap) {
super(reverseMappingCache, othermap);
}
public MultiProtocolURI referer() {
String referer = get(REFERER, null);
final String referer = get(REFERER, null);
if (referer == null) return null;
try {
return new MultiProtocolURI(referer);
} catch (MalformedURLException e) {
} catch (final MalformedURLException e) {
return null;
}
}
public String refererHost() {
final MultiProtocolURI url = referer();
if (url == null) return null;
return url.getHost();
}
public Date ifModifiedSince() {
return headerDate(IF_MODIFIED_SINCE);
}
public Object ifRange() {
if (containsKey(IF_RANGE)) {
final Date rangeDate = parseHTTPDate(get(IF_RANGE));
if (rangeDate != null)
if (rangeDate != null)
return rangeDate;
return get(IF_RANGE);
}
}
return null;
}
public String userAgent() {
return get(USER_AGENT, "");
}
public boolean acceptGzip() {
return ((containsKey(ACCEPT_ENCODING)) &&
((get(ACCEPT_ENCODING)).toUpperCase().indexOf("GZIP")) != -1);
((get(ACCEPT_ENCODING)).toUpperCase().indexOf("GZIP")) != -1);
}
public FileType fileType() {
String path = get(HeaderFramework.CONNECTION_PROP_PATH);
if (path == null) return FileType.HTML;
path = path.toLowerCase();
if (path.endsWith(".json")) return FileType.JSON;
if (path.endsWith(".xml")) return FileType.XML;
if (path.endsWith(".rss")) return FileType.XML;
return FileType.HTML;
}
}

Write Preview
Loading…
Cancel
Save