boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

code cleanup: call of static methods directly to the class

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6155 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent 222850414e
commit 154bbc3364
49 changed files with 226 additions and 187 deletions
Show Stats Download Patch File Download Diff File

5
htroot/Blog.java
Unescape View File

@ -36,6 +36,7 @@ import java.util.Iterator;
import de.anomic.data.blogBoard; import de.anomic.data.blogBoard;
import de.anomic.data.userDB; import de.anomic.data.userDB;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.util.DateFormatter; import de.anomic.kelondro.util.DateFormatter;
import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboard;
@ -67,7 +68,7 @@ public class Blog {
prop.put("display", 1); // Fixed to 1 prop.put("display", 1); // Fixed to 1
final boolean xml = (header.get(httpRequestHeader.CONNECTION_PROP_PATH)).endsWith(".xml"); final boolean xml = (header.get(httpHeader.CONNECTION_PROP_PATH)).endsWith(".xml");
final String address = sb.peers.mySeed().getPublicAddress(); final String address = sb.peers.mySeed().getPublicAddress();
if(hasRights) { if(hasRights) {
@ -96,7 +97,7 @@ public class Blog {
} }
String pagename = post.get("page", DEFAULT_PAGE); String pagename = post.get("page", DEFAULT_PAGE);
final String ip = (String)header.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP, "127.0.0.1"); final String ip = (String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "127.0.0.1");
String StrAuthor = post.get("author", ""); String StrAuthor = post.get("author", "");

3
htroot/BlogComments.java
Unescape View File

@ -40,6 +40,7 @@ import de.anomic.data.blogBoardComments;
import de.anomic.data.messageBoard; import de.anomic.data.messageBoard;
import de.anomic.data.userDB; import de.anomic.data.userDB;
import de.anomic.data.blogBoard.BlogEntry; import de.anomic.data.blogBoard.BlogEntry;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.FileUtils;
import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboard;
@ -83,7 +84,7 @@ public class BlogComments {
} }
final String pagename = post.get("page", "blog_default"); final String pagename = post.get("page", "blog_default");
final String ip = post.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP, "127.0.0.1"); final String ip = post.get(httpHeader.CONNECTION_PROP_CLIENTIP, "127.0.0.1");
String StrAuthor = post.get("author", "anonymous"); String StrAuthor = post.get("author", "anonymous");

3
htroot/ConfigAppearance_p.java
Unescape View File

@ -37,6 +37,7 @@ import java.util.List;
import de.anomic.crawler.HTTPLoader; import de.anomic.crawler.HTTPLoader;
import de.anomic.data.listManager; import de.anomic.data.listManager;
import de.anomic.http.httpClient; import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.FileUtils;
import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboard;
@ -90,7 +91,7 @@ public class ConfigAppearance_p {
try { try {
final yacyURL u = new yacyURL(url, null); final yacyURL u = new yacyURL(url, null);
final httpRequestHeader reqHeader = new httpRequestHeader(); final httpRequestHeader reqHeader = new httpRequestHeader();
reqHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.yacyUserAgent); reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
skinVector = FileUtils.strings(httpClient.wget(u.toString(), reqHeader, 10000), "UTF-8"); skinVector = FileUtils.strings(httpClient.wget(u.toString(), reqHeader, 10000), "UTF-8");
} catch (final IOException e) { } catch (final IOException e) {
prop.put("status", "1");// unable to get URL prop.put("status", "1");// unable to get URL

5
htroot/ConfigBasic.java
Unescape View File

@ -32,6 +32,7 @@ import java.io.File;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import de.anomic.data.translator; import de.anomic.data.translator;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.http.httpd; import de.anomic.http.httpd;
import de.anomic.http.httpdFileHandler; import de.anomic.http.httpdFileHandler;
@ -121,8 +122,8 @@ public class ConfigBasic {
if (upnp) UPnP.addPortMapping(); if (upnp) UPnP.addPortMapping();
String host = null; String host = null;
if (header.containsKey(httpRequestHeader.HOST)) { if (header.containsKey(httpHeader.HOST)) {
host = header.get(httpRequestHeader.HOST); host = header.get(httpHeader.HOST);
final int idx = host.indexOf(":"); final int idx = host.indexOf(":");
if (idx != -1) host = host.substring(0,idx); if (idx != -1) host = host.substring(0,idx);
} else { } else {

3
htroot/ConfigLanguage_p.java
Unescape View File

@ -40,6 +40,7 @@ import de.anomic.crawler.HTTPLoader;
import de.anomic.data.listManager; import de.anomic.data.listManager;
import de.anomic.data.translator; import de.anomic.data.translator;
import de.anomic.http.httpClient; import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.FileUtils;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
@ -80,7 +81,7 @@ public class ConfigLanguage_p {
try{ try{
final yacyURL u = new yacyURL(url, null); final yacyURL u = new yacyURL(url, null);
final httpRequestHeader reqHeader = new httpRequestHeader(); final httpRequestHeader reqHeader = new httpRequestHeader();
reqHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.yacyUserAgent); reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
langVector = FileUtils.strings(httpClient.wget(u.toString(), reqHeader, 10000), "UTF-8"); langVector = FileUtils.strings(httpClient.wget(u.toString(), reqHeader, 10000), "UTF-8");
}catch(final IOException e){ }catch(final IOException e){
prop.put("status", "1");//unable to get url prop.put("status", "1");//unable to get url

3
htroot/Connections_p.java
Unescape View File

@ -35,6 +35,7 @@ import java.util.Set;
import de.anomic.http.HttpConnectionInfo; import de.anomic.http.HttpConnectionInfo;
import de.anomic.http.httpClient; import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.http.httpd; import de.anomic.http.httpd;
import de.anomic.kelondro.util.DateFormatter; import de.anomic.kelondro.util.DateFormatter;
@ -151,7 +152,7 @@ public final class Connections_p {
final Properties conProp = (Properties) currentHttpd.getConProp().clone(); final Properties conProp = (Properties) currentHttpd.getConProp().clone();
// getting the destination host // getting the destination host
dest = conProp.getProperty(httpRequestHeader.CONNECTION_PROP_HOST); dest = conProp.getProperty(httpHeader.CONNECTION_PROP_HOST);
if (dest==null)continue; if (dest==null)continue;
} else if (cmdObj instanceof urlRedirectord) { } else if (cmdObj instanceof urlRedirectord) {
prot = "urlRedirector"; prot = "urlRedirector";

4
htroot/CrawlProfileEditor_p.java
Unescape View File

@ -25,7 +25,7 @@
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.io.IOException; import java.io.IOException;
import java.text.SimpleDateFormat; import java.text.DateFormat;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Iterator; import java.util.Iterator;
@ -217,7 +217,7 @@ public class CrawlProfileEditor_p {
prop.put("crawlProfiles_" + count + "_depth", profile.depth()); prop.put("crawlProfiles_" + count + "_depth", profile.depth());
prop.put("crawlProfiles_" + count + "_mustmatch", profile.mustMatchPattern().toString()); prop.put("crawlProfiles_" + count + "_mustmatch", profile.mustMatchPattern().toString());
prop.put("crawlProfiles_" + count + "_mustnotmatch", profile.mustNotMatchPattern().toString()); prop.put("crawlProfiles_" + count + "_mustnotmatch", profile.mustNotMatchPattern().toString());
prop.put("crawlProfiles_" + count + "_crawlingIfOlder", (profile.recrawlIfOlder() == 0L) ? "no re-crawl" : ""+ SimpleDateFormat.getDateTimeInstance().format(profile.recrawlIfOlder())); prop.put("crawlProfiles_" + count + "_crawlingIfOlder", (profile.recrawlIfOlder() == 0L) ? "no re-crawl" : ""+ DateFormat.getDateTimeInstance().format(profile.recrawlIfOlder()));
prop.put("crawlProfiles_" + count + "_crawlingDomFilterDepth", (profile.domFilterDepth() == Integer.MAX_VALUE) ? "inactive" : Integer.toString(profile.domFilterDepth())); prop.put("crawlProfiles_" + count + "_crawlingDomFilterDepth", (profile.domFilterDepth() == Integer.MAX_VALUE) ? "inactive" : Integer.toString(profile.domFilterDepth()));
// start contrib [MN] // start contrib [MN]

3
htroot/Messages_p.java
Unescape View File

@ -33,6 +33,7 @@ import java.util.Iterator;
import java.util.TreeMap; import java.util.TreeMap;
import de.anomic.data.messageBoard; import de.anomic.data.messageBoard;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.util.DateFormatter; import de.anomic.kelondro.util.DateFormatter;
import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.FileUtils;
@ -126,7 +127,7 @@ public class Messages_p {
prop.putXML("mode_messages_"+count+"_key", key); prop.putXML("mode_messages_"+count+"_key", key);
prop.put("mode_messages_"+count+"_hash", message.authorHash()); prop.put("mode_messages_"+count+"_hash", message.authorHash());
if ((header.get(httpRequestHeader.CONNECTION_PROP_PATH)).endsWith(".rss")) { if ((header.get(httpHeader.CONNECTION_PROP_PATH)).endsWith(".rss")) {
// set the peer address // set the peer address
prop.put("mode_messages_"+count+"_peerAddress", peerAddress); prop.put("mode_messages_"+count+"_peerAddress", peerAddress);

3
htroot/Network.java
Unescape View File

@ -48,7 +48,6 @@ import de.anomic.yacy.yacyClient;
import de.anomic.yacy.yacyNewsPool; import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacyNewsRecord; import de.anomic.yacy.yacyNewsRecord;
import de.anomic.yacy.yacySeed; import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyRelease;
import de.anomic.yacy.yacyVersion; import de.anomic.yacy.yacyVersion;
public class Network { public class Network {
@ -422,7 +421,7 @@ public class Network {
final String peertags = serverCodings.set2string(seed.getPeerTags(), ",", false); final String peertags = serverCodings.set2string(seed.getPeerTags(), ",", false);
prop.putHTML(STR_TABLE_LIST + conCount + "_dhtreceive_peertags", ((peertags == null) || (peertags.length() == 0)) ? "no tags given" : ("tags = " + peertags)); prop.putHTML(STR_TABLE_LIST + conCount + "_dhtreceive_peertags", ((peertags == null) || (peertags.length() == 0)) ? "no tags given" : ("tags = " + peertags));
} }
prop.putHTML(STR_TABLE_LIST + conCount + "_version", yacyRelease.combined2prettyVersion(seed.get(yacySeed.VERSION, "0.1"), shortname)); prop.putHTML(STR_TABLE_LIST + conCount + "_version", yacyVersion.combined2prettyVersion(seed.get(yacySeed.VERSION, "0.1"), shortname));
prop.putNum(STR_TABLE_LIST + conCount + "_lastSeen", /*seed.getLastSeenString() + " " +*/ lastseen); prop.putNum(STR_TABLE_LIST + conCount + "_lastSeen", /*seed.getLastSeenString() + " " +*/ lastseen);
prop.put(STR_TABLE_LIST + conCount + "_utc", seed.get(yacySeed.UTC, "-")); prop.put(STR_TABLE_LIST + conCount + "_utc", seed.get(yacySeed.UTC, "-"));
prop.putHTML(STR_TABLE_LIST + conCount + "_uptime", DateFormatter.formatInterval(60000 * Long.parseLong(seed.get(yacySeed.UPTIME, "0")))); prop.putHTML(STR_TABLE_LIST + conCount + "_uptime", DateFormatter.formatInterval(60000 * Long.parseLong(seed.get(yacySeed.UPTIME, "0"))));

3
htroot/QuickCrawlLink_p.java
Unescape View File

@ -36,6 +36,7 @@ import java.util.Date;
import de.anomic.crawler.CrawlEntry; import de.anomic.crawler.CrawlEntry;
import de.anomic.crawler.CrawlProfile; import de.anomic.crawler.CrawlProfile;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
@ -62,7 +63,7 @@ public class QuickCrawlLink_p {
prop.put("mode", "0"); prop.put("mode", "0");
// getting the http host header // getting the http host header
final String hostSocket = header.get(httpRequestHeader.CONNECTION_PROP_HOST); final String hostSocket = header.get(httpHeader.CONNECTION_PROP_HOST);
//String host = hostSocket; //String host = hostSocket;
int port = 80; int port = 80;

3
htroot/Settings_p.java
Unescape View File

@ -29,6 +29,7 @@ import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaParser; import de.anomic.plasma.plasmaParser;
import de.anomic.plasma.plasmaParserConfig; import de.anomic.plasma.plasmaParserConfig;
@ -162,7 +163,7 @@ public final class Settings_p {
} }
// clientIP // clientIP
prop.putXML("clientIP", (String) header.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP, "<unknown>")); // read an artificial header addendum prop.putXML("clientIP", (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "<unknown>")); // read an artificial header addendum
/* /*
* seed upload settings * seed upload settings

5
htroot/User.java
Unescape View File

@ -30,6 +30,7 @@
import java.io.IOException; import java.io.IOException;
import de.anomic.data.userDB; import de.anomic.data.userDB;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.http.httpResponseHeader; import de.anomic.http.httpResponseHeader;
import de.anomic.http.httpd; import de.anomic.http.httpd;
@ -61,7 +62,7 @@ public class User{
prop.put("logged-in_identified-by", "2"); prop.put("logged-in_identified-by", "2");
//try via ip //try via ip
if(entry == null){ if(entry == null){
entry=sb.userDB.ipAuth(((String)requestHeader.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP, "xxxxxx"))); entry=sb.userDB.ipAuth(((String)requestHeader.get(httpHeader.CONNECTION_PROP_CLIENTIP, "xxxxxx")));
if(entry != null){ if(entry != null){
prop.put("logged-in_identified-by", "0"); prop.put("logged-in_identified-by", "0");
} }
@ -144,7 +145,7 @@ public class User{
if(post!=null && post.containsKey("logout")){ if(post!=null && post.containsKey("logout")){
prop.put("logged-in", "0"); prop.put("logged-in", "0");
if(entry != null){ if(entry != null){
entry.logout(((String)requestHeader.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP, "xxxxxx")), userDB.getLoginToken(requestHeader.getHeaderCookies())); //todo: logout cookie entry.logout(((String)requestHeader.get(httpHeader.CONNECTION_PROP_CLIENTIP, "xxxxxx")), userDB.getLoginToken(requestHeader.getHeaderCookies())); //todo: logout cookie
}else{ }else{
sb.userDB.adminLogout(userDB.getLoginToken(requestHeader.getHeaderCookies())); sb.userDB.adminLogout(userDB.getLoginToken(requestHeader.getHeaderCookies()));
} }

3
htroot/ViewImage.java
Unescape View File

@ -29,6 +29,7 @@ import java.io.InputStream;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.util.HashMap; import java.util.HashMap;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.FileUtils;
import de.anomic.plasma.plasmaSnippetCache; import de.anomic.plasma.plasmaSnippetCache;
@ -61,7 +62,7 @@ public class ViewImage {
String urlString = post.get("url", ""); String urlString = post.get("url", "");
final String urlLicense = post.get("code", ""); final String urlLicense = post.get("code", "");
final boolean auth = ((String) header.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP, "")).equals("localhost") || sb.verifyAuthentication(header, true); // handle access rights final boolean auth = ((String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "")).equals("localhost") || sb.verifyAuthentication(header, true); // handle access rights
yacyURL url = null; yacyURL url = null;
if ((urlString.length() > 0) && (auth)) try { if ((urlString.length() > 0) && (auth)) try {

3
htroot/Wiki.java
Unescape View File

@ -36,6 +36,7 @@ import java.util.Iterator;
import de.anomic.data.diff; import de.anomic.data.diff;
import de.anomic.data.wiki.wikiBoard; import de.anomic.data.wiki.wikiBoard;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
@ -68,7 +69,7 @@ public class Wiki {
String access = sb.getConfig("WikiAccess", "admin"); String access = sb.getConfig("WikiAccess", "admin");
final String pagename = get(post, "page", "start"); final String pagename = get(post, "page", "start");
final String ip = get(post, httpRequestHeader.CONNECTION_PROP_CLIENTIP, "127.0.0.1"); final String ip = get(post, httpHeader.CONNECTION_PROP_CLIENTIP, "127.0.0.1");
String author = get(post, "author", "anonymous"); String author = get(post, "author", "anonymous");
if (author.equals("anonymous")) { if (author.equals("anonymous")) {
author = wikiBoard.guessAuthor(ip); author = wikiBoard.guessAuthor(ip);

3
htroot/YaCySearchPluginFF.java
Unescape View File

@ -24,6 +24,7 @@
//javac -classpath .:../Classes Status.java //javac -classpath .:../Classes Status.java
//if the shell's current path is HTROOT //if the shell's current path is HTROOT
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
@ -42,7 +43,7 @@ public class YaCySearchPluginFF {
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
// getting the http host header // getting the http host header
final String hostSocket = header.get(httpRequestHeader.CONNECTION_PROP_HOST); final String hostSocket = header.get(httpHeader.CONNECTION_PROP_HOST);
String host = hostSocket; String host = hostSocket;
int port = 80; int port = 80;

3
htroot/api/util/getpageinfo_p.java
Unescape View File

@ -5,6 +5,7 @@ import java.util.Set;
import de.anomic.crawler.HTTPLoader; import de.anomic.crawler.HTTPLoader;
import de.anomic.htmlFilter.htmlFilterContentScraper; import de.anomic.htmlFilter.htmlFilterContentScraper;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
@ -43,7 +44,7 @@ public class getpageinfo_p {
try { try {
final yacyURL u = new yacyURL(url, null); final yacyURL u = new yacyURL(url, null);
final httpRequestHeader reqHeader = new httpRequestHeader(); final httpRequestHeader reqHeader = new httpRequestHeader();
reqHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.yacyUserAgent); // do not set the crawler user agent, because this page was loaded by manual entering of the url reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.yacyUserAgent); // do not set the crawler user agent, because this page was loaded by manual entering of the url
final htmlFilterContentScraper scraper = htmlFilterContentScraper.parseResource(u, reqHeader); final htmlFilterContentScraper scraper = htmlFilterContentScraper.parseResource(u, reqHeader);
// put the document title // put the document title

3
htroot/autoconfig.java
Unescape View File

@ -24,6 +24,7 @@
//javac -classpath .:../Classes Status.java //javac -classpath .:../Classes Status.java
//if the shell's current path is HTROOT //if the shell's current path is HTROOT
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaSwitchboardConstants; import de.anomic.plasma.plasmaSwitchboardConstants;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
@ -47,7 +48,7 @@ public class autoconfig {
final boolean yacyonly = env.getConfigBool(plasmaSwitchboardConstants.PROXY_YACY_ONLY, false); final boolean yacyonly = env.getConfigBool(plasmaSwitchboardConstants.PROXY_YACY_ONLY, false);
// getting the http host header // getting the http host header
final String hostSocket = header.get(httpRequestHeader.CONNECTION_PROP_HOST); final String hostSocket = header.get(httpHeader.CONNECTION_PROP_HOST);
String host = hostSocket; String host = hostSocket;
int port = 80; int port = 80;

9
htroot/sharedBlacklist_p.java
Unescape View File

@ -45,6 +45,7 @@ import de.anomic.data.list.ListAccumulator;
import de.anomic.data.list.XMLBlacklistImporter; import de.anomic.data.list.XMLBlacklistImporter;
import de.anomic.htmlFilter.htmlFilterCharacterCoding; import de.anomic.htmlFilter.htmlFilterCharacterCoding;
import de.anomic.http.httpClient; import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.FileUtils;
import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboard;
@ -139,9 +140,9 @@ public class sharedBlacklist_p {
// download the blacklist // download the blacklist
try { try {
final httpRequestHeader reqHeader = new httpRequestHeader(); final httpRequestHeader reqHeader = new httpRequestHeader();
reqHeader.put(httpRequestHeader.PRAGMA,"no-cache"); reqHeader.put(httpHeader.PRAGMA,"no-cache");
reqHeader.put(httpRequestHeader.CACHE_CONTROL,"no-cache"); reqHeader.put(httpHeader.CACHE_CONTROL,"no-cache");
reqHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.yacyUserAgent); reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
// get List // get List
yacyURL u = new yacyURL(downloadURLOld, null); yacyURL u = new yacyURL(downloadURLOld, null);
@ -164,7 +165,7 @@ public class sharedBlacklist_p {
try { try {
final yacyURL u = new yacyURL(downloadURL, null); final yacyURL u = new yacyURL(downloadURL, null);
final httpRequestHeader reqHeader = new httpRequestHeader(); final httpRequestHeader reqHeader = new httpRequestHeader();
reqHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.yacyUserAgent); reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
otherBlacklist = FileUtils.strings(httpClient.wget(u.toString(), reqHeader, 10000), "UTF-8"); //get List otherBlacklist = FileUtils.strings(httpClient.wget(u.toString(), reqHeader, 10000), "UTF-8"); //get List
} catch (final Exception e) { } catch (final Exception e) {
prop.put("status", STATUS_URL_PROBLEM); prop.put("status", STATUS_URL_PROBLEM);

3
htroot/www/welcome.java
Unescape View File

@ -32,6 +32,7 @@ import java.io.File;
import java.net.InetAddress; import java.net.InetAddress;
import java.net.UnknownHostException; import java.net.UnknownHostException;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore; import de.anomic.server.serverCore;
@ -61,7 +62,7 @@ public class welcome {
prop.put("hostip", "Unknown Host Exception"); prop.put("hostip", "Unknown Host Exception");
} }
prop.put("port", serverCore.getPortNr(env.getConfig("port","8080"))); prop.put("port", serverCore.getPortNr(env.getConfig("port","8080")));
prop.put("clientip", (String) header.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP, "")); prop.put("clientip", (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP, ""));
final String peertype = (sb.peers.mySeed() == null) ? yacySeed.PEERTYPE_JUNIOR : sb.peers.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN); final String peertype = (sb.peers.mySeed() == null) ? yacySeed.PEERTYPE_JUNIOR : sb.peers.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN);
final boolean senior = (peertype.equals(yacySeed.PEERTYPE_SENIOR)) || (peertype.equals(yacySeed.PEERTYPE_PRINCIPAL)); final boolean senior = (peertype.equals(yacySeed.PEERTYPE_SENIOR)) || (peertype.equals(yacySeed.PEERTYPE_PRINCIPAL));

5
htroot/yacy/hello.java
Unescape View File

@ -32,6 +32,7 @@ import java.util.ConcurrentModificationException;
import java.util.Iterator; import java.util.Iterator;
import java.util.Map; import java.util.Map;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore; import de.anomic.server.serverCore;
@ -72,7 +73,7 @@ public final class hello {
int count = 0; int count = 0;
try {count = (countStr == null) ? 0 : Integer.parseInt(countStr);} catch (final NumberFormatException e) {count = 0;} try {count = (countStr == null) ? 0 : Integer.parseInt(countStr);} catch (final NumberFormatException e) {count = 0;}
// final Date remoteTime = yacyCore.parseUniversalDate((String) post.get(MYTIME)); // read remote time // final Date remoteTime = yacyCore.parseUniversalDate((String) post.get(MYTIME)); // read remote time
final String clientip = (String) header.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP, "<unknown>"); // read an artificial header addendum final String clientip = (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "<unknown>"); // read an artificial header addendum
final InetAddress ias = serverDomains.dnsResolve(clientip); final InetAddress ias = serverDomains.dnsResolve(clientip);
if (ias == null) { if (ias == null) {
prop.put("message", "cannot resolve your IP from your reported location " + clientip); prop.put("message", "cannot resolve your IP from your reported location " + clientip);
@ -96,7 +97,7 @@ public final class hello {
// if ((properTest != null) && (! properTest.substring(0,1).equals("IP"))) { return null; } // if ((properTest != null) && (! properTest.substring(0,1).equals("IP"))) { return null; }
// we easily know the caller's IP: // we easily know the caller's IP:
final String userAgent = (String) header.get(httpRequestHeader.USER_AGENT, "<unknown>"); final String userAgent = (String) header.get(httpHeader.USER_AGENT, "<unknown>");
final String reportedip = remoteSeed.getIP(); final String reportedip = remoteSeed.getIP();
final String reportedPeerType = remoteSeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR); final String reportedPeerType = remoteSeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR);
final float clientversion = remoteSeed.getVersion(); final float clientversion = remoteSeed.getVersion();

3
htroot/yacy/list.java
Unescape View File

@ -29,6 +29,7 @@
import java.io.File; import java.io.File;
import de.anomic.data.listManager; import de.anomic.data.listManager;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.plasma.plasmaSwitchboardConstants; import de.anomic.plasma.plasmaSwitchboardConstants;
@ -60,7 +61,7 @@ public final class list {
final yacySeed bla = sb.peers.get(post.get("iam", "")); final yacySeed bla = sb.peers.get(post.get("iam", ""));
if (bla != null) otherPeerName = bla.getName(); if (bla != null) otherPeerName = bla.getName();
} }
if (otherPeerName == null) otherPeerName = header.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP); if (otherPeerName == null) otherPeerName = header.get(httpHeader.CONNECTION_PROP_CLIENTIP);
if ((sb.isRobinsonMode()) && (!sb.isInMyCluster(otherPeerName))) { if ((sb.isRobinsonMode()) && (!sb.isInMyCluster(otherPeerName))) {
// if we are a robinson cluster, answer only if this client is known by our network definition // if we are a robinson cluster, answer only if this client is known by our network definition

3
htroot/yacy/message.java
Unescape View File

@ -35,6 +35,7 @@ import java.text.SimpleDateFormat;
import java.util.Date; import java.util.Date;
import de.anomic.data.messageBoard; import de.anomic.data.messageBoard;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.FileUtils;
import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboard;
@ -81,7 +82,7 @@ public final class message {
if ((sb.isRobinsonMode()) && if ((sb.isRobinsonMode()) &&
(!((sb.isPublicRobinson()) || (!((sb.isPublicRobinson()) ||
(sb.isInMyCluster(header.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP)))))) { (sb.isInMyCluster(header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))))) {
// if we are a robinson cluster, answer only if this client is known by our network definition // if we are a robinson cluster, answer only if this client is known by our network definition
prop.put("response", "-1"); // request rejected prop.put("response", "-1"); // request rejected
return prop; return prop;

3
htroot/yacy/profile.java
Unescape View File

@ -35,6 +35,7 @@ import java.io.IOException;
import java.util.Iterator; import java.util.Iterator;
import java.util.Properties; import java.util.Properties;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
@ -52,7 +53,7 @@ public final class profile {
if ((sb.isRobinsonMode()) && if ((sb.isRobinsonMode()) &&
(!sb.isPublicRobinson()) && (!sb.isPublicRobinson()) &&
(!sb.isInMyCluster(header.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP)))) { (!sb.isInMyCluster(header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))) {
// if we are a robinson cluster, answer only if this client is known by our network definition // if we are a robinson cluster, answer only if this client is known by our network definition
prop.put("list", "0"); prop.put("list", "0");
return prop; return prop;

3
htroot/yacy/query.java
Unescape View File

@ -29,6 +29,7 @@
import java.io.IOException; import java.io.IOException;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.util.DateFormatter; import de.anomic.kelondro.util.DateFormatter;
import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboard;
@ -54,7 +55,7 @@ public final class query {
if ((sb.isRobinsonMode()) && if ((sb.isRobinsonMode()) &&
(!sb.isPublicRobinson()) && (!sb.isPublicRobinson()) &&
(!sb.isInMyCluster(header.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP)))) { (!sb.isInMyCluster(header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))) {
// if we are a robinson cluster, answer only if we are public robinson peers, // if we are a robinson cluster, answer only if we are public robinson peers,
// or we are a private cluster and the requester is in our cluster. // or we are a private cluster and the requester is in our cluster.
// if we don't answer, the remote peer will recognize us as junior peer, // if we don't answer, the remote peer will recognize us as junior peer,

5
htroot/yacy/search.java
Unescape View File

@ -35,6 +35,7 @@ import java.util.Map;
import java.util.TreeSet; import java.util.TreeSet;
import de.anomic.content.RSSMessage; import de.anomic.content.RSSMessage;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.order.Base64Order; import de.anomic.kelondro.order.Base64Order;
import de.anomic.kelondro.order.Bitfield; import de.anomic.kelondro.order.Bitfield;
@ -71,7 +72,7 @@ public final class search {
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
if ((post == null) || (env == null)) return prop; if ((post == null) || (env == null)) return prop;
if (!yacyNetwork.authentifyRequest(post, env)) return prop; if (!yacyNetwork.authentifyRequest(post, env)) return prop;
final String client = header.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP); final String client = header.get(httpHeader.CONNECTION_PROP_CLIENTIP);
//System.out.println("yacy: search received request = " + post.toString()); //System.out.println("yacy: search received request = " + post.toString());
@ -126,7 +127,7 @@ public final class search {
if ((sb.isRobinsonMode()) && if ((sb.isRobinsonMode()) &&
(!((sb.isPublicRobinson()) || (!((sb.isPublicRobinson()) ||
(sb.isInMyCluster(header.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP)))))) { (sb.isInMyCluster(header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))))) {
// if we are a robinson cluster, answer only if this client is known by our network definition // if we are a robinson cluster, answer only if this client is known by our network definition
prop.put("links", ""); prop.put("links", "");
prop.put("linkcount", "0"); prop.put("linkcount", "0");

5
htroot/yacy/transfer.java
Unescape View File

@ -27,6 +27,7 @@
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.order.Base64Order; import de.anomic.kelondro.order.Base64Order;
import de.anomic.kelondro.order.Digest; import de.anomic.kelondro.order.Digest;
@ -71,8 +72,8 @@ public final class transfer {
// reject unknown peers: this does not appear fair, but anonymous senders are dangerous // reject unknown peers: this does not appear fair, but anonymous senders are dangerous
// reject paths that contain '..' because they are dangerous // reject paths that contain '..' because they are dangerous
if (sb.getLog().isFine()) { if (sb.getLog().isFine()) {
if (otherseed == null) sb.getLog().logFine("RankingTransmission: rejected unknown peer '" + otherpeer + "', current IP " + header.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP, "unknown")); if (otherseed == null) sb.getLog().logFine("RankingTransmission: rejected unknown peer '" + otherpeer + "', current IP " + header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "unknown"));
if (filename.indexOf("..") >= 0) sb.getLog().logFine("RankingTransmission: rejected wrong path '" + filename + "' from peer " + (otherseed == null ? "null" : otherseed.getName() + "/" + otherseed.getPublicAddress()) + ", current IP " + header.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP, "unknown")); if (filename.indexOf("..") >= 0) sb.getLog().logFine("RankingTransmission: rejected wrong path '" + filename + "' from peer " + (otherseed == null ? "null" : otherseed.getName() + "/" + otherseed.getPublicAddress()) + ", current IP " + header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "unknown"));
} }
return prop; return prop;
} }

5
htroot/yacysearch.java
Unescape View File

@ -33,6 +33,7 @@ import java.util.Iterator;
import java.util.TreeSet; import java.util.TreeSet;
import de.anomic.content.RSSMessage; import de.anomic.content.RSSMessage;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.order.Bitfield; import de.anomic.kelondro.order.Bitfield;
import de.anomic.kelondro.text.metadataPrototype.URLMetadataRow; import de.anomic.kelondro.text.metadataPrototype.URLMetadataRow;
@ -80,7 +81,7 @@ public class yacysearch {
} }
String promoteSearchPageGreeting = env.getConfig(plasmaSwitchboardConstants.GREETING, ""); String promoteSearchPageGreeting = env.getConfig(plasmaSwitchboardConstants.GREETING, "");
if (env.getConfigBool(plasmaSwitchboardConstants.GREETING_NETWORK_NAME, false)) promoteSearchPageGreeting = env.getConfig("network.unit.description", ""); if (env.getConfigBool(plasmaSwitchboardConstants.GREETING_NETWORK_NAME, false)) promoteSearchPageGreeting = env.getConfig("network.unit.description", "");
final String client = header.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP); // the search client who initiated the search final String client = header.get(httpHeader.CONNECTION_PROP_CLIENTIP); // the search client who initiated the search
// get query // get query
String originalquerystring = (post == null) ? "" : post.get("query", post.get("search", "")).trim(); // SRU compliance String originalquerystring = (post == null) ? "" : post.get("query", post.get("search", "")).trim(); // SRU compliance
@ -336,7 +337,7 @@ public class yacysearch {
if (language.startsWith("lang_")) language = language.substring(5); if (language.startsWith("lang_")) language = language.substring(5);
if (!iso639.exists(language)) { if (!iso639.exists(language)) {
// find out language of the user by reading of the user-agent string // find out language of the user by reading of the user-agent string
String agent = header.get(httpRequestHeader.ACCEPT_LANGUAGE); String agent = header.get(httpHeader.ACCEPT_LANGUAGE);
if (agent == null) agent = System.getProperty("user.language"); if (agent == null) agent = System.getProperty("user.language");
language = (agent == null) ? "en" : iso639.userAgentLanguageDetection(agent); language = (agent == null) ? "en" : iso639.userAgentLanguageDetection(agent);
if (language == null) language = "en"; if (language == null) language = "en";

3
htroot/yacysearchitem.java
Unescape View File

@ -30,6 +30,7 @@ import java.net.URLEncoder;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.TreeSet; import java.util.TreeSet;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaProfiling; import de.anomic.plasma.plasmaProfiling;
import de.anomic.plasma.plasmaSearchEvent; import de.anomic.plasma.plasmaSearchEvent;
@ -61,7 +62,7 @@ public class yacysearchitem {
final String eventID = post.get("eventID", ""); final String eventID = post.get("eventID", "");
final boolean authenticated = sb.adminAuthenticated(header) >= 2; final boolean authenticated = sb.adminAuthenticated(header) >= 2;
final int item = post.getInt("item", -1); final int item = post.getInt("item", -1);
final boolean auth = ((String) header.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP, "")).equals("localhost") || sb.verifyAuthentication(header, true); final boolean auth = ((String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "")).equals("localhost") || sb.verifyAuthentication(header, true);
final int display = (post == null) ? 0 : post.getInt("display", 0); final int display = (post == null) ? 0 : post.getInt("display", 0);
// default settings for blank item // default settings for blank item

5
source/de/anomic/crawler/FTPLoader.java
Unescape View File

@ -32,6 +32,7 @@ import java.io.IOException;
import java.io.PrintStream; import java.io.PrintStream;
import java.util.Date; import java.util.Date;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.http.httpResponseHeader; import de.anomic.http.httpResponseHeader;
import de.anomic.http.httpdProxyCacheEntry; import de.anomic.http.httpdProxyCacheEntry;
@ -61,8 +62,8 @@ public class FTPLoader {
httpRequestHeader requestHeader = new httpRequestHeader(); httpRequestHeader requestHeader = new httpRequestHeader();
if (entry.referrerhash() != null) requestHeader.put(httpRequestHeader.REFERER, sb.getURL(entry.referrerhash()).toNormalform(true, false)); if (entry.referrerhash() != null) requestHeader.put(httpRequestHeader.REFERER, sb.getURL(entry.referrerhash()).toNormalform(true, false));
httpResponseHeader responseHeader = new httpResponseHeader(); httpResponseHeader responseHeader = new httpResponseHeader();
responseHeader.put(httpResponseHeader.LAST_MODIFIED, DateFormatter.formatRFC1123(fileDate)); responseHeader.put(httpHeader.LAST_MODIFIED, DateFormatter.formatRFC1123(fileDate));
responseHeader.put(httpResponseHeader.CONTENT_TYPE, mimeType); responseHeader.put(httpHeader.CONTENT_TYPE, mimeType);
Document metadata = new httpdProxyCacheEntry( Document metadata = new httpdProxyCacheEntry(
entry.depth(), entry.url(), entry.name(), "OK", entry.depth(), entry.url(), entry.name(), "OK",
requestHeader, responseHeader, requestHeader, responseHeader,

13
source/de/anomic/crawler/HTTPLoader.java
Unescape View File

@ -30,6 +30,7 @@ import java.util.Date;
import de.anomic.data.Blacklist; import de.anomic.data.Blacklist;
import de.anomic.http.httpClient; import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpResponse; import de.anomic.http.httpResponse;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.http.httpResponseHeader; import de.anomic.http.httpResponseHeader;
@ -132,13 +133,13 @@ public final class HTTPLoader {
//try { //try {
// create a request header // create a request header
final httpRequestHeader requestHeader = new httpRequestHeader(); final httpRequestHeader requestHeader = new httpRequestHeader();
requestHeader.put(httpRequestHeader.USER_AGENT, crawlerUserAgent); requestHeader.put(httpHeader.USER_AGENT, crawlerUserAgent);
yacyURL refererURL = null; yacyURL refererURL = null;
if (entry.referrerhash() != null) refererURL = sb.getURL(entry.referrerhash()); if (entry.referrerhash() != null) refererURL = sb.getURL(entry.referrerhash());
if (refererURL != null) requestHeader.put(httpRequestHeader.REFERER, refererURL.toNormalform(true, true)); if (refererURL != null) requestHeader.put(httpRequestHeader.REFERER, refererURL.toNormalform(true, true));
requestHeader.put(httpRequestHeader.ACCEPT_LANGUAGE, sb.getConfig("crawler.http.acceptLanguage", DEFAULT_LANGUAGE)); requestHeader.put(httpHeader.ACCEPT_LANGUAGE, sb.getConfig("crawler.http.acceptLanguage", DEFAULT_LANGUAGE));
requestHeader.put(httpRequestHeader.ACCEPT_CHARSET, sb.getConfig("crawler.http.acceptCharset", DEFAULT_CHARSET)); requestHeader.put(httpHeader.ACCEPT_CHARSET, sb.getConfig("crawler.http.acceptCharset", DEFAULT_CHARSET));
requestHeader.put(httpRequestHeader.ACCEPT_ENCODING, sb.getConfig("crawler.http.acceptEncoding", DEFAULT_ENCODING)); requestHeader.put(httpHeader.ACCEPT_ENCODING, sb.getConfig("crawler.http.acceptEncoding", DEFAULT_ENCODING));
// HTTP-Client // HTTP-Client
final httpClient client = new httpClient(socketTimeout, requestHeader); final httpClient client = new httpClient(socketTimeout, requestHeader);
@ -195,9 +196,9 @@ public final class HTTPLoader {
htCache = null; htCache = null;
}*/ }*/
} else if (res.getStatusLine().startsWith("30")) { } else if (res.getStatusLine().startsWith("30")) {
if (res.getResponseHeader().containsKey(httpRequestHeader.LOCATION)) { if (res.getResponseHeader().containsKey(httpHeader.LOCATION)) {
// getting redirection URL // getting redirection URL
String redirectionUrlString = res.getResponseHeader().get(httpRequestHeader.LOCATION); String redirectionUrlString = res.getResponseHeader().get(httpHeader.LOCATION);
redirectionUrlString = redirectionUrlString.trim(); redirectionUrlString = redirectionUrlString.trim();
if (redirectionUrlString.length() == 0) { if (redirectionUrlString.length() == 0) {

9
source/de/anomic/crawler/IndexingStack.java
Unescape View File

@ -34,6 +34,7 @@ import java.util.Date;
import java.util.Iterator; import java.util.Iterator;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import de.anomic.http.httpHeader;
import de.anomic.http.httpResponseHeader; import de.anomic.http.httpResponseHeader;
import de.anomic.kelondro.index.Row; import de.anomic.kelondro.index.Row;
import de.anomic.kelondro.order.Base64Order; import de.anomic.kelondro.order.Base64Order;
@ -481,7 +482,7 @@ public class IndexingStack {
// -if-modified-since in request // -if-modified-since in request
// if the page is fresh at the very moment we can index it // if the page is fresh at the very moment we can index it
final Date ifModifiedSince = getModificationDate(); final Date ifModifiedSince = getModificationDate();
if ((ifModifiedSince != null) && (responseHeader.containsKey(httpResponseHeader.LAST_MODIFIED))) { if ((ifModifiedSince != null) && (responseHeader.containsKey(httpHeader.LAST_MODIFIED))) {
// parse date // parse date
Date d = responseHeader.lastModified(); Date d = responseHeader.lastModified();
if (d == null) { if (d == null) {
@ -495,8 +496,8 @@ public class IndexingStack {
} }
// -pragma in cached response // -pragma in cached response
if (responseHeader.containsKey(httpResponseHeader.PRAGMA) && if (responseHeader.containsKey(httpHeader.PRAGMA) &&
(responseHeader.get(httpResponseHeader.PRAGMA)).toUpperCase().equals("NO-CACHE")) { (responseHeader.get(httpHeader.PRAGMA)).toUpperCase().equals("NO-CACHE")) {
return "Denied_(pragma_no_cache)"; return "Denied_(pragma_no_cache)";
} }
@ -520,7 +521,7 @@ public class IndexingStack {
// -cache-control in cached response // -cache-control in cached response
// the cache-control has many value options. // the cache-control has many value options.
String cacheControl = responseHeader.get(httpResponseHeader.CACHE_CONTROL); String cacheControl = responseHeader.get(httpHeader.CACHE_CONTROL);
if (cacheControl != null) { if (cacheControl != null) {
cacheControl = cacheControl.trim().toUpperCase(); cacheControl = cacheControl.trim().toUpperCase();
/* we have the following cases for cache-control: /* we have the following cases for cache-control:

7
source/de/anomic/crawler/RobotsTxt.java
Unescape View File

@ -41,6 +41,7 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import de.anomic.http.httpClient; import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpResponse; import de.anomic.http.httpResponse;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.blob.Heap; import de.anomic.kelondro.blob.Heap;
@ -518,7 +519,7 @@ public class RobotsTxt {
httpRequestHeader reqHeaders = new httpRequestHeader(); httpRequestHeader reqHeaders = new httpRequestHeader();
// add yacybot user agent // add yacybot user agent
reqHeaders.put(httpRequestHeader.USER_AGENT, HTTPLoader.crawlerUserAgent); reqHeaders.put(httpHeader.USER_AGENT, HTTPLoader.crawlerUserAgent);
// adding referer // adding referer
reqHeaders.put(httpRequestHeader.REFERER, (yacyURL.newURL(robotsURL,"/")).toNormalform(true, true)); reqHeaders.put(httpRequestHeader.REFERER, (yacyURL.newURL(robotsURL,"/")).toNormalform(true, true));
@ -550,7 +551,7 @@ public class RobotsTxt {
} else { } else {
// getting some metadata // getting some metadata
eTag = res.getResponseHeader().containsKey(httpRequestHeader.ETAG)?(res.getResponseHeader().get(httpRequestHeader.ETAG)).trim():null; eTag = res.getResponseHeader().containsKey(httpHeader.ETAG)?(res.getResponseHeader().get(httpHeader.ETAG)).trim():null;
lastMod = res.getResponseHeader().lastModified(); lastMod = res.getResponseHeader().lastModified();
// if the robots.txt file was not changed we break here // if the robots.txt file was not changed we break here
@ -575,7 +576,7 @@ public class RobotsTxt {
return null; return null;
} else if (res.getStatusLine().startsWith("3")) { } else if (res.getStatusLine().startsWith("3")) {
// getting redirection URL // getting redirection URL
String redirectionUrlString = res.getResponseHeader().get(httpRequestHeader.LOCATION); String redirectionUrlString = res.getResponseHeader().get(httpHeader.LOCATION);
if (redirectionUrlString==null) { if (redirectionUrlString==null) {
if (log.isFinest()) log.logFinest("robots.txt could not be downloaded from URL '" + robotsURL + "' because of missing redirecton header. [" + res.getStatusLine() + "]."); if (log.isFinest()) log.logFinest("robots.txt could not be downloaded from URL '" + robotsURL + "' because of missing redirecton header. [" + res.getStatusLine() + "].");
robotsTxt = null; robotsTxt = null;

3
source/de/anomic/data/SitemapParser.java
Unescape View File

@ -42,6 +42,7 @@ import de.anomic.crawler.CrawlEntry;
import de.anomic.crawler.CrawlProfile; import de.anomic.crawler.CrawlProfile;
import de.anomic.crawler.HTTPLoader; import de.anomic.crawler.HTTPLoader;
import de.anomic.http.httpClient; import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpResponse; import de.anomic.http.httpResponse;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.http.httpdByteCountInputStream; import de.anomic.http.httpdByteCountInputStream;
@ -152,7 +153,7 @@ public class SitemapParser extends DefaultHandler {
public void parse() { public void parse() {
// download document // download document
final httpRequestHeader requestHeader = new httpRequestHeader(); final httpRequestHeader requestHeader = new httpRequestHeader();
requestHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.crawlerUserAgent); requestHeader.put(httpHeader.USER_AGENT, HTTPLoader.crawlerUserAgent);
final httpClient client = new httpClient(5000, requestHeader); final httpClient client = new httpClient(5000, requestHeader);
httpResponse res = null; httpResponse res = null;
try { try {

3
source/de/anomic/htmlFilter/htmlFilterContentScraper.java
Unescape View File

@ -46,6 +46,7 @@ import javax.swing.event.EventListenerList;
import de.anomic.crawler.HTTPLoader; import de.anomic.crawler.HTTPLoader;
import de.anomic.http.httpClient; import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.FileUtils;
import de.anomic.plasma.plasmaParser; import de.anomic.plasma.plasmaParser;
@ -512,7 +513,7 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
public static htmlFilterContentScraper parseResource(final yacyURL location) throws IOException { public static htmlFilterContentScraper parseResource(final yacyURL location) throws IOException {
// load page // load page
final httpRequestHeader reqHeader = new httpRequestHeader(); final httpRequestHeader reqHeader = new httpRequestHeader();
reqHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.crawlerUserAgent); reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.crawlerUserAgent);
return parseResource(location, reqHeader); return parseResource(location, reqHeader);
} }

4
source/de/anomic/http/httpClient.java
Unescape View File

@ -324,7 +324,7 @@ public class httpClient {
if (gzipBody) { if (gzipBody) {
data = zipRequest(data); data = zipRequest(data);
post.setRequestHeader(httpResponseHeader.CONTENT_ENCODING, httpHeader.CONTENT_ENCODING_GZIP); post.setRequestHeader(httpHeader.CONTENT_ENCODING, httpHeader.CONTENT_ENCODING_GZIP);
post.setContentChunked(true); post.setContentChunked(true);
} }
post.setRequestEntity(data); post.setRequestEntity(data);
@ -790,7 +790,7 @@ public class httpClient {
header = new httpRequestHeader(); header = new httpRequestHeader();
} }
// set host-header // set host-header
header.add(httpRequestHeader.HOST, vhost); header.add(httpHeader.HOST, vhost);
} }
} }

160
source/de/anomic/http/httpd.java
Unescape View File

@ -261,33 +261,33 @@ public final class httpd implements serverHandler, Cloneable {
private boolean handlePersistentConnection(final httpRequestHeader header) { private boolean handlePersistentConnection(final httpRequestHeader header) {
if (!keepAliveSupport) { if (!keepAliveSupport) {
this.prop.put(httpRequestHeader.CONNECTION_PROP_PERSISTENT,"close"); this.prop.put(httpHeader.CONNECTION_PROP_PERSISTENT,"close");
return false; return false;
} }
// getting the http version that is used by the client // getting the http version that is used by the client
final String httpVersion = this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_HTTP_VER, "HTTP/0.9"); final String httpVersion = this.prop.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER, "HTTP/0.9");
// managing keep-alive: in HTTP/0.9 and HTTP/1.0 every connection is closed // managing keep-alive: in HTTP/0.9 and HTTP/1.0 every connection is closed
// afterwards. In HTTP/1.1 (and above, in the future?) connections are // afterwards. In HTTP/1.1 (and above, in the future?) connections are
// persistent by default, but closed with the "Connection: close" // persistent by default, but closed with the "Connection: close"
// property. // property.
boolean persistent = !(httpVersion.equals(httpRequestHeader.HTTP_VERSION_0_9) || httpVersion.equals(httpRequestHeader.HTTP_VERSION_1_0)); boolean persistent = !(httpVersion.equals(httpHeader.HTTP_VERSION_0_9) || httpVersion.equals(httpHeader.HTTP_VERSION_1_0));
if (((String)header.get(httpRequestHeader.CONNECTION, "keep-alive")).toLowerCase().indexOf("close") != -1 || if (((String)header.get(httpRequestHeader.CONNECTION, "keep-alive")).toLowerCase().indexOf("close") != -1 ||
((String)header.get(httpRequestHeader.PROXY_CONNECTION, "keep-alive")).toLowerCase().indexOf("close") != -1) { ((String)header.get(httpRequestHeader.PROXY_CONNECTION, "keep-alive")).toLowerCase().indexOf("close") != -1) {
persistent = false; persistent = false;
} }
final String transferEncoding = (String) header.get(httpHeader.TRANSFER_ENCODING, "identity"); final String transferEncoding = (String) header.get(httpHeader.TRANSFER_ENCODING, "identity");
final boolean isPostRequest = this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_METHOD).equals(httpRequestHeader.METHOD_POST); final boolean isPostRequest = this.prop.getProperty(httpHeader.CONNECTION_PROP_METHOD).equals(httpHeader.METHOD_POST);
final boolean hasContentLength = header.containsKey(httpHeader.CONTENT_LENGTH); final boolean hasContentLength = header.containsKey(httpHeader.CONTENT_LENGTH);
final boolean hasTransferEncoding = header.containsKey(httpHeader.TRANSFER_ENCODING) && !transferEncoding.equalsIgnoreCase("identity"); final boolean hasTransferEncoding = header.containsKey(httpHeader.TRANSFER_ENCODING) && !transferEncoding.equalsIgnoreCase("identity");
// if the request does not contain a content-length we have to close the connection // if the request does not contain a content-length we have to close the connection
// independently of the value of the connection header // independently of the value of the connection header
if (persistent && isPostRequest && !(hasContentLength || hasTransferEncoding)) if (persistent && isPostRequest && !(hasContentLength || hasTransferEncoding))
this.prop.put(httpRequestHeader.CONNECTION_PROP_PERSISTENT,"close"); this.prop.put(httpHeader.CONNECTION_PROP_PERSISTENT,"close");
else this.prop.put(httpRequestHeader.CONNECTION_PROP_PERSISTENT,persistent?"keep-alive":"close"); else this.prop.put(httpHeader.CONNECTION_PROP_PERSISTENT,persistent?"keep-alive":"close");
return persistent; return persistent;
} }
@ -304,7 +304,7 @@ public final class httpd implements serverHandler, Cloneable {
private boolean handleServerAuthentication(final httpRequestHeader header) throws IOException { private boolean handleServerAuthentication(final httpRequestHeader header) throws IOException {
// getting the http version that is used by the client // getting the http version that is used by the client
final String httpVersion = this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_HTTP_VER, "HTTP/0.9"); final String httpVersion = this.prop.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER, "HTTP/0.9");
// reading the authentication settings from switchboard // reading the authentication settings from switchboard
if (this.serverAccountBase64MD5 == null) if (this.serverAccountBase64MD5 == null)
@ -317,7 +317,7 @@ public final class httpd implements serverHandler, Cloneable {
this.session.out.write((httpVersion + " 401 log-in required" + serverCore.CRLF_STRING + this.session.out.write((httpVersion + " 401 log-in required" + serverCore.CRLF_STRING +
httpRequestHeader.WWW_AUTHENTICATE + ": Basic realm=\"log-in\"" + serverCore.CRLF_STRING + httpRequestHeader.WWW_AUTHENTICATE + ": Basic realm=\"log-in\"" + serverCore.CRLF_STRING +
serverCore.CRLF_STRING).getBytes()); serverCore.CRLF_STRING).getBytes());
this.session.out.write((httpResponseHeader.CONTENT_LENGTH + ": 0\r\n").getBytes()); this.session.out.write((httpHeader.CONTENT_LENGTH + ": 0\r\n").getBytes());
this.session.out.write("\r\n".getBytes()); this.session.out.write("\r\n".getBytes());
return false; return false;
} else if (!this.serverAccountBase64MD5.equals(Digest.encodeMD5Hex(auth.trim().substring(6)))) { } else if (!this.serverAccountBase64MD5.equals(Digest.encodeMD5Hex(auth.trim().substring(6)))) {
@ -326,7 +326,7 @@ public final class httpd implements serverHandler, Cloneable {
this.session.out.write((httpVersion + " 401 log-in required" + serverCore.CRLF_STRING + this.session.out.write((httpVersion + " 401 log-in required" + serverCore.CRLF_STRING +
httpRequestHeader.WWW_AUTHENTICATE + ": Basic realm=\"log-in\"" + httpRequestHeader.WWW_AUTHENTICATE + ": Basic realm=\"log-in\"" +
serverCore.CRLF_STRING).getBytes()); serverCore.CRLF_STRING).getBytes());
this.session.out.write((httpResponseHeader.CONTENT_LENGTH + ": 0\r\n").getBytes()); this.session.out.write((httpHeader.CONTENT_LENGTH + ": 0\r\n").getBytes());
this.session.out.write("\r\n".getBytes()); this.session.out.write("\r\n".getBytes());
this.session.out.flush(); this.session.out.flush();
return false; return false;
@ -342,7 +342,7 @@ public final class httpd implements serverHandler, Cloneable {
// proxy hops must identify with 4 criteria: // proxy hops must identify with 4 criteria:
// the accessed port must not be port 80 // the accessed port must not be port 80
final String host = this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_HOST); final String host = this.prop.getProperty(httpHeader.CONNECTION_PROP_HOST);
if (host == null) return false; if (host == null) return false;
int pos; int pos;
if ((pos = host.indexOf(":")) < 0) { if ((pos = host.indexOf(":")) < 0) {
@ -352,7 +352,7 @@ public final class httpd implements serverHandler, Cloneable {
if (Integer.parseInt(host.substring(pos + 1)) == 80) return false; if (Integer.parseInt(host.substring(pos + 1)) == 80) return false;
// the access path must be into the yacy protocol path; it must start with 'yacy' // the access path must be into the yacy protocol path; it must start with 'yacy'
if (!(this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_PATH, "").startsWith("/yacy/"))) return false; if (!(this.prop.getProperty(httpHeader.CONNECTION_PROP_PATH, "").startsWith("/yacy/"))) return false;
// the accessing client must identify with user:password, where // the accessing client must identify with user:password, where
// user = addressed peer name // user = addressed peer name
@ -364,12 +364,12 @@ public final class httpd implements serverHandler, Cloneable {
} }
// the accessing client must use a yacy user-agent // the accessing client must use a yacy user-agent
if (!(((String) header.get(httpRequestHeader.USER_AGENT,"")).startsWith("yacy"))) return false; if (!(((String) header.get(httpHeader.USER_AGENT,"")).startsWith("yacy"))) return false;
// furthermore, YaCy hops must not exceed a specific access frequency // furthermore, YaCy hops must not exceed a specific access frequency
// check access requester frequency: protection against DoS against this peer // check access requester frequency: protection against DoS against this peer
final String requester = this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_CLIENTIP); final String requester = this.prop.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP);
if (requester == null) return false; if (requester == null) return false;
if (lastAccessDelta(YaCyHopAccessRequester, requester) < 10000) return false; if (lastAccessDelta(YaCyHopAccessRequester, requester) < 10000) return false;
YaCyHopAccessRequester.put(requester, Long.valueOf(System.currentTimeMillis())); YaCyHopAccessRequester.put(requester, Long.valueOf(System.currentTimeMillis()));
@ -422,8 +422,8 @@ public final class httpd implements serverHandler, Cloneable {
} }
// ask for authenticate // ask for authenticate
this.session.out.write((httpVersion + " 407 Proxy Authentication Required" + serverCore.CRLF_STRING + this.session.out.write((httpVersion + " 407 Proxy Authentication Required" + serverCore.CRLF_STRING +
httpRequestHeader.PROXY_AUTHENTICATE + ": Basic realm=\"log-in\"" + serverCore.CRLF_STRING).getBytes()); httpRequestHeader.PROXY_AUTHENTICATE + ": Basic realm=\"log-in\"" + serverCore.CRLF_STRING).getBytes());
this.session.out.write((httpResponseHeader.CONTENT_LENGTH + ": 0\r\n").getBytes()); this.session.out.write((httpHeader.CONTENT_LENGTH + ": 0\r\n").getBytes());
this.session.out.write("\r\n".getBytes()); this.session.out.write("\r\n".getBytes());
this.session.out.flush(); this.session.out.flush();
return false; return false;
@ -469,21 +469,21 @@ public final class httpd implements serverHandler, Cloneable {
public Boolean GET(final String arg) { public Boolean GET(final String arg) {
try { try {
// parsing the http request line // parsing the http request line
parseRequestLine(httpRequestHeader.METHOD_GET,arg); parseRequestLine(httpHeader.METHOD_GET,arg);
// we now know the HTTP version. depending on that, we read the header // we now know the HTTP version. depending on that, we read the header
final String httpVersion = this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_HTTP_VER, httpRequestHeader.HTTP_VERSION_0_9); final String httpVersion = this.prop.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER, httpHeader.HTTP_VERSION_0_9);
final httpRequestHeader header = (httpVersion.equals(httpRequestHeader.HTTP_VERSION_0_9)) final httpRequestHeader header = (httpVersion.equals(httpHeader.HTTP_VERSION_0_9))
? new httpRequestHeader(reverseMappingCache) ? new httpRequestHeader(reverseMappingCache)
: httpRequestHeader.readHeader(this.prop,this.session); : httpRequestHeader.readHeader(this.prop,this.session);
// handling transparent proxy support // handling transparent proxy support
httpRequestHeader.handleTransparentProxySupport(header, this.prop, virtualHost, httpdProxyHandler.isTransparentProxy); httpHeader.handleTransparentProxySupport(header, this.prop, virtualHost, httpdProxyHandler.isTransparentProxy);
// determines if the connection should be kept alive // determines if the connection should be kept alive
handlePersistentConnection(header); handlePersistentConnection(header);
if (this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_HOST).equals(virtualHost)) { if (this.prop.getProperty(httpHeader.CONNECTION_PROP_HOST).equals(virtualHost)) {
// pass to server // pass to server
if (this.allowServer) { if (this.allowServer) {
if (this.handleServerAuthentication(header)) { if (this.handleServerAuthentication(header)) {
@ -506,7 +506,7 @@ public final class httpd implements serverHandler, Cloneable {
} }
} }
return this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_PERSISTENT).equals("keep-alive") ? serverCore.RESUME_CONNECTION : serverCore.TERMINATE_CONNECTION; return this.prop.getProperty(httpHeader.CONNECTION_PROP_PERSISTENT).equals("keep-alive") ? serverCore.RESUME_CONNECTION : serverCore.TERMINATE_CONNECTION;
} catch (final Exception e) { } catch (final Exception e) {
logUnexpectedError(e); logUnexpectedError(e);
return serverCore.TERMINATE_CONNECTION; return serverCore.TERMINATE_CONNECTION;
@ -537,22 +537,22 @@ public final class httpd implements serverHandler, Cloneable {
public Boolean HEAD(final String arg) { public Boolean HEAD(final String arg) {
try { try {
parseRequestLine(httpRequestHeader.METHOD_HEAD,arg); parseRequestLine(httpHeader.METHOD_HEAD,arg);
// we now know the HTTP version. depending on that, we read the header // we now know the HTTP version. depending on that, we read the header
httpRequestHeader header; httpRequestHeader header;
final String httpVersion = this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_HTTP_VER, httpRequestHeader.HTTP_VERSION_0_9); final String httpVersion = this.prop.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER, httpHeader.HTTP_VERSION_0_9);
if (httpVersion.equals(httpRequestHeader.HTTP_VERSION_0_9)) header = new httpRequestHeader(reverseMappingCache); if (httpVersion.equals(httpHeader.HTTP_VERSION_0_9)) header = new httpRequestHeader(reverseMappingCache);
else header = httpRequestHeader.readHeader(this.prop,this.session); else header = httpRequestHeader.readHeader(this.prop,this.session);
// handle transparent proxy support // handle transparent proxy support
httpRequestHeader.handleTransparentProxySupport(header, this.prop, virtualHost, httpdProxyHandler.isTransparentProxy); httpHeader.handleTransparentProxySupport(header, this.prop, virtualHost, httpdProxyHandler.isTransparentProxy);
// determines if the connection should be kept alive // determines if the connection should be kept alive
handlePersistentConnection(header); handlePersistentConnection(header);
// return multi-line message // return multi-line message
if (this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_HOST).equals(virtualHost)) { if (this.prop.getProperty(httpHeader.CONNECTION_PROP_HOST).equals(virtualHost)) {
// pass to server // pass to server
if (allowServer) { if (allowServer) {
if (handleServerAuthentication(header)) { if (handleServerAuthentication(header)) {
@ -576,7 +576,7 @@ public final class httpd implements serverHandler, Cloneable {
return serverCore.TERMINATE_CONNECTION; return serverCore.TERMINATE_CONNECTION;
} }
} }
return this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_PERSISTENT).equals("keep-alive") ? serverCore.RESUME_CONNECTION : serverCore.TERMINATE_CONNECTION; return this.prop.getProperty(httpHeader.CONNECTION_PROP_PERSISTENT).equals("keep-alive") ? serverCore.RESUME_CONNECTION : serverCore.TERMINATE_CONNECTION;
} catch (final Exception e) { } catch (final Exception e) {
logUnexpectedError(e); logUnexpectedError(e);
return serverCore.TERMINATE_CONNECTION; return serverCore.TERMINATE_CONNECTION;
@ -585,22 +585,22 @@ public final class httpd implements serverHandler, Cloneable {
public Boolean POST(final String arg) { public Boolean POST(final String arg) {
try { try {
parseRequestLine(httpRequestHeader.METHOD_POST,arg); parseRequestLine(httpHeader.METHOD_POST,arg);
// we now know the HTTP version. depending on that, we read the header // we now know the HTTP version. depending on that, we read the header
httpRequestHeader header; httpRequestHeader header;
final String httpVersion = this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_HTTP_VER, httpRequestHeader.HTTP_VERSION_0_9); final String httpVersion = this.prop.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER, httpHeader.HTTP_VERSION_0_9);
if (httpVersion.equals(httpHeader.HTTP_VERSION_0_9)) header = new httpRequestHeader(reverseMappingCache); if (httpVersion.equals(httpHeader.HTTP_VERSION_0_9)) header = new httpRequestHeader(reverseMappingCache);
else header = httpRequestHeader.readHeader(this.prop,this.session); else header = httpRequestHeader.readHeader(this.prop,this.session);
// handle transfer-coding // handle transfer-coding
final InputStream sessionIn; final InputStream sessionIn;
final String transferEncoding = header.get(httpRequestHeader.TRANSFER_ENCODING); final String transferEncoding = header.get(httpHeader.TRANSFER_ENCODING);
if (transferEncoding != null) { if (transferEncoding != null) {
if (!httpRequestHeader.HTTP_VERSION_1_1.equals(httpVersion)) { if (!httpHeader.HTTP_VERSION_1_1.equals(httpVersion)) {
log.logWarning("client "+ session.getName() +" uses transfer-coding with HTTP version "+ httpVersion +"!"); log.logWarning("client "+ session.getName() +" uses transfer-coding with HTTP version "+ httpVersion +"!");
} }
if("chunked".equalsIgnoreCase(header.get(httpRequestHeader.TRANSFER_ENCODING))) { if("chunked".equalsIgnoreCase(header.get(httpHeader.TRANSFER_ENCODING))) {
sessionIn = new ChunkedInputStream(this.session.in); sessionIn = new ChunkedInputStream(this.session.in);
} else { } else {
// "A server which receives an entity-body with a transfer-coding it does // "A server which receives an entity-body with a transfer-coding it does
@ -614,13 +614,13 @@ public final class httpd implements serverHandler, Cloneable {
} }
// handle transparent proxy support // handle transparent proxy support
httpRequestHeader.handleTransparentProxySupport(header, this.prop, virtualHost, httpdProxyHandler.isTransparentProxy); httpHeader.handleTransparentProxySupport(header, this.prop, virtualHost, httpdProxyHandler.isTransparentProxy);
// determines if the connection should be kept alive // determines if the connection should be kept alive
handlePersistentConnection(header); handlePersistentConnection(header);
// return multi-line message // return multi-line message
if (prop.getProperty(httpRequestHeader.CONNECTION_PROP_HOST).equals(virtualHost)) { if (prop.getProperty(httpHeader.CONNECTION_PROP_HOST).equals(virtualHost)) {
// pass to server // pass to server
if (allowServer) { if (allowServer) {
if (handleServerAuthentication(header)) { if (handleServerAuthentication(header)) {
@ -644,7 +644,7 @@ public final class httpd implements serverHandler, Cloneable {
} }
if(sessionIn instanceof ChunkedInputStream) sessionIn.close(); // read to end, but do not close the stream (maybe HTTP/1.1 persistent) if(sessionIn instanceof ChunkedInputStream) sessionIn.close(); // read to end, but do not close the stream (maybe HTTP/1.1 persistent)
//return serverCore.RESUME_CONNECTION; //return serverCore.RESUME_CONNECTION;
return this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_PERSISTENT).equals("keep-alive") ? serverCore.RESUME_CONNECTION : serverCore.TERMINATE_CONNECTION; return this.prop.getProperty(httpHeader.CONNECTION_PROP_PERSISTENT).equals("keep-alive") ? serverCore.RESUME_CONNECTION : serverCore.TERMINATE_CONNECTION;
} catch (final Exception e) { } catch (final Exception e) {
logUnexpectedError(e); logUnexpectedError(e);
return serverCore.TERMINATE_CONNECTION; return serverCore.TERMINATE_CONNECTION;
@ -663,10 +663,10 @@ public final class httpd implements serverHandler, Cloneable {
httpVersion = arg.substring(pos + 1); httpVersion = arg.substring(pos + 1);
arg = arg.substring(0, pos); arg = arg.substring(0, pos);
} }
prop.setProperty(httpRequestHeader.CONNECTION_PROP_HTTP_VER, httpVersion); prop.setProperty(httpHeader.CONNECTION_PROP_HTTP_VER, httpVersion);
// parse hostname and port // parse hostname and port
prop.setProperty(httpRequestHeader.CONNECTION_PROP_HOST, arg); prop.setProperty(httpHeader.CONNECTION_PROP_HOST, arg);
pos = arg.indexOf(":"); pos = arg.indexOf(":");
int port = 443; int port = 443;
if (pos >= 0) { if (pos >= 0) {
@ -675,11 +675,11 @@ public final class httpd implements serverHandler, Cloneable {
} }
// setting other connection properties // setting other connection properties
prop.setProperty(httpRequestHeader.CONNECTION_PROP_CLIENTIP, this.clientIP); prop.setProperty(httpHeader.CONNECTION_PROP_CLIENTIP, this.clientIP);
prop.setProperty(httpRequestHeader.CONNECTION_PROP_METHOD, httpHeader.METHOD_CONNECT); prop.setProperty(httpHeader.CONNECTION_PROP_METHOD, httpHeader.METHOD_CONNECT);
prop.setProperty(httpRequestHeader.CONNECTION_PROP_PATH, "/"); prop.setProperty(httpHeader.CONNECTION_PROP_PATH, "/");
prop.setProperty(httpRequestHeader.CONNECTION_PROP_EXT, ""); prop.setProperty(httpHeader.CONNECTION_PROP_EXT, "");
prop.setProperty(httpRequestHeader.CONNECTION_PROP_URL, ""); prop.setProperty(httpHeader.CONNECTION_PROP_URL, "");
// parse remaining lines // parse remaining lines
final httpRequestHeader header = httpRequestHeader.readHeader(this.prop,this.session); final httpRequestHeader header = httpRequestHeader.readHeader(this.prop,this.session);
@ -716,18 +716,18 @@ public final class httpd implements serverHandler, Cloneable {
httpRequestHeader.parseRequestLine(cmd,s,this.prop,virtualHost); httpRequestHeader.parseRequestLine(cmd,s,this.prop,virtualHost);
// track the request // track the request
final String path = this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_URL); final String path = this.prop.getProperty(httpHeader.CONNECTION_PROP_URL);
final String args = this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_ARGS, ""); final String args = this.prop.getProperty(httpHeader.CONNECTION_PROP_ARGS, "");
switchboard.track(this.userAddress.getHostName(), (args.length() > 0) ? path + "?" + args : path); switchboard.track(this.userAddress.getHostName(), (args.length() > 0) ? path + "?" + args : path);
// reseting the empty request counter // reseting the empty request counter
this.emptyRequestCount = 0; this.emptyRequestCount = 0;
// counting the amount of received requests within this permanent connection // counting the amount of received requests within this permanent connection
this.prop.setProperty(httpRequestHeader.CONNECTION_PROP_KEEP_ALIVE_COUNT, Integer.toString(++this.keepAliveRequestCount)); this.prop.setProperty(httpHeader.CONNECTION_PROP_KEEP_ALIVE_COUNT, Integer.toString(++this.keepAliveRequestCount));
// setting the client-IP // setting the client-IP
this.prop.setProperty(httpRequestHeader.CONNECTION_PROP_CLIENTIP, this.clientIP); this.prop.setProperty(httpHeader.CONNECTION_PROP_CLIENTIP, this.clientIP);
} }
// some static methods that needs to be used from any CGI // some static methods that needs to be used from any CGI
@ -1109,20 +1109,20 @@ public final class httpd implements serverHandler, Cloneable {
ByteArrayOutputStream o = null; ByteArrayOutputStream o = null;
try { try {
// setting the proper http status message // setting the proper http status message
final String httpVersion = conProp.getProperty(httpRequestHeader.CONNECTION_PROP_HTTP_VER,"HTTP/1.1"); final String httpVersion = conProp.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER,"HTTP/1.1");
if ((httpStatusText == null)||(httpStatusText.length()==0)) { if ((httpStatusText == null)||(httpStatusText.length()==0)) {
if (httpVersion.equals("HTTP/1.0") && httpRequestHeader.http1_0.containsKey(Integer.toString(httpStatusCode))) if (httpVersion.equals("HTTP/1.0") && httpHeader.http1_0.containsKey(Integer.toString(httpStatusCode)))
httpStatusText = httpRequestHeader.http1_0.get(Integer.toString(httpStatusCode)); httpStatusText = httpHeader.http1_0.get(Integer.toString(httpStatusCode));
else if (httpVersion.equals("HTTP/1.1") && httpRequestHeader.http1_1.containsKey(Integer.toString(httpStatusCode))) else if (httpVersion.equals("HTTP/1.1") && httpHeader.http1_1.containsKey(Integer.toString(httpStatusCode)))
httpStatusText = httpRequestHeader.http1_1.get(Integer.toString(httpStatusCode)); httpStatusText = httpHeader.http1_1.get(Integer.toString(httpStatusCode));
else httpStatusText = "Unknown"; else httpStatusText = "Unknown";
} }
// generating the desired request url // generating the desired request url
String host = conProp.getProperty(httpRequestHeader.CONNECTION_PROP_HOST); String host = conProp.getProperty(httpHeader.CONNECTION_PROP_HOST);
final String path = conProp.getProperty(httpRequestHeader.CONNECTION_PROP_PATH,"/"); final String path = conProp.getProperty(httpHeader.CONNECTION_PROP_PATH,"/");
final String args = conProp.getProperty(httpRequestHeader.CONNECTION_PROP_ARGS); final String args = conProp.getProperty(httpHeader.CONNECTION_PROP_ARGS);
final String method = conProp.getProperty(httpRequestHeader.CONNECTION_PROP_METHOD); final String method = conProp.getProperty(httpHeader.CONNECTION_PROP_METHOD);
int port = 80; int port = 80;
final int pos = host.indexOf(":"); final int pos = host.indexOf(":");
@ -1133,7 +1133,7 @@ public final class httpd implements serverHandler, Cloneable {
String urlString; String urlString;
try { try {
urlString = (new yacyURL((method.equals(httpRequestHeader.METHOD_CONNECT)?"https":"http"), host, port, (args == null) ? path : path + "?" + args)).toString(); urlString = (new yacyURL((method.equals(httpHeader.METHOD_CONNECT)?"https":"http"), host, port, (args == null) ? path : path + "?" + args)).toString();
} catch (final MalformedURLException e) { } catch (final MalformedURLException e) {
urlString = "invalid URL"; urlString = "invalid URL";
} }
@ -1144,7 +1144,7 @@ public final class httpd implements serverHandler, Cloneable {
// tp.put("host", serverCore.publicIP().getHostAddress()); // tp.put("host", serverCore.publicIP().getHostAddress());
// tp.put("port", switchboard.getConfig("port", "8080")); // tp.put("port", switchboard.getConfig("port", "8080"));
final String clientIP = conProp.getProperty(httpRequestHeader.CONNECTION_PROP_CLIENTIP, "127.0.0.1"); final String clientIP = conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP, "127.0.0.1");
// check if ip is local ip address // check if ip is local ip address
final InetAddress hostAddress = serverDomains.dnsResolve(clientIP); final InetAddress hostAddress = serverDomains.dnsResolve(clientIP);
@ -1171,7 +1171,7 @@ public final class httpd implements serverHandler, Cloneable {
tp.put("peerName", (getAlternativeResolver() == null) ? "" : getAlternativeResolver().myName()); tp.put("peerName", (getAlternativeResolver() == null) ? "" : getAlternativeResolver().myName());
tp.put("errorMessageType", errorcase); tp.put("errorMessageType", errorcase);
tp.put("httpStatus", Integer.toString(httpStatusCode) + " " + httpStatusText); tp.put("httpStatus", Integer.toString(httpStatusCode) + " " + httpStatusText);
tp.put("requestMethod", conProp.getProperty(httpRequestHeader.CONNECTION_PROP_METHOD)); tp.put("requestMethod", conProp.getProperty(httpHeader.CONNECTION_PROP_METHOD));
tp.put("requestURL", urlString); tp.put("requestURL", urlString);
switch (errorcase) { switch (errorcase) {
@ -1220,11 +1220,11 @@ public final class httpd implements serverHandler, Cloneable {
if(header == null) if(header == null)
header = new httpResponseHeader(); header = new httpResponseHeader();
header.put(httpResponseHeader.CONNECTION_PROP_PROXY_RESPOND_STATUS, Integer.toString(httpStatusCode)); header.put(httpHeader.CONNECTION_PROP_PROXY_RESPOND_STATUS, Integer.toString(httpStatusCode));
header.put(httpResponseHeader.DATE, systemDate); header.put(httpHeader.DATE, systemDate);
header.put(httpResponseHeader.CONTENT_TYPE, "text/html"); header.put(httpHeader.CONTENT_TYPE, "text/html");
header.put(httpResponseHeader.CONTENT_LENGTH, Integer.toString(result.length)); header.put(httpHeader.CONTENT_LENGTH, Integer.toString(result.length));
header.put(httpResponseHeader.PRAGMA, "no-cache"); header.put(httpHeader.PRAGMA, "no-cache");
sendRespondHeader(conProp,respond,httpVersion,httpStatusCode,httpStatusText,header); sendRespondHeader(conProp,respond,httpVersion,httpStatusCode,httpStatusText,header);
if (! method.equals(httpHeader.METHOD_HEAD)) { if (! method.equals(httpHeader.METHOD_HEAD)) {
@ -1274,17 +1274,17 @@ public final class httpd implements serverHandler, Cloneable {
if(headers==null) headers = new httpResponseHeader(); if(headers==null) headers = new httpResponseHeader();
final Date now = new Date(System.currentTimeMillis()); final Date now = new Date(System.currentTimeMillis());
headers.put(httpResponseHeader.SERVER, "AnomicHTTPD (www.anomic.de)"); headers.put(httpHeader.SERVER, "AnomicHTTPD (www.anomic.de)");
headers.put(httpResponseHeader.DATE, DateFormatter.formatRFC1123(now)); headers.put(httpHeader.DATE, DateFormatter.formatRFC1123(now));
if (moddate.after(now)) { if (moddate.after(now)) {
System.out.println("*** DEBUG: correcting moddate = " + moddate.toString() + " to now = " + now.toString()); System.out.println("*** DEBUG: correcting moddate = " + moddate.toString() + " to now = " + now.toString());
moddate = now; moddate = now;
} }
headers.put(httpResponseHeader.LAST_MODIFIED, DateFormatter.formatRFC1123(moddate)); headers.put(httpHeader.LAST_MODIFIED, DateFormatter.formatRFC1123(moddate));
if (nocache) { if (nocache) {
if (httpVersion.toUpperCase().equals(httpHeader.HTTP_VERSION_1_1)) headers.put(httpResponseHeader.CACHE_CONTROL, "no-cache"); if (httpVersion.toUpperCase().equals(httpHeader.HTTP_VERSION_1_1)) headers.put(httpHeader.CACHE_CONTROL, "no-cache");
else headers.put(httpResponseHeader.PRAGMA, "no-cache"); else headers.put(httpHeader.PRAGMA, "no-cache");
} }
if (contentType == null) if (contentType == null)
@ -1292,11 +1292,11 @@ public final class httpd implements serverHandler, Cloneable {
else if (contentType.startsWith("text/") && contentType.toLowerCase().indexOf("charset=")==-1) else if (contentType.startsWith("text/") && contentType.toLowerCase().indexOf("charset=")==-1)
contentType +="; charset=" + Charset.defaultCharset().name(); contentType +="; charset=" + Charset.defaultCharset().name();
headers.put(httpHeader.CONTENT_TYPE, contentType); headers.put(httpHeader.CONTENT_TYPE, contentType);
if (contentLength > 0) headers.put(httpResponseHeader.CONTENT_LENGTH, Long.toString(contentLength)); if (contentLength > 0) headers.put(httpHeader.CONTENT_LENGTH, Long.toString(contentLength));
//if (cookie != null) headers.put(httpResponseHeader.SET_COOKIE, cookie); //if (cookie != null) headers.put(httpHeader.SET_COOKIE, cookie);
if (expires != null) headers.put(httpResponseHeader.EXPIRES, DateFormatter.formatRFC1123(expires)); if (expires != null) headers.put(httpHeader.EXPIRES, DateFormatter.formatRFC1123(expires));
if (contentEnc != null) headers.put(httpResponseHeader.CONTENT_ENCODING, contentEnc); if (contentEnc != null) headers.put(httpHeader.CONTENT_ENCODING, contentEnc);
if (transferEnc != null) headers.put(httpResponseHeader.TRANSFER_ENCODING, transferEnc); if (transferEnc != null) headers.put(httpHeader.TRANSFER_ENCODING, transferEnc);
sendRespondHeader(conProp, respond, httpVersion, httpStatusCode, httpStatusText, headers); sendRespondHeader(conProp, respond, httpVersion, httpStatusCode, httpStatusText, headers);
} }
@ -1327,10 +1327,10 @@ public final class httpd implements serverHandler, Cloneable {
try { try {
if ((httpStatusText == null)||(httpStatusText.length()==0)) { if ((httpStatusText == null)||(httpStatusText.length()==0)) {
if (httpVersion.equals(httpResponseHeader.HTTP_VERSION_1_0) && httpHeader.http1_0.containsKey(Integer.toString(httpStatusCode))) if (httpVersion.equals(httpHeader.HTTP_VERSION_1_0) && httpHeader.http1_0.containsKey(Integer.toString(httpStatusCode)))
httpStatusText = httpResponseHeader.http1_0.get(Integer.toString(httpStatusCode)); httpStatusText = httpHeader.http1_0.get(Integer.toString(httpStatusCode));
else if (httpVersion.equals(httpResponseHeader.HTTP_VERSION_1_1) && httpHeader.http1_1.containsKey(Integer.toString(httpStatusCode))) else if (httpVersion.equals(httpHeader.HTTP_VERSION_1_1) && httpHeader.http1_1.containsKey(Integer.toString(httpStatusCode)))
httpStatusText = httpResponseHeader.http1_1.get(Integer.toString(httpStatusCode)); httpStatusText = httpHeader.http1_1.get(Integer.toString(httpStatusCode));
else httpStatusText = "Unknown"; else httpStatusText = "Unknown";
} }
@ -1360,9 +1360,9 @@ public final class httpd implements serverHandler, Cloneable {
responseHeader.put(httpHeader.CONTENT_LENGTH, "0"); responseHeader.put(httpHeader.CONTENT_LENGTH, "0");
// adding some yacy specific headers // adding some yacy specific headers
responseHeader.put(httpResponseHeader.X_YACY_KEEP_ALIVE_REQUEST_COUNT,conProp.getProperty(httpRequestHeader.CONNECTION_PROP_KEEP_ALIVE_COUNT)); responseHeader.put(httpHeader.X_YACY_KEEP_ALIVE_REQUEST_COUNT,conProp.getProperty(httpHeader.CONNECTION_PROP_KEEP_ALIVE_COUNT));
responseHeader.put(httpResponseHeader.X_YACY_ORIGINAL_REQUEST_LINE,conProp.getProperty(httpRequestHeader.CONNECTION_PROP_REQUESTLINE)); responseHeader.put(httpHeader.X_YACY_ORIGINAL_REQUEST_LINE,conProp.getProperty(httpHeader.CONNECTION_PROP_REQUESTLINE));
responseHeader.put(httpResponseHeader.X_YACY_PREVIOUS_REQUEST_LINE,conProp.getProperty(httpRequestHeader.CONNECTION_PROP_PREV_REQUESTLINE)); responseHeader.put(httpHeader.X_YACY_PREVIOUS_REQUEST_LINE,conProp.getProperty(httpHeader.CONNECTION_PROP_PREV_REQUESTLINE));
//read custom headers //read custom headers
/* /*

8
source/de/anomic/http/httpdFileHandler.java
Unescape View File

@ -228,10 +228,10 @@ public final class httpdFileHandler {
} else { } else {
ext = path.substring(pos + 1).toLowerCase(); ext = path.substring(pos + 1).toLowerCase();
} }
headers.put(httpResponseHeader.SERVER, "AnomicHTTPD (www.anomic.de)"); headers.put(httpHeader.SERVER, "AnomicHTTPD (www.anomic.de)");
headers.put(httpResponseHeader.DATE, DateFormatter.formatRFC1123(new Date())); headers.put(httpHeader.DATE, DateFormatter.formatRFC1123(new Date()));
if(!(plasmaParser.mediaExtContains(ext))){ if(!(plasmaParser.mediaExtContains(ext))){
headers.put(httpResponseHeader.PRAGMA, "no-cache"); headers.put(httpHeader.PRAGMA, "no-cache");
} }
return headers; return headers;
} }
@ -580,7 +580,7 @@ public final class httpdFileHandler {
// set environment variables // set environment variables
Map<String, String> env = pb.environment(); Map<String, String> env = pb.environment();
env.put("SERVER_SOFTWARE", getDefaultHeaders(path).get(httpResponseHeader.SERVER)); env.put("SERVER_SOFTWARE", getDefaultHeaders(path).get(httpHeader.SERVER));
env.put("SERVER_NAME", switchboard.getConfig("peerName", "<nameless>")); env.put("SERVER_NAME", switchboard.getConfig("peerName", "<nameless>"));
env.put("GATEWAY_INTERFACE", "CGI/1.1"); env.put("GATEWAY_INTERFACE", "CGI/1.1");
if (httpVersion != null) { if (httpVersion != null) {

18
source/de/anomic/http/httpdProxyCacheEntry.java
Unescape View File

@ -304,7 +304,7 @@ public class httpdProxyCacheEntry implements Document {
// -pragma in response // -pragma in response
// if we have a pragma non-cache, we don't cache. usually if this is wanted from // if we have a pragma non-cache, we don't cache. usually if this is wanted from
// the server, it makes sense // the server, it makes sense
String cacheControl = responseHeader.get(httpResponseHeader.PRAGMA); String cacheControl = responseHeader.get(httpHeader.PRAGMA);
if (cacheControl != null && cacheControl.trim().toUpperCase().equals("NO-CACHE")) { return "controlled_no_cache"; } if (cacheControl != null && cacheControl.trim().toUpperCase().equals("NO-CACHE")) { return "controlled_no_cache"; }
// -expires in response // -expires in response
@ -313,7 +313,7 @@ public class httpdProxyCacheEntry implements Document {
// -cache-control in response // -cache-control in response
// the cache-control has many value options. // the cache-control has many value options.
cacheControl = responseHeader.get(httpResponseHeader.CACHE_CONTROL); cacheControl = responseHeader.get(httpHeader.CACHE_CONTROL);
if (cacheControl != null) { if (cacheControl != null) {
cacheControl = cacheControl.trim().toUpperCase(); cacheControl = cacheControl.trim().toUpperCase();
if (cacheControl.startsWith("MAX-AGE=")) { if (cacheControl.startsWith("MAX-AGE=")) {
@ -364,10 +364,10 @@ public class httpdProxyCacheEntry implements Document {
if (requestHeader.containsKey(httpHeader.RANGE)) { return false; } if (requestHeader.containsKey(httpHeader.RANGE)) { return false; }
// if the client requests a un-cached copy of the resource ... // if the client requests a un-cached copy of the resource ...
cacheControl = requestHeader.get(httpResponseHeader.PRAGMA); cacheControl = requestHeader.get(httpHeader.PRAGMA);
if (cacheControl != null && cacheControl.trim().toUpperCase().equals("NO-CACHE")) { return false; } if (cacheControl != null && cacheControl.trim().toUpperCase().equals("NO-CACHE")) { return false; }
cacheControl = requestHeader.get(httpResponseHeader.CACHE_CONTROL); cacheControl = requestHeader.get(httpHeader.CACHE_CONTROL);
if (cacheControl != null) { if (cacheControl != null) {
cacheControl = cacheControl.trim().toUpperCase(); cacheControl = cacheControl.trim().toUpperCase();
if (cacheControl.startsWith("NO-CACHE") || cacheControl.startsWith("MAX-AGE=0")) { return false; } if (cacheControl.startsWith("NO-CACHE") || cacheControl.startsWith("MAX-AGE=0")) { return false; }
@ -379,7 +379,7 @@ public class httpdProxyCacheEntry implements Document {
if (requestHeader.containsKey(httpRequestHeader.IF_MODIFIED_SINCE)) { if (requestHeader.containsKey(httpRequestHeader.IF_MODIFIED_SINCE)) {
// checking this makes only sense if the cached response contains // checking this makes only sense if the cached response contains
// a Last-Modified field. If the field does not exist, we go the safe way // a Last-Modified field. If the field does not exist, we go the safe way
if (!responseHeader.containsKey(httpResponseHeader.LAST_MODIFIED)) { return false; } if (!responseHeader.containsKey(httpHeader.LAST_MODIFIED)) { return false; }
// parse date // parse date
Date d1, d2; Date d1, d2;
d2 = responseHeader.lastModified(); if (d2 == null) { d2 = new Date(DateFormatter.correctedUTCTime()); } d2 = responseHeader.lastModified(); if (d2 == null) { d2 = new Date(DateFormatter.correctedUTCTime()); }
@ -396,8 +396,8 @@ public class httpdProxyCacheEntry implements Document {
// -set-cookie in cached response // -set-cookie in cached response
// this is a similar case as for COOKIE. // this is a similar case as for COOKIE.
if (requestHeader.containsKey(httpRequestHeader.COOKIE) || if (requestHeader.containsKey(httpRequestHeader.COOKIE) ||
responseHeader.containsKey(httpResponseHeader.SET_COOKIE) || responseHeader.containsKey(httpHeader.SET_COOKIE) ||
responseHeader.containsKey(httpResponseHeader.SET_COOKIE2)) { responseHeader.containsKey(httpHeader.SET_COOKIE2)) {
return false; // too strong return false; // too strong
} }
} }
@ -408,7 +408,7 @@ public class httpdProxyCacheEntry implements Document {
// logically, we would not need to care about no-cache pragmas in cached response headers, // logically, we would not need to care about no-cache pragmas in cached response headers,
// because they cannot exist since they are not written to the cache. // because they cannot exist since they are not written to the cache.
// So this IF should always fail.. // So this IF should always fail..
cacheControl = responseHeader.get(httpResponseHeader.PRAGMA); cacheControl = responseHeader.get(httpHeader.PRAGMA);
if (cacheControl != null && cacheControl.trim().toUpperCase().equals("NO-CACHE")) { return false; } if (cacheControl != null && cacheControl.trim().toUpperCase().equals("NO-CACHE")) { return false; }
// see for documentation also: // see for documentation also:
@ -427,7 +427,7 @@ public class httpdProxyCacheEntry implements Document {
if (expires.before(new Date(DateFormatter.correctedUTCTime()))) { return false; } if (expires.before(new Date(DateFormatter.correctedUTCTime()))) { return false; }
} }
final Date lastModified = responseHeader.lastModified(); final Date lastModified = responseHeader.lastModified();
cacheControl = responseHeader.get(httpResponseHeader.CACHE_CONTROL); cacheControl = responseHeader.get(httpHeader.CACHE_CONTROL);
if (cacheControl == null && lastModified == null && expires == null) { return false; } if (cacheControl == null && lastModified == null && expires == null) { return false; }
// -lastModified in cached response // -lastModified in cached response

26
source/de/anomic/http/httpdProxyHandler.java
Unescape View File

@ -259,8 +259,8 @@ public final class httpdProxyHandler {
| "Version" "=" 1*DIGIT | "Version" "=" 1*DIGIT
*/ */
if (sb.getConfigBool("proxy.monitorCookies", false)) { if (sb.getConfigBool("proxy.monitorCookies", false)) {
if (respondHeader.containsKey(httpResponseHeader.SET_COOKIE)) { if (respondHeader.containsKey(httpHeader.SET_COOKIE)) {
final Object[] entry = new Object[]{new Date(), targetclient, respondHeader.getMultiple(httpResponseHeader.SET_COOKIE)}; final Object[] entry = new Object[]{new Date(), targetclient, respondHeader.getMultiple(httpHeader.SET_COOKIE)};
synchronized(sb.incomingCookies) { synchronized(sb.incomingCookies) {
sb.incomingCookies.put(serverhost, entry); sb.incomingCookies.put(serverhost, entry);
} }
@ -511,7 +511,7 @@ public final class httpdProxyHandler {
// sending the respond header back to the client // sending the respond header back to the client
if (chunkedOut != null) { if (chunkedOut != null) {
responseHeader.put(httpResponseHeader.TRANSFER_ENCODING, "chunked"); responseHeader.put(httpHeader.TRANSFER_ENCODING, "chunked");
} }
if (theLogger.isFinest()) theLogger.logFinest(reqID +" sending response header: "+ responseHeader); if (theLogger.isFinest()) theLogger.logFinest(reqID +" sending response header: "+ responseHeader);
@ -672,7 +672,7 @@ public final class httpdProxyHandler {
conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_REFRESH_HIT"); conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_REFRESH_HIT");
// setting the content length header to 0 // setting the content length header to 0
cachedResponseHeader.put(httpResponseHeader.CONTENT_LENGTH, Integer.toString(0)); cachedResponseHeader.put(httpHeader.CONTENT_LENGTH, Integer.toString(0));
// send cached header with replaced date and added length // send cached header with replaced date and added length
httpd.sendRespondHeader(conProp,respond,httpVer,304,cachedResponseHeader); httpd.sendRespondHeader(conProp,respond,httpVer,304,cachedResponseHeader);
@ -683,7 +683,7 @@ public final class httpdProxyHandler {
conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_HIT"); conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_HIT");
// setting the content header to the proper length // setting the content header to the proper length
cachedResponseHeader.put(httpResponseHeader.CONTENT_LENGTH, Long.toString(cacheEntry.length)); cachedResponseHeader.put(httpHeader.CONTENT_LENGTH, Long.toString(cacheEntry.length));
// send cached header with replaced date and added length // send cached header with replaced date and added length
httpd.sendRespondHeader(conProp,respond,httpVer,203,cachedResponseHeader); httpd.sendRespondHeader(conProp,respond,httpVer,203,cachedResponseHeader);
@ -920,7 +920,7 @@ public final class httpdProxyHandler {
// sending the respond header back to the client // sending the respond header back to the client
if (chunked != null) { if (chunked != null) {
responseHeader.put(httpResponseHeader.TRANSFER_ENCODING, "chunked"); responseHeader.put(httpHeader.TRANSFER_ENCODING, "chunked");
} }
// sending response headers // sending response headers
@ -1011,10 +1011,10 @@ public final class httpdProxyHandler {
} }
// only gzip-encoding is supported, remove other encodings (e. g. deflate) // only gzip-encoding is supported, remove other encodings (e. g. deflate)
if (((String)requestHeader.get(httpRequestHeader.ACCEPT_ENCODING,"")).indexOf("gzip") != -1) { if (((String)requestHeader.get(httpHeader.ACCEPT_ENCODING,"")).indexOf("gzip") != -1) {
requestHeader.put(httpRequestHeader.ACCEPT_ENCODING, "gzip"); requestHeader.put(httpHeader.ACCEPT_ENCODING, "gzip");
} else { } else {
requestHeader.put(httpRequestHeader.ACCEPT_ENCODING, ""); requestHeader.put(httpHeader.ACCEPT_ENCODING, "");
} }
addXForwardedForHeader(conProp, requestHeader); addXForwardedForHeader(conProp, requestHeader);
@ -1111,7 +1111,7 @@ public final class httpdProxyHandler {
private static void correctContentEncoding(final httpResponseHeader responseHeader) { private static void correctContentEncoding(final httpResponseHeader responseHeader) {
// TODO gzip again? set "correct" encoding? // TODO gzip again? set "correct" encoding?
if(responseHeader.gzip()) { if(responseHeader.gzip()) {
responseHeader.remove(httpResponseHeader.CONTENT_ENCODING); responseHeader.remove(httpHeader.CONTENT_ENCODING);
responseHeader.remove(httpHeader.CONTENT_LENGTH); // remove gziped length responseHeader.remove(httpHeader.CONTENT_LENGTH); // remove gziped length
} }
} }
@ -1158,11 +1158,11 @@ public final class httpdProxyHandler {
headers.remove(httpRequestHeader.X_CACHE_LOOKUP); headers.remove(httpRequestHeader.X_CACHE_LOOKUP);
// remove transfer encoding header // remove transfer encoding header
headers.remove(httpResponseHeader.TRANSFER_ENCODING); headers.remove(httpHeader.TRANSFER_ENCODING);
//removing yacy status headers //removing yacy status headers
headers.remove(httpResponseHeader.X_YACY_KEEP_ALIVE_REQUEST_COUNT); headers.remove(httpHeader.X_YACY_KEEP_ALIVE_REQUEST_COUNT);
headers.remove(httpResponseHeader.X_YACY_ORIGINAL_REQUEST_LINE); headers.remove(httpHeader.X_YACY_ORIGINAL_REQUEST_LINE);
} }
private static void setViaHeader(final httpHeader header, final String httpVer) { private static void setViaHeader(final httpHeader header, final String httpVer) {

9
source/de/anomic/icap/icapd.java
Unescape View File

@ -37,6 +37,7 @@ import java.util.Date;
import java.util.Properties; import java.util.Properties;
import de.anomic.http.httpChunkedInputStream; import de.anomic.http.httpChunkedInputStream;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.http.httpResponseHeader; import de.anomic.http.httpResponseHeader;
import de.anomic.http.httpdProxyCacheEntry; import de.anomic.http.httpdProxyCacheEntry;
@ -310,9 +311,9 @@ public class icapd implements serverHandler, Cloneable {
final Properties httpReqProps = new Properties(); final Properties httpReqProps = new Properties();
httpRequestHeader.parseRequestLine(httpRequestLine,httpReqProps,virtualHost); httpRequestHeader.parseRequestLine(httpRequestLine,httpReqProps,virtualHost);
if (!httpReqProps.getProperty(httpRequestHeader.CONNECTION_PROP_METHOD).equals(httpRequestHeader.METHOD_GET)) { if (!httpReqProps.getProperty(httpHeader.CONNECTION_PROP_METHOD).equals(httpHeader.METHOD_GET)) {
log.logInfo("Wrong http request method for indexing:" + log.logInfo("Wrong http request method for indexing:" +
"\nRequest Method: " + httpReqProps.getProperty(httpRequestHeader.CONNECTION_PROP_METHOD) + "\nRequest Method: " + httpReqProps.getProperty(httpHeader.CONNECTION_PROP_METHOD) +
"\nRequest Line: " + httpRequestLine); "\nRequest Line: " + httpRequestLine);
reader.close(); reader.close();
if(reqHdrStream != null) { if(reqHdrStream != null) {
@ -330,10 +331,10 @@ public class icapd implements serverHandler, Cloneable {
} }
// handle transparent proxy support: this function call is needed to set the host property properly // handle transparent proxy support: this function call is needed to set the host property properly
httpRequestHeader.handleTransparentProxySupport(httpReqHeader,httpReqProps,virtualHost,true); httpHeader.handleTransparentProxySupport(httpReqHeader,httpReqProps,virtualHost,true);
// getting the request URL // getting the request URL
final yacyURL httpRequestURL = httpRequestHeader.getRequestURL(httpReqProps); final yacyURL httpRequestURL = httpHeader.getRequestURL(httpReqProps);
/* ========================================================================= /* =========================================================================
* Parsing response data * Parsing response data

3
source/de/anomic/plasma/parser/odt/odtParser.java
Unescape View File

@ -44,6 +44,7 @@ import com.catcode.odf.OpenDocumentTextInputStream;
import de.anomic.crawler.HTTPLoader; import de.anomic.crawler.HTTPLoader;
import de.anomic.http.httpClient; import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.FileUtils;
import de.anomic.plasma.plasmaParserDocument; import de.anomic.plasma.plasmaParserDocument;
@ -245,7 +246,7 @@ public class odtParser extends AbstractParser implements Parser {
// downloading the document content // downloading the document content
final httpRequestHeader reqHeader = new httpRequestHeader(); final httpRequestHeader reqHeader = new httpRequestHeader();
reqHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.crawlerUserAgent); reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.crawlerUserAgent);
final byte[] content = httpClient.wget(contentUrl.toString(), reqHeader, 10000); final byte[] content = httpClient.wget(contentUrl.toString(), reqHeader, 10000);
final ByteArrayInputStream input = new ByteArrayInputStream(content); final ByteArrayInputStream input = new ByteArrayInputStream(content);

3
source/de/anomic/plasma/parser/rpm/rpmParser.java
Unescape View File

@ -35,6 +35,7 @@ import com.jguild.jrpm.io.datatype.DataTypeIf;
import de.anomic.crawler.HTTPLoader; import de.anomic.crawler.HTTPLoader;
import de.anomic.http.httpClient; import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.FileUtils;
import de.anomic.plasma.plasmaParserDocument; import de.anomic.plasma.plasmaParserDocument;
@ -165,7 +166,7 @@ public class rpmParser extends AbstractParser implements Parser {
final rpmParser testParser = new rpmParser(); final rpmParser testParser = new rpmParser();
final httpRequestHeader reqHeader = new httpRequestHeader(); final httpRequestHeader reqHeader = new httpRequestHeader();
reqHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.crawlerUserAgent); reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.crawlerUserAgent);
final byte[] content = httpClient.wget(contentUrl.toString(), reqHeader, 10000); final byte[] content = httpClient.wget(contentUrl.toString(), reqHeader, 10000);
final ByteArrayInputStream input = new ByteArrayInputStream(content); final ByteArrayInputStream input = new ByteArrayInputStream(content);
testParser.parse(contentUrl, "application/x-rpm", null, input); testParser.parse(contentUrl, "application/x-rpm", null, input);

3
source/de/anomic/plasma/parser/vcf/vcfParser.java
Unescape View File

@ -36,6 +36,7 @@ import java.util.LinkedList;
import de.anomic.crawler.HTTPLoader; import de.anomic.crawler.HTTPLoader;
import de.anomic.http.httpClient; import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.order.Base64Order; import de.anomic.kelondro.order.Base64Order;
import de.anomic.plasma.plasmaParserDocument; import de.anomic.plasma.plasmaParserDocument;
@ -276,7 +277,7 @@ public class vcfParser extends AbstractParser implements Parser {
final vcfParser testParser = new vcfParser(); final vcfParser testParser = new vcfParser();
final httpRequestHeader reqHeader = new httpRequestHeader(); final httpRequestHeader reqHeader = new httpRequestHeader();
reqHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.crawlerUserAgent); reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.crawlerUserAgent);
final byte[] content = httpClient.wget(contentUrl.toString(), reqHeader, 10000); final byte[] content = httpClient.wget(contentUrl.toString(), reqHeader, 10000);
final ByteArrayInputStream input = new ByteArrayInputStream(content); final ByteArrayInputStream input = new ByteArrayInputStream(content);
testParser.parse(contentUrl, "text/x-vcard", "UTF-8",input); testParser.parse(contentUrl, "text/x-vcard", "UTF-8",input);

11
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -144,6 +144,7 @@ import de.anomic.data.wiki.wikiBoard;
import de.anomic.data.wiki.wikiCode; import de.anomic.data.wiki.wikiCode;
import de.anomic.data.wiki.wikiParser; import de.anomic.data.wiki.wikiParser;
import de.anomic.http.httpClient; import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRemoteProxyConfig; import de.anomic.http.httpRemoteProxyConfig;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.http.httpResponseHeader; import de.anomic.http.httpResponseHeader;
@ -1931,7 +1932,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
public int adminAuthenticated(final httpRequestHeader requestHeader) { public int adminAuthenticated(final httpRequestHeader requestHeader) {
// authorization for localhost, only if flag is set to grant localhost access as admin // authorization for localhost, only if flag is set to grant localhost access as admin
final String clientIP = (String) requestHeader.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP, ""); final String clientIP = (String) requestHeader.get(httpHeader.CONNECTION_PROP_CLIENTIP, "");
final String refererHost = requestHeader.refererHost(); final String refererHost = requestHeader.refererHost();
final boolean accessFromLocalhost = serverCore.isLocalhost(clientIP) && (refererHost.length() == 0 || serverCore.isLocalhost(refererHost)); final boolean accessFromLocalhost = serverCore.isLocalhost(clientIP) && (refererHost.length() == 0 || serverCore.isLocalhost(refererHost));
if (getConfigBool("adminAccountForLocalhost", false) && accessFromLocalhost) return 3; // soft-authenticated for localhost if (getConfigBool("adminAccountForLocalhost", false) && accessFromLocalhost) return 3; // soft-authenticated for localhost
@ -2188,9 +2189,9 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
// load the seed list // load the seed list
try { try {
final httpRequestHeader reqHeader = new httpRequestHeader(); final httpRequestHeader reqHeader = new httpRequestHeader();
reqHeader.put(httpRequestHeader.PRAGMA, "no-cache"); reqHeader.put(httpHeader.PRAGMA, "no-cache");
reqHeader.put(httpRequestHeader.CACHE_CONTROL, "no-cache"); reqHeader.put(httpHeader.CACHE_CONTROL, "no-cache");
reqHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.yacyUserAgent); reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
url = new yacyURL(seedListFileURL, null); url = new yacyURL(seedListFileURL, null);
final long start = System.currentTimeMillis(); final long start = System.currentTimeMillis();
@ -2275,7 +2276,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
try { try {
// sending request // sending request
final httpRequestHeader reqHeader = new httpRequestHeader(); final httpRequestHeader reqHeader = new httpRequestHeader();
reqHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.yacyUserAgent); reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
final HashMap<String, String> result = FileUtils.table(httpClient.wget(url.toString(), reqHeader, 10000), "UTF-8"); final HashMap<String, String> result = FileUtils.table(httpClient.wget(url.toString(), reqHeader, 10000), "UTF-8");
if (result == null) return new HashMap<String, String>(); if (result == null) return new HashMap<String, String>();
return result; return result;

3
source/de/anomic/tools/loaderThreads.java
Unescape View File

@ -26,6 +26,7 @@ import java.util.Hashtable;
import de.anomic.crawler.HTTPLoader; import de.anomic.crawler.HTTPLoader;
import de.anomic.http.httpClient; import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRemoteProxyConfig; import de.anomic.http.httpRemoteProxyConfig;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.FileUtils;
@ -124,7 +125,7 @@ public class loaderThreads {
public void run() { public void run() {
try { try {
final httpRequestHeader reqHeader = new httpRequestHeader(); final httpRequestHeader reqHeader = new httpRequestHeader();
reqHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.crawlerUserAgent); reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.crawlerUserAgent);
page = httpClient.wget(url.toString(), reqHeader, timeout); page = httpClient.wget(url.toString(), reqHeader, timeout);
loaded = true; loaded = true;
process.feed(page); process.feed(page);

7
source/de/anomic/yacy/yacyClient.java
Unescape View File

@ -64,6 +64,7 @@ import de.anomic.data.Blacklist;
import de.anomic.http.DefaultCharsetFilePart; import de.anomic.http.DefaultCharsetFilePart;
import de.anomic.http.DefaultCharsetStringPart; import de.anomic.http.DefaultCharsetStringPart;
import de.anomic.http.httpClient; import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpResponse; import de.anomic.http.httpResponse;
import de.anomic.http.httpRemoteProxyConfig; import de.anomic.http.httpRemoteProxyConfig;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
@ -269,8 +270,8 @@ public final class yacyClient {
*/ */
private static byte[] wput(final String url, final String vhost, final List<Part> post, final int timeout, final boolean gzipBody) throws IOException { private static byte[] wput(final String url, final String vhost, final List<Part> post, final int timeout, final boolean gzipBody) throws IOException {
final httpRequestHeader header = new httpRequestHeader(); final httpRequestHeader header = new httpRequestHeader();
header.put(httpRequestHeader.USER_AGENT, HTTPLoader.yacyUserAgent); header.put(httpHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
header.put(httpRequestHeader.HOST, vhost); header.put(httpHeader.HOST, vhost);
final httpClient client = new httpClient(timeout, header); final httpClient client = new httpClient(timeout, header);
client.setProxy(proxyConfig()); client.setProxy(proxyConfig());
@ -1086,7 +1087,7 @@ public final class yacyClient {
//System.out.println("permission=" + permissionMessage(args[1])); //System.out.println("permission=" + permissionMessage(args[1]));
final httpRequestHeader reqHeader = new httpRequestHeader(); final httpRequestHeader reqHeader = new httpRequestHeader();
reqHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.crawlerUserAgent); reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.crawlerUserAgent);
final byte[] content = httpClient.wget( final byte[] content = httpClient.wget(
"http://" + target.getPublicAddress() + "/yacy/search.html" + "http://" + target.getPublicAddress() + "/yacy/search.html" +
"?myseed=" + sb.peers.mySeed().genSeedStr(null) + "?myseed=" + sb.peers.mySeed().genSeedStr(null) +

4
source/de/anomic/yacy/yacyRelease.java
Unescape View File

@ -47,8 +47,8 @@ import java.util.TreeSet;
import de.anomic.crawler.HTTPLoader; import de.anomic.crawler.HTTPLoader;
import de.anomic.htmlFilter.htmlFilterContentScraper; import de.anomic.htmlFilter.htmlFilterContentScraper;
import de.anomic.http.httpClient; import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpResponse; import de.anomic.http.httpResponse;
import de.anomic.http.httpResponseHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.order.Base64Order; import de.anomic.kelondro.order.Base64Order;
import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.FileUtils;
@ -273,7 +273,7 @@ public final class yacyRelease extends yacyVersion {
File download = null; File download = null;
// setup httpClient // setup httpClient
final httpRequestHeader reqHeader = new httpRequestHeader(); final httpRequestHeader reqHeader = new httpRequestHeader();
reqHeader.put(httpResponseHeader.USER_AGENT, HTTPLoader.yacyUserAgent); reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
httpResponse res = null; httpResponse res = null;
final String name = this.getUrl().getFileName(); final String name = this.getUrl().getFileName();

7
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -42,6 +42,7 @@ import java.util.TreeMap;
import de.anomic.crawler.HTTPLoader; import de.anomic.crawler.HTTPLoader;
import de.anomic.http.httpClient; import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpResponse; import de.anomic.http.httpResponse;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.http.httpd; import de.anomic.http.httpd;
@ -817,9 +818,9 @@ public final class yacySeedDB implements httpdAlternativeDomainNames {
private ArrayList<String> downloadSeedFile(final yacyURL seedURL) throws IOException { private ArrayList<String> downloadSeedFile(final yacyURL seedURL) throws IOException {
// Configure http headers // Configure http headers
final httpRequestHeader reqHeader = new httpRequestHeader(); final httpRequestHeader reqHeader = new httpRequestHeader();
reqHeader.put(httpRequestHeader.PRAGMA, "no-cache"); reqHeader.put(httpHeader.PRAGMA, "no-cache");
reqHeader.put(httpRequestHeader.CACHE_CONTROL, "no-cache"); // httpc uses HTTP/1.0 is this necessary? reqHeader.put(httpHeader.CACHE_CONTROL, "no-cache"); // httpc uses HTTP/1.0 is this necessary?
reqHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.yacyUserAgent); reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
// init http-client // init http-client
final httpClient client = new httpClient(10000, reqHeader); final httpClient client = new httpClient(10000, reqHeader);

2
source/yacy.java
Unescape View File

@ -252,7 +252,7 @@ public final class yacy {
final String svrReleaseNr = matcher.group(1); final String svrReleaseNr = matcher.group(1);
try { try {
try {version = Double.parseDouble(vString);} catch (final NumberFormatException e) {version = (float) 0.1;} try {version = Double.parseDouble(vString);} catch (final NumberFormatException e) {version = (float) 0.1;}
version = yacyRelease.versvn2combinedVersion(version, Integer.parseInt(svrReleaseNr)); version = yacyVersion.versvn2combinedVersion(version, Integer.parseInt(svrReleaseNr));
} catch (final NumberFormatException e) {} } catch (final NumberFormatException e) {}
sb.setConfig("svnRevision", svrReleaseNr); sb.setConfig("svnRevision", svrReleaseNr);
} }

Write Preview
Loading…
Cancel
Save