boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

code cleanup: call of static methods directly to the class

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6155 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent 222850414e
commit 154bbc3364
49 changed files with 226 additions and 187 deletions
Show Stats Download Patch File Download Diff File

5
htroot/Blog.java
Unescape View File

@ -36,6 +36,7 @@ import java.util.Iterator;
import de.anomic.data.blogBoard;
import de.anomic.data.userDB;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.util.DateFormatter;
import de.anomic.plasma.plasmaSwitchboard;
@ -67,7 +68,7 @@ public class Blog {
prop.put("display", 1); // Fixed to 1
final boolean xml = (header.get(httpRequestHeader.CONNECTION_PROP_PATH)).endsWith(".xml");
final boolean xml = (header.get(httpHeader.CONNECTION_PROP_PATH)).endsWith(".xml");
final String address = sb.peers.mySeed().getPublicAddress();
if(hasRights) {
@ -96,7 +97,7 @@ public class Blog {
}
String pagename = post.get("page", DEFAULT_PAGE);
final String ip = (String)header.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP, "127.0.0.1");
final String ip = (String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "127.0.0.1");
String StrAuthor = post.get("author", "");

3
htroot/BlogComments.java
Unescape View File

@ -40,6 +40,7 @@ import de.anomic.data.blogBoardComments;
import de.anomic.data.messageBoard;
import de.anomic.data.userDB;
import de.anomic.data.blogBoard.BlogEntry;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.plasma.plasmaSwitchboard;
@ -83,7 +84,7 @@ public class BlogComments {
}
final String pagename = post.get("page", "blog_default");
final String ip = post.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP, "127.0.0.1");
final String ip = post.get(httpHeader.CONNECTION_PROP_CLIENTIP, "127.0.0.1");
String StrAuthor = post.get("author", "anonymous");

3
htroot/ConfigAppearance_p.java
Unescape View File

@ -37,6 +37,7 @@ import java.util.List;
import de.anomic.crawler.HTTPLoader;
import de.anomic.data.listManager;
import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.plasma.plasmaSwitchboard;
@ -90,7 +91,7 @@ public class ConfigAppearance_p {
try {
final yacyURL u = new yacyURL(url, null);
final httpRequestHeader reqHeader = new httpRequestHeader();
reqHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
skinVector = FileUtils.strings(httpClient.wget(u.toString(), reqHeader, 10000), "UTF-8");
} catch (final IOException e) {
prop.put("status", "1");// unable to get URL

5
htroot/ConfigBasic.java
Unescape View File

@ -32,6 +32,7 @@ import java.io.File;
import java.util.regex.Pattern;
import de.anomic.data.translator;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.http.httpd;
import de.anomic.http.httpdFileHandler;
@ -121,8 +122,8 @@ public class ConfigBasic {
if (upnp) UPnP.addPortMapping();
String host = null;
if (header.containsKey(httpRequestHeader.HOST)) {
host = header.get(httpRequestHeader.HOST);
if (header.containsKey(httpHeader.HOST)) {
host = header.get(httpHeader.HOST);
final int idx = host.indexOf(":");
if (idx != -1) host = host.substring(0,idx);
} else {

3
htroot/ConfigLanguage_p.java
Unescape View File

@ -40,6 +40,7 @@ import de.anomic.crawler.HTTPLoader;
import de.anomic.data.listManager;
import de.anomic.data.translator;
import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.server.serverObjects;
@ -80,7 +81,7 @@ public class ConfigLanguage_p {
try{
final yacyURL u = new yacyURL(url, null);
final httpRequestHeader reqHeader = new httpRequestHeader();
reqHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
langVector = FileUtils.strings(httpClient.wget(u.toString(), reqHeader, 10000), "UTF-8");
}catch(final IOException e){
prop.put("status", "1");//unable to get url

3
htroot/Connections_p.java
Unescape View File

@ -35,6 +35,7 @@ import java.util.Set;
import de.anomic.http.HttpConnectionInfo;
import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.http.httpd;
import de.anomic.kelondro.util.DateFormatter;
@ -151,7 +152,7 @@ public final class Connections_p {
final Properties conProp = (Properties) currentHttpd.getConProp().clone();
// getting the destination host
dest = conProp.getProperty(httpRequestHeader.CONNECTION_PROP_HOST);
dest = conProp.getProperty(httpHeader.CONNECTION_PROP_HOST);
if (dest==null)continue;
} else if (cmdObj instanceof urlRedirectord) {
prot = "urlRedirector";

4
htroot/CrawlProfileEditor_p.java
Unescape View File

@ -25,7 +25,7 @@
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.text.DateFormat;
import java.util.ArrayList;
import java.util.Iterator;
@ -217,7 +217,7 @@ public class CrawlProfileEditor_p {
prop.put("crawlProfiles_" + count + "_depth", profile.depth());
prop.put("crawlProfiles_" + count + "_mustmatch", profile.mustMatchPattern().toString());
prop.put("crawlProfiles_" + count + "_mustnotmatch", profile.mustNotMatchPattern().toString());
prop.put("crawlProfiles_" + count + "_crawlingIfOlder", (profile.recrawlIfOlder() == 0L) ? "no re-crawl" : ""+ SimpleDateFormat.getDateTimeInstance().format(profile.recrawlIfOlder()));
prop.put("crawlProfiles_" + count + "_crawlingIfOlder", (profile.recrawlIfOlder() == 0L) ? "no re-crawl" : ""+ DateFormat.getDateTimeInstance().format(profile.recrawlIfOlder()));
prop.put("crawlProfiles_" + count + "_crawlingDomFilterDepth", (profile.domFilterDepth() == Integer.MAX_VALUE) ? "inactive" : Integer.toString(profile.domFilterDepth()));
// start contrib [MN]

3
htroot/Messages_p.java
Unescape View File

@ -33,6 +33,7 @@ import java.util.Iterator;
import java.util.TreeMap;
import de.anomic.data.messageBoard;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.util.DateFormatter;
import de.anomic.kelondro.util.FileUtils;
@ -126,7 +127,7 @@ public class Messages_p {
prop.putXML("mode_messages_"+count+"_key", key);
prop.put("mode_messages_"+count+"_hash", message.authorHash());
if ((header.get(httpRequestHeader.CONNECTION_PROP_PATH)).endsWith(".rss")) {
if ((header.get(httpHeader.CONNECTION_PROP_PATH)).endsWith(".rss")) {
// set the peer address
prop.put("mode_messages_"+count+"_peerAddress", peerAddress);

3
htroot/Network.java
Unescape View File

@ -48,7 +48,6 @@ import de.anomic.yacy.yacyClient;
import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacyNewsRecord;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyRelease;
import de.anomic.yacy.yacyVersion;
public class Network {
@ -422,7 +421,7 @@ public class Network {
final String peertags = serverCodings.set2string(seed.getPeerTags(), ",", false);
prop.putHTML(STR_TABLE_LIST + conCount + "_dhtreceive_peertags", ((peertags == null) || (peertags.length() == 0)) ? "no tags given" : ("tags = " + peertags));
}
prop.putHTML(STR_TABLE_LIST + conCount + "_version", yacyRelease.combined2prettyVersion(seed.get(yacySeed.VERSION, "0.1"), shortname));
prop.putHTML(STR_TABLE_LIST + conCount + "_version", yacyVersion.combined2prettyVersion(seed.get(yacySeed.VERSION, "0.1"), shortname));
prop.putNum(STR_TABLE_LIST + conCount + "_lastSeen", /*seed.getLastSeenString() + " " +*/ lastseen);
prop.put(STR_TABLE_LIST + conCount + "_utc", seed.get(yacySeed.UTC, "-"));
prop.putHTML(STR_TABLE_LIST + conCount + "_uptime", DateFormatter.formatInterval(60000 * Long.parseLong(seed.get(yacySeed.UPTIME, "0"))));

3
htroot/QuickCrawlLink_p.java
Unescape View File

@ -36,6 +36,7 @@ import java.util.Date;
import de.anomic.crawler.CrawlEntry;
import de.anomic.crawler.CrawlProfile;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
@ -62,7 +63,7 @@ public class QuickCrawlLink_p {
prop.put("mode", "0");
// getting the http host header
final String hostSocket = header.get(httpRequestHeader.CONNECTION_PROP_HOST);
final String hostSocket = header.get(httpHeader.CONNECTION_PROP_HOST);
//String host = hostSocket;
int port = 80;

3
htroot/Settings_p.java
Unescape View File

@ -29,6 +29,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaParser;
import de.anomic.plasma.plasmaParserConfig;
@ -162,7 +163,7 @@ public final class Settings_p {
}
// clientIP
prop.putXML("clientIP", (String) header.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP, "<unknown>")); // read an artificial header addendum
prop.putXML("clientIP", (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "<unknown>")); // read an artificial header addendum
/*
* seed upload settings

5
htroot/User.java
Unescape View File

@ -30,6 +30,7 @@
import java.io.IOException;
import de.anomic.data.userDB;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.http.httpResponseHeader;
import de.anomic.http.httpd;
@ -61,7 +62,7 @@ public class User{
prop.put("logged-in_identified-by", "2");
//try via ip
if(entry == null){
entry=sb.userDB.ipAuth(((String)requestHeader.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP, "xxxxxx")));
entry=sb.userDB.ipAuth(((String)requestHeader.get(httpHeader.CONNECTION_PROP_CLIENTIP, "xxxxxx")));
if(entry != null){
prop.put("logged-in_identified-by", "0");
}
@ -144,7 +145,7 @@ public class User{
if(post!=null && post.containsKey("logout")){
prop.put("logged-in", "0");
if(entry != null){
entry.logout(((String)requestHeader.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP, "xxxxxx")), userDB.getLoginToken(requestHeader.getHeaderCookies())); //todo: logout cookie
entry.logout(((String)requestHeader.get(httpHeader.CONNECTION_PROP_CLIENTIP, "xxxxxx")), userDB.getLoginToken(requestHeader.getHeaderCookies())); //todo: logout cookie
}else{
sb.userDB.adminLogout(userDB.getLoginToken(requestHeader.getHeaderCookies()));
}

3
htroot/ViewImage.java
Unescape View File

@ -29,6 +29,7 @@ import java.io.InputStream;
import java.net.MalformedURLException;
import java.util.HashMap;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.plasma.plasmaSnippetCache;
@ -61,7 +62,7 @@ public class ViewImage {
String urlString = post.get("url", "");
final String urlLicense = post.get("code", "");
final boolean auth = ((String) header.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP, "")).equals("localhost") || sb.verifyAuthentication(header, true); // handle access rights
final boolean auth = ((String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "")).equals("localhost") || sb.verifyAuthentication(header, true); // handle access rights
yacyURL url = null;
if ((urlString.length() > 0) && (auth)) try {

3
htroot/Wiki.java
Unescape View File

@ -36,6 +36,7 @@ import java.util.Iterator;
import de.anomic.data.diff;
import de.anomic.data.wiki.wikiBoard;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
@ -68,7 +69,7 @@ public class Wiki {
String access = sb.getConfig("WikiAccess", "admin");
final String pagename = get(post, "page", "start");
final String ip = get(post, httpRequestHeader.CONNECTION_PROP_CLIENTIP, "127.0.0.1");
final String ip = get(post, httpHeader.CONNECTION_PROP_CLIENTIP, "127.0.0.1");
String author = get(post, "author", "anonymous");
if (author.equals("anonymous")) {
author = wikiBoard.guessAuthor(ip);

3
htroot/YaCySearchPluginFF.java
Unescape View File

@ -24,6 +24,7 @@
//javac -classpath .:../Classes Status.java
//if the shell's current path is HTROOT
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
@ -42,7 +43,7 @@ public class YaCySearchPluginFF {
final serverObjects prop = new serverObjects();
// getting the http host header
final String hostSocket = header.get(httpRequestHeader.CONNECTION_PROP_HOST);
final String hostSocket = header.get(httpHeader.CONNECTION_PROP_HOST);
String host = hostSocket;
int port = 80;

3
htroot/api/util/getpageinfo_p.java
Unescape View File

@ -5,6 +5,7 @@ import java.util.Set;
import de.anomic.crawler.HTTPLoader;
import de.anomic.htmlFilter.htmlFilterContentScraper;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
@ -43,7 +44,7 @@ public class getpageinfo_p {
try {
final yacyURL u = new yacyURL(url, null);
final httpRequestHeader reqHeader = new httpRequestHeader();
reqHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.yacyUserAgent); // do not set the crawler user agent, because this page was loaded by manual entering of the url
reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.yacyUserAgent); // do not set the crawler user agent, because this page was loaded by manual entering of the url
final htmlFilterContentScraper scraper = htmlFilterContentScraper.parseResource(u, reqHeader);
// put the document title

3
htroot/autoconfig.java
Unescape View File

@ -24,6 +24,7 @@
//javac -classpath .:../Classes Status.java
//if the shell's current path is HTROOT
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaSwitchboardConstants;
import de.anomic.server.serverObjects;
@ -47,7 +48,7 @@ public class autoconfig {
final boolean yacyonly = env.getConfigBool(plasmaSwitchboardConstants.PROXY_YACY_ONLY, false);
// getting the http host header
final String hostSocket = header.get(httpRequestHeader.CONNECTION_PROP_HOST);
final String hostSocket = header.get(httpHeader.CONNECTION_PROP_HOST);
String host = hostSocket;
int port = 80;

9
htroot/sharedBlacklist_p.java
Unescape View File

@ -45,6 +45,7 @@ import de.anomic.data.list.ListAccumulator;
import de.anomic.data.list.XMLBlacklistImporter;
import de.anomic.htmlFilter.htmlFilterCharacterCoding;
import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.plasma.plasmaSwitchboard;
@ -139,9 +140,9 @@ public class sharedBlacklist_p {
// download the blacklist
try {
final httpRequestHeader reqHeader = new httpRequestHeader();
reqHeader.put(httpRequestHeader.PRAGMA,"no-cache");
reqHeader.put(httpRequestHeader.CACHE_CONTROL,"no-cache");
reqHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
reqHeader.put(httpHeader.PRAGMA,"no-cache");
reqHeader.put(httpHeader.CACHE_CONTROL,"no-cache");
reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
// get List
yacyURL u = new yacyURL(downloadURLOld, null);
@ -164,7 +165,7 @@ public class sharedBlacklist_p {
try {
final yacyURL u = new yacyURL(downloadURL, null);
final httpRequestHeader reqHeader = new httpRequestHeader();
reqHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
otherBlacklist = FileUtils.strings(httpClient.wget(u.toString(), reqHeader, 10000), "UTF-8"); //get List
} catch (final Exception e) {
prop.put("status", STATUS_URL_PROBLEM);

3
htroot/www/welcome.java
Unescape View File

@ -32,6 +32,7 @@ import java.io.File;
import java.net.InetAddress;
import java.net.UnknownHostException;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore;
@ -61,7 +62,7 @@ public class welcome {
prop.put("hostip", "Unknown Host Exception");
}
prop.put("port", serverCore.getPortNr(env.getConfig("port","8080")));
prop.put("clientip", (String) header.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP, ""));
prop.put("clientip", (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP, ""));
final String peertype = (sb.peers.mySeed() == null) ? yacySeed.PEERTYPE_JUNIOR : sb.peers.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN);
final boolean senior = (peertype.equals(yacySeed.PEERTYPE_SENIOR)) || (peertype.equals(yacySeed.PEERTYPE_PRINCIPAL));

5
htroot/yacy/hello.java
Unescape View File

@ -32,6 +32,7 @@ import java.util.ConcurrentModificationException;
import java.util.Iterator;
import java.util.Map;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore;
@ -72,7 +73,7 @@ public final class hello {
int count = 0;
try {count = (countStr == null) ? 0 : Integer.parseInt(countStr);} catch (final NumberFormatException e) {count = 0;}
// final Date remoteTime = yacyCore.parseUniversalDate((String) post.get(MYTIME)); // read remote time
final String clientip = (String) header.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP, "<unknown>"); // read an artificial header addendum
final String clientip = (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "<unknown>"); // read an artificial header addendum
final InetAddress ias = serverDomains.dnsResolve(clientip);
if (ias == null) {
prop.put("message", "cannot resolve your IP from your reported location " + clientip);
@ -96,7 +97,7 @@ public final class hello {
// if ((properTest != null) && (! properTest.substring(0,1).equals("IP"))) { return null; }
// we easily know the caller's IP:
final String userAgent = (String) header.get(httpRequestHeader.USER_AGENT, "<unknown>");
final String userAgent = (String) header.get(httpHeader.USER_AGENT, "<unknown>");
final String reportedip = remoteSeed.getIP();
final String reportedPeerType = remoteSeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR);
final float clientversion = remoteSeed.getVersion();

3
htroot/yacy/list.java
Unescape View File

@ -29,6 +29,7 @@
import java.io.File;
import de.anomic.data.listManager;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.plasma.plasmaSwitchboardConstants;
@ -60,7 +61,7 @@ public final class list {
final yacySeed bla = sb.peers.get(post.get("iam", ""));
if (bla != null) otherPeerName = bla.getName();
}
if (otherPeerName == null) otherPeerName = header.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP);
if (otherPeerName == null) otherPeerName = header.get(httpHeader.CONNECTION_PROP_CLIENTIP);
if ((sb.isRobinsonMode()) && (!sb.isInMyCluster(otherPeerName))) {
// if we are a robinson cluster, answer only if this client is known by our network definition

3
htroot/yacy/message.java
Unescape View File

@ -35,6 +35,7 @@ import java.text.SimpleDateFormat;
import java.util.Date;
import de.anomic.data.messageBoard;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.plasma.plasmaSwitchboard;
@ -81,7 +82,7 @@ public final class message {
if ((sb.isRobinsonMode()) &&
(!((sb.isPublicRobinson()) ||
(sb.isInMyCluster(header.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP)))))) {
(sb.isInMyCluster(header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))))) {
// if we are a robinson cluster, answer only if this client is known by our network definition
prop.put("response", "-1"); // request rejected
return prop;

3
htroot/yacy/profile.java
Unescape View File

@ -35,6 +35,7 @@ import java.io.IOException;
import java.util.Iterator;
import java.util.Properties;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
@ -52,7 +53,7 @@ public final class profile {
if ((sb.isRobinsonMode()) &&
(!sb.isPublicRobinson()) &&
(!sb.isInMyCluster(header.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP)))) {
(!sb.isInMyCluster(header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))) {
// if we are a robinson cluster, answer only if this client is known by our network definition
prop.put("list", "0");
return prop;

3
htroot/yacy/query.java
Unescape View File

@ -29,6 +29,7 @@
import java.io.IOException;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.util.DateFormatter;
import de.anomic.plasma.plasmaSwitchboard;
@ -54,7 +55,7 @@ public final class query {
if ((sb.isRobinsonMode()) &&
(!sb.isPublicRobinson()) &&
(!sb.isInMyCluster(header.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP)))) {
(!sb.isInMyCluster(header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))) {
// if we are a robinson cluster, answer only if we are public robinson peers,
// or we are a private cluster and the requester is in our cluster.
// if we don't answer, the remote peer will recognize us as junior peer,

5
htroot/yacy/search.java
Unescape View File

@ -35,6 +35,7 @@ import java.util.Map;
import java.util.TreeSet;
import de.anomic.content.RSSMessage;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.order.Base64Order;
import de.anomic.kelondro.order.Bitfield;
@ -71,7 +72,7 @@ public final class search {
final serverObjects prop = new serverObjects();
if ((post == null) || (env == null)) return prop;
if (!yacyNetwork.authentifyRequest(post, env)) return prop;
final String client = header.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP);
final String client = header.get(httpHeader.CONNECTION_PROP_CLIENTIP);
//System.out.println("yacy: search received request = " + post.toString());
@ -126,7 +127,7 @@ public final class search {
if ((sb.isRobinsonMode()) &&
(!((sb.isPublicRobinson()) ||
(sb.isInMyCluster(header.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP)))))) {
(sb.isInMyCluster(header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))))) {
// if we are a robinson cluster, answer only if this client is known by our network definition
prop.put("links", "");
prop.put("linkcount", "0");

5
htroot/yacy/transfer.java
Unescape View File

@ -27,6 +27,7 @@
import java.io.File;
import java.io.IOException;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.order.Base64Order;
import de.anomic.kelondro.order.Digest;
@ -71,8 +72,8 @@ public final class transfer {
// reject unknown peers: this does not appear fair, but anonymous senders are dangerous
// reject paths that contain '..' because they are dangerous
if (sb.getLog().isFine()) {
if (otherseed == null) sb.getLog().logFine("RankingTransmission: rejected unknown peer '" + otherpeer + "', current IP " + header.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP, "unknown"));
if (filename.indexOf("..") >= 0) sb.getLog().logFine("RankingTransmission: rejected wrong path '" + filename + "' from peer " + (otherseed == null ? "null" : otherseed.getName() + "/" + otherseed.getPublicAddress()) + ", current IP " + header.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP, "unknown"));
if (otherseed == null) sb.getLog().logFine("RankingTransmission: rejected unknown peer '" + otherpeer + "', current IP " + header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "unknown"));
if (filename.indexOf("..") >= 0) sb.getLog().logFine("RankingTransmission: rejected wrong path '" + filename + "' from peer " + (otherseed == null ? "null" : otherseed.getName() + "/" + otherseed.getPublicAddress()) + ", current IP " + header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "unknown"));
}
return prop;
}

5
htroot/yacysearch.java
Unescape View File

@ -33,6 +33,7 @@ import java.util.Iterator;
import java.util.TreeSet;
import de.anomic.content.RSSMessage;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.order.Bitfield;
import de.anomic.kelondro.text.metadataPrototype.URLMetadataRow;
@ -80,7 +81,7 @@ public class yacysearch {
}
String promoteSearchPageGreeting = env.getConfig(plasmaSwitchboardConstants.GREETING, "");
if (env.getConfigBool(plasmaSwitchboardConstants.GREETING_NETWORK_NAME, false)) promoteSearchPageGreeting = env.getConfig("network.unit.description", "");
final String client = header.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP); // the search client who initiated the search
final String client = header.get(httpHeader.CONNECTION_PROP_CLIENTIP); // the search client who initiated the search
// get query
String originalquerystring = (post == null) ? "" : post.get("query", post.get("search", "")).trim(); // SRU compliance
@ -336,7 +337,7 @@ public class yacysearch {
if (language.startsWith("lang_")) language = language.substring(5);
if (!iso639.exists(language)) {
// find out language of the user by reading of the user-agent string
String agent = header.get(httpRequestHeader.ACCEPT_LANGUAGE);
String agent = header.get(httpHeader.ACCEPT_LANGUAGE);
if (agent == null) agent = System.getProperty("user.language");
language = (agent == null) ? "en" : iso639.userAgentLanguageDetection(agent);
if (language == null) language = "en";

3
htroot/yacysearchitem.java
Unescape View File

@ -30,6 +30,7 @@ import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.TreeSet;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaProfiling;
import de.anomic.plasma.plasmaSearchEvent;
@ -61,7 +62,7 @@ public class yacysearchitem {
final String eventID = post.get("eventID", "");
final boolean authenticated = sb.adminAuthenticated(header) >= 2;
final int item = post.getInt("item", -1);
final boolean auth = ((String) header.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP, "")).equals("localhost") || sb.verifyAuthentication(header, true);
final boolean auth = ((String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "")).equals("localhost") || sb.verifyAuthentication(header, true);
final int display = (post == null) ? 0 : post.getInt("display", 0);
// default settings for blank item

5
source/de/anomic/crawler/FTPLoader.java
Unescape View File

@ -32,6 +32,7 @@ import java.io.IOException;
import java.io.PrintStream;
import java.util.Date;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.http.httpResponseHeader;
import de.anomic.http.httpdProxyCacheEntry;
@ -61,8 +62,8 @@ public class FTPLoader {
httpRequestHeader requestHeader = new httpRequestHeader();
if (entry.referrerhash() != null) requestHeader.put(httpRequestHeader.REFERER, sb.getURL(entry.referrerhash()).toNormalform(true, false));
httpResponseHeader responseHeader = new httpResponseHeader();
responseHeader.put(httpResponseHeader.LAST_MODIFIED, DateFormatter.formatRFC1123(fileDate));
responseHeader.put(httpResponseHeader.CONTENT_TYPE, mimeType);
responseHeader.put(httpHeader.LAST_MODIFIED, DateFormatter.formatRFC1123(fileDate));
responseHeader.put(httpHeader.CONTENT_TYPE, mimeType);
Document metadata = new httpdProxyCacheEntry(
entry.depth(), entry.url(), entry.name(), "OK",
requestHeader, responseHeader,

13
source/de/anomic/crawler/HTTPLoader.java
Unescape View File

@ -30,6 +30,7 @@ import java.util.Date;
import de.anomic.data.Blacklist;
import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpResponse;
import de.anomic.http.httpRequestHeader;
import de.anomic.http.httpResponseHeader;
@ -132,13 +133,13 @@ public final class HTTPLoader {
//try {
// create a request header
final httpRequestHeader requestHeader = new httpRequestHeader();
requestHeader.put(httpRequestHeader.USER_AGENT, crawlerUserAgent);
requestHeader.put(httpHeader.USER_AGENT, crawlerUserAgent);
yacyURL refererURL = null;
if (entry.referrerhash() != null) refererURL = sb.getURL(entry.referrerhash());
if (refererURL != null) requestHeader.put(httpRequestHeader.REFERER, refererURL.toNormalform(true, true));
requestHeader.put(httpRequestHeader.ACCEPT_LANGUAGE, sb.getConfig("crawler.http.acceptLanguage", DEFAULT_LANGUAGE));
requestHeader.put(httpRequestHeader.ACCEPT_CHARSET, sb.getConfig("crawler.http.acceptCharset", DEFAULT_CHARSET));
requestHeader.put(httpRequestHeader.ACCEPT_ENCODING, sb.getConfig("crawler.http.acceptEncoding", DEFAULT_ENCODING));
requestHeader.put(httpHeader.ACCEPT_LANGUAGE, sb.getConfig("crawler.http.acceptLanguage", DEFAULT_LANGUAGE));
requestHeader.put(httpHeader.ACCEPT_CHARSET, sb.getConfig("crawler.http.acceptCharset", DEFAULT_CHARSET));
requestHeader.put(httpHeader.ACCEPT_ENCODING, sb.getConfig("crawler.http.acceptEncoding", DEFAULT_ENCODING));
// HTTP-Client
final httpClient client = new httpClient(socketTimeout, requestHeader);
@ -195,9 +196,9 @@ public final class HTTPLoader {
htCache = null;
}*/
} else if (res.getStatusLine().startsWith("30")) {
if (res.getResponseHeader().containsKey(httpRequestHeader.LOCATION)) {
if (res.getResponseHeader().containsKey(httpHeader.LOCATION)) {
// getting redirection URL
String redirectionUrlString = res.getResponseHeader().get(httpRequestHeader.LOCATION);
String redirectionUrlString = res.getResponseHeader().get(httpHeader.LOCATION);
redirectionUrlString = redirectionUrlString.trim();
if (redirectionUrlString.length() == 0) {

9
source/de/anomic/crawler/IndexingStack.java
Unescape View File

@ -34,6 +34,7 @@ import java.util.Date;
import java.util.Iterator;
import java.util.concurrent.ConcurrentHashMap;
import de.anomic.http.httpHeader;
import de.anomic.http.httpResponseHeader;
import de.anomic.kelondro.index.Row;
import de.anomic.kelondro.order.Base64Order;
@ -481,7 +482,7 @@ public class IndexingStack {
// -if-modified-since in request
// if the page is fresh at the very moment we can index it
final Date ifModifiedSince = getModificationDate();
if ((ifModifiedSince != null) && (responseHeader.containsKey(httpResponseHeader.LAST_MODIFIED))) {
if ((ifModifiedSince != null) && (responseHeader.containsKey(httpHeader.LAST_MODIFIED))) {
// parse date
Date d = responseHeader.lastModified();
if (d == null) {
@ -495,8 +496,8 @@ public class IndexingStack {
}
// -pragma in cached response
if (responseHeader.containsKey(httpResponseHeader.PRAGMA) &&
(responseHeader.get(httpResponseHeader.PRAGMA)).toUpperCase().equals("NO-CACHE")) {
if (responseHeader.containsKey(httpHeader.PRAGMA) &&
(responseHeader.get(httpHeader.PRAGMA)).toUpperCase().equals("NO-CACHE")) {
return "Denied_(pragma_no_cache)";
}
@ -520,7 +521,7 @@ public class IndexingStack {
// -cache-control in cached response
// the cache-control has many value options.
String cacheControl = responseHeader.get(httpResponseHeader.CACHE_CONTROL);
String cacheControl = responseHeader.get(httpHeader.CACHE_CONTROL);
if (cacheControl != null) {
cacheControl = cacheControl.trim().toUpperCase();
/* we have the following cases for cache-control:

7
source/de/anomic/crawler/RobotsTxt.java
Unescape View File

@ -41,6 +41,7 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpResponse;
import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.blob.Heap;
@ -518,7 +519,7 @@ public class RobotsTxt {
httpRequestHeader reqHeaders = new httpRequestHeader();
// add yacybot user agent
reqHeaders.put(httpRequestHeader.USER_AGENT, HTTPLoader.crawlerUserAgent);
reqHeaders.put(httpHeader.USER_AGENT, HTTPLoader.crawlerUserAgent);
// adding referer
reqHeaders.put(httpRequestHeader.REFERER, (yacyURL.newURL(robotsURL,"/")).toNormalform(true, true));
@ -550,7 +551,7 @@ public class RobotsTxt {
} else {
// getting some metadata
eTag = res.getResponseHeader().containsKey(httpRequestHeader.ETAG)?(res.getResponseHeader().get(httpRequestHeader.ETAG)).trim():null;
eTag = res.getResponseHeader().containsKey(httpHeader.ETAG)?(res.getResponseHeader().get(httpHeader.ETAG)).trim():null;
lastMod = res.getResponseHeader().lastModified();
// if the robots.txt file was not changed we break here
@ -575,7 +576,7 @@ public class RobotsTxt {
return null;
} else if (res.getStatusLine().startsWith("3")) {
// getting redirection URL
String redirectionUrlString = res.getResponseHeader().get(httpRequestHeader.LOCATION);
String redirectionUrlString = res.getResponseHeader().get(httpHeader.LOCATION);
if (redirectionUrlString==null) {
if (log.isFinest()) log.logFinest("robots.txt could not be downloaded from URL '" + robotsURL + "' because of missing redirecton header. [" + res.getStatusLine() + "].");
robotsTxt = null;

3
source/de/anomic/data/SitemapParser.java
Unescape View File

@ -42,6 +42,7 @@ import de.anomic.crawler.CrawlEntry;
import de.anomic.crawler.CrawlProfile;
import de.anomic.crawler.HTTPLoader;
import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpResponse;
import de.anomic.http.httpRequestHeader;
import de.anomic.http.httpdByteCountInputStream;
@ -152,7 +153,7 @@ public class SitemapParser extends DefaultHandler {
public void parse() {
// download document
final httpRequestHeader requestHeader = new httpRequestHeader();
requestHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.crawlerUserAgent);
requestHeader.put(httpHeader.USER_AGENT, HTTPLoader.crawlerUserAgent);
final httpClient client = new httpClient(5000, requestHeader);
httpResponse res = null;
try {

3
source/de/anomic/htmlFilter/htmlFilterContentScraper.java
Unescape View File

@ -46,6 +46,7 @@ import javax.swing.event.EventListenerList;
import de.anomic.crawler.HTTPLoader;
import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.plasma.plasmaParser;
@ -512,7 +513,7 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
public static htmlFilterContentScraper parseResource(final yacyURL location) throws IOException {
// load page
final httpRequestHeader reqHeader = new httpRequestHeader();
reqHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.crawlerUserAgent);
reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.crawlerUserAgent);
return parseResource(location, reqHeader);
}

4
source/de/anomic/http/httpClient.java
Unescape View File

@ -324,7 +324,7 @@ public class httpClient {
if (gzipBody) {
data = zipRequest(data);
post.setRequestHeader(httpResponseHeader.CONTENT_ENCODING, httpHeader.CONTENT_ENCODING_GZIP);
post.setRequestHeader(httpHeader.CONTENT_ENCODING, httpHeader.CONTENT_ENCODING_GZIP);
post.setContentChunked(true);
}
post.setRequestEntity(data);
@ -790,7 +790,7 @@ public class httpClient {
header = new httpRequestHeader();
}
// set host-header
header.add(httpRequestHeader.HOST, vhost);
header.add(httpHeader.HOST, vhost);
}
}

158
source/de/anomic/http/httpd.java
Unescape View File

@ -261,33 +261,33 @@ public final class httpd implements serverHandler, Cloneable {
private boolean handlePersistentConnection(final httpRequestHeader header) {
if (!keepAliveSupport) {
this.prop.put(httpRequestHeader.CONNECTION_PROP_PERSISTENT,"close");
this.prop.put(httpHeader.CONNECTION_PROP_PERSISTENT,"close");
return false;
}
// getting the http version that is used by the client
final String httpVersion = this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_HTTP_VER, "HTTP/0.9");
final String httpVersion = this.prop.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER, "HTTP/0.9");
// managing keep-alive: in HTTP/0.9 and HTTP/1.0 every connection is closed
// afterwards. In HTTP/1.1 (and above, in the future?) connections are
// persistent by default, but closed with the "Connection: close"
// property.
boolean persistent = !(httpVersion.equals(httpRequestHeader.HTTP_VERSION_0_9) || httpVersion.equals(httpRequestHeader.HTTP_VERSION_1_0));
boolean persistent = !(httpVersion.equals(httpHeader.HTTP_VERSION_0_9) || httpVersion.equals(httpHeader.HTTP_VERSION_1_0));
if (((String)header.get(httpRequestHeader.CONNECTION, "keep-alive")).toLowerCase().indexOf("close") != -1 ||
((String)header.get(httpRequestHeader.PROXY_CONNECTION, "keep-alive")).toLowerCase().indexOf("close") != -1) {
persistent = false;
}
final String transferEncoding = (String) header.get(httpHeader.TRANSFER_ENCODING, "identity");
final boolean isPostRequest = this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_METHOD).equals(httpRequestHeader.METHOD_POST);
final boolean isPostRequest = this.prop.getProperty(httpHeader.CONNECTION_PROP_METHOD).equals(httpHeader.METHOD_POST);
final boolean hasContentLength = header.containsKey(httpHeader.CONTENT_LENGTH);
final boolean hasTransferEncoding = header.containsKey(httpHeader.TRANSFER_ENCODING) && !transferEncoding.equalsIgnoreCase("identity");
// if the request does not contain a content-length we have to close the connection
// independently of the value of the connection header
if (persistent && isPostRequest && !(hasContentLength || hasTransferEncoding))
this.prop.put(httpRequestHeader.CONNECTION_PROP_PERSISTENT,"close");
else this.prop.put(httpRequestHeader.CONNECTION_PROP_PERSISTENT,persistent?"keep-alive":"close");
this.prop.put(httpHeader.CONNECTION_PROP_PERSISTENT,"close");
else this.prop.put(httpHeader.CONNECTION_PROP_PERSISTENT,persistent?"keep-alive":"close");
return persistent;
}
@ -304,7 +304,7 @@ public final class httpd implements serverHandler, Cloneable {
private boolean handleServerAuthentication(final httpRequestHeader header) throws IOException {
// getting the http version that is used by the client
final String httpVersion = this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_HTTP_VER, "HTTP/0.9");
final String httpVersion = this.prop.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER, "HTTP/0.9");
// reading the authentication settings from switchboard
if (this.serverAccountBase64MD5 == null)
@ -317,7 +317,7 @@ public final class httpd implements serverHandler, Cloneable {
this.session.out.write((httpVersion + " 401 log-in required" + serverCore.CRLF_STRING +
httpRequestHeader.WWW_AUTHENTICATE + ": Basic realm=\"log-in\"" + serverCore.CRLF_STRING +
serverCore.CRLF_STRING).getBytes());
this.session.out.write((httpResponseHeader.CONTENT_LENGTH + ": 0\r\n").getBytes());
this.session.out.write((httpHeader.CONTENT_LENGTH + ": 0\r\n").getBytes());
this.session.out.write("\r\n".getBytes());
return false;
} else if (!this.serverAccountBase64MD5.equals(Digest.encodeMD5Hex(auth.trim().substring(6)))) {
@ -326,7 +326,7 @@ public final class httpd implements serverHandler, Cloneable {
this.session.out.write((httpVersion + " 401 log-in required" + serverCore.CRLF_STRING +
httpRequestHeader.WWW_AUTHENTICATE + ": Basic realm=\"log-in\"" +
serverCore.CRLF_STRING).getBytes());
this.session.out.write((httpResponseHeader.CONTENT_LENGTH + ": 0\r\n").getBytes());
this.session.out.write((httpHeader.CONTENT_LENGTH + ": 0\r\n").getBytes());
this.session.out.write("\r\n".getBytes());
this.session.out.flush();
return false;
@ -342,7 +342,7 @@ public final class httpd implements serverHandler, Cloneable {
// proxy hops must identify with 4 criteria:
// the accessed port must not be port 80
final String host = this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_HOST);
final String host = this.prop.getProperty(httpHeader.CONNECTION_PROP_HOST);
if (host == null) return false;
int pos;
if ((pos = host.indexOf(":")) < 0) {
@ -352,7 +352,7 @@ public final class httpd implements serverHandler, Cloneable {
if (Integer.parseInt(host.substring(pos + 1)) == 80) return false;
// the access path must be into the yacy protocol path; it must start with 'yacy'
if (!(this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_PATH, "").startsWith("/yacy/"))) return false;
if (!(this.prop.getProperty(httpHeader.CONNECTION_PROP_PATH, "").startsWith("/yacy/"))) return false;
// the accessing client must identify with user:password, where
// user = addressed peer name
@ -364,12 +364,12 @@ public final class httpd implements serverHandler, Cloneable {
}
// the accessing client must use a yacy user-agent
if (!(((String) header.get(httpRequestHeader.USER_AGENT,"")).startsWith("yacy"))) return false;
if (!(((String) header.get(httpHeader.USER_AGENT,"")).startsWith("yacy"))) return false;
// furthermore, YaCy hops must not exceed a specific access frequency
// check access requester frequency: protection against DoS against this peer
final String requester = this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_CLIENTIP);
final String requester = this.prop.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP);
if (requester == null) return false;
if (lastAccessDelta(YaCyHopAccessRequester, requester) < 10000) return false;
YaCyHopAccessRequester.put(requester, Long.valueOf(System.currentTimeMillis()));
@ -423,7 +423,7 @@ public final class httpd implements serverHandler, Cloneable {
// ask for authenticate
this.session.out.write((httpVersion + " 407 Proxy Authentication Required" + serverCore.CRLF_STRING +
httpRequestHeader.PROXY_AUTHENTICATE + ": Basic realm=\"log-in\"" + serverCore.CRLF_STRING).getBytes());
this.session.out.write((httpResponseHeader.CONTENT_LENGTH + ": 0\r\n").getBytes());
this.session.out.write((httpHeader.CONTENT_LENGTH + ": 0\r\n").getBytes());
this.session.out.write("\r\n".getBytes());
this.session.out.flush();
return false;
@ -469,21 +469,21 @@ public final class httpd implements serverHandler, Cloneable {
public Boolean GET(final String arg) {
try {
// parsing the http request line
parseRequestLine(httpRequestHeader.METHOD_GET,arg);
parseRequestLine(httpHeader.METHOD_GET,arg);
// we now know the HTTP version. depending on that, we read the header
final String httpVersion = this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_HTTP_VER, httpRequestHeader.HTTP_VERSION_0_9);
final httpRequestHeader header = (httpVersion.equals(httpRequestHeader.HTTP_VERSION_0_9))
final String httpVersion = this.prop.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER, httpHeader.HTTP_VERSION_0_9);
final httpRequestHeader header = (httpVersion.equals(httpHeader.HTTP_VERSION_0_9))
? new httpRequestHeader(reverseMappingCache)
: httpRequestHeader.readHeader(this.prop,this.session);
// handling transparent proxy support
httpRequestHeader.handleTransparentProxySupport(header, this.prop, virtualHost, httpdProxyHandler.isTransparentProxy);
httpHeader.handleTransparentProxySupport(header, this.prop, virtualHost, httpdProxyHandler.isTransparentProxy);
// determines if the connection should be kept alive
handlePersistentConnection(header);
if (this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_HOST).equals(virtualHost)) {
if (this.prop.getProperty(httpHeader.CONNECTION_PROP_HOST).equals(virtualHost)) {
// pass to server
if (this.allowServer) {
if (this.handleServerAuthentication(header)) {
@ -506,7 +506,7 @@ public final class httpd implements serverHandler, Cloneable {
}
}
return this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_PERSISTENT).equals("keep-alive") ? serverCore.RESUME_CONNECTION : serverCore.TERMINATE_CONNECTION;
return this.prop.getProperty(httpHeader.CONNECTION_PROP_PERSISTENT).equals("keep-alive") ? serverCore.RESUME_CONNECTION : serverCore.TERMINATE_CONNECTION;
} catch (final Exception e) {
logUnexpectedError(e);
return serverCore.TERMINATE_CONNECTION;
@ -537,22 +537,22 @@ public final class httpd implements serverHandler, Cloneable {
public Boolean HEAD(final String arg) {
try {
parseRequestLine(httpRequestHeader.METHOD_HEAD,arg);
parseRequestLine(httpHeader.METHOD_HEAD,arg);
// we now know the HTTP version. depending on that, we read the header
httpRequestHeader header;
final String httpVersion = this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_HTTP_VER, httpRequestHeader.HTTP_VERSION_0_9);
if (httpVersion.equals(httpRequestHeader.HTTP_VERSION_0_9)) header = new httpRequestHeader(reverseMappingCache);
final String httpVersion = this.prop.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER, httpHeader.HTTP_VERSION_0_9);
if (httpVersion.equals(httpHeader.HTTP_VERSION_0_9)) header = new httpRequestHeader(reverseMappingCache);
else header = httpRequestHeader.readHeader(this.prop,this.session);
// handle transparent proxy support
httpRequestHeader.handleTransparentProxySupport(header, this.prop, virtualHost, httpdProxyHandler.isTransparentProxy);
httpHeader.handleTransparentProxySupport(header, this.prop, virtualHost, httpdProxyHandler.isTransparentProxy);
// determines if the connection should be kept alive
handlePersistentConnection(header);
// return multi-line message
if (this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_HOST).equals(virtualHost)) {
if (this.prop.getProperty(httpHeader.CONNECTION_PROP_HOST).equals(virtualHost)) {
// pass to server
if (allowServer) {
if (handleServerAuthentication(header)) {
@ -576,7 +576,7 @@ public final class httpd implements serverHandler, Cloneable {
return serverCore.TERMINATE_CONNECTION;
}
}
return this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_PERSISTENT).equals("keep-alive") ? serverCore.RESUME_CONNECTION : serverCore.TERMINATE_CONNECTION;
return this.prop.getProperty(httpHeader.CONNECTION_PROP_PERSISTENT).equals("keep-alive") ? serverCore.RESUME_CONNECTION : serverCore.TERMINATE_CONNECTION;
} catch (final Exception e) {
logUnexpectedError(e);
return serverCore.TERMINATE_CONNECTION;
@ -585,22 +585,22 @@ public final class httpd implements serverHandler, Cloneable {
public Boolean POST(final String arg) {
try {
parseRequestLine(httpRequestHeader.METHOD_POST,arg);
parseRequestLine(httpHeader.METHOD_POST,arg);
// we now know the HTTP version. depending on that, we read the header
httpRequestHeader header;
final String httpVersion = this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_HTTP_VER, httpRequestHeader.HTTP_VERSION_0_9);
final String httpVersion = this.prop.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER, httpHeader.HTTP_VERSION_0_9);
if (httpVersion.equals(httpHeader.HTTP_VERSION_0_9)) header = new httpRequestHeader(reverseMappingCache);
else header = httpRequestHeader.readHeader(this.prop,this.session);
// handle transfer-coding
final InputStream sessionIn;
final String transferEncoding = header.get(httpRequestHeader.TRANSFER_ENCODING);
final String transferEncoding = header.get(httpHeader.TRANSFER_ENCODING);
if (transferEncoding != null) {
if (!httpRequestHeader.HTTP_VERSION_1_1.equals(httpVersion)) {
if (!httpHeader.HTTP_VERSION_1_1.equals(httpVersion)) {
log.logWarning("client "+ session.getName() +" uses transfer-coding with HTTP version "+ httpVersion +"!");
}
if("chunked".equalsIgnoreCase(header.get(httpRequestHeader.TRANSFER_ENCODING))) {
if("chunked".equalsIgnoreCase(header.get(httpHeader.TRANSFER_ENCODING))) {
sessionIn = new ChunkedInputStream(this.session.in);
} else {
// "A server which receives an entity-body with a transfer-coding it does
@ -614,13 +614,13 @@ public final class httpd implements serverHandler, Cloneable {
}
// handle transparent proxy support
httpRequestHeader.handleTransparentProxySupport(header, this.prop, virtualHost, httpdProxyHandler.isTransparentProxy);
httpHeader.handleTransparentProxySupport(header, this.prop, virtualHost, httpdProxyHandler.isTransparentProxy);
// determines if the connection should be kept alive
handlePersistentConnection(header);
// return multi-line message
if (prop.getProperty(httpRequestHeader.CONNECTION_PROP_HOST).equals(virtualHost)) {
if (prop.getProperty(httpHeader.CONNECTION_PROP_HOST).equals(virtualHost)) {
// pass to server
if (allowServer) {
if (handleServerAuthentication(header)) {
@ -644,7 +644,7 @@ public final class httpd implements serverHandler, Cloneable {
}
if(sessionIn instanceof ChunkedInputStream) sessionIn.close(); // read to end, but do not close the stream (maybe HTTP/1.1 persistent)
//return serverCore.RESUME_CONNECTION;
return this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_PERSISTENT).equals("keep-alive") ? serverCore.RESUME_CONNECTION : serverCore.TERMINATE_CONNECTION;
return this.prop.getProperty(httpHeader.CONNECTION_PROP_PERSISTENT).equals("keep-alive") ? serverCore.RESUME_CONNECTION : serverCore.TERMINATE_CONNECTION;
} catch (final Exception e) {
logUnexpectedError(e);
return serverCore.TERMINATE_CONNECTION;
@ -663,10 +663,10 @@ public final class httpd implements serverHandler, Cloneable {
httpVersion = arg.substring(pos + 1);
arg = arg.substring(0, pos);
}
prop.setProperty(httpRequestHeader.CONNECTION_PROP_HTTP_VER, httpVersion);
prop.setProperty(httpHeader.CONNECTION_PROP_HTTP_VER, httpVersion);
// parse hostname and port
prop.setProperty(httpRequestHeader.CONNECTION_PROP_HOST, arg);
prop.setProperty(httpHeader.CONNECTION_PROP_HOST, arg);
pos = arg.indexOf(":");
int port = 443;
if (pos >= 0) {
@ -675,11 +675,11 @@ public final class httpd implements serverHandler, Cloneable {
}
// setting other connection properties
prop.setProperty(httpRequestHeader.CONNECTION_PROP_CLIENTIP, this.clientIP);
prop.setProperty(httpRequestHeader.CONNECTION_PROP_METHOD, httpHeader.METHOD_CONNECT);
prop.setProperty(httpRequestHeader.CONNECTION_PROP_PATH, "/");
prop.setProperty(httpRequestHeader.CONNECTION_PROP_EXT, "");
prop.setProperty(httpRequestHeader.CONNECTION_PROP_URL, "");
prop.setProperty(httpHeader.CONNECTION_PROP_CLIENTIP, this.clientIP);
prop.setProperty(httpHeader.CONNECTION_PROP_METHOD, httpHeader.METHOD_CONNECT);
prop.setProperty(httpHeader.CONNECTION_PROP_PATH, "/");
prop.setProperty(httpHeader.CONNECTION_PROP_EXT, "");
prop.setProperty(httpHeader.CONNECTION_PROP_URL, "");
// parse remaining lines
final httpRequestHeader header = httpRequestHeader.readHeader(this.prop,this.session);
@ -716,18 +716,18 @@ public final class httpd implements serverHandler, Cloneable {
httpRequestHeader.parseRequestLine(cmd,s,this.prop,virtualHost);
// track the request
final String path = this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_URL);
final String args = this.prop.getProperty(httpRequestHeader.CONNECTION_PROP_ARGS, "");
final String path = this.prop.getProperty(httpHeader.CONNECTION_PROP_URL);
final String args = this.prop.getProperty(httpHeader.CONNECTION_PROP_ARGS, "");
switchboard.track(this.userAddress.getHostName(), (args.length() > 0) ? path + "?" + args : path);
// reseting the empty request counter
this.emptyRequestCount = 0;
// counting the amount of received requests within this permanent connection
this.prop.setProperty(httpRequestHeader.CONNECTION_PROP_KEEP_ALIVE_COUNT, Integer.toString(++this.keepAliveRequestCount));
this.prop.setProperty(httpHeader.CONNECTION_PROP_KEEP_ALIVE_COUNT, Integer.toString(++this.keepAliveRequestCount));
// setting the client-IP
this.prop.setProperty(httpRequestHeader.CONNECTION_PROP_CLIENTIP, this.clientIP);
this.prop.setProperty(httpHeader.CONNECTION_PROP_CLIENTIP, this.clientIP);
}
// some static methods that needs to be used from any CGI
@ -1109,20 +1109,20 @@ public final class httpd implements serverHandler, Cloneable {
ByteArrayOutputStream o = null;
try {
// setting the proper http status message
final String httpVersion = conProp.getProperty(httpRequestHeader.CONNECTION_PROP_HTTP_VER,"HTTP/1.1");
final String httpVersion = conProp.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER,"HTTP/1.1");
if ((httpStatusText == null)||(httpStatusText.length()==0)) {
if (httpVersion.equals("HTTP/1.0") && httpRequestHeader.http1_0.containsKey(Integer.toString(httpStatusCode)))
httpStatusText = httpRequestHeader.http1_0.get(Integer.toString(httpStatusCode));
else if (httpVersion.equals("HTTP/1.1") && httpRequestHeader.http1_1.containsKey(Integer.toString(httpStatusCode)))
httpStatusText = httpRequestHeader.http1_1.get(Integer.toString(httpStatusCode));
if (httpVersion.equals("HTTP/1.0") && httpHeader.http1_0.containsKey(Integer.toString(httpStatusCode)))
httpStatusText = httpHeader.http1_0.get(Integer.toString(httpStatusCode));
else if (httpVersion.equals("HTTP/1.1") && httpHeader.http1_1.containsKey(Integer.toString(httpStatusCode)))
httpStatusText = httpHeader.http1_1.get(Integer.toString(httpStatusCode));
else httpStatusText = "Unknown";
}
// generating the desired request url
String host = conProp.getProperty(httpRequestHeader.CONNECTION_PROP_HOST);
final String path = conProp.getProperty(httpRequestHeader.CONNECTION_PROP_PATH,"/");
final String args = conProp.getProperty(httpRequestHeader.CONNECTION_PROP_ARGS);
final String method = conProp.getProperty(httpRequestHeader.CONNECTION_PROP_METHOD);
String host = conProp.getProperty(httpHeader.CONNECTION_PROP_HOST);
final String path = conProp.getProperty(httpHeader.CONNECTION_PROP_PATH,"/");
final String args = conProp.getProperty(httpHeader.CONNECTION_PROP_ARGS);
final String method = conProp.getProperty(httpHeader.CONNECTION_PROP_METHOD);
int port = 80;
final int pos = host.indexOf(":");
@ -1133,7 +1133,7 @@ public final class httpd implements serverHandler, Cloneable {
String urlString;
try {
urlString = (new yacyURL((method.equals(httpRequestHeader.METHOD_CONNECT)?"https":"http"), host, port, (args == null) ? path : path + "?" + args)).toString();
urlString = (new yacyURL((method.equals(httpHeader.METHOD_CONNECT)?"https":"http"), host, port, (args == null) ? path : path + "?" + args)).toString();
} catch (final MalformedURLException e) {
urlString = "invalid URL";
}
@ -1144,7 +1144,7 @@ public final class httpd implements serverHandler, Cloneable {
// tp.put("host", serverCore.publicIP().getHostAddress());
// tp.put("port", switchboard.getConfig("port", "8080"));
final String clientIP = conProp.getProperty(httpRequestHeader.CONNECTION_PROP_CLIENTIP, "127.0.0.1");
final String clientIP = conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP, "127.0.0.1");
// check if ip is local ip address
final InetAddress hostAddress = serverDomains.dnsResolve(clientIP);
@ -1171,7 +1171,7 @@ public final class httpd implements serverHandler, Cloneable {
tp.put("peerName", (getAlternativeResolver() == null) ? "" : getAlternativeResolver().myName());
tp.put("errorMessageType", errorcase);
tp.put("httpStatus", Integer.toString(httpStatusCode) + " " + httpStatusText);
tp.put("requestMethod", conProp.getProperty(httpRequestHeader.CONNECTION_PROP_METHOD));
tp.put("requestMethod", conProp.getProperty(httpHeader.CONNECTION_PROP_METHOD));
tp.put("requestURL", urlString);
switch (errorcase) {
@ -1220,11 +1220,11 @@ public final class httpd implements serverHandler, Cloneable {
if(header == null)
header = new httpResponseHeader();
header.put(httpResponseHeader.CONNECTION_PROP_PROXY_RESPOND_STATUS, Integer.toString(httpStatusCode));
header.put(httpResponseHeader.DATE, systemDate);
header.put(httpResponseHeader.CONTENT_TYPE, "text/html");
header.put(httpResponseHeader.CONTENT_LENGTH, Integer.toString(result.length));
header.put(httpResponseHeader.PRAGMA, "no-cache");
header.put(httpHeader.CONNECTION_PROP_PROXY_RESPOND_STATUS, Integer.toString(httpStatusCode));
header.put(httpHeader.DATE, systemDate);
header.put(httpHeader.CONTENT_TYPE, "text/html");
header.put(httpHeader.CONTENT_LENGTH, Integer.toString(result.length));
header.put(httpHeader.PRAGMA, "no-cache");
sendRespondHeader(conProp,respond,httpVersion,httpStatusCode,httpStatusText,header);
if (! method.equals(httpHeader.METHOD_HEAD)) {
@ -1274,17 +1274,17 @@ public final class httpd implements serverHandler, Cloneable {
if(headers==null) headers = new httpResponseHeader();
final Date now = new Date(System.currentTimeMillis());
headers.put(httpResponseHeader.SERVER, "AnomicHTTPD (www.anomic.de)");
headers.put(httpResponseHeader.DATE, DateFormatter.formatRFC1123(now));
headers.put(httpHeader.SERVER, "AnomicHTTPD (www.anomic.de)");
headers.put(httpHeader.DATE, DateFormatter.formatRFC1123(now));
if (moddate.after(now)) {
System.out.println("*** DEBUG: correcting moddate = " + moddate.toString() + " to now = " + now.toString());
moddate = now;
}
headers.put(httpResponseHeader.LAST_MODIFIED, DateFormatter.formatRFC1123(moddate));
headers.put(httpHeader.LAST_MODIFIED, DateFormatter.formatRFC1123(moddate));
if (nocache) {
if (httpVersion.toUpperCase().equals(httpHeader.HTTP_VERSION_1_1)) headers.put(httpResponseHeader.CACHE_CONTROL, "no-cache");
else headers.put(httpResponseHeader.PRAGMA, "no-cache");
if (httpVersion.toUpperCase().equals(httpHeader.HTTP_VERSION_1_1)) headers.put(httpHeader.CACHE_CONTROL, "no-cache");
else headers.put(httpHeader.PRAGMA, "no-cache");
}
if (contentType == null)
@ -1292,11 +1292,11 @@ public final class httpd implements serverHandler, Cloneable {
else if (contentType.startsWith("text/") && contentType.toLowerCase().indexOf("charset=")==-1)
contentType +="; charset=" + Charset.defaultCharset().name();
headers.put(httpHeader.CONTENT_TYPE, contentType);
if (contentLength > 0) headers.put(httpResponseHeader.CONTENT_LENGTH, Long.toString(contentLength));
//if (cookie != null) headers.put(httpResponseHeader.SET_COOKIE, cookie);
if (expires != null) headers.put(httpResponseHeader.EXPIRES, DateFormatter.formatRFC1123(expires));
if (contentEnc != null) headers.put(httpResponseHeader.CONTENT_ENCODING, contentEnc);
if (transferEnc != null) headers.put(httpResponseHeader.TRANSFER_ENCODING, transferEnc);
if (contentLength > 0) headers.put(httpHeader.CONTENT_LENGTH, Long.toString(contentLength));
//if (cookie != null) headers.put(httpHeader.SET_COOKIE, cookie);
if (expires != null) headers.put(httpHeader.EXPIRES, DateFormatter.formatRFC1123(expires));
if (contentEnc != null) headers.put(httpHeader.CONTENT_ENCODING, contentEnc);
if (transferEnc != null) headers.put(httpHeader.TRANSFER_ENCODING, transferEnc);
sendRespondHeader(conProp, respond, httpVersion, httpStatusCode, httpStatusText, headers);
}
@ -1327,10 +1327,10 @@ public final class httpd implements serverHandler, Cloneable {
try {
if ((httpStatusText == null)||(httpStatusText.length()==0)) {
if (httpVersion.equals(httpResponseHeader.HTTP_VERSION_1_0) && httpHeader.http1_0.containsKey(Integer.toString(httpStatusCode)))
httpStatusText = httpResponseHeader.http1_0.get(Integer.toString(httpStatusCode));
else if (httpVersion.equals(httpResponseHeader.HTTP_VERSION_1_1) && httpHeader.http1_1.containsKey(Integer.toString(httpStatusCode)))
httpStatusText = httpResponseHeader.http1_1.get(Integer.toString(httpStatusCode));
if (httpVersion.equals(httpHeader.HTTP_VERSION_1_0) && httpHeader.http1_0.containsKey(Integer.toString(httpStatusCode)))
httpStatusText = httpHeader.http1_0.get(Integer.toString(httpStatusCode));
else if (httpVersion.equals(httpHeader.HTTP_VERSION_1_1) && httpHeader.http1_1.containsKey(Integer.toString(httpStatusCode)))
httpStatusText = httpHeader.http1_1.get(Integer.toString(httpStatusCode));
else httpStatusText = "Unknown";
}
@ -1360,9 +1360,9 @@ public final class httpd implements serverHandler, Cloneable {
responseHeader.put(httpHeader.CONTENT_LENGTH, "0");
// adding some yacy specific headers
responseHeader.put(httpResponseHeader.X_YACY_KEEP_ALIVE_REQUEST_COUNT,conProp.getProperty(httpRequestHeader.CONNECTION_PROP_KEEP_ALIVE_COUNT));
responseHeader.put(httpResponseHeader.X_YACY_ORIGINAL_REQUEST_LINE,conProp.getProperty(httpRequestHeader.CONNECTION_PROP_REQUESTLINE));
responseHeader.put(httpResponseHeader.X_YACY_PREVIOUS_REQUEST_LINE,conProp.getProperty(httpRequestHeader.CONNECTION_PROP_PREV_REQUESTLINE));
responseHeader.put(httpHeader.X_YACY_KEEP_ALIVE_REQUEST_COUNT,conProp.getProperty(httpHeader.CONNECTION_PROP_KEEP_ALIVE_COUNT));
responseHeader.put(httpHeader.X_YACY_ORIGINAL_REQUEST_LINE,conProp.getProperty(httpHeader.CONNECTION_PROP_REQUESTLINE));
responseHeader.put(httpHeader.X_YACY_PREVIOUS_REQUEST_LINE,conProp.getProperty(httpHeader.CONNECTION_PROP_PREV_REQUESTLINE));
//read custom headers
/*

8
source/de/anomic/http/httpdFileHandler.java
Unescape View File

@ -228,10 +228,10 @@ public final class httpdFileHandler {
} else {
ext = path.substring(pos + 1).toLowerCase();
}
headers.put(httpResponseHeader.SERVER, "AnomicHTTPD (www.anomic.de)");
headers.put(httpResponseHeader.DATE, DateFormatter.formatRFC1123(new Date()));
headers.put(httpHeader.SERVER, "AnomicHTTPD (www.anomic.de)");
headers.put(httpHeader.DATE, DateFormatter.formatRFC1123(new Date()));
if(!(plasmaParser.mediaExtContains(ext))){
headers.put(httpResponseHeader.PRAGMA, "no-cache");
headers.put(httpHeader.PRAGMA, "no-cache");
}
return headers;
}
@ -580,7 +580,7 @@ public final class httpdFileHandler {
// set environment variables
Map<String, String> env = pb.environment();
env.put("SERVER_SOFTWARE", getDefaultHeaders(path).get(httpResponseHeader.SERVER));
env.put("SERVER_SOFTWARE", getDefaultHeaders(path).get(httpHeader.SERVER));
env.put("SERVER_NAME", switchboard.getConfig("peerName", "<nameless>"));
env.put("GATEWAY_INTERFACE", "CGI/1.1");
if (httpVersion != null) {

18
source/de/anomic/http/httpdProxyCacheEntry.java
Unescape View File

@ -304,7 +304,7 @@ public class httpdProxyCacheEntry implements Document {
// -pragma in response
// if we have a pragma non-cache, we don't cache. usually if this is wanted from
// the server, it makes sense
String cacheControl = responseHeader.get(httpResponseHeader.PRAGMA);
String cacheControl = responseHeader.get(httpHeader.PRAGMA);
if (cacheControl != null && cacheControl.trim().toUpperCase().equals("NO-CACHE")) { return "controlled_no_cache"; }
// -expires in response
@ -313,7 +313,7 @@ public class httpdProxyCacheEntry implements Document {
// -cache-control in response
// the cache-control has many value options.
cacheControl = responseHeader.get(httpResponseHeader.CACHE_CONTROL);
cacheControl = responseHeader.get(httpHeader.CACHE_CONTROL);
if (cacheControl != null) {
cacheControl = cacheControl.trim().toUpperCase();
if (cacheControl.startsWith("MAX-AGE=")) {
@ -364,10 +364,10 @@ public class httpdProxyCacheEntry implements Document {
if (requestHeader.containsKey(httpHeader.RANGE)) { return false; }
// if the client requests a un-cached copy of the resource ...
cacheControl = requestHeader.get(httpResponseHeader.PRAGMA);
cacheControl = requestHeader.get(httpHeader.PRAGMA);
if (cacheControl != null && cacheControl.trim().toUpperCase().equals("NO-CACHE")) { return false; }
cacheControl = requestHeader.get(httpResponseHeader.CACHE_CONTROL);
cacheControl = requestHeader.get(httpHeader.CACHE_CONTROL);
if (cacheControl != null) {
cacheControl = cacheControl.trim().toUpperCase();
if (cacheControl.startsWith("NO-CACHE") || cacheControl.startsWith("MAX-AGE=0")) { return false; }
@ -379,7 +379,7 @@ public class httpdProxyCacheEntry implements Document {
if (requestHeader.containsKey(httpRequestHeader.IF_MODIFIED_SINCE)) {
// checking this makes only sense if the cached response contains
// a Last-Modified field. If the field does not exist, we go the safe way
if (!responseHeader.containsKey(httpResponseHeader.LAST_MODIFIED)) { return false; }
if (!responseHeader.containsKey(httpHeader.LAST_MODIFIED)) { return false; }
// parse date
Date d1, d2;
d2 = responseHeader.lastModified(); if (d2 == null) { d2 = new Date(DateFormatter.correctedUTCTime()); }
@ -396,8 +396,8 @@ public class httpdProxyCacheEntry implements Document {
// -set-cookie in cached response
// this is a similar case as for COOKIE.
if (requestHeader.containsKey(httpRequestHeader.COOKIE) ||
responseHeader.containsKey(httpResponseHeader.SET_COOKIE) ||
responseHeader.containsKey(httpResponseHeader.SET_COOKIE2)) {
responseHeader.containsKey(httpHeader.SET_COOKIE) ||
responseHeader.containsKey(httpHeader.SET_COOKIE2)) {
return false; // too strong
}
}
@ -408,7 +408,7 @@ public class httpdProxyCacheEntry implements Document {
// logically, we would not need to care about no-cache pragmas in cached response headers,
// because they cannot exist since they are not written to the cache.
// So this IF should always fail..
cacheControl = responseHeader.get(httpResponseHeader.PRAGMA);
cacheControl = responseHeader.get(httpHeader.PRAGMA);
if (cacheControl != null && cacheControl.trim().toUpperCase().equals("NO-CACHE")) { return false; }
// see for documentation also:
@ -427,7 +427,7 @@ public class httpdProxyCacheEntry implements Document {
if (expires.before(new Date(DateFormatter.correctedUTCTime()))) { return false; }
}
final Date lastModified = responseHeader.lastModified();
cacheControl = responseHeader.get(httpResponseHeader.CACHE_CONTROL);
cacheControl = responseHeader.get(httpHeader.CACHE_CONTROL);
if (cacheControl == null && lastModified == null && expires == null) { return false; }
// -lastModified in cached response

26
source/de/anomic/http/httpdProxyHandler.java
Unescape View File

@ -259,8 +259,8 @@ public final class httpdProxyHandler {
| "Version" "=" 1*DIGIT
*/
if (sb.getConfigBool("proxy.monitorCookies", false)) {
if (respondHeader.containsKey(httpResponseHeader.SET_COOKIE)) {
final Object[] entry = new Object[]{new Date(), targetclient, respondHeader.getMultiple(httpResponseHeader.SET_COOKIE)};
if (respondHeader.containsKey(httpHeader.SET_COOKIE)) {
final Object[] entry = new Object[]{new Date(), targetclient, respondHeader.getMultiple(httpHeader.SET_COOKIE)};
synchronized(sb.incomingCookies) {
sb.incomingCookies.put(serverhost, entry);
}
@ -511,7 +511,7 @@ public final class httpdProxyHandler {
// sending the respond header back to the client
if (chunkedOut != null) {
responseHeader.put(httpResponseHeader.TRANSFER_ENCODING, "chunked");
responseHeader.put(httpHeader.TRANSFER_ENCODING, "chunked");
}
if (theLogger.isFinest()) theLogger.logFinest(reqID +" sending response header: "+ responseHeader);
@ -672,7 +672,7 @@ public final class httpdProxyHandler {
conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_REFRESH_HIT");
// setting the content length header to 0
cachedResponseHeader.put(httpResponseHeader.CONTENT_LENGTH, Integer.toString(0));
cachedResponseHeader.put(httpHeader.CONTENT_LENGTH, Integer.toString(0));
// send cached header with replaced date and added length
httpd.sendRespondHeader(conProp,respond,httpVer,304,cachedResponseHeader);
@ -683,7 +683,7 @@ public final class httpdProxyHandler {
conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_HIT");
// setting the content header to the proper length
cachedResponseHeader.put(httpResponseHeader.CONTENT_LENGTH, Long.toString(cacheEntry.length));
cachedResponseHeader.put(httpHeader.CONTENT_LENGTH, Long.toString(cacheEntry.length));
// send cached header with replaced date and added length
httpd.sendRespondHeader(conProp,respond,httpVer,203,cachedResponseHeader);
@ -920,7 +920,7 @@ public final class httpdProxyHandler {
// sending the respond header back to the client
if (chunked != null) {
responseHeader.put(httpResponseHeader.TRANSFER_ENCODING, "chunked");
responseHeader.put(httpHeader.TRANSFER_ENCODING, "chunked");
}
// sending response headers
@ -1011,10 +1011,10 @@ public final class httpdProxyHandler {
}
// only gzip-encoding is supported, remove other encodings (e. g. deflate)
if (((String)requestHeader.get(httpRequestHeader.ACCEPT_ENCODING,"")).indexOf("gzip") != -1) {
requestHeader.put(httpRequestHeader.ACCEPT_ENCODING, "gzip");
if (((String)requestHeader.get(httpHeader.ACCEPT_ENCODING,"")).indexOf("gzip") != -1) {
requestHeader.put(httpHeader.ACCEPT_ENCODING, "gzip");
} else {
requestHeader.put(httpRequestHeader.ACCEPT_ENCODING, "");
requestHeader.put(httpHeader.ACCEPT_ENCODING, "");
}
addXForwardedForHeader(conProp, requestHeader);
@ -1111,7 +1111,7 @@ public final class httpdProxyHandler {
private static void correctContentEncoding(final httpResponseHeader responseHeader) {
// TODO gzip again? set "correct" encoding?
if(responseHeader.gzip()) {
responseHeader.remove(httpResponseHeader.CONTENT_ENCODING);
responseHeader.remove(httpHeader.CONTENT_ENCODING);
responseHeader.remove(httpHeader.CONTENT_LENGTH); // remove gziped length
}
}
@ -1158,11 +1158,11 @@ public final class httpdProxyHandler {
headers.remove(httpRequestHeader.X_CACHE_LOOKUP);
// remove transfer encoding header
headers.remove(httpResponseHeader.TRANSFER_ENCODING);
headers.remove(httpHeader.TRANSFER_ENCODING);
//removing yacy status headers
headers.remove(httpResponseHeader.X_YACY_KEEP_ALIVE_REQUEST_COUNT);
headers.remove(httpResponseHeader.X_YACY_ORIGINAL_REQUEST_LINE);
headers.remove(httpHeader.X_YACY_KEEP_ALIVE_REQUEST_COUNT);
headers.remove(httpHeader.X_YACY_ORIGINAL_REQUEST_LINE);
}
private static void setViaHeader(final httpHeader header, final String httpVer) {

9
source/de/anomic/icap/icapd.java
Unescape View File

@ -37,6 +37,7 @@ import java.util.Date;
import java.util.Properties;
import de.anomic.http.httpChunkedInputStream;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.http.httpResponseHeader;
import de.anomic.http.httpdProxyCacheEntry;
@ -310,9 +311,9 @@ public class icapd implements serverHandler, Cloneable {
final Properties httpReqProps = new Properties();
httpRequestHeader.parseRequestLine(httpRequestLine,httpReqProps,virtualHost);
if (!httpReqProps.getProperty(httpRequestHeader.CONNECTION_PROP_METHOD).equals(httpRequestHeader.METHOD_GET)) {
if (!httpReqProps.getProperty(httpHeader.CONNECTION_PROP_METHOD).equals(httpHeader.METHOD_GET)) {
log.logInfo("Wrong http request method for indexing:" +
"\nRequest Method: " + httpReqProps.getProperty(httpRequestHeader.CONNECTION_PROP_METHOD) +
"\nRequest Method: " + httpReqProps.getProperty(httpHeader.CONNECTION_PROP_METHOD) +
"\nRequest Line: " + httpRequestLine);
reader.close();
if(reqHdrStream != null) {
@ -330,10 +331,10 @@ public class icapd implements serverHandler, Cloneable {
}
// handle transparent proxy support: this function call is needed to set the host property properly
httpRequestHeader.handleTransparentProxySupport(httpReqHeader,httpReqProps,virtualHost,true);
httpHeader.handleTransparentProxySupport(httpReqHeader,httpReqProps,virtualHost,true);
// getting the request URL
final yacyURL httpRequestURL = httpRequestHeader.getRequestURL(httpReqProps);
final yacyURL httpRequestURL = httpHeader.getRequestURL(httpReqProps);
/* =========================================================================
* Parsing response data

3
source/de/anomic/plasma/parser/odt/odtParser.java
Unescape View File

@ -44,6 +44,7 @@ import com.catcode.odf.OpenDocumentTextInputStream;
import de.anomic.crawler.HTTPLoader;
import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.plasma.plasmaParserDocument;
@ -245,7 +246,7 @@ public class odtParser extends AbstractParser implements Parser {
// downloading the document content
final httpRequestHeader reqHeader = new httpRequestHeader();
reqHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.crawlerUserAgent);
reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.crawlerUserAgent);
final byte[] content = httpClient.wget(contentUrl.toString(), reqHeader, 10000);
final ByteArrayInputStream input = new ByteArrayInputStream(content);

3
source/de/anomic/plasma/parser/rpm/rpmParser.java
Unescape View File

@ -35,6 +35,7 @@ import com.jguild.jrpm.io.datatype.DataTypeIf;
import de.anomic.crawler.HTTPLoader;
import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.plasma.plasmaParserDocument;
@ -165,7 +166,7 @@ public class rpmParser extends AbstractParser implements Parser {
final rpmParser testParser = new rpmParser();
final httpRequestHeader reqHeader = new httpRequestHeader();
reqHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.crawlerUserAgent);
reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.crawlerUserAgent);
final byte[] content = httpClient.wget(contentUrl.toString(), reqHeader, 10000);
final ByteArrayInputStream input = new ByteArrayInputStream(content);
testParser.parse(contentUrl, "application/x-rpm", null, input);

3
source/de/anomic/plasma/parser/vcf/vcfParser.java
Unescape View File

@ -36,6 +36,7 @@ import java.util.LinkedList;
import de.anomic.crawler.HTTPLoader;
import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.order.Base64Order;
import de.anomic.plasma.plasmaParserDocument;
@ -276,7 +277,7 @@ public class vcfParser extends AbstractParser implements Parser {
final vcfParser testParser = new vcfParser();
final httpRequestHeader reqHeader = new httpRequestHeader();
reqHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.crawlerUserAgent);
reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.crawlerUserAgent);
final byte[] content = httpClient.wget(contentUrl.toString(), reqHeader, 10000);
final ByteArrayInputStream input = new ByteArrayInputStream(content);
testParser.parse(contentUrl, "text/x-vcard", "UTF-8",input);

11
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -144,6 +144,7 @@ import de.anomic.data.wiki.wikiBoard;
import de.anomic.data.wiki.wikiCode;
import de.anomic.data.wiki.wikiParser;
import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRemoteProxyConfig;
import de.anomic.http.httpRequestHeader;
import de.anomic.http.httpResponseHeader;
@ -1931,7 +1932,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
public int adminAuthenticated(final httpRequestHeader requestHeader) {
// authorization for localhost, only if flag is set to grant localhost access as admin
final String clientIP = (String) requestHeader.get(httpRequestHeader.CONNECTION_PROP_CLIENTIP, "");
final String clientIP = (String) requestHeader.get(httpHeader.CONNECTION_PROP_CLIENTIP, "");
final String refererHost = requestHeader.refererHost();
final boolean accessFromLocalhost = serverCore.isLocalhost(clientIP) && (refererHost.length() == 0 || serverCore.isLocalhost(refererHost));
if (getConfigBool("adminAccountForLocalhost", false) && accessFromLocalhost) return 3; // soft-authenticated for localhost
@ -2188,9 +2189,9 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
// load the seed list
try {
final httpRequestHeader reqHeader = new httpRequestHeader();
reqHeader.put(httpRequestHeader.PRAGMA, "no-cache");
reqHeader.put(httpRequestHeader.CACHE_CONTROL, "no-cache");
reqHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
reqHeader.put(httpHeader.PRAGMA, "no-cache");
reqHeader.put(httpHeader.CACHE_CONTROL, "no-cache");
reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
url = new yacyURL(seedListFileURL, null);
final long start = System.currentTimeMillis();
@ -2275,7 +2276,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
try {
// sending request
final httpRequestHeader reqHeader = new httpRequestHeader();
reqHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
final HashMap<String, String> result = FileUtils.table(httpClient.wget(url.toString(), reqHeader, 10000), "UTF-8");
if (result == null) return new HashMap<String, String>();
return result;

3
source/de/anomic/tools/loaderThreads.java
Unescape View File

@ -26,6 +26,7 @@ import java.util.Hashtable;
import de.anomic.crawler.HTTPLoader;
import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRemoteProxyConfig;
import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.util.FileUtils;
@ -124,7 +125,7 @@ public class loaderThreads {
public void run() {
try {
final httpRequestHeader reqHeader = new httpRequestHeader();
reqHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.crawlerUserAgent);
reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.crawlerUserAgent);
page = httpClient.wget(url.toString(), reqHeader, timeout);
loaded = true;
process.feed(page);

7
source/de/anomic/yacy/yacyClient.java
Unescape View File

@ -64,6 +64,7 @@ import de.anomic.data.Blacklist;
import de.anomic.http.DefaultCharsetFilePart;
import de.anomic.http.DefaultCharsetStringPart;
import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpResponse;
import de.anomic.http.httpRemoteProxyConfig;
import de.anomic.http.httpRequestHeader;
@ -269,8 +270,8 @@ public final class yacyClient {
*/
private static byte[] wput(final String url, final String vhost, final List<Part> post, final int timeout, final boolean gzipBody) throws IOException {
final httpRequestHeader header = new httpRequestHeader();
header.put(httpRequestHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
header.put(httpRequestHeader.HOST, vhost);
header.put(httpHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
header.put(httpHeader.HOST, vhost);
final httpClient client = new httpClient(timeout, header);
client.setProxy(proxyConfig());
@ -1086,7 +1087,7 @@ public final class yacyClient {
//System.out.println("permission=" + permissionMessage(args[1]));
final httpRequestHeader reqHeader = new httpRequestHeader();
reqHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.crawlerUserAgent);
reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.crawlerUserAgent);
final byte[] content = httpClient.wget(
"http://" + target.getPublicAddress() + "/yacy/search.html" +
"?myseed=" + sb.peers.mySeed().genSeedStr(null) +

4
source/de/anomic/yacy/yacyRelease.java
Unescape View File

@ -47,8 +47,8 @@ import java.util.TreeSet;
import de.anomic.crawler.HTTPLoader;
import de.anomic.htmlFilter.htmlFilterContentScraper;
import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpResponse;
import de.anomic.http.httpResponseHeader;
import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.order.Base64Order;
import de.anomic.kelondro.util.FileUtils;
@ -273,7 +273,7 @@ public final class yacyRelease extends yacyVersion {
File download = null;
// setup httpClient
final httpRequestHeader reqHeader = new httpRequestHeader();
reqHeader.put(httpResponseHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
httpResponse res = null;
final String name = this.getUrl().getFileName();

7
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -42,6 +42,7 @@ import java.util.TreeMap;
import de.anomic.crawler.HTTPLoader;
import de.anomic.http.httpClient;
import de.anomic.http.httpHeader;
import de.anomic.http.httpResponse;
import de.anomic.http.httpRequestHeader;
import de.anomic.http.httpd;
@ -817,9 +818,9 @@ public final class yacySeedDB implements httpdAlternativeDomainNames {
private ArrayList<String> downloadSeedFile(final yacyURL seedURL) throws IOException {
// Configure http headers
final httpRequestHeader reqHeader = new httpRequestHeader();
reqHeader.put(httpRequestHeader.PRAGMA, "no-cache");
reqHeader.put(httpRequestHeader.CACHE_CONTROL, "no-cache"); // httpc uses HTTP/1.0 is this necessary?
reqHeader.put(httpRequestHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
reqHeader.put(httpHeader.PRAGMA, "no-cache");
reqHeader.put(httpHeader.CACHE_CONTROL, "no-cache"); // httpc uses HTTP/1.0 is this necessary?
reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
// init http-client
final httpClient client = new httpClient(10000, reqHeader);

2
source/yacy.java
Unescape View File

@ -252,7 +252,7 @@ public final class yacy {
final String svrReleaseNr = matcher.group(1);
try {
try {version = Double.parseDouble(vString);} catch (final NumberFormatException e) {version = (float) 0.1;}
version = yacyRelease.versvn2combinedVersion(version, Integer.parseInt(svrReleaseNr));
version = yacyVersion.versvn2combinedVersion(version, Integer.parseInt(svrReleaseNr));
} catch (final NumberFormatException e) {}
sb.setConfig("svnRevision", svrReleaseNr);
}

Write Preview
Loading…
Cancel
Save