- added SSI parser and include directive for <!--# include virtual="<file>" -->

- added chunked file transfer for non-yacy clients
- SSIs are streamed using chunked transfer, partly delivered pages can be seen in browser before transmission is finished
- added client-side network unit identification
- cleaned up code

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3926 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 18 years ago
parent 32640044c0
commit 1782ef57e5

@ -3,7 +3,7 @@ javacSource=1.4
javacTarget=1.4
# Release Configuration
releaseVersion=0.532
releaseVersion=0.533
releaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
proReleaseFile=yacy_pro_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
releaseFileParentDir=yacy

Binary file not shown.

Before

Width:  |  Height:  |  Size: 41 KiB

@ -216,9 +216,8 @@ public class BlacklistCleaner_p {
if (ok.contains(s)) {
r.put(s, new Integer(ERR_DOUBLE_OCCURANCE));
continue;
} else {
ok.add(s);
}
ok.add(s);
if ((slashPos = s.indexOf("/")) == -1) {
host = s;
@ -235,10 +234,9 @@ public class BlacklistCleaner_p {
if (i == 0 && host.length() > 1 && host.charAt(1) != '.') {
r.put(s, new Integer(ERR_SUBDOMAIN_XOR_WILDCARD));
continue;
} else {
r.put(s, new Integer(ERR_HOST_WRONG_CHARS));
continue;
}
r.put(s, new Integer(ERR_HOST_WRONG_CHARS));
continue;
}
// in host-part only full sub-domains may be wildcards

@ -93,11 +93,12 @@ public class ConfigBasic {
if ((post != null)) config.reinitPortForwarding(post, env);
// starting a peer ping
boolean doPeerPing = false;
//boolean doPeerPing = false;
if ((yacyCore.seedDB.mySeed.isVirgin()) || (yacyCore.seedDB.mySeed.isJunior())) {
serverInstantThread.oneTimeJob(sb.yc, "peerPing", null, 0);
doPeerPing = true;
}
//doPeerPing = true;
}
// scan for Upnp routers
/*

@ -151,80 +151,79 @@ public class CrawlURLFetchStack_p {
} catch (IOException e) { e.printStackTrace(); }
outw.flush();
return null;
} else {
/* =================================================================
* 'normal' request
* ================================================================= */
if (post != null) {
if (post.containsKey("addurls")) {
prop.put("addedUrls", 1);
prop.put("addedUrls_added", addURLs(post, post.getInt("addurls", -1), getURLFetcherStack(env)));
}
else if (post.containsKey("setMaxSize")) {
final int count = post.getInt("maxSize", maxURLsPerFetch);
if (count > 0) {
maxURLsPerFetch = count;
prop.put("set", 1);
prop.put("set_value", maxURLsPerFetch);
} else {
prop.put("set", 2);
prop.put("set_value", count);
}
}
else if (post.containsKey("shiftlcq")) {
final int count = Math.min(post.getInt("shiftloc", 0), sb.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE));
final int failed = shiftFromNotice(sb.noticeURL, plasmaCrawlNURL.STACK_TYPE_CORE, getURLFetcherStack(env), count);
prop.put("shiftloc", 1);
prop.put("shiftloc_value", count - failed);
prop.put("shiftloc_failed", failed);
}
else if (post.containsKey("shiftrcq")) {
final int count = post.getInt("shiftrem", 0);
final int failed = shiftFromNotice(sb.noticeURL, plasmaCrawlNURL.STACK_TYPE_LIMIT, getURLFetcherStack(env), count);
prop.put("shiftrem", 1);
prop.put("shiftrem_value", count - failed);
prop.put("shiftrem_failed", failed);
}
/* =================================================================
* 'normal' request
* ================================================================= */
if (post != null) {
if (post.containsKey("addurls")) {
prop.put("addedUrls", 1);
prop.put("addedUrls_added", addURLs(post, post.getInt("addurls", -1), getURLFetcherStack(env)));
}
else if (post.containsKey("setMaxSize")) {
final int count = post.getInt("maxSize", maxURLsPerFetch);
if (count > 0) {
maxURLsPerFetch = count;
prop.put("set", 1);
prop.put("set_value", maxURLsPerFetch);
} else {
prop.put("set", 2);
prop.put("set_value", count);
}
else if (post.containsKey("subupload")) {
if (post.get("upload", "").length() == 0) {
prop.put("uploadError", 1);
} else {
final File file = new File(post.get("upload", ""));
final String content = new String((byte[])post.get("upload$file"));
final String type = post.get("uploadType", "");
final boolean blCheck = post.containsKey("blacklistCheck");
if (type.equals("plain")) {
prop.put("upload_added", addURLs(content.split("\n"), blCheck, getURLFetcherStack(env)));
prop.put("upload_failed", 0);
}
else if (post.containsKey("shiftlcq")) {
final int count = Math.min(post.getInt("shiftloc", 0), sb.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE));
final int failed = shiftFromNotice(sb.noticeURL, plasmaCrawlNURL.STACK_TYPE_CORE, getURLFetcherStack(env), count);
prop.put("shiftloc", 1);
prop.put("shiftloc_value", count - failed);
prop.put("shiftloc_failed", failed);
}
else if (post.containsKey("shiftrcq")) {
final int count = post.getInt("shiftrem", 0);
final int failed = shiftFromNotice(sb.noticeURL, plasmaCrawlNURL.STACK_TYPE_LIMIT, getURLFetcherStack(env), count);
prop.put("shiftrem", 1);
prop.put("shiftrem_value", count - failed);
prop.put("shiftrem_failed", failed);
}
else if (post.containsKey("subupload")) {
if (post.get("upload", "").length() == 0) {
prop.put("uploadError", 1);
} else {
final File file = new File(post.get("upload", ""));
final String content = new String((byte[])post.get("upload$file"));
final String type = post.get("uploadType", "");
final boolean blCheck = post.containsKey("blacklistCheck");
if (type.equals("plain")) {
prop.put("upload_added", addURLs(content.split("\n"), blCheck, getURLFetcherStack(env)));
prop.put("upload_failed", 0);
prop.put("upload", 1);
} else if (type.equals("html")) {
try {
final htmlFilterContentScraper scraper = new htmlFilterContentScraper(new URL(file));
final Writer writer = new htmlFilterWriter(null, null, scraper, null, false);
serverFileUtils.write(content, writer);
writer.close();
final Iterator it = ((HashMap)scraper.getAnchors()).keySet().iterator();
int added = 0, failed = 0;
URL url;
while (it.hasNext()) try {
url = new URL((String)it.next());
if (blCheck && plasmaSwitchboard.urlBlacklist.isListed(plasmaURLPattern.BLACKLIST_CRAWLER, url)) {
failed++;
continue;
}
getURLFetcherStack(env).push(url);
added++;
} catch (MalformedURLException e) { failed++; }
prop.put("upload", 1);
} else if (type.equals("html")) {
try {
final htmlFilterContentScraper scraper = new htmlFilterContentScraper(new URL(file));
final Writer writer = new htmlFilterWriter(null, null, scraper, null, false);
serverFileUtils.write(content, writer);
writer.close();
final Iterator it = ((HashMap)scraper.getAnchors()).keySet().iterator();
int added = 0, failed = 0;
URL url;
while (it.hasNext()) try {
url = new URL((String)it.next());
if (blCheck && plasmaSwitchboard.urlBlacklist.isListed(plasmaURLPattern.BLACKLIST_CRAWLER, url)) {
failed++;
continue;
}
getURLFetcherStack(env).push(url);
added++;
} catch (MalformedURLException e) { failed++; }
prop.put("upload", 1);
prop.put("upload_added", added);
prop.put("upload_failed", failed);
} catch (Exception e) {
e.printStackTrace();
prop.put("upload", 2);
prop.put("upload_error", e.getMessage());
}
prop.put("upload_added", added);
prop.put("upload_failed", failed);
} catch (Exception e) {
e.printStackTrace();
prop.put("upload", 2);
prop.put("upload_error", e.getMessage());
}
}
}

@ -130,7 +130,6 @@ public class CrawlURLFetch_p {
if (fetcher != null) fetcher.interrupt();
fetcher = null;
try {
if (post.get("source", "").equals("peer") &&
post.get("peerhash", "").equals("random")) {
fetcher = new URLFetcher(
@ -184,11 +183,7 @@ public class CrawlURLFetch_p {
frequency);
}
}
if (fetcher != null)
fetcher.start();
} catch (IOException e) {
e.printStackTrace();
}
if (fetcher != null) fetcher.start();
}
else if (post.containsKey("stop")) {
if (fetcher != null) {
@ -199,7 +194,6 @@ public class CrawlURLFetch_p {
}
else if (post.containsKey("restart")) {
if (fetcher != null) {
try {
fetcher.interrupt();
if (fetcher.url == null) {
fetcher = new URLFetcher(
@ -216,9 +210,6 @@ public class CrawlURLFetch_p {
fetcher.delay);
}
fetcher.start();
} catch (IOException e) {
e.printStackTrace();
}
} else {
prop.put("threadError", ERR_THREAD_RESUME);
}
@ -397,7 +388,7 @@ public class CrawlURLFetch_p {
plasmaCrawlProfile.entry profile,
URL url,
int count,
long delayMs) throws IOException {
long delayMs) {
if (env == null || profile == null || url == null)
throw new NullPointerException("env, profile or url must not be null");
this.sb = (plasmaSwitchboard)env;
@ -412,7 +403,7 @@ public class CrawlURLFetch_p {
serverSwitch env,
plasmaCrawlProfile.entry profile,
int count,
long delayMs) throws IOException {
long delayMs) {
if (env == null || profile == null)
throw new NullPointerException("env or profile must not be null");
this.sb = (plasmaSwitchboard)env;

@ -1,6 +1,11 @@
<!--HEADER for FeedReader_p.html
<title>#[clientname]#'s feed reader</title>
end of HEADER-->
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>YaCy '#[clientname]#': Feed Reader</title>
#%env/templates/metas.template%#
</head>
<body>
#%env/templates/header.template%#
#(page)#
please select your feed with ?url=Feedurl&max=5&offset=1 (to be implemented in html ;))
::
@ -28,4 +33,7 @@ You need to install libx
Problem with url
#(/error)#
test
#(/page)#
#(/page)#
#%env/templates/footer.template%#
</body>
</html>

@ -28,7 +28,7 @@ import java.net.URL;
import java.util.Collection;
import java.util.Iterator;
import de.anomic.data.rssReaderItem;
import de.anomic.data.rssReader;
import de.anomic.http.httpHeader;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
@ -86,7 +86,6 @@ public class FeedReader_p {
}
public static servletProperties respond(httpHeader header, serverObjects post, serverSwitch env) {
servletProperties prop = new servletProperties();
prop.put("SUPERTEMPLATE", "/env/page.html");
URL url;
prop.put("page", 0);
@ -122,7 +121,7 @@ public class FeedReader_p {
Iterator it=feedItems.iterator();
int count=0;
while(it.hasNext() && (maxitems==0 || count<maxitems)){
rssReaderItem item=(rssReaderItem)it.next();
rssReader.Item item=(rssReader.Item)it.next();
prop.put("page_items_"+count+"_author", item.getCreator());
prop.put("page_items_"+count+"_title", item.getTitle());
prop.put("page_items_"+count+"_link", item.getLink().toString());

@ -1,24 +0,0 @@
#(mode)#
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>Gettext Locales</title>
#%env/templates/metas.template%#
</head>
<body marginheight="0" marginwidth="0" leftmargin="0" topmargin="0">
#%env/templates/header.template%#
<h2>Gettext Locales</h2>
<p>
<a href="Gettext_p.html?mode=1">Get empty gettext file</a>
<form action="Gettext_p.html" method="GET">
<input type="hidden" name="mode" value="1" />
old file: <input type="text" name="oldfile" value="DATA/LOCALE" />
<input type="submit" value="get updated file" />
</form>
</p>
#%env/templates/footer.template%#
</body>
</html>
::
#[gettext]#
#(/mode)#

@ -1,78 +0,0 @@
//Gettext_p.java
//------------
// part of YACY
//
// (C) 2006 Alexander Schier
//
// last change: $LastChangedDate: $ by $LastChangedBy: $
// $LastChangedRevision: $
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.io.File;
import java.io.FileNotFoundException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Map;
import de.anomic.data.gettext;
import de.anomic.http.httpHeader;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
public class Gettext_p{
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
serverObjects prop = new serverObjects();
if(post != null && post.get("mode").equals("1")){
prop.put("mode", "1");
File oldfile=null;
String oldfilename;
if(post.containsKey("oldfile")){
oldfilename=(String) post.get("oldfile");
oldfile=new File(env.getRootPath(), oldfilename);
if(!oldfile.exists())
//TODO: display warning?
oldfile=null;
}
String htRootPath = env.getConfig("htRootPath", "htroot");
File sourceDir = new File(env.getRootPath(), htRootPath);
ArrayList list;
try {
list = gettext.createGettextRecursive(sourceDir, "html,template,inc", "locale", oldfile);
} catch (FileNotFoundException e) {
// TODO warn the user
list = gettext.createGettextRecursive(sourceDir, "html,template,inc", "locale", (Map)null);
}
Iterator it=list.iterator();
String out="";
while(it.hasNext()){
out+=(String)it.next()+"\n";
}
//this does not work
/*httpHeader outheader=new httpHeader();
outheader.put("Content-Type", "text/plain");
prop.setOutgoingHeader(outheader);*/
prop.put("mode_gettext", out);
}
return prop;
}
}

@ -93,9 +93,7 @@ public class IndexCleaner_p {
prop.put("LOCATION","");
return prop;
}
else {
prop.put("bla", "post==null");
}
prop.put("bla", "post==null");
if (urldbCleanerThread!=null) {
prop.put("urldb", 1);
prop.put("urldb_percentUrls", ((double)urldbCleanerThread.totalSearchedUrls/sb.wordIndex.loadedURL.size())*100 + "");

@ -1,14 +0,0 @@
<!--HEADER for Lab.html
<title>#[clientname]#'s Lab</title>
end of HEADER-->
<h2>The YaCy Lab</h2>
<p>
This is the place where we try new functions of the YaCy search engine.
All these things here are to be considered as probably unstable, and/or experimental.
You may try out these things but please do not care about bugs.</p>
<ul>
<li>The <a href="Wiki.html">Wiki</a></li>
<li>Advanced <a href="/Config_p.html">Configuration</a></li>
</ul>

@ -1,13 +0,0 @@
import de.anomic.http.httpHeader;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
public class Lab {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
serverObjects prop = new serverObjects();
//XXX: Should we use Constants like DEFAULT_PAGE, PAGE_WITHOUT_MENU and so on,
//or is it nice enough to set the real path in the servlets?
prop.put("SUPERTEMPLATE", "/env/page.html");
return prop;
}
}

@ -84,100 +84,99 @@ public class LogStatistics_p {
if (r == null) {
prop.put("results", 0);
return prop;
}
prop.put("results", 1);
String[] t;
float l;
prop.put(RESULTS + LogParserPLASMA.DHT_DISTANCE_AVERAGE, r.get(LogParserPLASMA.DHT_DISTANCE_AVERAGE));
prop.put(RESULTS + LogParserPLASMA.DHT_DISTANCE_MAX, r.get(LogParserPLASMA.DHT_DISTANCE_MAX));
prop.put(RESULTS + LogParserPLASMA.DHT_DISTANCE_MIN, r.get(LogParserPLASMA.DHT_DISTANCE_MIN));
prop.put(RESULTS + LogParserPLASMA.DHT_REJECTED, r.get(LogParserPLASMA.DHT_REJECTED));
prop.put(RESULTS + LogParserPLASMA.DHT_SELECTED, r.get(LogParserPLASMA.DHT_SELECTED));
prop.put(RESULTS + LogParserPLASMA.DHT_SENT_FAILED, r.get(LogParserPLASMA.DHT_SENT_FAILED));
t = transformMem(((Long)r.get(LogParserPLASMA.DHT_TRAFFIC_SENT)).longValue());
prop.put(RESULTS + LogParserPLASMA.DHT_TRAFFIC_SENT, t[0]);
prop.put(RESULTS + LogParserPLASMA.DHT_TRAFFIC_SENT + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.DHT_URLS_SENT, r.get(LogParserPLASMA.DHT_URLS_SENT));
prop.put(RESULTS + LogParserPLASMA.DHT_WORDS_SELECTED, r.get(LogParserPLASMA.DHT_WORDS_SELECTED));
t = transformTime(((Integer)r.get(LogParserPLASMA.DHT_WORDS_SELECTED_TIME)).longValue() * 1000L);
prop.put(RESULTS + LogParserPLASMA.DHT_WORDS_SELECTED_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.DHT_WORDS_SELECTED_TIME + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.ERROR_CHILD_TWICE_LEFT, r.get(LogParserPLASMA.ERROR_CHILD_TWICE_LEFT));
prop.put(RESULTS + LogParserPLASMA.ERROR_CHILD_TWICE_RIGHT, r.get(LogParserPLASMA.ERROR_CHILD_TWICE_RIGHT));
prop.put(RESULTS + LogParserPLASMA.ERROR_MALFORMED_URL, r.get(LogParserPLASMA.ERROR_MALFORMED_URL));
prop.put(RESULTS + LogParserPLASMA.INDEXED_ANCHORS, r.get(LogParserPLASMA.INDEXED_ANCHORS));
t = transformTime(((Integer)r.get(LogParserPLASMA.INDEXED_INDEX_TIME)).longValue());
prop.put(RESULTS + LogParserPLASMA.INDEXED_INDEX_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.INDEXED_INDEX_TIME + "Unit", t[1]);
t = transformTime(((Integer)r.get(LogParserPLASMA.INDEXED_PARSE_TIME)).longValue());
prop.put(RESULTS + LogParserPLASMA.INDEXED_PARSE_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.INDEXED_PARSE_TIME + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.INDEXED_SITES, r.get(LogParserPLASMA.INDEXED_SITES));
t = transformMem(((Integer)r.get(LogParserPLASMA.INDEXED_SITES_SIZE)).longValue());
prop.put(RESULTS + LogParserPLASMA.INDEXED_SITES_SIZE, t[0]);
prop.put(RESULTS + LogParserPLASMA.INDEXED_SITES_SIZE + "Unit", t[1]);
t = transformTime(((Integer)r.get(LogParserPLASMA.INDEXED_STACK_TIME)).longValue());
prop.put(RESULTS + LogParserPLASMA.INDEXED_STACK_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.INDEXED_STACK_TIME + "Unit", t[1]);
t = transformTime(((Integer)r.get(LogParserPLASMA.INDEXED_STORE_TIME)).longValue());
prop.put(RESULTS + LogParserPLASMA.INDEXED_STORE_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.INDEXED_STORE_TIME + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.INDEXED_WORDS, r.get(LogParserPLASMA.INDEXED_WORDS));
prop.put(RESULTS + LogParserPLASMA.PEERS_BUSY, r.get(LogParserPLASMA.PEERS_BUSY));
prop.put(RESULTS + LogParserPLASMA.PEERS_TOO_LESS, r.get(LogParserPLASMA.PEERS_TOO_LESS));
prop.put(RESULTS + LogParserPLASMA.RANKING_DIST, r.get(LogParserPLASMA.RANKING_DIST));
prop.put(RESULTS + LogParserPLASMA.RANKING_DIST_FAILED, r.get(LogParserPLASMA.RANKING_DIST_FAILED));
t = transformTime(((Integer)r.get(LogParserPLASMA.RANKING_DIST_TIME)).longValue());
prop.put(RESULTS + LogParserPLASMA.RANKING_DIST_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.RANKING_DIST_TIME + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.RWIS_BLOCKED, r.get(LogParserPLASMA.RWIS_BLOCKED));
prop.put(RESULTS + LogParserPLASMA.RWIS_RECEIVED, r.get(LogParserPLASMA.RWIS_RECEIVED));
t = transformTime(((Long)r.get(LogParserPLASMA.RWIS_RECEIVED_TIME)).longValue());
prop.put(RESULTS + LogParserPLASMA.RWIS_RECEIVED_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.RWIS_RECEIVED_TIME + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.URLS_BLOCKED, r.get(LogParserPLASMA.URLS_BLOCKED));
prop.put(RESULTS + LogParserPLASMA.URLS_RECEIVED, r.get(LogParserPLASMA.URLS_RECEIVED));
t = transformTime(((Long)r.get(LogParserPLASMA.URLS_RECEIVED_TIME)).longValue());
prop.put(RESULTS + LogParserPLASMA.URLS_RECEIVED_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.URLS_RECEIVED_TIME + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.URLS_REQUESTED, r.get(LogParserPLASMA.URLS_REQUESTED));
prop.put(RESULTS + LogParserPLASMA.WORDS_RECEIVED, r.get(LogParserPLASMA.WORDS_RECEIVED));
l = ((Long)r.get(LogParserPLASMA.TOTAL_PARSER_TIME)).floatValue();
t = transformTime((long)l);
prop.put(RESULTS + LogParserPLASMA.TOTAL_PARSER_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.TOTAL_PARSER_TIME + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.TOTAL_PARSER_RUNS, r.get(LogParserPLASMA.TOTAL_PARSER_RUNS));
if ((l /= 1000) == 0) {
prop.put(RESULTS + "avgExists", 0);
} else {
prop.put("results", 1);
String[] t;
float l;
prop.put(RESULTS + LogParserPLASMA.DHT_DISTANCE_AVERAGE, r.get(LogParserPLASMA.DHT_DISTANCE_AVERAGE));
prop.put(RESULTS + LogParserPLASMA.DHT_DISTANCE_MAX, r.get(LogParserPLASMA.DHT_DISTANCE_MAX));
prop.put(RESULTS + LogParserPLASMA.DHT_DISTANCE_MIN, r.get(LogParserPLASMA.DHT_DISTANCE_MIN));
prop.put(RESULTS + LogParserPLASMA.DHT_REJECTED, r.get(LogParserPLASMA.DHT_REJECTED));
prop.put(RESULTS + LogParserPLASMA.DHT_SELECTED, r.get(LogParserPLASMA.DHT_SELECTED));
prop.put(RESULTS + LogParserPLASMA.DHT_SENT_FAILED, r.get(LogParserPLASMA.DHT_SENT_FAILED));
t = transformMem(((Long)r.get(LogParserPLASMA.DHT_TRAFFIC_SENT)).longValue());
prop.put(RESULTS + LogParserPLASMA.DHT_TRAFFIC_SENT, t[0]);
prop.put(RESULTS + LogParserPLASMA.DHT_TRAFFIC_SENT + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.DHT_URLS_SENT, r.get(LogParserPLASMA.DHT_URLS_SENT));
prop.put(RESULTS + LogParserPLASMA.DHT_WORDS_SELECTED, r.get(LogParserPLASMA.DHT_WORDS_SELECTED));
t = transformTime(((Integer)r.get(LogParserPLASMA.DHT_WORDS_SELECTED_TIME)).longValue() * 1000L);
prop.put(RESULTS + LogParserPLASMA.DHT_WORDS_SELECTED_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.DHT_WORDS_SELECTED_TIME + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.ERROR_CHILD_TWICE_LEFT, r.get(LogParserPLASMA.ERROR_CHILD_TWICE_LEFT));
prop.put(RESULTS + LogParserPLASMA.ERROR_CHILD_TWICE_RIGHT, r.get(LogParserPLASMA.ERROR_CHILD_TWICE_RIGHT));
prop.put(RESULTS + LogParserPLASMA.ERROR_MALFORMED_URL, r.get(LogParserPLASMA.ERROR_MALFORMED_URL));
prop.put(RESULTS + LogParserPLASMA.INDEXED_ANCHORS, r.get(LogParserPLASMA.INDEXED_ANCHORS));
t = transformTime(((Integer)r.get(LogParserPLASMA.INDEXED_INDEX_TIME)).longValue());
prop.put(RESULTS + LogParserPLASMA.INDEXED_INDEX_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.INDEXED_INDEX_TIME + "Unit", t[1]);
t = transformTime(((Integer)r.get(LogParserPLASMA.INDEXED_PARSE_TIME)).longValue());
prop.put(RESULTS + LogParserPLASMA.INDEXED_PARSE_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.INDEXED_PARSE_TIME + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.INDEXED_SITES, r.get(LogParserPLASMA.INDEXED_SITES));
t = transformMem(((Integer)r.get(LogParserPLASMA.INDEXED_SITES_SIZE)).longValue());
prop.put(RESULTS + LogParserPLASMA.INDEXED_SITES_SIZE, t[0]);
prop.put(RESULTS + LogParserPLASMA.INDEXED_SITES_SIZE + "Unit", t[1]);
t = transformTime(((Integer)r.get(LogParserPLASMA.INDEXED_STACK_TIME)).longValue());
prop.put(RESULTS + LogParserPLASMA.INDEXED_STACK_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.INDEXED_STACK_TIME + "Unit", t[1]);
t = transformTime(((Integer)r.get(LogParserPLASMA.INDEXED_STORE_TIME)).longValue());
prop.put(RESULTS + LogParserPLASMA.INDEXED_STORE_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.INDEXED_STORE_TIME + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.INDEXED_WORDS, r.get(LogParserPLASMA.INDEXED_WORDS));
prop.put(RESULTS + LogParserPLASMA.PEERS_BUSY, r.get(LogParserPLASMA.PEERS_BUSY));
prop.put(RESULTS + LogParserPLASMA.PEERS_TOO_LESS, r.get(LogParserPLASMA.PEERS_TOO_LESS));
prop.put(RESULTS + LogParserPLASMA.RANKING_DIST, r.get(LogParserPLASMA.RANKING_DIST));
prop.put(RESULTS + LogParserPLASMA.RANKING_DIST_FAILED, r.get(LogParserPLASMA.RANKING_DIST_FAILED));
t = transformTime(((Integer)r.get(LogParserPLASMA.RANKING_DIST_TIME)).longValue());
prop.put(RESULTS + LogParserPLASMA.RANKING_DIST_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.RANKING_DIST_TIME + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.RWIS_BLOCKED, r.get(LogParserPLASMA.RWIS_BLOCKED));
prop.put(RESULTS + LogParserPLASMA.RWIS_RECEIVED, r.get(LogParserPLASMA.RWIS_RECEIVED));
t = transformTime(((Long)r.get(LogParserPLASMA.RWIS_RECEIVED_TIME)).longValue());
prop.put(RESULTS + LogParserPLASMA.RWIS_RECEIVED_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.RWIS_RECEIVED_TIME + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.URLS_BLOCKED, r.get(LogParserPLASMA.URLS_BLOCKED));
prop.put(RESULTS + LogParserPLASMA.URLS_RECEIVED, r.get(LogParserPLASMA.URLS_RECEIVED));
t = transformTime(((Long)r.get(LogParserPLASMA.URLS_RECEIVED_TIME)).longValue());
prop.put(RESULTS + LogParserPLASMA.URLS_RECEIVED_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.URLS_RECEIVED_TIME + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.URLS_REQUESTED, r.get(LogParserPLASMA.URLS_REQUESTED));
prop.put(RESULTS + LogParserPLASMA.WORDS_RECEIVED, r.get(LogParserPLASMA.WORDS_RECEIVED));
l = ((Long)r.get(LogParserPLASMA.TOTAL_PARSER_TIME)).floatValue();
t = transformTime((long)l);
prop.put(RESULTS + LogParserPLASMA.TOTAL_PARSER_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.TOTAL_PARSER_TIME + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.TOTAL_PARSER_RUNS, r.get(LogParserPLASMA.TOTAL_PARSER_RUNS));
if ((l /= 1000) == 0) {
prop.put(RESULTS + "avgExists", 0);
} else {
prop.put(RESULTS + "avgExists", 1);
prop.put(RESULTS + "avgExists_avgParserRunsPerMinute", (int) (((Integer) r.get(LogParserPLASMA.TOTAL_PARSER_RUNS)).floatValue() / l));
}
Object[] names = ((HashSet)r.get(LogParserPLASMA.DHT_REJECTED_PEERS_NAME)).toArray();
Object[] hashes = ((HashSet)r.get(LogParserPLASMA.DHT_REJECTED_PEERS_HASH)).toArray();
int i = 0;
for (; i<names.length && i<hashes.length; i++) {
prop.put(RESULTS + "useDHTRejectPeers_DHTRejectPeers_" + i + "_name", names[i]);
prop.put(RESULTS + "useDHTRejectPeers_DHTRejectPeers_" + i + "_hash", hashes[i]);
}
prop.put(RESULTS + "DHTRejectPeers", i);
prop.put(RESULTS + "useDHTRejectPeers", (i > 0) ? 1 : 0);
prop.put(RESULTS + "useDHTRejectPeers_DHTRejectPeers", i);
names = ((HashSet)r.get(LogParserPLASMA.DHT_SENT_PEERS_NAME)).toArray();
hashes = ((HashSet)r.get(LogParserPLASMA.DHT_SENT_PEERS_HASH)).toArray();
i = 0;
for (; i<names.length && i<hashes.length; i++) {
prop.put(RESULTS + "useDHTPeers_DHTPeers_" + i + "_name", names[i]);
prop.put(RESULTS + "useDHTPeers_DHTPeers_" + i + "_hash", hashes[i]);
}
prop.put(RESULTS + "DHTPeers", i);
prop.put(RESULTS + "useDHTPeers", (i > 0) ? 1 : 0);
prop.put(RESULTS + "useDHTPeers_DHTPeers", i);
return prop;
prop.put(RESULTS + "avgExists", 1);
prop.put(RESULTS + "avgExists_avgParserRunsPerMinute", (int) (((Integer) r.get(LogParserPLASMA.TOTAL_PARSER_RUNS)).floatValue() / l));
}
Object[] names = ((HashSet)r.get(LogParserPLASMA.DHT_REJECTED_PEERS_NAME)).toArray();
Object[] hashes = ((HashSet)r.get(LogParserPLASMA.DHT_REJECTED_PEERS_HASH)).toArray();
int i = 0;
for (; i<names.length && i<hashes.length; i++) {
prop.put(RESULTS + "useDHTRejectPeers_DHTRejectPeers_" + i + "_name", names[i]);
prop.put(RESULTS + "useDHTRejectPeers_DHTRejectPeers_" + i + "_hash", hashes[i]);
}
prop.put(RESULTS + "DHTRejectPeers", i);
prop.put(RESULTS + "useDHTRejectPeers", (i > 0) ? 1 : 0);
prop.put(RESULTS + "useDHTRejectPeers_DHTRejectPeers", i);
names = ((HashSet)r.get(LogParserPLASMA.DHT_SENT_PEERS_NAME)).toArray();
hashes = ((HashSet)r.get(LogParserPLASMA.DHT_SENT_PEERS_HASH)).toArray();
i = 0;
for (; i<names.length && i<hashes.length; i++) {
prop.put(RESULTS + "useDHTPeers_DHTPeers_" + i + "_name", names[i]);
prop.put(RESULTS + "useDHTPeers_DHTPeers_" + i + "_hash", hashes[i]);
}
prop.put(RESULTS + "DHTPeers", i);
prop.put(RESULTS + "useDHTPeers", (i > 0) ? 1 : 0);
prop.put(RESULTS + "useDHTPeers_DHTPeers", i);
return prop;
}
private static final String MILLISECONDS = "ms";

@ -18,16 +18,10 @@
// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.lang.reflect.InvocationTargetException;
import java.util.Iterator;
import de.anomic.http.httpHeader;
import de.anomic.http.httpTemplate;
import de.anomic.http.httpdFileHandler;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverByteBuffer;
@ -56,19 +50,18 @@ public class TestApplet {
}
prop.put("mode", "1");
File templatefile=filehandler.getOverlayedFile((String)post.get("url"));
//File templatefile=filehandler.getOverlayedFile((String)post.get("url"));
File classfile=filehandler.getOverlayedClass((String)post.get("url"));
httpHeader header2=new httpHeader();
header2.put("CLIENTIP", "127.0.0.1");
header2.put("PATH", (String)post.get("url"));
header2.put("PATH", post.get("url"));
serverObjects tp=null;
try {
if(classfile==null || !classfile.exists()){
prop.put("mode_templates", "classfile does not exist");
return prop;
}else{
tp=(serverObjects)filehandler.invokeServlet(classfile, header2, args);
}
tp=(serverObjects)filehandler.invokeServlet(classfile, header2, args);
}
catch (IllegalArgumentException e) {}
catch (IllegalAccessException e) {}
@ -87,22 +80,22 @@ public class TestApplet {
tmp.append(key).append("=").append(tp.get(key)).append("\n");
}
prop.put("mode_templates", tmp.toString());
FileInputStream fis=null;
try {
fis=new FileInputStream(templatefile);
//FileInputStream fis=null;
//try {
//fis=new FileInputStream(templatefile);
serverByteBuffer o=new serverByteBuffer();
byte[] structure=httpTemplate.writeTemplate(fis, (OutputStream)o, tp, "-UNRESOLVED_PATTERN-".getBytes("UTF-8"));
prop.put("mode_structure", structure);
//byte[] structure=httpTemplate.writeTemplate(fis, o, tp, "-UNRESOLVED_PATTERN-".getBytes("UTF-8"));
//prop.put("mode_structure", structure);
prop.put("mode_text", o.toString());
return prop;
}
catch (FileNotFoundException e) {}
catch (UnsupportedEncodingException e) {}
catch (IOException e) {}
//}
//catch (FileNotFoundException e) {}
//catch (UnsupportedEncodingException e) {}
//catch (IOException e) {}
prop.put("mode_text", "could not finish correctly"); //very informative errormessage
return prop;
//prop.put("mode_text", "could not finish correctly"); //very informative errormessage
//return prop;
}

@ -4,10 +4,8 @@
<title>YaCy '#[clientname]#': User Page</title>
#%env/templates/metas.template%#
</head>
<body>
<body>
#%env/templates/header.template%#
<br />
<br />
<h2>User Page</h2>
#(logged-in)#
<p>

@ -1,6 +1,11 @@
<!--HEADER for Lab.html
<title>#[clientname]#'s User Administration</title>
end of HEADER-->
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>YaCy '#[clientname]#': User Page</title>
#%env/templates/metas.template%#
</head>
<body>
#%env/templates/header.template%#
<h2>User Administration</h2>
<!-- Page 1: Results -->
@ -70,3 +75,6 @@
</dl>
</fieldset>
</form>
#%env/templates/footer.template%#
</body>
</html>

@ -63,7 +63,6 @@ public class User_p {
serverObjects prop = new serverObjects();
plasmaSwitchboard sb = plasmaSwitchboard.getSwitchboard();
userDB.Entry entry=null;
prop.put("SUPERTEMPLATE", "/env/page.html"); //user Supertemplates
//default values
prop.put("current_user", "newuser");

@ -53,7 +53,7 @@ import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import de.anomic.data.Diff;
import de.anomic.data.diff;
import de.anomic.data.wikiBoard;
import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaSwitchboard;
@ -244,10 +244,10 @@ public class Wiki {
if (nentry == null) nentry = entry;
if (post.containsKey("compare") && oentry != null && nentry != null) {
// TODO: split into paragraphs and compare them with the same diff-algo
Diff diff = new Diff(
diff diff = new diff(
new String(oentry.page(), "UTF-8"),
new String(nentry.page(), "UTF-8"), 3);
prop.putASIS("mode_versioning_diff", Diff.toHTML(new Diff[] { diff }));
prop.putASIS("mode_versioning_diff", de.anomic.data.diff.toHTML(new diff[] { diff }));
prop.put("mode_versioning", 1);
} else if (post.containsKey("viewold") && oentry != null) {
prop.put("mode_versioning", 2);

@ -1,16 +0,0 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
#[header]#
<!--<title>YaCy '#[clientname]#': #[title]#</title>-->
<!-- TODO: title cannot be set, yet. -->
<!-- clientname needs to be set in page.java (this will remove the code in httpdFileHandler,
if we use supertemplates for everything -->
#%env/templates/metas.template%#
</head>
<body>
#%env/templates/header.template%#
#[page]#
#%env/templates/footer.template%#
</body>
</html>

@ -1,41 +0,0 @@
//page.java - super template, to allow a scripted layout.
//----------------------------------------------------------
//part of YaCy
//
// (C) 2006 by Alexander Schier
//
// last change: $LastChangedDate: $ by $LastChangedBy: $
// $LastChangedRevision: $
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//note: this isn't useful, yet.
//this is only a basic demonstration, what can be done with the supertemplates.
import de.anomic.http.httpHeader;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
public class page {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
//plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
//prop.put("test1", "testit!");
//prop.put("test2", 1);
return prop;
}
}

@ -0,0 +1,11 @@
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<!-- aus http://de.selfhtml.org/servercgi/server/ssi.htm -->
<html>
<head>
<title>Dynamisches HTML mit Server Side Includes</title>
</head>
<body>
<h1>Dynamisches HTML mit Server Side Includes</h1>
<!--#include virtual="ssitest.inc" -->
</body>
</html>

@ -0,0 +1,7 @@
<!-- aus http://de.selfhtml.org/servercgi/server/ssi.htm -->
<p>Auf einer Meierei,<br>
da lebte einst ein braves Huhn,<br>
das legte, wie die HŸhner tun,<br>
an jedem Tag ein Ei.<br>
Und kakelte, mirakelte, spektakelte,<br>
als obs ein Wunder sei.</p>

@ -640,7 +640,7 @@ final class dbTable implements kelondroIndex {
throw new UnsupportedOperationException();
}
public synchronized void addUnique(kelondroRow.Entry row, Date entryDate) throws IOException {
public synchronized void addUnique(kelondroRow.Entry row, Date entryDate) {
throw new UnsupportedOperationException();
}

@ -783,9 +783,8 @@ public class bookmarksDB {
public String getOwner(){
if(entry.containsKey(BOOKMARK_OWNER)){
return (String) entry.get(BOOKMARK_OWNER);
}else{
return null; //null means admin
}
return null; //null means admin
}
public void setOwner(String owner){
entry.put(BOOKMARK_OWNER, owner);
@ -793,16 +792,14 @@ public class bookmarksDB {
public boolean getPublic(){
if(entry.containsKey(BOOKMARK_PUBLIC)){
return ((String) entry.get(BOOKMARK_PUBLIC)).equals("public");
}else{
return false;
}
return false;
}
public boolean getFeed(){
if(entry.containsKey(BOOKMARK_IS_FEED)){
return ((String) entry.get(BOOKMARK_IS_FEED)).equals("true");
}else{
return false;
}
return false;
}
public void setPublic(boolean isPublic){
if(isPublic){
@ -888,8 +885,8 @@ public class bookmarksDB {
public void remove() {
if (this.nextEntry != null) {
try {
String tagHash = (String)this.nextEntry.getTagHash();
if (tagHash != null) removeTag((String) tagHash);
String tagHash = this.nextEntry.getTagHash();
if (tagHash != null) removeTag(tagHash);
} catch (kelondroException e) {
//resetDatabase();
}
@ -949,16 +946,11 @@ public class bookmarksDB {
if(bm1==null || bm2==null)
return 0; //XXX: i think this should not happen? maybe this needs further tracing of the bug
if(this.newestFirst){
if(bm2.getTimeStamp() - bm1.getTimeStamp() >0)
return 1;
else
return -1;
}else{
if(bm1.getTimeStamp() - bm2.getTimeStamp() >0)
return 1;
else
return -1;
if(bm2.getTimeStamp() - bm1.getTimeStamp() >0) return 1;
return -1;
}
if(bm1.getTimeStamp() - bm2.getTimeStamp() >0) return 1;
return -1;
}
}
/**

@ -6,7 +6,7 @@
// Frankfurt, Germany, 2007
// Created 03.02.2007
//
// This file is contributed by Franz Brauße
// This file is contributed by Franz Brau§e
//
// $LastChangedDate: $
// $LastChangedRevision: $
@ -52,7 +52,7 @@ import java.util.ArrayList;
/**
* This class provides a diff-functionality.
*/
public class Diff {
public class diff {
private final ArrayList /* of Part */ parts = new ArrayList();
private final Object[] o;
@ -63,7 +63,7 @@ public class Diff {
* @param n the new <code>String</code>
* @throws NullPointerException if one of the arguments is <code>null</code>
*/
public Diff(String o, String n) {
public diff(String o, String n) {
this(o, n, 1);
}
@ -76,7 +76,7 @@ public class Diff {
* @throws NullPointerException if <code>o</code> or <code>n</code> is
* <code>null</code>
*/
public Diff(String o, String n, int minConsecutive) {
public diff(String o, String n, int minConsecutive) {
if (o == null || n == null) throw new NullPointerException("neither o nor n must be null");
this.o = new Comparable[o.length()];
for (int i=0; i<o.length(); i++)
@ -87,7 +87,7 @@ public class Diff {
parse((minConsecutive > 0) ? minConsecutive : 1);
}
public Diff(Object[] o, Object[] n, int minConsecutive) {
public diff(Object[] o, Object[] n, int minConsecutive) {
if (o == null || n == null) throw new NullPointerException("neither o nor n must be null");
this.o = o;
this.n = n;
@ -237,10 +237,10 @@ public class Diff {
final StringBuffer sb = new StringBuffer(this.posNew - this.posOld);
if (this.action == ADDED) {
for (int i=this.posOld; i<this.posNew; i++)
sb.append(Diff.this.n[i]);
sb.append(diff.this.n[i]);
} else {
for (int i=this.posOld; i<this.posNew; i++)
sb.append(Diff.this.o[i]);
sb.append(diff.this.o[i]);
}
return new String(sb);
}
@ -259,18 +259,18 @@ public class Diff {
}
}
public static String toHTML(Diff[] diffs) {
public static String toHTML(diff[] diffs) {
StringBuffer sb = new StringBuffer(diffs.length * 60);
Diff.Part[] ps;
diff.Part[] ps;
for (int i=0; i<diffs.length; i++) {
sb.append("<p class=\"diff\">\n");
ps = diffs[i].getParts();
for (int j=0; j<ps.length; j++) {
sb.append("<span\nclass=\"");
switch (ps[j].getAction()) {
case Diff.Part.UNCHANGED: sb.append("unchanged"); break;
case Diff.Part.ADDED: sb.append("added"); break;
case Diff.Part.DELETED: sb.append("deleted"); break;
case diff.Part.UNCHANGED: sb.append("unchanged"); break;
case diff.Part.ADDED: sb.append("added"); break;
case diff.Part.DELETED: sb.append("deleted"); break;
}
sb.append("\">").append(htmlTools.replaceXMLEntities(ps[j].getString()).replaceAll("\n", "<br />"));
sb.append("</span>");
@ -279,4 +279,4 @@ public class Diff {
}
return new String(sb);
}
}
}

@ -1,333 +0,0 @@
//gettext.java - translations in a simplified gettext-format
//----------------------------------------------------------
//part of YaCy
//
// (C) 2006 by Alexander Schier
//
// last change: $LastChangedDate: $ by $LastChangedBy: $
// $LastChangedRevision: $
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.data;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import de.anomic.htmlFilter.htmlFilterContentTransformer;
import de.anomic.server.logging.serverLog;
public class gettext{
public static ArrayList createGettextRecursive(File sourceDir, String extensions, String notdir, File oldgettextfile) throws FileNotFoundException{
if(oldgettextfile==null)
return createGettextRecursive(sourceDir, extensions, notdir, (Map)null); //no old file
return createGettextRecursive(sourceDir, extensions, notdir, parseGettext(oldgettextfile));
}
public static ArrayList createGettextRecursive(File sourceDir, String extensions, String notdir, Map oldgettext){
ArrayList list=new ArrayList();
ArrayList exts=listManager.string2arraylist(extensions);
Iterator it2;
String filename;
ArrayList filenames=new ArrayList();
ArrayList dirList=listManager.getDirsRecursive(sourceDir, notdir);
dirList.add(sourceDir);
Iterator it=dirList.iterator();
File dir=null;
File[] files;
//this looks a lot more complicated, than it is ...
while(it.hasNext()){
dir=(File)it.next();
if(dir.isDirectory() && !dir.getName().equals(notdir)){
files=dir.listFiles();
for(int i=0;i<files.length;i++){
if(!files[i].isDirectory()){
it2=exts.iterator();
filename=files[i].getName();
while(it2.hasNext()){
if(filename.endsWith((String)it2.next())){
try {
filenames.add(files[i].getCanonicalPath());
} catch (IOException e) {
// TODO Auto-generated catch block
}
break;
}
}
}
}
}
}
list=createGettext(filenames, oldgettext);
return list;
}
private static ArrayList getGettextHeader(){
return getGettextHeader("UNKNOWN", "EMAIL");
}
private static ArrayList getGettextHeader(String translator, String email){
ArrayList list=new ArrayList();
list.add("#yacy translation");
list.add("msgid \"\"");
list.add("msgstr \"\"");
list.add("\"Content-Type: text/plain; charset=UTF-8\\n\"");
list.add("\"Content-Transfer-Encoding: 8bit\\n\"");
list.add("\"Last-Translator: "+translator+"\\n\"");
SimpleDateFormat dateformat=new SimpleDateFormat("yyyy-mm-dd HH:MMZ");
list.add("\"PO-Revision-Date: "+dateformat.format(new Date())+"\\n\"");
list.add("\"Project-Id-Version: YaCy\\n\"");
list.add("\"Language-Team: <"+email+">\\n\"");
list.add("\"X-Generator: YaCy\\n\"");
list.add("\"Mime-Version: 1.0\\n\"");
list.add("");
return list;
}
public static ArrayList createGettext(ArrayList filenames, File oldgettextfile) throws FileNotFoundException{
return createGettext(filenames, parseGettext(oldgettextfile));
}
/*
* create a list of gettext file for some textfiles
* @param filenames the ArrayList with the Filenames
* @param oldgettextmap a map with the old translations.
*/
public static ArrayList createGettext(ArrayList filenames, Map oldgettext){
ArrayList list=new ArrayList();
ArrayList tmp=null;
String filename=null;
Iterator it=filenames.iterator();
list.addAll(getGettextHeader());
while(it.hasNext()){
try {
filename=(String)it.next();
//TODO: better possibility to switch the behaviour
//tmp=getGettextSource(new File(filename), oldgettext);
tmp=getGettextSourceFromHTML(new File(filename), oldgettext);
serverLog.logFinest("Gettext", "Extracting Strings from: "+filename);
} catch (FileNotFoundException e) {
System.out.println("File \""+filename+"\" not found.");
}
if(tmp!=null)
list.addAll(tmp);
}
return list;
}
public static ArrayList getGettextSource(File inputfile, File oldmapfile) throws FileNotFoundException{
if(oldmapfile != null && oldmapfile.exists())
return getGettextSource(inputfile, parseGettext(oldmapfile));
return getGettextSource(inputfile);
}
public static ArrayList getGettextSource(File inputfile) throws FileNotFoundException{
return getGettextSource(inputfile, new HashMap());
}
public static ArrayList getGettextSource(File inputfile, Map oldgettextmap) throws FileNotFoundException{
ArrayList strings=getGettextItems(inputfile);
return getGettextSource(inputfile, oldgettextmap, strings);
}
public static ArrayList getGettextSourceFromHTML(File inputfile, Map oldgettextmap) throws FileNotFoundException{
htmlFilterContentTransformer transformer=new htmlFilterContentTransformer();
BufferedReader br=new BufferedReader(new FileReader(inputfile));
StringBuffer content=new StringBuffer();
String line="";
try {
while((line=br.readLine())!=null){
content.append(line).append("\n");
}
} catch (IOException e) {}
ArrayList strings = transformer.getStrings(content.toString().getBytes());
return getGettextSource(inputfile, oldgettextmap, strings);
}
public static ArrayList getGettextSource(File inputfile, Map oldgettextmap, ArrayList strings) {
if(oldgettextmap==null)
oldgettextmap=new HashMap();
ArrayList list=new ArrayList();
Iterator it=strings.iterator();
if(strings.isEmpty())
return null;
list.add("#"+inputfile.getName());
String key;
while(it.hasNext()){
key=((String)it.next()).replaceAll("\"", "\\\\\"").replaceAll("\n", "\\\\n");
list.add("msgid \""+key+"\"");
if(oldgettextmap.containsKey(key))
list.add("msgstr \""+oldgettextmap.get(key)+"\"");
else
list.add("msgstr \"\"");
list.add("");
}
return list;
}
/*
* create a list of gettext Strings ( _() ) from a file
* @param inputfile the file, which contains the raw Strings.
*/
public static ArrayList getGettextItems(File inputfile) throws FileNotFoundException{
ArrayList list=new ArrayList();
int character;
InputStreamReader reader;
int state=0; //0=no gettext macro 1= _ found, 2= ( found and in the string 3=\ found
String untranslatedString="";
reader = new InputStreamReader(new FileInputStream(inputfile));
try {
character=reader.read();
while(character >=0) {
if(state==0 && (char)character=='_')
state=1;
else if(state==1){
if((char)character=='('){
state=2;
untranslatedString="";
}else{
state=0;
untranslatedString+=(char)character;
}
}else if(state==2){
if((char)character=='\\')
state=3;
else if((char)character==')'){
state=0;
list.add(untranslatedString);
}else{
untranslatedString+=(char)character;
}
}else if(state==3){
state=2;
if((char)character==')')
untranslatedString+=")";
else
untranslatedString+="\\"+(char)character;
}else{
untranslatedString+=(char)character;
}
character=reader.read();
}
} catch (IOException e) {}
return list;
}
public static HashMap parseGettext(File gettextfile) throws FileNotFoundException{
ArrayList gettext=new ArrayList();
String line;
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(gettextfile)));
try {
line = br.readLine();
while (line != null) {
gettext.add(line);
line = br.readLine();
}
} catch (IOException e) {}
return parseGettext(gettext);
}
public static HashMap parseGettext(ArrayList gettext){
HashMap map = new HashMap();
int mode=0; //1= in msgid, 2= in msgstr
String msgid = "", msgstr = "", tmp = "";
Iterator it=gettext.iterator();
while(it.hasNext()){
tmp=(String) it.next();
if(tmp.startsWith("msgid \"")){
if(mode==2)
map.put(msgid, msgstr);
msgid=tmp.substring(7,tmp.length()-1).replaceAll("\\\"", "\"");
msgstr="";
mode=1;
}else if(tmp.startsWith("msgstr \"")){
mode=2;
msgstr=tmp.substring(8,tmp.length()-1);
}else if(tmp.startsWith("\"")){
//multiline strings with "..." on each line
if(mode==1){
msgid+="\n"+tmp.substring(1,tmp.length()-1).replaceAll("\\\"", "\"");
}else if(mode==2){
msgstr+="\n"+tmp.substring(1,tmp.length()-1).replaceAll("\\\"", "\"");
}
}
}
map.put(msgid, msgstr); //the last one cannot be put, on the next msgid ;-)
return map;
}
public static void main(String[] argv){
if(argv.length < 2){
System.out.println("Syntax: java de.anomic.data.gettext creategettext [inputfile] ... [inputfile]");
System.out.println("Syntax: java de.anomic.data.gettext parsegettext [gettextfile]");
System.out.println("Syntax: java de.anomic.data.gettext updategettext [gettextfile] [inputfile] ... [inputfile]");
System.exit(1);
}
if(argv[0].equals("creategettext")){
ArrayList filelist=new ArrayList();
for(int i=1;i<argv.length;i++){
filelist.add(argv[i]);
}
ArrayList list = createGettext(filelist, (Map)null);
Iterator it=list.iterator();
while(it.hasNext())
System.out.println((String)it.next());
}else if(argv[0].equals("parsegettext")){
if(argv.length >2){
System.out.println("only one file allowed for parsegettext");
System.exit(1);
}
try {
HashMap translations=parseGettext(new File(argv[1]));
Iterator it=translations.keySet().iterator();
String key="";
while(it.hasNext()){
key=(String)it.next();
System.out.println("key: "+key);
System.out.println("value: "+translations.get(key));
}
} catch (FileNotFoundException e) {
System.exit(1);
}
}else if(argv[0].equals("updategettext")){
if(argv.length < 3){
System.out.println("Too less arguments");
System.exit(1);
}
ArrayList filelist=new ArrayList();
for(int i=2;i<argv.length;i++){
filelist.add(argv[i]);
}
try {
ArrayList list=createGettext(filelist, new File(argv[1]));
Iterator it=list.iterator();
while(it.hasNext())
System.out.println((String)it.next());
} catch (FileNotFoundException e) {
System.out.println("File not found.");
System.exit(1);
}
}else{
System.out.println("unknown Mode ...");
System.exit(1);
}
}
}

@ -27,6 +27,8 @@ import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Collection;
import java.util.Comparator;
import java.util.Date;
import java.util.Iterator;
import java.util.TreeSet;
@ -51,12 +53,12 @@ public class rssReader {
try {
channel=FeedParser.parse(builder, this.url);
Collection oldfeedItems=channel.getItems();
feedItems=new TreeSet(new rssReaderItemComparator());
feedItems=new TreeSet(new ItemComparator());
Iterator it=oldfeedItems.iterator();
int count=0;
while(it.hasNext()){
de.nava.informa.impl.basic.Item item=(de.nava.informa.impl.basic.Item) it.next();
rssReaderItem newItem=new rssReaderItem(count++, item.getLink(), item.getTitle(), item.getDescription(), item.getDate(), item.getCreator());
Item newItem=new Item(count++, item.getLink(), item.getTitle(), item.getDescription(), item.getDate(), item.getCreator());
feedItems.add(newItem);
}
}
@ -75,5 +77,50 @@ public class rssReader {
public Collection getFeedItems(){
return feedItems;
}
public class Item{
String creator, title, description;
Date date;
URL link;
int num;
public Item(int num, URL link, String title, String description, Date date, String creator){
this.link=link;
this.title=title;
this.description=description;
this.date=date;
this.creator=creator;
this.num=num;
}
public URL getLink(){
return link;
}
public String getTitle(){
return (title!=null)? title: "";
}
public String getDescription(){
return (description!=null)? description: "";
}
public Date getDate(){
return (date!=null)? date: new Date();
}
public String getCreator(){
return (creator!=null)? creator: "";
}
public int getNum(){
return num;
}
}
public class ItemComparator implements Comparator {
public int compare(Object o1, Object o2){
int num1=((Item)o1).getNum();
int num2=((Item)o2).getNum();
return num2-num1;
}
public boolean equals(Object o1, Object o2){
return compare(o1, o2)==0;
}
}
}

@ -1,62 +0,0 @@
//rssReaderItem.java
//------------
// part of YACY
//
// (C) 2007 Alexander Schier
//
// last change: $LastChangedDate: $ by $LastChangedBy: $
// $LastChangedRevision: $
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.data;
import java.net.URL;
import java.util.Date;
public class rssReaderItem{
String creator, title, description;
Date date;
URL link;
int num;
public rssReaderItem(int num, URL link, String title, String description, Date date, String creator){
this.link=link;
this.title=title;
this.description=description;
this.date=date;
this.creator=creator;
this.num=num;
}
public URL getLink(){
return link;
}
public String getTitle(){
return (title!=null)? title: "";
}
public String getDescription(){
return (description!=null)? description: "";
}
public Date getDate(){
return (date!=null)? date: new Date();
}
public String getCreator(){
return (creator!=null)? creator: "";
}
public int getNum(){
return num;
}
}

@ -1,36 +0,0 @@
//rssReaderItemComparator.java
//------------
// part of YACY
//
// (C) 2007 Alexander Schier
//
// last change: $LastChangedDate: $ by $LastChangedBy: $
// $LastChangedRevision: $
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.data;
import java.util.Comparator;
public class rssReaderItemComparator implements Comparator{
public int compare(Object o1, Object o2){
int num1=((rssReaderItem)o1).getNum();
int num2=((rssReaderItem)o2).getNum();
return num2-num1;
}
public boolean equals(Object o1, Object o2){
return compare(o1, o2)==0;
}
}

@ -439,10 +439,8 @@ public final class userDB {
return PROXY_ALLOK;
}
public boolean canSurf(){
if(this.surfRight()==PROXY_ALLOK)
return true;
else
return false;
if(this.surfRight()==PROXY_ALLOK) return true;
return false;
}
public long updateLastAccess(boolean incrementTimeUsed) {
return updateLastAccess(System.currentTimeMillis(), incrementTimeUsed);

@ -78,17 +78,13 @@ public class htmlFilterContentTransformer extends htmlFilterAbstractTransformer
}
private ArrayList bluelist = null;
private boolean gettext = false;
public htmlFilterContentTransformer() {
super(linkTags0, linkTags1);
}
public void init(String initarg) {
if (initarg.equals("gettext")) {
// the initarg declares that the transformer applies a gettext-quotation on strings
gettext = true;
} else if (bluelist == null) {
if (bluelist == null) {
// here, the initarg is used to load a list of bluelisted words
bluelist = new ArrayList();
File f = new File(initarg);
@ -108,7 +104,7 @@ public class htmlFilterContentTransformer extends htmlFilterAbstractTransformer
}
public boolean isIdentityTransformer() {
return (bluelist.size() == 0) && (!gettext);
return (bluelist.size() == 0);
}
private static char[] genBlueLetters(int length) {
@ -154,33 +150,6 @@ public class htmlFilterContentTransformer extends htmlFilterAbstractTransformer
return result;
}
public char[] transformText(char[] text) {
if (gettext) {
// serverCharBuffer sbb = new serverCharBuffer(text);
// //if (sbb.length() > 0) System.out.println(" TEXT: " + sbb.toString());
// serverCharBuffer[] sbbs = httpTemplate.splitQuotations(sbb);
// sbb = new serverCharBuffer();
// for (int i = 0; i < sbbs.length; i++) {
// if (sbbs[i].isWhitespace(true)) {
// sbb.append(sbbs[i]);
// } else if ((sbbs[i].byteAt(0) == httpTemplate.hash) ||
// (sbbs[i].startsWith(httpTemplate.dpdpa))) {
// // this is a template or a part of a template
// sbb.append(sbbs[i]);
// } else {
// // this is a text fragment, generate gettext quotation
// int ws = sbbs[i].whitespaceStart(true);
// int we = sbbs[i].whitespaceEnd(true);
// sbb.append(sbbs[i].getBytes(0, ws));
// sbb.append('_');
// sbb.append('(');
// sbb.append(sbbs[i].getBytes(ws, we));
// sbb.append(')');
// sbb.append(sbbs[i].getBytes(we));
// }
// }
// //if (sbb.length() > 0) System.out.println("GETTEXT: " + sbb.toString());
// return sbb.getChars();
}
if (bluelist != null) {
if (bluelistHit(text)) {
// System.out.println("FILTERHIT: " + text);

@ -512,7 +512,6 @@ public final class htmlFilterWriter extends Writer {
htmlFilterTransformer transformer = new htmlFilterContentTransformer();
// TODO: this does not work at the moment
System.exit(0);
transformer.init("gettext");
Reader is = new FileReader(args[0]);
FileOutputStream fos = new FileOutputStream(new File(args[0] + ".out"));
Writer os = new htmlFilterWriter(fos, "UTF-8",scraper, transformer, false);

@ -45,12 +45,15 @@ package de.anomic.http;
import java.io.FilterOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
public final class httpChunkedOutputStream extends FilterOutputStream
{
private boolean finished = false;
private static final byte[] crlf = {(byte)13,(byte)10};
import de.anomic.server.serverByteBuffer;
import de.anomic.server.serverCore;
import de.anomic.server.serverFileUtils;
public final class httpChunkedOutputStream extends FilterOutputStream {
private boolean finished = false;
public httpChunkedOutputStream(OutputStream out) {
super(out);
@ -63,7 +66,9 @@ public final class httpChunkedOutputStream extends FilterOutputStream
public void finish() throws IOException {
if (!this.finished) {
this.out.write("0\r\n\r\n".getBytes());
this.out.write((byte) 48);
this.out.write(serverCore.crlf);
this.out.write(serverCore.crlf);
this.out.flush();
this.finished = true;
}
@ -74,9 +79,9 @@ public final class httpChunkedOutputStream extends FilterOutputStream
if (b.length == 0) return;
this.out.write(Integer.toHexString(b.length).getBytes());
this.out.write(crlf);
this.out.write(serverCore.crlf);
this.out.write(b);
this.out.write(crlf);
this.out.write(serverCore.crlf);
this.out.flush();
}
@ -85,9 +90,32 @@ public final class httpChunkedOutputStream extends FilterOutputStream
if (len == 0) return;
this.out.write(Integer.toHexString(len).getBytes());
this.out.write(crlf);
this.out.write(serverCore.crlf);
this.out.write(b, off, len);
this.out.write(crlf);
this.out.write(serverCore.crlf);
this.out.flush();
}
public void write(serverByteBuffer b, int off, int len) throws IOException {
if (this.finished) throw new IOException("ChunkedOutputStream already finalized.");
if (len == 0) return;
this.out.write(Integer.toHexString(len).getBytes());
this.out.write(serverCore.crlf);
this.out.write(b.getBytes(off, off + len));
this.out.write(serverCore.crlf);
this.out.flush();
}
public void write(InputStream b) throws IOException {
if (this.finished) throw new IOException("ChunkedOutputStream already finalized.");
int len = b.available();
if (len == 0) return;
this.out.write(Integer.toHexString(len).getBytes());
this.out.write(serverCore.crlf);
serverFileUtils.copy(b, out, len);
this.out.write(serverCore.crlf);
this.out.flush();
}
@ -95,9 +123,9 @@ public final class httpChunkedOutputStream extends FilterOutputStream
if (this.finished) throw new IOException("ChunkedOutputStream already finalized.");
this.out.write("1".getBytes());
this.out.write(crlf);
this.out.write(serverCore.crlf);
this.out.write(b);
this.out.write(crlf);
this.out.write(serverCore.crlf);
this.out.flush();
}
}

@ -491,6 +491,10 @@ public final class httpHeader extends TreeMap implements Map {
return null;
}
public String userAgent() {
return (String) get(USER_AGENT);
}
public long age() {
Date lm = lastModified();
Date sd = date();

@ -0,0 +1,76 @@
// httpSSI.java
// -----------------------------
// (C) 2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 26.06.2007 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.http;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import de.anomic.server.serverByteBuffer;
public class httpSSI {
public static void writeSSI(File referenceFile, serverByteBuffer in, httpChunkedOutputStream out) throws IOException {
writeSSI(referenceFile, in, 0, out);
}
public static void writeSSI(File referenceFile, serverByteBuffer in, int start, httpChunkedOutputStream out) throws IOException {
int p = in.indexOf("<!--#".getBytes(), start);
if (p == 0) {
int q = in.indexOf("-->".getBytes(), start + 10);
assert q >= 0;
parseSSI(referenceFile, in, start, q + 3 - start, out);
writeSSI(referenceFile, in, start + q + 3, out);
} else if (p > 0) {
int q = in.indexOf("-->".getBytes(), start + 10);
out.write(in, start, p - start);
parseSSI(referenceFile, in, start + p, q + 3 - start - p, out);
writeSSI(referenceFile, in, start + q + 3, out);
} else /* p < 0 */ {
out.write(in, start, in.length() - start);
}
}
private static void parseSSI(File referenceFile, serverByteBuffer in, int start, int length, httpChunkedOutputStream out) {
if (in.startsWith("<!--#include virtual=\"".getBytes(), start)) {
int q = in.indexOf("\"".getBytes(), start + 22);
if (q > 0) {
String path = in.toString(start + 22, q);
File loadFile = new File(referenceFile.getParentFile(), path);
try {
out.write(new FileInputStream(loadFile));
} catch (FileNotFoundException e) {
// do nothing
} catch (IOException e) {
// do nothing
}
}
}
}
}

@ -229,8 +229,8 @@ public final class httpTemplate {
return false;
}
public static byte[] writeTemplate(InputStream in, OutputStream out, Hashtable pattern, byte[] dflt) throws IOException {
return writeTemplate(in, out, pattern, dflt, new byte[0]);
public static void writeTemplate(InputStream in, OutputStream out, Hashtable pattern, byte[] dflt) throws IOException {
writeTemplate(in, out, pattern, dflt, new byte[0]);
}
/**

@ -340,7 +340,7 @@ public final class httpd implements serverHandler {
return true;
}
private boolean handleYaCyHopAuthentication(httpHeader header) throws IOException {
private boolean handleYaCyHopAuthentication(httpHeader header) {
// check if the user has allowed that his/her peer is used for hops
if (!this.allowYaCyHop) return false;
@ -1364,7 +1364,7 @@ public final class httpd implements serverHandler {
if (!reqMethod.equals(httpHeader.METHOD_HEAD)){
if (!conProp.getProperty(httpHeader.CONNECTION_PROP_PERSISTENT,"close").equals("close")) {
if (transferEnc == null && contentLength < 0) {
throw new IllegalArgumentException("Message MUST contain a Content-Length or a non-identity transfer-coding heder field.");
throw new IllegalArgumentException("Message MUST contain a Content-Length or a non-identity transfer-coding header field.");
}
}
if (transferEnc != null && contentLength >= 0) {

@ -77,13 +77,11 @@ package de.anomic.http;
import java.awt.Image;
import java.awt.image.BufferedImage;
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.PushbackInputStream;
import java.io.UnsupportedEncodingException;
@ -91,8 +89,6 @@ import java.lang.ref.SoftReference;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.URLDecoder;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Date;
import java.util.Enumeration;
import java.util.HashMap;
@ -134,9 +130,6 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
private static String[] defaultFiles = null;
private static File htDefaultPath = null;
private static File htLocalePath = null;
private MessageDigest md5Digest = null;
private static final HashMap templateCache;
private static final HashMap templateMethodCache;
@ -205,12 +198,6 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
//htLocaleSelection = switchboard.getConfig("htLocaleSelection","default");
}
// initialise an message digest for Content-MD5 support ...
try {
this.md5Digest = MessageDigest.getInstance("MD5");
} catch (NoSuchAlgorithmException e) {
serverLog.logWarning("HTTPDFileHandler", "Content-MD5 support not availabel ...");
}
}
public static final void initDefaultPath() {
@ -585,7 +572,6 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
// if this file uses templates, then we use the template
// re-write - method to create an result
String mimeType = mimeTable.getProperty(targetExt,"text/html");
byte[] result;
boolean zipContent = requestHeader.acceptGzip() && httpd.shallTransportZipped("." + conProp.getProperty("EXT",""));
if (path.endsWith("html") ||
path.endsWith("xml") ||
@ -675,148 +661,96 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
tp.putAll(templates);
// rewrite the file
serverByteBuffer o = null;
InputStream fis = null;
GZIPOutputStream zippedOut = null;
try {
// do fileCaching here
byte[] templateContent = null;
if (useTemplateCache) {
long fileSize = targetFile.length();
if (fileSize <= 512*1024) {
SoftReference ref = (SoftReference) templateCache.get(targetFile);
if (ref != null) {
templateContent = (byte[]) ref.get();
if (templateContent == null)
templateCache.remove(targetFile);
}
if (templateContent == null) {
// loading the content of the template file into a byte array
templateContent = serverFileUtils.read(targetFile);
// storing the content into the cache
ref = new SoftReference(templateContent);
templateCache.put(targetFile,ref);
if (this.theLogger.isLoggable(Level.FINEST))
this.theLogger.logFinest("Cache MISS for file " + targetFile);
} else {
if (this.theLogger.isLoggable(Level.FINEST))
this.theLogger.logFinest("Cache HIT for file " + targetFile);
}
// creating an inputstream needed by the template rewrite function
fis = new ByteArrayInputStream(templateContent);
templateContent = null;
// read the file/template
byte[] templateContent = null;
if (useTemplateCache) {
long fileSize = targetFile.length();
if (fileSize <= 512 * 1024) {
// read from cache
SoftReference ref = (SoftReference) templateCache.get(targetFile);
if (ref != null) {
templateContent = (byte[]) ref.get();
if (templateContent == null) templateCache.remove(targetFile);
}
if (templateContent == null) {
// loading the content of the template file into
// a byte array
templateContent = serverFileUtils.read(targetFile);
// storing the content into the cache
ref = new SoftReference(templateContent);
templateCache.put(targetFile, ref);
if (this.theLogger.isLoggable(Level.FINEST))
this.theLogger.logFinest("Cache MISS for file " + targetFile);
} else {
fis = new BufferedInputStream(new FileInputStream(targetFile));
if (this.theLogger.isLoggable(Level.FINEST))
this.theLogger.logFinest("Cache HIT for file " + targetFile);
}
// creating an inputstream needed by the template
// rewrite function
fis = new ByteArrayInputStream(templateContent);
templateContent = null;
} else {
// read from file directly
fis = new BufferedInputStream(new FileInputStream(targetFile));
}
//use the page template
String supertemplate="";
if(tp.containsKey("SUPERTEMPLATE")){
supertemplate=(String) tp.get("SUPERTEMPLATE");
}
//if(sb.getConfig("usePageTemplate", "false").equals("true")){
if(!supertemplate.equals("")){
File pageFile=getOverlayedFile(supertemplate); //typically /env/page.html
File pageClass=getOverlayedClass(supertemplate);
if(pageFile != null && pageFile.exists()){
//warning: o,tp and fis are reused
//this replaces the normal template function (below), and the
//normal function is used for supertemplates.
//search for header Data
//XXX: This part may be slow :-(
BufferedReader br=new BufferedReader(new InputStreamReader(fis));
String line;
boolean inheader=false;
StringBuffer header=new StringBuffer();
StringBuffer content=new StringBuffer();
String content_s,header_s;
while((line=br.readLine())!=null){
if(!inheader){
if(line.startsWith("<!--HEADER")){
inheader=true;
}else{
content.append(line);
}
}else{
if(line.endsWith("HEADER-->")){
inheader=false;
}else{
header.append(line);
}
}
}
//tp is from the servlet, fis is the content-only from the servlet
//this resolvs templates in both header and content
o = new serverByteBuffer();
fis=new ByteArrayInputStream((new String(content)).getBytes());
httpTemplate.writeTemplate(fis, o, tp, "-UNRESOLVED_PATTERN-".getBytes());
content_s=o.toString();
o = new serverByteBuffer();
fis=new ByteArrayInputStream(header.toString().getBytes());
httpTemplate.writeTemplate(fis, o, tp, "-UNRESOLVED_PATTERN-".getBytes());
header_s=o.toString();
//further processing of page.html (via page.class)?
if (pageClass != null && pageClass.exists()){
Object tmp = invokeServlet(pageClass, requestHeader, args);
if(tmp instanceof serverObjects){
tp=new servletProperties((serverObjects)tmp);
}else{
tp=(servletProperties)tmp;
}
}else
tp = new servletProperties();
tp.putASIS("header", header_s);
tp.putASIS("page", content_s);
fis=new BufferedInputStream(new FileInputStream(pageFile));
}
}
o = new serverByteBuffer();
if (zipContent) zippedOut = new GZIPOutputStream(o);
httpTemplate.writeTemplate(fis, (zipContent) ? (OutputStream)zippedOut: (OutputStream)o, tp, "-UNRESOLVED_PATTERN-".getBytes("UTF-8"));
} else {
fis = new BufferedInputStream(new FileInputStream(targetFile));
}
// write the array to the client
// we can do that either in standard mode (whole thing completely) or in chunked mode
// since yacy clients do not understand chunked mode, we use this only for communication with the administrator
boolean yacyClient = requestHeader.userAgent().startsWith("yacy");
boolean chunked = !method.equals(httpHeader.METHOD_HEAD) && !yacyClient;
if (chunked) {
// send page in chunks and parse SSIs
serverByteBuffer o = new serverByteBuffer();
// apply templates
httpTemplate.writeTemplate(fis, o, tp, "-UNRESOLVED_PATTERN-".getBytes("UTF-8"));
httpd.sendRespondHeader(this.connectionProperties, out,
httpVersion, 200, null, mimeType, -1,
targetDate, null, tp.getOutgoingHeader(),
null, "chunked", nocache);
// send the content in chunked parts, see RFC 2616 section 3.6.1
//byte[] result = o.toByteArray(); // this interrupts streaming (bad idea!)
httpChunkedOutputStream chos = new httpChunkedOutputStream(out);
httpSSI.writeSSI(targetFile, o, chos);
//chos.write(result);
chos.finish();
} else {
// send page as whole thing, SSIs are not possible
String contentEncoding = (zipContent) ? "gzip" : null;
// apply templates
serverByteBuffer o = new serverByteBuffer();
if (zipContent) {
GZIPOutputStream zippedOut = new GZIPOutputStream(o);
httpTemplate.writeTemplate(fis, zippedOut, tp, "-UNRESOLVED_PATTERN-".getBytes("UTF-8"));
zippedOut.finish();
zippedOut.flush();
zippedOut.close();
zippedOut = null;
} else {
httpTemplate.writeTemplate(fis, o, tp, "-UNRESOLVED_PATTERN-".getBytes("UTF-8"));
}
result = o.toByteArray();
if (this.md5Digest != null) {
this.md5Digest.reset();
this.md5Digest.update(result);
byte[] digest = this.md5Digest.digest();
StringBuffer digestString = new StringBuffer();
for ( int i = 0; i < digest.length; i++ )
digestString.append(Integer.toHexString( digest[i]&0xff));
}
} finally {
if (zippedOut != null) try {zippedOut.close();} catch(Exception e) {}
if (o != null) try {o.close(); o = null;} catch(Exception e) {}
if (fis != null) try {fis.close(); fis=null;} catch(Exception e) {}
if (method.equals(httpHeader.METHOD_HEAD)) {
httpd.sendRespondHeader(this.connectionProperties, out,
httpVersion, 200, null, mimeType, o.length(),
targetDate, null, tp.getOutgoingHeader(),
contentEncoding, null, nocache);
} else {
byte[] result = o.toByteArray(); // this interrupts streaming (bad idea!)
httpd.sendRespondHeader(this.connectionProperties, out,
httpVersion, 200, null, mimeType, result.length,
targetDate, null, tp.getOutgoingHeader(),
contentEncoding, null, nocache);
serverFileUtils.write(result, out);
}
}
// write the array to the client
long contentLength = result.length;
String contentEncoding = (zipContent)?"gzip":null;
httpd.sendRespondHeader(this.connectionProperties, out, httpVersion, 200, null, mimeType, contentLength, targetDate, null, tp.getOutgoingHeader(), contentEncoding, null, nocache);
if (! method.equals(httpHeader.METHOD_HEAD)) {
//Thread.sleep(200); // this solved the message problem (!!)
serverFileUtils.write(result, out);
}
} else { // no html
int statusCode = 200;

@ -147,7 +147,8 @@ public class icapHeader extends TreeMap implements Map {
// to make the occurrence of multiple keys possible, we add them using a counter
public Object add(Object key, Object value) {
int c = keyCount((String) key);
if (c == 0) return put(key, value); else return put("*" + key + "-" + c, value);
if (c == 0) return put(key, value);
return put("*" + key + "-" + c, value);
}
public int keyCount(String key) {
@ -160,7 +161,8 @@ public class icapHeader extends TreeMap implements Map {
// a convenience method to access the map with fail-over defaults
public Object get(Object key, Object dflt) {
Object result = get(key);
if (result == null) return dflt; else return result;
if (result == null) return dflt;
return result;
}
// return multiple results

@ -68,12 +68,7 @@ public class indexCollectionRI implements indexRI {
}
public int size() {
try {
return collectionIndex.size();
} catch (IOException e) {
e.printStackTrace();
return 0;
}
return collectionIndex.size();
}
public synchronized int indexSize(String wordHash) {
@ -196,11 +191,7 @@ public class indexCollectionRI implements indexRI {
}
public synchronized void close() {
try {
collectionIndex.close();
} catch (IOException e) {
e.printStackTrace();
}
collectionIndex.close();
}
}

@ -27,7 +27,6 @@
package de.anomic.index;
import java.io.IOException;
import java.net.MalformedURLException;
import java.text.ParseException;
import java.util.ArrayList;
@ -285,7 +284,7 @@ public class indexURLEntry {
}
}
public kelondroRow.Entry toRowEntry() throws IOException {
public kelondroRow.Entry toRowEntry() {
return this.entry;
}

@ -32,7 +32,7 @@ public class kelondroBytesIntMap {
private kelondroRow rowdef;
private kelondroIndex index0, index1;
public kelondroBytesIntMap(kelondroIndex ki) throws IOException {
public kelondroBytesIntMap(kelondroIndex ki) {
assert (ki.row().columns() == 2); // must be a key/index relation
assert (ki.row().width(1) == 4); // the value must be a b256-encoded int, 4 bytes long
this.index0 = null; // not used
@ -46,7 +46,7 @@ public class kelondroBytesIntMap {
this.index1 = null; // to show that this is the initialization phase
}
public kelondroRow row() throws IOException {
public kelondroRow row() {
return index0.row();
}

@ -264,7 +264,7 @@ public class kelondroCollectionIndex {
}
}
public int size() throws IOException {
public int size() {
return index.size();
}
@ -1000,7 +1000,7 @@ public class kelondroCollectionIndex {
}
public synchronized void close() throws IOException {
public synchronized void close() {
this.index.close();
Iterator i = arrays.values().iterator();
while (i.hasNext()) {

@ -123,7 +123,7 @@ public class kelondroDyn {
return this.rowdef;
}
public synchronized int sizeDyn() throws IOException {
public synchronized int sizeDyn() {
//this.segmentCount = 0;
//Iterator i = keys(true); while (i.hasNext()) segmentCount++;
//return segmentCount;

@ -68,7 +68,7 @@ public class kelondroDynTree {
private Hashtable buffer, cache;
private long cycleBuffer;
public kelondroDynTree(File file, long preloadTime, int keylength, int nodesize, kelondroRow rowdef, char fillChar, boolean resetOnFail) throws IOException {
public kelondroDynTree(File file, long preloadTime, int keylength, int nodesize, kelondroRow rowdef, char fillChar, boolean resetOnFail) {
// creates or opens a DynTree
this.file = file;
this.preloadTime = preloadTime;

@ -46,14 +46,14 @@ public class kelondroFlexSplitTable implements kelondroIndex {
private File path;
private String tablename;
public kelondroFlexSplitTable(File path, String tablename, long preloadTime, kelondroRow rowdef, boolean resetOnFail) throws IOException {
public kelondroFlexSplitTable(File path, String tablename, long preloadTime, kelondroRow rowdef, boolean resetOnFail) {
this.path = path;
this.tablename = tablename;
this.rowdef = rowdef;
init(preloadTime, resetOnFail);
}
public void init(long preloadTime, boolean resetOnFail) throws IOException {
public void init(long preloadTime, boolean resetOnFail) {
// initialized tables map
this.tables = new HashMap();

@ -423,12 +423,9 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
private final Map memoryStats() {
// returns statistical data about this object
HashMap map = new HashMap();
try {
map.put("tableIndexChunkSize", (!RAMIndex) ? "0" : Integer.toString(index.row().objectsize));
map.put("tableIndexCount", (!RAMIndex) ? "0" : Integer.toString(index.size()));
map.put("tableIndexMem", (!RAMIndex) ? "0" : Integer.toString((int) (index.row().objectsize * index.size() * kelondroRowCollection.growfactor)));
} catch (IOException e) {
}
map.put("tableIndexChunkSize", (!RAMIndex) ? "0" : Integer.toString(index.row().objectsize));
map.put("tableIndexCount", (!RAMIndex) ? "0" : Integer.toString(index.size()));
map.put("tableIndexMem", (!RAMIndex) ? "0" : Integer.toString((int) (index.row().objectsize * index.size() * kelondroRowCollection.growfactor)));
return map;
}

@ -24,7 +24,6 @@
package de.anomic.kelondro;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
@ -55,7 +54,7 @@ public class kelondroIntBytesMap {
return index1.memoryNeededForGrow();
}
public kelondroRow row() throws IOException {
public kelondroRow row() {
return index0.row();
}

@ -63,20 +63,20 @@ public class kelondroMapTable {
public void declareMaps(
String tablename, int keysize, kelondroOrder objectOrder, int nodesize, int cacheslots,
char fillChar, boolean resetOnFail) throws IOException {
char fillChar, boolean resetOnFail) {
declareMaps(tablename, keysize, objectOrder, nodesize, cacheslots, null, null, null, fillChar, resetOnFail);
}
public void declareMaps(
String tablename, int keysize, kelondroOrder objectOrder, int nodesize, int cacheslots,
String[] sortfields, String[] longaccfields, String[] doubleaccfields, char fillChar, boolean resetOnFail) throws IOException {
String[] sortfields, String[] longaccfields, String[] doubleaccfields, char fillChar, boolean resetOnFail) {
declareMaps(tablename, keysize, objectOrder, nodesize, cacheslots, sortfields, longaccfields, doubleaccfields, fillChar, 0, resetOnFail);
}
public void declareMaps(
String tablename, int keysize, kelondroOrder objectOrder, int nodesize, int cacheslots,
String[] sortfields, String[] longaccfields, String[] doubleaccfields, char fillChar,
long preloadTime, boolean resetOnFail) throws IOException {
long preloadTime, boolean resetOnFail) {
if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared twice.");
if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared already in other context.");
File tablefile = new File(tablesPath, "table." + tablename + ".mdb");
@ -87,7 +87,7 @@ public class kelondroMapTable {
mTables.put(tablename, map);
}
public void declareTree(String tablename, kelondroRow rowdef, long buffersize /*bytes*/, long preloadTime) throws IOException {
public void declareTree(String tablename, kelondroRow rowdef, long buffersize /*bytes*/, long preloadTime) {
if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareTree: table '" + tablename + "' declared already in other context.");
if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareTree: table '" + tablename + "' declared twice.");
File tablefile = new File(tablesPath, "table." + tablename + ".tdb");
@ -110,7 +110,7 @@ public class kelondroMapTable {
tTables.put(tablename, tree);
}
public synchronized Map selectMap(String tablename, String key) throws IOException {
public synchronized Map selectMap(String tablename, String key) {
kelondroMapObjects table = (kelondroMapObjects) mTables.get(tablename);
if (table == null) throw new RuntimeException("kelondroTables.selectMap: map table '" + tablename + "' does not exist.");
if (key.length() > table.keySize()) key = key.substring(0, table.keySize());

@ -149,11 +149,7 @@ public class kelondroObjects {
}
public synchronized int size() {
try {
return dyn.sizeDyn();
} catch (IOException e) {
return 0;
}
return dyn.sizeDyn();
}
public void close() {

@ -301,7 +301,7 @@ public class kelondroRowCollection {
addUnique(r, 0, r.length);
}
public synchronized void addUniqueMultiple(List rows) throws IOException {
public synchronized void addUniqueMultiple(List rows) {
assert this.sortBound == 0 : "sortBound = " + this.sortBound + ", chunkcount = " + this.chunkcount;
Iterator i = rows.iterator();
while (i.hasNext()) addUnique((kelondroRow.Entry) i.next());

@ -138,7 +138,7 @@ public class kelondroSplittedTree implements kelondroIndex {
throw new UnsupportedOperationException();
}
public synchronized void addUnique(kelondroRow.Entry row, Date entryDate) throws IOException {
public synchronized void addUnique(kelondroRow.Entry row, Date entryDate) {
throw new UnsupportedOperationException();
}

@ -76,7 +76,7 @@ public class sevenzipParser extends AbstractParser implements Parser {
*/
private static final String[] LIBX_DEPENDENCIES = new String[] { "J7Zip-modified.jar" };
public sevenzipParser() throws IOException {
public sevenzipParser() {
super(LIBX_DEPENDENCIES);
super.parserName = "7zip Archive Parser";
}

@ -95,12 +95,7 @@ public final class plasmaCrawlLURL {
public plasmaCrawlLURL(File indexPath, long preloadTime) {
super();
try {
urlIndexFile = new kelondroFlexSplitTable(new File(indexPath, "PUBLIC/TEXT"), "urls", preloadTime, indexURLEntry.rowdef, false);
} catch (IOException e) {
e.printStackTrace();
System.exit(-1);
}
urlIndexFile = new kelondroFlexSplitTable(new File(indexPath, "PUBLIC/TEXT"), "urls", preloadTime, indexURLEntry.rowdef, false);
// init result stacks
externResultStack = new LinkedList();

@ -143,13 +143,9 @@ public final class plasmaCrawlStacker {
this.log.logSevere("Unable to shutdown all remaining stackCrawl threads", e1);
}
try {
this.log.logFine("Shutdown. Closing stackCrawl queue.");
if (this.queue != null) this.queue.close();
this.queue = null;
} catch (IOException e) {
this.log.logSevere("DB could not be closed properly.", e);
}
this.log.logFine("Shutdown. Closing stackCrawl queue.");
if (this.queue != null) this.queue.close();
this.queue = null;
}
public int getNumActiveWorker() {
@ -540,7 +536,7 @@ public final class plasmaCrawlStacker {
}
}
public void close() throws IOException {
public void close() {
// closing the db
this.urlEntryCache.close();

@ -1076,7 +1076,7 @@ public final class plasmaHTCache {
return this.resInfo;
}
public boolean writeResourceInfo() throws IOException {
public boolean writeResourceInfo() {
assert(this.nomalizedURLHash != null) : "URL Hash is null";
if (this.resInfo == null) return false;
try {

@ -1609,14 +1609,14 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
return cacheManager;
}
synchronized public void htEntryStoreEnqueued(plasmaHTCache.Entry entry) throws IOException {
synchronized public void htEntryStoreEnqueued(plasmaHTCache.Entry entry) {
if (cacheManager.full())
htEntryStoreProcess(entry);
else
cacheManager.push(entry);
}
synchronized public boolean htEntryStoreProcess(plasmaHTCache.Entry entry) throws IOException {
synchronized public boolean htEntryStoreProcess(plasmaHTCache.Entry entry) {
if (entry == null) return false;
@ -1721,11 +1721,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
public boolean htEntryStoreJob() {
if (cacheManager.empty()) return false;
try {
return htEntryStoreProcess(cacheManager.pop());
} catch (IOException e) {
return false;
}
return htEntryStoreProcess(cacheManager.pop());
}
public int htEntrySize() {

@ -155,7 +155,7 @@ public class plasmaSwitchboardQueue {
}
}
public void clear() throws IOException {
public void clear() {
sbQueueStack = kelondroStack.reset(sbQueueStack);
}

@ -59,7 +59,7 @@ public class plasmaWordConnotation {
private static final int nodesize = 4048;
private kelondroDynTree refDB;
public plasmaWordConnotation(File refDBfile, long preloadTime, char fillChar) throws IOException {
public plasmaWordConnotation(File refDBfile, long preloadTime, char fillChar) {
refDB = new kelondroDynTree(refDBfile, preloadTime, wordlength, nodesize, new kelondroRow("byte[] word-" + wordlength + ", Cardinal count-" + countlength, kelondroNaturalOrder.naturalOrder, 0), fillChar, true);
}

@ -277,9 +277,13 @@ public final class serverByteBuffer extends OutputStream {
}
public boolean startsWith(byte[] bs) {
if (length < bs.length) return false;
return startsWith(bs, 0);
}
public boolean startsWith(byte[] bs, int start) {
if (length - start < bs.length) return false;
for (int i = 0; i < bs.length; i++) {
if (buffer[offset + i] != bs[i]) return false;
if (buffer[offset + i + start] != bs[i]) return false;
}
return true;
}

@ -126,6 +126,7 @@ public final class yacyClient {
obj.put("mytime", yacyCore.universalDateShortString(new Date()));
obj.put("myUTC", System.currentTimeMillis());
obj.put("seed", yacyCore.seedDB.mySeed.genSeedStr(key));
obj.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
// sending request
result = nxTools.table(
@ -272,7 +273,8 @@ public final class yacyClient {
"&youare=" + target.hash +
"&key=" + key +
"&object=seed" +
"&env=" + seedHash
"&env=" + seedHash +
"&network.unit.name=" + plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT)
),
target.getHexHash() + ".yacyh",
8000,
@ -309,7 +311,8 @@ public final class yacyClient {
"&key=" +
"&object=rwicount" +
"&env=" + wordHash +
"&ttl=0"
"&ttl=0" +
"&network.unit.name=" + plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT)
),
target.getHexHash() + ".yacyh",
10000,
@ -345,7 +348,8 @@ public final class yacyClient {
"&key=" +
"&object=lurlcount" +
"&env=" +
"&ttl=0";
"&ttl=0" +
"&network.unit.name=" + plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT);
// seinding request
try {
@ -452,6 +456,7 @@ public final class yacyClient {
obj.put("profile", crypt.simpleEncode(rankingProfile.toExternalString()));
obj.put("constraint", constraint.exportB64());
obj.put("mytime", yacyCore.universalDateShortString(new Date()));
obj.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
if (abstractCache != null) obj.put("abstracts", "auto");
//yacyCore.log.logDebug("yacyClient.search url=" + url);
@ -622,6 +627,7 @@ public final class yacyClient {
post.put("iam", yacyCore.seedDB.mySeed.hash);
post.put("youare", targetHash);
post.put("mytime", yacyCore.universalDateShortString(new Date()));
post.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
// get target address
String address = targetAddress(targetHash);
@ -664,6 +670,7 @@ public final class yacyClient {
post.put("youare", targetHash);
post.put("subject", subject);
post.put("mytime", yacyCore.universalDateShortString(new Date()));
post.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
try {
post.put("message", new String(message, "UTF-8"));
} catch (UnsupportedEncodingException e) {
@ -722,6 +729,7 @@ public final class yacyClient {
post.put("filename", filename);
post.put("filesize", Long.toString(filesize));
post.put("can-send-protocol", "http");
post.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
// send request
try {
@ -763,6 +771,7 @@ public final class yacyClient {
post.put("filesize", Long.toString(file.length));
post.put("md5", serverCodings.encodeMD5Hex(file));
post.put("access", access);
post.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
HashMap files = new HashMap();
files.put("filename", file);
@ -845,7 +854,8 @@ public final class yacyClient {
}
post.put("depth", "0");
post.put("ttl", "0");
post.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
// determining target address
final String address = targetSeed.getClusterAddress();
if (address == null) { return null; }
@ -928,7 +938,8 @@ public final class yacyClient {
"&result=" + result +
"&reason=" + reason +
"&wordh=" + wordhashes +
"&lurlEntry=" + ((entry == null) ? "" : crypt.simpleEncode(entry.toString(), key))
"&lurlEntry=" + ((entry == null) ? "" : crypt.simpleEncode(entry.toString(), key)) +
"&network.unit.name=" + plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT)
),
targetSeed.getHexHash() + ".yacyh",
60000,
@ -1055,7 +1066,7 @@ public final class yacyClient {
post.put("iam", yacyCore.seedDB.mySeed.hash);
post.put("youare", targetSeed.hash);
post.put("wordc", Integer.toString(indexes.length));
post.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
int indexcount = 0;
final StringBuffer entrypost = new StringBuffer(indexes.length*73);
@ -1131,6 +1142,8 @@ public final class yacyClient {
post.put("key", key);
post.put("iam", yacyCore.seedDB.mySeed.hash);
post.put("youare", targetSeed.hash);
post.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
String resource = "";
int urlc = 0;
int urlPayloadSize = 0;
@ -1183,6 +1196,8 @@ public final class yacyClient {
final serverObjects post = new serverObjects(2);
post.put("iam", yacyCore.seedDB.mySeed.hash);
post.put("youare", targetSeed.hash);
post.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
String address = targetSeed.getClusterAddress();
if (address == null) { address = "localhost:8080"; }
try {
@ -1226,7 +1241,8 @@ public final class yacyClient {
"&myseed=" + yacyCore.seedDB.mySeed.genSeedStr(null) +
"&count=10" +
"&resource=global" +
"&query=" + wordhashe),
"&query=" + wordhashe +
"&network.unit.name=" + plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT)),
target.getHexHash() + ".yacyh",
5000,
null,

@ -204,12 +204,7 @@ public class yacyCore {
// create or init news database
long memNews_time = Long.parseLong(switchboard.getConfig("ramCacheNews_time", "1000"));
try {
newsPool = new yacyNewsPool(yacyDBPath, memNews_time);
} catch (IOException e) {
e.printStackTrace();
System.exit(-1);
}
newsPool = new yacyNewsPool(yacyDBPath, memNews_time);
loadSeedUploadMethods();

@ -64,13 +64,13 @@ public class yacyNewsDB {
private long preloadTime;
protected kelondroIndex news;
public yacyNewsDB(File path, long preloadTime) throws IOException {
public yacyNewsDB(File path, long preloadTime) {
this.path = path;
this.preloadTime = preloadTime;
this.news = new kelondroCache(kelondroTree.open(path, true, preloadTime, yacyNewsRecord.rowdef), true, false);
}
private void resetDB() throws IOException {
private void resetDB() {
try {close();} catch (Exception e) {}
if (path.exists()) path.delete();
this.news = new kelondroCache(kelondroTree.open(path, true, preloadTime, yacyNewsRecord.rowdef), true, false);
@ -85,7 +85,7 @@ public class yacyNewsDB {
close();
}
public int size() throws IOException {
public int size() {
return news.size();
}
@ -148,7 +148,7 @@ public class yacyNewsDB {
);
}
protected final kelondroRow.Entry r2b(yacyNewsRecord r) throws IOException {
protected final kelondroRow.Entry r2b(yacyNewsRecord r) {
try {
if (r == null) return null;
String attributes = r.attributes().toString();

@ -265,7 +265,7 @@ public class yacyNewsPool {
private int maxDistribution;
public yacyNewsPool(File yacyDBPath, long preloadTime) throws IOException {
public yacyNewsPool(File yacyDBPath, long preloadTime) {
newsDB = new yacyNewsDB(new File(yacyDBPath, "news1.db"), preloadTime);
outgoingNews = new yacyNewsQueue(new File(yacyDBPath, "newsOut1.stack"), newsDB);
publishedNews = new yacyNewsQueue(new File(yacyDBPath, "newsPublished1.stack"), newsDB);
@ -283,12 +283,7 @@ public class yacyNewsPool {
}
public int dbSize() {
try {
return newsDB.size();
} catch (IOException e) {
e.printStackTrace();
return 0;
}
return newsDB.size();
}
public void publishMyNews(yacyNewsRecord record) {

@ -168,6 +168,9 @@ public class yacySeed {
/** zero-value */
public static final String ZERO = "0";
public static final String DFLT_NETWORK_UNIT = "freeworld";
public static final String DFLT_NETWORK_GROUP = "";
// class variables
/** the peer-hash */
public String hash;

@ -750,12 +750,6 @@ CRDist1Target = kaskelix.de:8080,yacy.dyndns.org:8000,suma-lab.de:8080
# Hash of the peer, you would like to store to the data your installation collected.
storagePeerHash =
#use /env/page.html (and if it exists page.class) as super template,
# which includes the generated page in #[page]#.
#This option is for developers only, because it may be integrated by default, or not,
#but its not useful to mix configurations with and without this supertemplate.
usePageTemplate = false
# Search sequence settings
# collection:
# time = time to get a RWI out of RAM cache, assortments and WORDS files

Loading…
Cancel
Save