- added SSI parser and include directive for <!--# include virtual="<file>" -->

- added chunked file transfer for non-yacy clients
- SSIs are streamed using chunked transfer, partly delivered pages can be seen in browser before transmission is finished
- added client-side network unit identification
- cleaned up code

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3926 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 18 years ago
parent 32640044c0
commit 1782ef57e5

@ -3,7 +3,7 @@ javacSource=1.4
javacTarget=1.4 javacTarget=1.4
# Release Configuration # Release Configuration
releaseVersion=0.532 releaseVersion=0.533
releaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz releaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
proReleaseFile=yacy_pro_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz proReleaseFile=yacy_pro_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
releaseFileParentDir=yacy releaseFileParentDir=yacy

Binary file not shown.

Before

Width:  |  Height:  |  Size: 41 KiB

@ -216,9 +216,8 @@ public class BlacklistCleaner_p {
if (ok.contains(s)) { if (ok.contains(s)) {
r.put(s, new Integer(ERR_DOUBLE_OCCURANCE)); r.put(s, new Integer(ERR_DOUBLE_OCCURANCE));
continue; continue;
} else {
ok.add(s);
} }
ok.add(s);
if ((slashPos = s.indexOf("/")) == -1) { if ((slashPos = s.indexOf("/")) == -1) {
host = s; host = s;
@ -235,10 +234,9 @@ public class BlacklistCleaner_p {
if (i == 0 && host.length() > 1 && host.charAt(1) != '.') { if (i == 0 && host.length() > 1 && host.charAt(1) != '.') {
r.put(s, new Integer(ERR_SUBDOMAIN_XOR_WILDCARD)); r.put(s, new Integer(ERR_SUBDOMAIN_XOR_WILDCARD));
continue; continue;
} else {
r.put(s, new Integer(ERR_HOST_WRONG_CHARS));
continue;
} }
r.put(s, new Integer(ERR_HOST_WRONG_CHARS));
continue;
} }
// in host-part only full sub-domains may be wildcards // in host-part only full sub-domains may be wildcards

@ -93,11 +93,12 @@ public class ConfigBasic {
if ((post != null)) config.reinitPortForwarding(post, env); if ((post != null)) config.reinitPortForwarding(post, env);
// starting a peer ping // starting a peer ping
boolean doPeerPing = false;
//boolean doPeerPing = false;
if ((yacyCore.seedDB.mySeed.isVirgin()) || (yacyCore.seedDB.mySeed.isJunior())) { if ((yacyCore.seedDB.mySeed.isVirgin()) || (yacyCore.seedDB.mySeed.isJunior())) {
serverInstantThread.oneTimeJob(sb.yc, "peerPing", null, 0); serverInstantThread.oneTimeJob(sb.yc, "peerPing", null, 0);
doPeerPing = true; //doPeerPing = true;
} }
// scan for Upnp routers // scan for Upnp routers
/* /*

@ -151,80 +151,79 @@ public class CrawlURLFetchStack_p {
} catch (IOException e) { e.printStackTrace(); } } catch (IOException e) { e.printStackTrace(); }
outw.flush(); outw.flush();
return null; return null;
} else { }
/* ================================================================= /* =================================================================
* 'normal' request * 'normal' request
* ================================================================= */ * ================================================================= */
if (post != null) { if (post != null) {
if (post.containsKey("addurls")) { if (post.containsKey("addurls")) {
prop.put("addedUrls", 1); prop.put("addedUrls", 1);
prop.put("addedUrls_added", addURLs(post, post.getInt("addurls", -1), getURLFetcherStack(env))); prop.put("addedUrls_added", addURLs(post, post.getInt("addurls", -1), getURLFetcherStack(env)));
} }
else if (post.containsKey("setMaxSize")) { else if (post.containsKey("setMaxSize")) {
final int count = post.getInt("maxSize", maxURLsPerFetch); final int count = post.getInt("maxSize", maxURLsPerFetch);
if (count > 0) { if (count > 0) {
maxURLsPerFetch = count; maxURLsPerFetch = count;
prop.put("set", 1); prop.put("set", 1);
prop.put("set_value", maxURLsPerFetch); prop.put("set_value", maxURLsPerFetch);
} else { } else {
prop.put("set", 2); prop.put("set", 2);
prop.put("set_value", count); prop.put("set_value", count);
}
}
else if (post.containsKey("shiftlcq")) {
final int count = Math.min(post.getInt("shiftloc", 0), sb.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE));
final int failed = shiftFromNotice(sb.noticeURL, plasmaCrawlNURL.STACK_TYPE_CORE, getURLFetcherStack(env), count);
prop.put("shiftloc", 1);
prop.put("shiftloc_value", count - failed);
prop.put("shiftloc_failed", failed);
}
else if (post.containsKey("shiftrcq")) {
final int count = post.getInt("shiftrem", 0);
final int failed = shiftFromNotice(sb.noticeURL, plasmaCrawlNURL.STACK_TYPE_LIMIT, getURLFetcherStack(env), count);
prop.put("shiftrem", 1);
prop.put("shiftrem_value", count - failed);
prop.put("shiftrem_failed", failed);
} }
else if (post.containsKey("subupload")) { }
if (post.get("upload", "").length() == 0) { else if (post.containsKey("shiftlcq")) {
prop.put("uploadError", 1); final int count = Math.min(post.getInt("shiftloc", 0), sb.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE));
} else { final int failed = shiftFromNotice(sb.noticeURL, plasmaCrawlNURL.STACK_TYPE_CORE, getURLFetcherStack(env), count);
final File file = new File(post.get("upload", "")); prop.put("shiftloc", 1);
final String content = new String((byte[])post.get("upload$file")); prop.put("shiftloc_value", count - failed);
prop.put("shiftloc_failed", failed);
final String type = post.get("uploadType", ""); }
final boolean blCheck = post.containsKey("blacklistCheck"); else if (post.containsKey("shiftrcq")) {
if (type.equals("plain")) { final int count = post.getInt("shiftrem", 0);
prop.put("upload_added", addURLs(content.split("\n"), blCheck, getURLFetcherStack(env))); final int failed = shiftFromNotice(sb.noticeURL, plasmaCrawlNURL.STACK_TYPE_LIMIT, getURLFetcherStack(env), count);
prop.put("upload_failed", 0); prop.put("shiftrem", 1);
prop.put("shiftrem_value", count - failed);
prop.put("shiftrem_failed", failed);
}
else if (post.containsKey("subupload")) {
if (post.get("upload", "").length() == 0) {
prop.put("uploadError", 1);
} else {
final File file = new File(post.get("upload", ""));
final String content = new String((byte[])post.get("upload$file"));
final String type = post.get("uploadType", "");
final boolean blCheck = post.containsKey("blacklistCheck");
if (type.equals("plain")) {
prop.put("upload_added", addURLs(content.split("\n"), blCheck, getURLFetcherStack(env)));
prop.put("upload_failed", 0);
prop.put("upload", 1);
} else if (type.equals("html")) {
try {
final htmlFilterContentScraper scraper = new htmlFilterContentScraper(new URL(file));
final Writer writer = new htmlFilterWriter(null, null, scraper, null, false);
serverFileUtils.write(content, writer);
writer.close();
final Iterator it = ((HashMap)scraper.getAnchors()).keySet().iterator();
int added = 0, failed = 0;
URL url;
while (it.hasNext()) try {
url = new URL((String)it.next());
if (blCheck && plasmaSwitchboard.urlBlacklist.isListed(plasmaURLPattern.BLACKLIST_CRAWLER, url)) {
failed++;
continue;
}
getURLFetcherStack(env).push(url);
added++;
} catch (MalformedURLException e) { failed++; }
prop.put("upload", 1); prop.put("upload", 1);
} else if (type.equals("html")) { prop.put("upload_added", added);
try { prop.put("upload_failed", failed);
final htmlFilterContentScraper scraper = new htmlFilterContentScraper(new URL(file)); } catch (Exception e) {
final Writer writer = new htmlFilterWriter(null, null, scraper, null, false); e.printStackTrace();
serverFileUtils.write(content, writer); prop.put("upload", 2);
writer.close(); prop.put("upload_error", e.getMessage());
final Iterator it = ((HashMap)scraper.getAnchors()).keySet().iterator();
int added = 0, failed = 0;
URL url;
while (it.hasNext()) try {
url = new URL((String)it.next());
if (blCheck && plasmaSwitchboard.urlBlacklist.isListed(plasmaURLPattern.BLACKLIST_CRAWLER, url)) {
failed++;
continue;
}
getURLFetcherStack(env).push(url);
added++;
} catch (MalformedURLException e) { failed++; }
prop.put("upload", 1);
prop.put("upload_added", added);
prop.put("upload_failed", failed);
} catch (Exception e) {
e.printStackTrace();
prop.put("upload", 2);
prop.put("upload_error", e.getMessage());
}
} }
} }
} }

@ -130,7 +130,6 @@ public class CrawlURLFetch_p {
if (fetcher != null) fetcher.interrupt(); if (fetcher != null) fetcher.interrupt();
fetcher = null; fetcher = null;
try {
if (post.get("source", "").equals("peer") && if (post.get("source", "").equals("peer") &&
post.get("peerhash", "").equals("random")) { post.get("peerhash", "").equals("random")) {
fetcher = new URLFetcher( fetcher = new URLFetcher(
@ -184,11 +183,7 @@ public class CrawlURLFetch_p {
frequency); frequency);
} }
} }
if (fetcher != null) if (fetcher != null) fetcher.start();
fetcher.start();
} catch (IOException e) {
e.printStackTrace();
}
} }
else if (post.containsKey("stop")) { else if (post.containsKey("stop")) {
if (fetcher != null) { if (fetcher != null) {
@ -199,7 +194,6 @@ public class CrawlURLFetch_p {
} }
else if (post.containsKey("restart")) { else if (post.containsKey("restart")) {
if (fetcher != null) { if (fetcher != null) {
try {
fetcher.interrupt(); fetcher.interrupt();
if (fetcher.url == null) { if (fetcher.url == null) {
fetcher = new URLFetcher( fetcher = new URLFetcher(
@ -216,9 +210,6 @@ public class CrawlURLFetch_p {
fetcher.delay); fetcher.delay);
} }
fetcher.start(); fetcher.start();
} catch (IOException e) {
e.printStackTrace();
}
} else { } else {
prop.put("threadError", ERR_THREAD_RESUME); prop.put("threadError", ERR_THREAD_RESUME);
} }
@ -397,7 +388,7 @@ public class CrawlURLFetch_p {
plasmaCrawlProfile.entry profile, plasmaCrawlProfile.entry profile,
URL url, URL url,
int count, int count,
long delayMs) throws IOException { long delayMs) {
if (env == null || profile == null || url == null) if (env == null || profile == null || url == null)
throw new NullPointerException("env, profile or url must not be null"); throw new NullPointerException("env, profile or url must not be null");
this.sb = (plasmaSwitchboard)env; this.sb = (plasmaSwitchboard)env;
@ -412,7 +403,7 @@ public class CrawlURLFetch_p {
serverSwitch env, serverSwitch env,
plasmaCrawlProfile.entry profile, plasmaCrawlProfile.entry profile,
int count, int count,
long delayMs) throws IOException { long delayMs) {
if (env == null || profile == null) if (env == null || profile == null)
throw new NullPointerException("env or profile must not be null"); throw new NullPointerException("env or profile must not be null");
this.sb = (plasmaSwitchboard)env; this.sb = (plasmaSwitchboard)env;

@ -1,6 +1,11 @@
<!--HEADER for FeedReader_p.html <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<title>#[clientname]#'s feed reader</title> <html xmlns="http://www.w3.org/1999/xhtml">
end of HEADER--> <head>
<title>YaCy '#[clientname]#': Feed Reader</title>
#%env/templates/metas.template%#
</head>
<body>
#%env/templates/header.template%#
#(page)# #(page)#
please select your feed with ?url=Feedurl&max=5&offset=1 (to be implemented in html ;)) please select your feed with ?url=Feedurl&max=5&offset=1 (to be implemented in html ;))
:: ::
@ -28,4 +33,7 @@ You need to install libx
Problem with url Problem with url
#(/error)# #(/error)#
test test
#(/page)# #(/page)#
#%env/templates/footer.template%#
</body>
</html>

@ -28,7 +28,7 @@ import java.net.URL;
import java.util.Collection; import java.util.Collection;
import java.util.Iterator; import java.util.Iterator;
import de.anomic.data.rssReaderItem; import de.anomic.data.rssReader;
import de.anomic.http.httpHeader; import de.anomic.http.httpHeader;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
@ -86,7 +86,6 @@ public class FeedReader_p {
} }
public static servletProperties respond(httpHeader header, serverObjects post, serverSwitch env) { public static servletProperties respond(httpHeader header, serverObjects post, serverSwitch env) {
servletProperties prop = new servletProperties(); servletProperties prop = new servletProperties();
prop.put("SUPERTEMPLATE", "/env/page.html");
URL url; URL url;
prop.put("page", 0); prop.put("page", 0);
@ -122,7 +121,7 @@ public class FeedReader_p {
Iterator it=feedItems.iterator(); Iterator it=feedItems.iterator();
int count=0; int count=0;
while(it.hasNext() && (maxitems==0 || count<maxitems)){ while(it.hasNext() && (maxitems==0 || count<maxitems)){
rssReaderItem item=(rssReaderItem)it.next(); rssReader.Item item=(rssReader.Item)it.next();
prop.put("page_items_"+count+"_author", item.getCreator()); prop.put("page_items_"+count+"_author", item.getCreator());
prop.put("page_items_"+count+"_title", item.getTitle()); prop.put("page_items_"+count+"_title", item.getTitle());
prop.put("page_items_"+count+"_link", item.getLink().toString()); prop.put("page_items_"+count+"_link", item.getLink().toString());

@ -1,24 +0,0 @@
#(mode)#
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>Gettext Locales</title>
#%env/templates/metas.template%#
</head>
<body marginheight="0" marginwidth="0" leftmargin="0" topmargin="0">
#%env/templates/header.template%#
<h2>Gettext Locales</h2>
<p>
<a href="Gettext_p.html?mode=1">Get empty gettext file</a>
<form action="Gettext_p.html" method="GET">
<input type="hidden" name="mode" value="1" />
old file: <input type="text" name="oldfile" value="DATA/LOCALE" />
<input type="submit" value="get updated file" />
</form>
</p>
#%env/templates/footer.template%#
</body>
</html>
::
#[gettext]#
#(/mode)#

@ -1,78 +0,0 @@
//Gettext_p.java
//------------
// part of YACY
//
// (C) 2006 Alexander Schier
//
// last change: $LastChangedDate: $ by $LastChangedBy: $
// $LastChangedRevision: $
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.io.File;
import java.io.FileNotFoundException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Map;
import de.anomic.data.gettext;
import de.anomic.http.httpHeader;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
public class Gettext_p{
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
serverObjects prop = new serverObjects();
if(post != null && post.get("mode").equals("1")){
prop.put("mode", "1");
File oldfile=null;
String oldfilename;
if(post.containsKey("oldfile")){
oldfilename=(String) post.get("oldfile");
oldfile=new File(env.getRootPath(), oldfilename);
if(!oldfile.exists())
//TODO: display warning?
oldfile=null;
}
String htRootPath = env.getConfig("htRootPath", "htroot");
File sourceDir = new File(env.getRootPath(), htRootPath);
ArrayList list;
try {
list = gettext.createGettextRecursive(sourceDir, "html,template,inc", "locale", oldfile);
} catch (FileNotFoundException e) {
// TODO warn the user
list = gettext.createGettextRecursive(sourceDir, "html,template,inc", "locale", (Map)null);
}
Iterator it=list.iterator();
String out="";
while(it.hasNext()){
out+=(String)it.next()+"\n";
}
//this does not work
/*httpHeader outheader=new httpHeader();
outheader.put("Content-Type", "text/plain");
prop.setOutgoingHeader(outheader);*/
prop.put("mode_gettext", out);
}
return prop;
}
}

@ -93,9 +93,7 @@ public class IndexCleaner_p {
prop.put("LOCATION",""); prop.put("LOCATION","");
return prop; return prop;
} }
else { prop.put("bla", "post==null");
prop.put("bla", "post==null");
}
if (urldbCleanerThread!=null) { if (urldbCleanerThread!=null) {
prop.put("urldb", 1); prop.put("urldb", 1);
prop.put("urldb_percentUrls", ((double)urldbCleanerThread.totalSearchedUrls/sb.wordIndex.loadedURL.size())*100 + ""); prop.put("urldb_percentUrls", ((double)urldbCleanerThread.totalSearchedUrls/sb.wordIndex.loadedURL.size())*100 + "");

@ -1,14 +0,0 @@
<!--HEADER for Lab.html
<title>#[clientname]#'s Lab</title>
end of HEADER-->
<h2>The YaCy Lab</h2>
<p>
This is the place where we try new functions of the YaCy search engine.
All these things here are to be considered as probably unstable, and/or experimental.
You may try out these things but please do not care about bugs.</p>
<ul>
<li>The <a href="Wiki.html">Wiki</a></li>
<li>Advanced <a href="/Config_p.html">Configuration</a></li>
</ul>

@ -1,13 +0,0 @@
import de.anomic.http.httpHeader;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
public class Lab {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
serverObjects prop = new serverObjects();
//XXX: Should we use Constants like DEFAULT_PAGE, PAGE_WITHOUT_MENU and so on,
//or is it nice enough to set the real path in the servlets?
prop.put("SUPERTEMPLATE", "/env/page.html");
return prop;
}
}

@ -84,100 +84,99 @@ public class LogStatistics_p {
if (r == null) { if (r == null) {
prop.put("results", 0); prop.put("results", 0);
return prop; return prop;
}
prop.put("results", 1);
String[] t;
float l;
prop.put(RESULTS + LogParserPLASMA.DHT_DISTANCE_AVERAGE, r.get(LogParserPLASMA.DHT_DISTANCE_AVERAGE));
prop.put(RESULTS + LogParserPLASMA.DHT_DISTANCE_MAX, r.get(LogParserPLASMA.DHT_DISTANCE_MAX));
prop.put(RESULTS + LogParserPLASMA.DHT_DISTANCE_MIN, r.get(LogParserPLASMA.DHT_DISTANCE_MIN));
prop.put(RESULTS + LogParserPLASMA.DHT_REJECTED, r.get(LogParserPLASMA.DHT_REJECTED));
prop.put(RESULTS + LogParserPLASMA.DHT_SELECTED, r.get(LogParserPLASMA.DHT_SELECTED));
prop.put(RESULTS + LogParserPLASMA.DHT_SENT_FAILED, r.get(LogParserPLASMA.DHT_SENT_FAILED));
t = transformMem(((Long)r.get(LogParserPLASMA.DHT_TRAFFIC_SENT)).longValue());
prop.put(RESULTS + LogParserPLASMA.DHT_TRAFFIC_SENT, t[0]);
prop.put(RESULTS + LogParserPLASMA.DHT_TRAFFIC_SENT + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.DHT_URLS_SENT, r.get(LogParserPLASMA.DHT_URLS_SENT));
prop.put(RESULTS + LogParserPLASMA.DHT_WORDS_SELECTED, r.get(LogParserPLASMA.DHT_WORDS_SELECTED));
t = transformTime(((Integer)r.get(LogParserPLASMA.DHT_WORDS_SELECTED_TIME)).longValue() * 1000L);
prop.put(RESULTS + LogParserPLASMA.DHT_WORDS_SELECTED_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.DHT_WORDS_SELECTED_TIME + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.ERROR_CHILD_TWICE_LEFT, r.get(LogParserPLASMA.ERROR_CHILD_TWICE_LEFT));
prop.put(RESULTS + LogParserPLASMA.ERROR_CHILD_TWICE_RIGHT, r.get(LogParserPLASMA.ERROR_CHILD_TWICE_RIGHT));
prop.put(RESULTS + LogParserPLASMA.ERROR_MALFORMED_URL, r.get(LogParserPLASMA.ERROR_MALFORMED_URL));
prop.put(RESULTS + LogParserPLASMA.INDEXED_ANCHORS, r.get(LogParserPLASMA.INDEXED_ANCHORS));
t = transformTime(((Integer)r.get(LogParserPLASMA.INDEXED_INDEX_TIME)).longValue());
prop.put(RESULTS + LogParserPLASMA.INDEXED_INDEX_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.INDEXED_INDEX_TIME + "Unit", t[1]);
t = transformTime(((Integer)r.get(LogParserPLASMA.INDEXED_PARSE_TIME)).longValue());
prop.put(RESULTS + LogParserPLASMA.INDEXED_PARSE_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.INDEXED_PARSE_TIME + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.INDEXED_SITES, r.get(LogParserPLASMA.INDEXED_SITES));
t = transformMem(((Integer)r.get(LogParserPLASMA.INDEXED_SITES_SIZE)).longValue());
prop.put(RESULTS + LogParserPLASMA.INDEXED_SITES_SIZE, t[0]);
prop.put(RESULTS + LogParserPLASMA.INDEXED_SITES_SIZE + "Unit", t[1]);
t = transformTime(((Integer)r.get(LogParserPLASMA.INDEXED_STACK_TIME)).longValue());
prop.put(RESULTS + LogParserPLASMA.INDEXED_STACK_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.INDEXED_STACK_TIME + "Unit", t[1]);
t = transformTime(((Integer)r.get(LogParserPLASMA.INDEXED_STORE_TIME)).longValue());
prop.put(RESULTS + LogParserPLASMA.INDEXED_STORE_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.INDEXED_STORE_TIME + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.INDEXED_WORDS, r.get(LogParserPLASMA.INDEXED_WORDS));
prop.put(RESULTS + LogParserPLASMA.PEERS_BUSY, r.get(LogParserPLASMA.PEERS_BUSY));
prop.put(RESULTS + LogParserPLASMA.PEERS_TOO_LESS, r.get(LogParserPLASMA.PEERS_TOO_LESS));
prop.put(RESULTS + LogParserPLASMA.RANKING_DIST, r.get(LogParserPLASMA.RANKING_DIST));
prop.put(RESULTS + LogParserPLASMA.RANKING_DIST_FAILED, r.get(LogParserPLASMA.RANKING_DIST_FAILED));
t = transformTime(((Integer)r.get(LogParserPLASMA.RANKING_DIST_TIME)).longValue());
prop.put(RESULTS + LogParserPLASMA.RANKING_DIST_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.RANKING_DIST_TIME + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.RWIS_BLOCKED, r.get(LogParserPLASMA.RWIS_BLOCKED));
prop.put(RESULTS + LogParserPLASMA.RWIS_RECEIVED, r.get(LogParserPLASMA.RWIS_RECEIVED));
t = transformTime(((Long)r.get(LogParserPLASMA.RWIS_RECEIVED_TIME)).longValue());
prop.put(RESULTS + LogParserPLASMA.RWIS_RECEIVED_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.RWIS_RECEIVED_TIME + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.URLS_BLOCKED, r.get(LogParserPLASMA.URLS_BLOCKED));
prop.put(RESULTS + LogParserPLASMA.URLS_RECEIVED, r.get(LogParserPLASMA.URLS_RECEIVED));
t = transformTime(((Long)r.get(LogParserPLASMA.URLS_RECEIVED_TIME)).longValue());
prop.put(RESULTS + LogParserPLASMA.URLS_RECEIVED_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.URLS_RECEIVED_TIME + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.URLS_REQUESTED, r.get(LogParserPLASMA.URLS_REQUESTED));
prop.put(RESULTS + LogParserPLASMA.WORDS_RECEIVED, r.get(LogParserPLASMA.WORDS_RECEIVED));
l = ((Long)r.get(LogParserPLASMA.TOTAL_PARSER_TIME)).floatValue();
t = transformTime((long)l);
prop.put(RESULTS + LogParserPLASMA.TOTAL_PARSER_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.TOTAL_PARSER_TIME + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.TOTAL_PARSER_RUNS, r.get(LogParserPLASMA.TOTAL_PARSER_RUNS));
if ((l /= 1000) == 0) {
prop.put(RESULTS + "avgExists", 0);
} else { } else {
prop.put("results", 1); prop.put(RESULTS + "avgExists", 1);
String[] t; prop.put(RESULTS + "avgExists_avgParserRunsPerMinute", (int) (((Integer) r.get(LogParserPLASMA.TOTAL_PARSER_RUNS)).floatValue() / l));
float l;
prop.put(RESULTS + LogParserPLASMA.DHT_DISTANCE_AVERAGE, r.get(LogParserPLASMA.DHT_DISTANCE_AVERAGE));
prop.put(RESULTS + LogParserPLASMA.DHT_DISTANCE_MAX, r.get(LogParserPLASMA.DHT_DISTANCE_MAX));
prop.put(RESULTS + LogParserPLASMA.DHT_DISTANCE_MIN, r.get(LogParserPLASMA.DHT_DISTANCE_MIN));
prop.put(RESULTS + LogParserPLASMA.DHT_REJECTED, r.get(LogParserPLASMA.DHT_REJECTED));
prop.put(RESULTS + LogParserPLASMA.DHT_SELECTED, r.get(LogParserPLASMA.DHT_SELECTED));
prop.put(RESULTS + LogParserPLASMA.DHT_SENT_FAILED, r.get(LogParserPLASMA.DHT_SENT_FAILED));
t = transformMem(((Long)r.get(LogParserPLASMA.DHT_TRAFFIC_SENT)).longValue());
prop.put(RESULTS + LogParserPLASMA.DHT_TRAFFIC_SENT, t[0]);
prop.put(RESULTS + LogParserPLASMA.DHT_TRAFFIC_SENT + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.DHT_URLS_SENT, r.get(LogParserPLASMA.DHT_URLS_SENT));
prop.put(RESULTS + LogParserPLASMA.DHT_WORDS_SELECTED, r.get(LogParserPLASMA.DHT_WORDS_SELECTED));
t = transformTime(((Integer)r.get(LogParserPLASMA.DHT_WORDS_SELECTED_TIME)).longValue() * 1000L);
prop.put(RESULTS + LogParserPLASMA.DHT_WORDS_SELECTED_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.DHT_WORDS_SELECTED_TIME + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.ERROR_CHILD_TWICE_LEFT, r.get(LogParserPLASMA.ERROR_CHILD_TWICE_LEFT));
prop.put(RESULTS + LogParserPLASMA.ERROR_CHILD_TWICE_RIGHT, r.get(LogParserPLASMA.ERROR_CHILD_TWICE_RIGHT));
prop.put(RESULTS + LogParserPLASMA.ERROR_MALFORMED_URL, r.get(LogParserPLASMA.ERROR_MALFORMED_URL));
prop.put(RESULTS + LogParserPLASMA.INDEXED_ANCHORS, r.get(LogParserPLASMA.INDEXED_ANCHORS));
t = transformTime(((Integer)r.get(LogParserPLASMA.INDEXED_INDEX_TIME)).longValue());
prop.put(RESULTS + LogParserPLASMA.INDEXED_INDEX_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.INDEXED_INDEX_TIME + "Unit", t[1]);
t = transformTime(((Integer)r.get(LogParserPLASMA.INDEXED_PARSE_TIME)).longValue());
prop.put(RESULTS + LogParserPLASMA.INDEXED_PARSE_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.INDEXED_PARSE_TIME + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.INDEXED_SITES, r.get(LogParserPLASMA.INDEXED_SITES));
t = transformMem(((Integer)r.get(LogParserPLASMA.INDEXED_SITES_SIZE)).longValue());
prop.put(RESULTS + LogParserPLASMA.INDEXED_SITES_SIZE, t[0]);
prop.put(RESULTS + LogParserPLASMA.INDEXED_SITES_SIZE + "Unit", t[1]);
t = transformTime(((Integer)r.get(LogParserPLASMA.INDEXED_STACK_TIME)).longValue());
prop.put(RESULTS + LogParserPLASMA.INDEXED_STACK_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.INDEXED_STACK_TIME + "Unit", t[1]);
t = transformTime(((Integer)r.get(LogParserPLASMA.INDEXED_STORE_TIME)).longValue());
prop.put(RESULTS + LogParserPLASMA.INDEXED_STORE_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.INDEXED_STORE_TIME + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.INDEXED_WORDS, r.get(LogParserPLASMA.INDEXED_WORDS));
prop.put(RESULTS + LogParserPLASMA.PEERS_BUSY, r.get(LogParserPLASMA.PEERS_BUSY));
prop.put(RESULTS + LogParserPLASMA.PEERS_TOO_LESS, r.get(LogParserPLASMA.PEERS_TOO_LESS));
prop.put(RESULTS + LogParserPLASMA.RANKING_DIST, r.get(LogParserPLASMA.RANKING_DIST));
prop.put(RESULTS + LogParserPLASMA.RANKING_DIST_FAILED, r.get(LogParserPLASMA.RANKING_DIST_FAILED));
t = transformTime(((Integer)r.get(LogParserPLASMA.RANKING_DIST_TIME)).longValue());
prop.put(RESULTS + LogParserPLASMA.RANKING_DIST_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.RANKING_DIST_TIME + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.RWIS_BLOCKED, r.get(LogParserPLASMA.RWIS_BLOCKED));
prop.put(RESULTS + LogParserPLASMA.RWIS_RECEIVED, r.get(LogParserPLASMA.RWIS_RECEIVED));
t = transformTime(((Long)r.get(LogParserPLASMA.RWIS_RECEIVED_TIME)).longValue());
prop.put(RESULTS + LogParserPLASMA.RWIS_RECEIVED_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.RWIS_RECEIVED_TIME + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.URLS_BLOCKED, r.get(LogParserPLASMA.URLS_BLOCKED));
prop.put(RESULTS + LogParserPLASMA.URLS_RECEIVED, r.get(LogParserPLASMA.URLS_RECEIVED));
t = transformTime(((Long)r.get(LogParserPLASMA.URLS_RECEIVED_TIME)).longValue());
prop.put(RESULTS + LogParserPLASMA.URLS_RECEIVED_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.URLS_RECEIVED_TIME + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.URLS_REQUESTED, r.get(LogParserPLASMA.URLS_REQUESTED));
prop.put(RESULTS + LogParserPLASMA.WORDS_RECEIVED, r.get(LogParserPLASMA.WORDS_RECEIVED));
l = ((Long)r.get(LogParserPLASMA.TOTAL_PARSER_TIME)).floatValue();
t = transformTime((long)l);
prop.put(RESULTS + LogParserPLASMA.TOTAL_PARSER_TIME, t[0]);
prop.put(RESULTS + LogParserPLASMA.TOTAL_PARSER_TIME + "Unit", t[1]);
prop.put(RESULTS + LogParserPLASMA.TOTAL_PARSER_RUNS, r.get(LogParserPLASMA.TOTAL_PARSER_RUNS));
if ((l /= 1000) == 0) {
prop.put(RESULTS + "avgExists", 0);
} else {
prop.put(RESULTS + "avgExists", 1);
prop.put(RESULTS + "avgExists_avgParserRunsPerMinute", (int) (((Integer) r.get(LogParserPLASMA.TOTAL_PARSER_RUNS)).floatValue() / l));
}
Object[] names = ((HashSet)r.get(LogParserPLASMA.DHT_REJECTED_PEERS_NAME)).toArray();
Object[] hashes = ((HashSet)r.get(LogParserPLASMA.DHT_REJECTED_PEERS_HASH)).toArray();
int i = 0;
for (; i<names.length && i<hashes.length; i++) {
prop.put(RESULTS + "useDHTRejectPeers_DHTRejectPeers_" + i + "_name", names[i]);
prop.put(RESULTS + "useDHTRejectPeers_DHTRejectPeers_" + i + "_hash", hashes[i]);
}
prop.put(RESULTS + "DHTRejectPeers", i);
prop.put(RESULTS + "useDHTRejectPeers", (i > 0) ? 1 : 0);
prop.put(RESULTS + "useDHTRejectPeers_DHTRejectPeers", i);
names = ((HashSet)r.get(LogParserPLASMA.DHT_SENT_PEERS_NAME)).toArray();
hashes = ((HashSet)r.get(LogParserPLASMA.DHT_SENT_PEERS_HASH)).toArray();
i = 0;
for (; i<names.length && i<hashes.length; i++) {
prop.put(RESULTS + "useDHTPeers_DHTPeers_" + i + "_name", names[i]);
prop.put(RESULTS + "useDHTPeers_DHTPeers_" + i + "_hash", hashes[i]);
}
prop.put(RESULTS + "DHTPeers", i);
prop.put(RESULTS + "useDHTPeers", (i > 0) ? 1 : 0);
prop.put(RESULTS + "useDHTPeers_DHTPeers", i);
return prop;
} }
Object[] names = ((HashSet)r.get(LogParserPLASMA.DHT_REJECTED_PEERS_NAME)).toArray();
Object[] hashes = ((HashSet)r.get(LogParserPLASMA.DHT_REJECTED_PEERS_HASH)).toArray();
int i = 0;
for (; i<names.length && i<hashes.length; i++) {
prop.put(RESULTS + "useDHTRejectPeers_DHTRejectPeers_" + i + "_name", names[i]);
prop.put(RESULTS + "useDHTRejectPeers_DHTRejectPeers_" + i + "_hash", hashes[i]);
}
prop.put(RESULTS + "DHTRejectPeers", i);
prop.put(RESULTS + "useDHTRejectPeers", (i > 0) ? 1 : 0);
prop.put(RESULTS + "useDHTRejectPeers_DHTRejectPeers", i);
names = ((HashSet)r.get(LogParserPLASMA.DHT_SENT_PEERS_NAME)).toArray();
hashes = ((HashSet)r.get(LogParserPLASMA.DHT_SENT_PEERS_HASH)).toArray();
i = 0;
for (; i<names.length && i<hashes.length; i++) {
prop.put(RESULTS + "useDHTPeers_DHTPeers_" + i + "_name", names[i]);
prop.put(RESULTS + "useDHTPeers_DHTPeers_" + i + "_hash", hashes[i]);
}
prop.put(RESULTS + "DHTPeers", i);
prop.put(RESULTS + "useDHTPeers", (i > 0) ? 1 : 0);
prop.put(RESULTS + "useDHTPeers_DHTPeers", i);
return prop;
} }
private static final String MILLISECONDS = "ms"; private static final String MILLISECONDS = "ms";

@ -18,16 +18,10 @@
// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA // Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
import java.io.File; import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.lang.reflect.InvocationTargetException; import java.lang.reflect.InvocationTargetException;
import java.util.Iterator; import java.util.Iterator;
import de.anomic.http.httpHeader; import de.anomic.http.httpHeader;
import de.anomic.http.httpTemplate;
import de.anomic.http.httpdFileHandler; import de.anomic.http.httpdFileHandler;
import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverByteBuffer; import de.anomic.server.serverByteBuffer;
@ -56,19 +50,18 @@ public class TestApplet {
} }
prop.put("mode", "1"); prop.put("mode", "1");
File templatefile=filehandler.getOverlayedFile((String)post.get("url")); //File templatefile=filehandler.getOverlayedFile((String)post.get("url"));
File classfile=filehandler.getOverlayedClass((String)post.get("url")); File classfile=filehandler.getOverlayedClass((String)post.get("url"));
httpHeader header2=new httpHeader(); httpHeader header2=new httpHeader();
header2.put("CLIENTIP", "127.0.0.1"); header2.put("CLIENTIP", "127.0.0.1");
header2.put("PATH", (String)post.get("url")); header2.put("PATH", post.get("url"));
serverObjects tp=null; serverObjects tp=null;
try { try {
if(classfile==null || !classfile.exists()){ if(classfile==null || !classfile.exists()){
prop.put("mode_templates", "classfile does not exist"); prop.put("mode_templates", "classfile does not exist");
return prop; return prop;
}else{
tp=(serverObjects)filehandler.invokeServlet(classfile, header2, args);
} }
tp=(serverObjects)filehandler.invokeServlet(classfile, header2, args);
} }
catch (IllegalArgumentException e) {} catch (IllegalArgumentException e) {}
catch (IllegalAccessException e) {} catch (IllegalAccessException e) {}
@ -87,22 +80,22 @@ public class TestApplet {
tmp.append(key).append("=").append(tp.get(key)).append("\n"); tmp.append(key).append("=").append(tp.get(key)).append("\n");
} }
prop.put("mode_templates", tmp.toString()); prop.put("mode_templates", tmp.toString());
FileInputStream fis=null; //FileInputStream fis=null;
try { //try {
fis=new FileInputStream(templatefile); //fis=new FileInputStream(templatefile);
serverByteBuffer o=new serverByteBuffer(); serverByteBuffer o=new serverByteBuffer();
byte[] structure=httpTemplate.writeTemplate(fis, (OutputStream)o, tp, "-UNRESOLVED_PATTERN-".getBytes("UTF-8")); //byte[] structure=httpTemplate.writeTemplate(fis, o, tp, "-UNRESOLVED_PATTERN-".getBytes("UTF-8"));
prop.put("mode_structure", structure); //prop.put("mode_structure", structure);
prop.put("mode_text", o.toString()); prop.put("mode_text", o.toString());
return prop; return prop;
} //}
catch (FileNotFoundException e) {} //catch (FileNotFoundException e) {}
catch (UnsupportedEncodingException e) {} //catch (UnsupportedEncodingException e) {}
catch (IOException e) {} //catch (IOException e) {}
prop.put("mode_text", "could not finish correctly"); //very informative errormessage //prop.put("mode_text", "could not finish correctly"); //very informative errormessage
return prop; //return prop;
} }

@ -4,10 +4,8 @@
<title>YaCy '#[clientname]#': User Page</title> <title>YaCy '#[clientname]#': User Page</title>
#%env/templates/metas.template%# #%env/templates/metas.template%#
</head> </head>
<body> <body>
#%env/templates/header.template%# #%env/templates/header.template%#
<br />
<br />
<h2>User Page</h2> <h2>User Page</h2>
#(logged-in)# #(logged-in)#
<p> <p>

@ -1,6 +1,11 @@
<!--HEADER for Lab.html <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<title>#[clientname]#'s User Administration</title> <html xmlns="http://www.w3.org/1999/xhtml">
end of HEADER--> <head>
<title>YaCy '#[clientname]#': User Page</title>
#%env/templates/metas.template%#
</head>
<body>
#%env/templates/header.template%#
<h2>User Administration</h2> <h2>User Administration</h2>
<!-- Page 1: Results --> <!-- Page 1: Results -->
@ -70,3 +75,6 @@
</dl> </dl>
</fieldset> </fieldset>
</form> </form>
#%env/templates/footer.template%#
</body>
</html>

@ -63,7 +63,6 @@ public class User_p {
serverObjects prop = new serverObjects(); serverObjects prop = new serverObjects();
plasmaSwitchboard sb = plasmaSwitchboard.getSwitchboard(); plasmaSwitchboard sb = plasmaSwitchboard.getSwitchboard();
userDB.Entry entry=null; userDB.Entry entry=null;
prop.put("SUPERTEMPLATE", "/env/page.html"); //user Supertemplates
//default values //default values
prop.put("current_user", "newuser"); prop.put("current_user", "newuser");

@ -53,7 +53,7 @@ import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import de.anomic.data.Diff; import de.anomic.data.diff;
import de.anomic.data.wikiBoard; import de.anomic.data.wikiBoard;
import de.anomic.http.httpHeader; import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboard;
@ -244,10 +244,10 @@ public class Wiki {
if (nentry == null) nentry = entry; if (nentry == null) nentry = entry;
if (post.containsKey("compare") && oentry != null && nentry != null) { if (post.containsKey("compare") && oentry != null && nentry != null) {
// TODO: split into paragraphs and compare them with the same diff-algo // TODO: split into paragraphs and compare them with the same diff-algo
Diff diff = new Diff( diff diff = new diff(
new String(oentry.page(), "UTF-8"), new String(oentry.page(), "UTF-8"),
new String(nentry.page(), "UTF-8"), 3); new String(nentry.page(), "UTF-8"), 3);
prop.putASIS("mode_versioning_diff", Diff.toHTML(new Diff[] { diff })); prop.putASIS("mode_versioning_diff", de.anomic.data.diff.toHTML(new diff[] { diff }));
prop.put("mode_versioning", 1); prop.put("mode_versioning", 1);
} else if (post.containsKey("viewold") && oentry != null) { } else if (post.containsKey("viewold") && oentry != null) {
prop.put("mode_versioning", 2); prop.put("mode_versioning", 2);

@ -1,16 +0,0 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
#[header]#
<!--<title>YaCy '#[clientname]#': #[title]#</title>-->
<!-- TODO: title cannot be set, yet. -->
<!-- clientname needs to be set in page.java (this will remove the code in httpdFileHandler,
if we use supertemplates for everything -->
#%env/templates/metas.template%#
</head>
<body>
#%env/templates/header.template%#
#[page]#
#%env/templates/footer.template%#
</body>
</html>

@ -1,41 +0,0 @@
//page.java - super template, to allow a scripted layout.
//----------------------------------------------------------
//part of YaCy
//
// (C) 2006 by Alexander Schier
//
// last change: $LastChangedDate: $ by $LastChangedBy: $
// $LastChangedRevision: $
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//note: this isn't useful, yet.
//this is only a basic demonstration, what can be done with the supertemplates.
import de.anomic.http.httpHeader;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
public class page {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
//plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
//prop.put("test1", "testit!");
//prop.put("test2", 1);
return prop;
}
}

@ -0,0 +1,11 @@
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<!-- aus http://de.selfhtml.org/servercgi/server/ssi.htm -->
<html>
<head>
<title>Dynamisches HTML mit Server Side Includes</title>
</head>
<body>
<h1>Dynamisches HTML mit Server Side Includes</h1>
<!--#include virtual="ssitest.inc" -->
</body>
</html>

@ -0,0 +1,7 @@
<!-- aus http://de.selfhtml.org/servercgi/server/ssi.htm -->
<p>Auf einer Meierei,<br>
da lebte einst ein braves Huhn,<br>
das legte, wie die HŸhner tun,<br>
an jedem Tag ein Ei.<br>
Und kakelte, mirakelte, spektakelte,<br>
als obs ein Wunder sei.</p>

@ -640,7 +640,7 @@ final class dbTable implements kelondroIndex {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
public synchronized void addUnique(kelondroRow.Entry row, Date entryDate) throws IOException { public synchronized void addUnique(kelondroRow.Entry row, Date entryDate) {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }

@ -783,9 +783,8 @@ public class bookmarksDB {
public String getOwner(){ public String getOwner(){
if(entry.containsKey(BOOKMARK_OWNER)){ if(entry.containsKey(BOOKMARK_OWNER)){
return (String) entry.get(BOOKMARK_OWNER); return (String) entry.get(BOOKMARK_OWNER);
}else{
return null; //null means admin
} }
return null; //null means admin
} }
public void setOwner(String owner){ public void setOwner(String owner){
entry.put(BOOKMARK_OWNER, owner); entry.put(BOOKMARK_OWNER, owner);
@ -793,16 +792,14 @@ public class bookmarksDB {
public boolean getPublic(){ public boolean getPublic(){
if(entry.containsKey(BOOKMARK_PUBLIC)){ if(entry.containsKey(BOOKMARK_PUBLIC)){
return ((String) entry.get(BOOKMARK_PUBLIC)).equals("public"); return ((String) entry.get(BOOKMARK_PUBLIC)).equals("public");
}else{
return false;
} }
return false;
} }
public boolean getFeed(){ public boolean getFeed(){
if(entry.containsKey(BOOKMARK_IS_FEED)){ if(entry.containsKey(BOOKMARK_IS_FEED)){
return ((String) entry.get(BOOKMARK_IS_FEED)).equals("true"); return ((String) entry.get(BOOKMARK_IS_FEED)).equals("true");
}else{
return false;
} }
return false;
} }
public void setPublic(boolean isPublic){ public void setPublic(boolean isPublic){
if(isPublic){ if(isPublic){
@ -888,8 +885,8 @@ public class bookmarksDB {
public void remove() { public void remove() {
if (this.nextEntry != null) { if (this.nextEntry != null) {
try { try {
String tagHash = (String)this.nextEntry.getTagHash(); String tagHash = this.nextEntry.getTagHash();
if (tagHash != null) removeTag((String) tagHash); if (tagHash != null) removeTag(tagHash);
} catch (kelondroException e) { } catch (kelondroException e) {
//resetDatabase(); //resetDatabase();
} }
@ -949,16 +946,11 @@ public class bookmarksDB {
if(bm1==null || bm2==null) if(bm1==null || bm2==null)
return 0; //XXX: i think this should not happen? maybe this needs further tracing of the bug return 0; //XXX: i think this should not happen? maybe this needs further tracing of the bug
if(this.newestFirst){ if(this.newestFirst){
if(bm2.getTimeStamp() - bm1.getTimeStamp() >0) if(bm2.getTimeStamp() - bm1.getTimeStamp() >0) return 1;
return 1; return -1;
else
return -1;
}else{
if(bm1.getTimeStamp() - bm2.getTimeStamp() >0)
return 1;
else
return -1;
} }
if(bm1.getTimeStamp() - bm2.getTimeStamp() >0) return 1;
return -1;
} }
} }
/** /**

@ -6,7 +6,7 @@
// Frankfurt, Germany, 2007 // Frankfurt, Germany, 2007
// Created 03.02.2007 // Created 03.02.2007
// //
// This file is contributed by Franz Brauße // This file is contributed by Franz Brau§e
// //
// $LastChangedDate: $ // $LastChangedDate: $
// $LastChangedRevision: $ // $LastChangedRevision: $
@ -52,7 +52,7 @@ import java.util.ArrayList;
/** /**
* This class provides a diff-functionality. * This class provides a diff-functionality.
*/ */
public class Diff { public class diff {
private final ArrayList /* of Part */ parts = new ArrayList(); private final ArrayList /* of Part */ parts = new ArrayList();
private final Object[] o; private final Object[] o;
@ -63,7 +63,7 @@ public class Diff {
* @param n the new <code>String</code> * @param n the new <code>String</code>
* @throws NullPointerException if one of the arguments is <code>null</code> * @throws NullPointerException if one of the arguments is <code>null</code>
*/ */
public Diff(String o, String n) { public diff(String o, String n) {
this(o, n, 1); this(o, n, 1);
} }
@ -76,7 +76,7 @@ public class Diff {
* @throws NullPointerException if <code>o</code> or <code>n</code> is * @throws NullPointerException if <code>o</code> or <code>n</code> is
* <code>null</code> * <code>null</code>
*/ */
public Diff(String o, String n, int minConsecutive) { public diff(String o, String n, int minConsecutive) {
if (o == null || n == null) throw new NullPointerException("neither o nor n must be null"); if (o == null || n == null) throw new NullPointerException("neither o nor n must be null");
this.o = new Comparable[o.length()]; this.o = new Comparable[o.length()];
for (int i=0; i<o.length(); i++) for (int i=0; i<o.length(); i++)
@ -87,7 +87,7 @@ public class Diff {
parse((minConsecutive > 0) ? minConsecutive : 1); parse((minConsecutive > 0) ? minConsecutive : 1);
} }
public Diff(Object[] o, Object[] n, int minConsecutive) { public diff(Object[] o, Object[] n, int minConsecutive) {
if (o == null || n == null) throw new NullPointerException("neither o nor n must be null"); if (o == null || n == null) throw new NullPointerException("neither o nor n must be null");
this.o = o; this.o = o;
this.n = n; this.n = n;
@ -237,10 +237,10 @@ public class Diff {
final StringBuffer sb = new StringBuffer(this.posNew - this.posOld); final StringBuffer sb = new StringBuffer(this.posNew - this.posOld);
if (this.action == ADDED) { if (this.action == ADDED) {
for (int i=this.posOld; i<this.posNew; i++) for (int i=this.posOld; i<this.posNew; i++)
sb.append(Diff.this.n[i]); sb.append(diff.this.n[i]);
} else { } else {
for (int i=this.posOld; i<this.posNew; i++) for (int i=this.posOld; i<this.posNew; i++)
sb.append(Diff.this.o[i]); sb.append(diff.this.o[i]);
} }
return new String(sb); return new String(sb);
} }
@ -259,18 +259,18 @@ public class Diff {
} }
} }
public static String toHTML(Diff[] diffs) { public static String toHTML(diff[] diffs) {
StringBuffer sb = new StringBuffer(diffs.length * 60); StringBuffer sb = new StringBuffer(diffs.length * 60);
Diff.Part[] ps; diff.Part[] ps;
for (int i=0; i<diffs.length; i++) { for (int i=0; i<diffs.length; i++) {
sb.append("<p class=\"diff\">\n"); sb.append("<p class=\"diff\">\n");
ps = diffs[i].getParts(); ps = diffs[i].getParts();
for (int j=0; j<ps.length; j++) { for (int j=0; j<ps.length; j++) {
sb.append("<span\nclass=\""); sb.append("<span\nclass=\"");
switch (ps[j].getAction()) { switch (ps[j].getAction()) {
case Diff.Part.UNCHANGED: sb.append("unchanged"); break; case diff.Part.UNCHANGED: sb.append("unchanged"); break;
case Diff.Part.ADDED: sb.append("added"); break; case diff.Part.ADDED: sb.append("added"); break;
case Diff.Part.DELETED: sb.append("deleted"); break; case diff.Part.DELETED: sb.append("deleted"); break;
} }
sb.append("\">").append(htmlTools.replaceXMLEntities(ps[j].getString()).replaceAll("\n", "<br />")); sb.append("\">").append(htmlTools.replaceXMLEntities(ps[j].getString()).replaceAll("\n", "<br />"));
sb.append("</span>"); sb.append("</span>");
@ -279,4 +279,4 @@ public class Diff {
} }
return new String(sb); return new String(sb);
} }
} }

@ -1,333 +0,0 @@
//gettext.java - translations in a simplified gettext-format
//----------------------------------------------------------
//part of YaCy
//
// (C) 2006 by Alexander Schier
//
// last change: $LastChangedDate: $ by $LastChangedBy: $
// $LastChangedRevision: $
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.data;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import de.anomic.htmlFilter.htmlFilterContentTransformer;
import de.anomic.server.logging.serverLog;
public class gettext{
public static ArrayList createGettextRecursive(File sourceDir, String extensions, String notdir, File oldgettextfile) throws FileNotFoundException{
if(oldgettextfile==null)
return createGettextRecursive(sourceDir, extensions, notdir, (Map)null); //no old file
return createGettextRecursive(sourceDir, extensions, notdir, parseGettext(oldgettextfile));
}
public static ArrayList createGettextRecursive(File sourceDir, String extensions, String notdir, Map oldgettext){
ArrayList list=new ArrayList();
ArrayList exts=listManager.string2arraylist(extensions);
Iterator it2;
String filename;
ArrayList filenames=new ArrayList();
ArrayList dirList=listManager.getDirsRecursive(sourceDir, notdir);
dirList.add(sourceDir);
Iterator it=dirList.iterator();
File dir=null;
File[] files;
//this looks a lot more complicated, than it is ...
while(it.hasNext()){
dir=(File)it.next();
if(dir.isDirectory() && !dir.getName().equals(notdir)){
files=dir.listFiles();
for(int i=0;i<files.length;i++){
if(!files[i].isDirectory()){
it2=exts.iterator();
filename=files[i].getName();
while(it2.hasNext()){
if(filename.endsWith((String)it2.next())){
try {
filenames.add(files[i].getCanonicalPath());
} catch (IOException e) {
// TODO Auto-generated catch block
}
break;
}
}
}
}
}
}
list=createGettext(filenames, oldgettext);
return list;
}
private static ArrayList getGettextHeader(){
return getGettextHeader("UNKNOWN", "EMAIL");
}
private static ArrayList getGettextHeader(String translator, String email){
ArrayList list=new ArrayList();
list.add("#yacy translation");
list.add("msgid \"\"");
list.add("msgstr \"\"");
list.add("\"Content-Type: text/plain; charset=UTF-8\\n\"");
list.add("\"Content-Transfer-Encoding: 8bit\\n\"");
list.add("\"Last-Translator: "+translator+"\\n\"");
SimpleDateFormat dateformat=new SimpleDateFormat("yyyy-mm-dd HH:MMZ");
list.add("\"PO-Revision-Date: "+dateformat.format(new Date())+"\\n\"");
list.add("\"Project-Id-Version: YaCy\\n\"");
list.add("\"Language-Team: <"+email+">\\n\"");
list.add("\"X-Generator: YaCy\\n\"");
list.add("\"Mime-Version: 1.0\\n\"");
list.add("");
return list;
}
public static ArrayList createGettext(ArrayList filenames, File oldgettextfile) throws FileNotFoundException{
return createGettext(filenames, parseGettext(oldgettextfile));
}
/*
* create a list of gettext file for some textfiles
* @param filenames the ArrayList with the Filenames
* @param oldgettextmap a map with the old translations.
*/
public static ArrayList createGettext(ArrayList filenames, Map oldgettext){
ArrayList list=new ArrayList();
ArrayList tmp=null;
String filename=null;
Iterator it=filenames.iterator();
list.addAll(getGettextHeader());
while(it.hasNext()){
try {
filename=(String)it.next();
//TODO: better possibility to switch the behaviour
//tmp=getGettextSource(new File(filename), oldgettext);
tmp=getGettextSourceFromHTML(new File(filename), oldgettext);
serverLog.logFinest("Gettext", "Extracting Strings from: "+filename);
} catch (FileNotFoundException e) {
System.out.println("File \""+filename+"\" not found.");
}
if(tmp!=null)
list.addAll(tmp);
}
return list;
}
public static ArrayList getGettextSource(File inputfile, File oldmapfile) throws FileNotFoundException{
if(oldmapfile != null && oldmapfile.exists())
return getGettextSource(inputfile, parseGettext(oldmapfile));
return getGettextSource(inputfile);
}
public static ArrayList getGettextSource(File inputfile) throws FileNotFoundException{
return getGettextSource(inputfile, new HashMap());
}
public static ArrayList getGettextSource(File inputfile, Map oldgettextmap) throws FileNotFoundException{
ArrayList strings=getGettextItems(inputfile);
return getGettextSource(inputfile, oldgettextmap, strings);
}
public static ArrayList getGettextSourceFromHTML(File inputfile, Map oldgettextmap) throws FileNotFoundException{
htmlFilterContentTransformer transformer=new htmlFilterContentTransformer();
BufferedReader br=new BufferedReader(new FileReader(inputfile));
StringBuffer content=new StringBuffer();
String line="";
try {
while((line=br.readLine())!=null){
content.append(line).append("\n");
}
} catch (IOException e) {}
ArrayList strings = transformer.getStrings(content.toString().getBytes());
return getGettextSource(inputfile, oldgettextmap, strings);
}
public static ArrayList getGettextSource(File inputfile, Map oldgettextmap, ArrayList strings) {
if(oldgettextmap==null)
oldgettextmap=new HashMap();
ArrayList list=new ArrayList();
Iterator it=strings.iterator();
if(strings.isEmpty())
return null;
list.add("#"+inputfile.getName());
String key;
while(it.hasNext()){
key=((String)it.next()).replaceAll("\"", "\\\\\"").replaceAll("\n", "\\\\n");
list.add("msgid \""+key+"\"");
if(oldgettextmap.containsKey(key))
list.add("msgstr \""+oldgettextmap.get(key)+"\"");
else
list.add("msgstr \"\"");
list.add("");
}
return list;
}
/*
* create a list of gettext Strings ( _() ) from a file
* @param inputfile the file, which contains the raw Strings.
*/
public static ArrayList getGettextItems(File inputfile) throws FileNotFoundException{
ArrayList list=new ArrayList();
int character;
InputStreamReader reader;
int state=0; //0=no gettext macro 1= _ found, 2= ( found and in the string 3=\ found
String untranslatedString="";
reader = new InputStreamReader(new FileInputStream(inputfile));
try {
character=reader.read();
while(character >=0) {
if(state==0 && (char)character=='_')
state=1;
else if(state==1){
if((char)character=='('){
state=2;
untranslatedString="";
}else{
state=0;
untranslatedString+=(char)character;
}
}else if(state==2){
if((char)character=='\\')
state=3;
else if((char)character==')'){
state=0;
list.add(untranslatedString);
}else{
untranslatedString+=(char)character;
}
}else if(state==3){
state=2;
if((char)character==')')
untranslatedString+=")";
else
untranslatedString+="\\"+(char)character;
}else{
untranslatedString+=(char)character;
}
character=reader.read();
}
} catch (IOException e) {}
return list;
}
public static HashMap parseGettext(File gettextfile) throws FileNotFoundException{
ArrayList gettext=new ArrayList();
String line;
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(gettextfile)));
try {
line = br.readLine();
while (line != null) {
gettext.add(line);
line = br.readLine();
}
} catch (IOException e) {}
return parseGettext(gettext);
}
public static HashMap parseGettext(ArrayList gettext){
HashMap map = new HashMap();
int mode=0; //1= in msgid, 2= in msgstr
String msgid = "", msgstr = "", tmp = "";
Iterator it=gettext.iterator();
while(it.hasNext()){
tmp=(String) it.next();
if(tmp.startsWith("msgid \"")){
if(mode==2)
map.put(msgid, msgstr);
msgid=tmp.substring(7,tmp.length()-1).replaceAll("\\\"", "\"");
msgstr="";
mode=1;
}else if(tmp.startsWith("msgstr \"")){
mode=2;
msgstr=tmp.substring(8,tmp.length()-1);
}else if(tmp.startsWith("\"")){
//multiline strings with "..." on each line
if(mode==1){
msgid+="\n"+tmp.substring(1,tmp.length()-1).replaceAll("\\\"", "\"");
}else if(mode==2){
msgstr+="\n"+tmp.substring(1,tmp.length()-1).replaceAll("\\\"", "\"");
}
}
}
map.put(msgid, msgstr); //the last one cannot be put, on the next msgid ;-)
return map;
}
public static void main(String[] argv){
if(argv.length < 2){
System.out.println("Syntax: java de.anomic.data.gettext creategettext [inputfile] ... [inputfile]");
System.out.println("Syntax: java de.anomic.data.gettext parsegettext [gettextfile]");
System.out.println("Syntax: java de.anomic.data.gettext updategettext [gettextfile] [inputfile] ... [inputfile]");
System.exit(1);
}
if(argv[0].equals("creategettext")){
ArrayList filelist=new ArrayList();
for(int i=1;i<argv.length;i++){
filelist.add(argv[i]);
}
ArrayList list = createGettext(filelist, (Map)null);
Iterator it=list.iterator();
while(it.hasNext())
System.out.println((String)it.next());
}else if(argv[0].equals("parsegettext")){
if(argv.length >2){
System.out.println("only one file allowed for parsegettext");
System.exit(1);
}
try {
HashMap translations=parseGettext(new File(argv[1]));
Iterator it=translations.keySet().iterator();
String key="";
while(it.hasNext()){
key=(String)it.next();
System.out.println("key: "+key);
System.out.println("value: "+translations.get(key));
}
} catch (FileNotFoundException e) {
System.exit(1);
}
}else if(argv[0].equals("updategettext")){
if(argv.length < 3){
System.out.println("Too less arguments");
System.exit(1);
}
ArrayList filelist=new ArrayList();
for(int i=2;i<argv.length;i++){
filelist.add(argv[i]);
}
try {
ArrayList list=createGettext(filelist, new File(argv[1]));
Iterator it=list.iterator();
while(it.hasNext())
System.out.println((String)it.next());
} catch (FileNotFoundException e) {
System.out.println("File not found.");
System.exit(1);
}
}else{
System.out.println("unknown Mode ...");
System.exit(1);
}
}
}

@ -27,6 +27,8 @@ import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.util.Collection; import java.util.Collection;
import java.util.Comparator;
import java.util.Date;
import java.util.Iterator; import java.util.Iterator;
import java.util.TreeSet; import java.util.TreeSet;
@ -51,12 +53,12 @@ public class rssReader {
try { try {
channel=FeedParser.parse(builder, this.url); channel=FeedParser.parse(builder, this.url);
Collection oldfeedItems=channel.getItems(); Collection oldfeedItems=channel.getItems();
feedItems=new TreeSet(new rssReaderItemComparator()); feedItems=new TreeSet(new ItemComparator());
Iterator it=oldfeedItems.iterator(); Iterator it=oldfeedItems.iterator();
int count=0; int count=0;
while(it.hasNext()){ while(it.hasNext()){
de.nava.informa.impl.basic.Item item=(de.nava.informa.impl.basic.Item) it.next(); de.nava.informa.impl.basic.Item item=(de.nava.informa.impl.basic.Item) it.next();
rssReaderItem newItem=new rssReaderItem(count++, item.getLink(), item.getTitle(), item.getDescription(), item.getDate(), item.getCreator()); Item newItem=new Item(count++, item.getLink(), item.getTitle(), item.getDescription(), item.getDate(), item.getCreator());
feedItems.add(newItem); feedItems.add(newItem);
} }
} }
@ -75,5 +77,50 @@ public class rssReader {
public Collection getFeedItems(){ public Collection getFeedItems(){
return feedItems; return feedItems;
} }
public class Item{
String creator, title, description;
Date date;
URL link;
int num;
public Item(int num, URL link, String title, String description, Date date, String creator){
this.link=link;
this.title=title;
this.description=description;
this.date=date;
this.creator=creator;
this.num=num;
}
public URL getLink(){
return link;
}
public String getTitle(){
return (title!=null)? title: "";
}
public String getDescription(){
return (description!=null)? description: "";
}
public Date getDate(){
return (date!=null)? date: new Date();
}
public String getCreator(){
return (creator!=null)? creator: "";
}
public int getNum(){
return num;
}
}
public class ItemComparator implements Comparator {
public int compare(Object o1, Object o2){
int num1=((Item)o1).getNum();
int num2=((Item)o2).getNum();
return num2-num1;
}
public boolean equals(Object o1, Object o2){
return compare(o1, o2)==0;
}
}
} }

@ -1,62 +0,0 @@
//rssReaderItem.java
//------------
// part of YACY
//
// (C) 2007 Alexander Schier
//
// last change: $LastChangedDate: $ by $LastChangedBy: $
// $LastChangedRevision: $
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.data;
import java.net.URL;
import java.util.Date;
public class rssReaderItem{
String creator, title, description;
Date date;
URL link;
int num;
public rssReaderItem(int num, URL link, String title, String description, Date date, String creator){
this.link=link;
this.title=title;
this.description=description;
this.date=date;
this.creator=creator;
this.num=num;
}
public URL getLink(){
return link;
}
public String getTitle(){
return (title!=null)? title: "";
}
public String getDescription(){
return (description!=null)? description: "";
}
public Date getDate(){
return (date!=null)? date: new Date();
}
public String getCreator(){
return (creator!=null)? creator: "";
}
public int getNum(){
return num;
}
}

@ -1,36 +0,0 @@
//rssReaderItemComparator.java
//------------
// part of YACY
//
// (C) 2007 Alexander Schier
//
// last change: $LastChangedDate: $ by $LastChangedBy: $
// $LastChangedRevision: $
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.data;
import java.util.Comparator;
public class rssReaderItemComparator implements Comparator{
public int compare(Object o1, Object o2){
int num1=((rssReaderItem)o1).getNum();
int num2=((rssReaderItem)o2).getNum();
return num2-num1;
}
public boolean equals(Object o1, Object o2){
return compare(o1, o2)==0;
}
}

@ -439,10 +439,8 @@ public final class userDB {
return PROXY_ALLOK; return PROXY_ALLOK;
} }
public boolean canSurf(){ public boolean canSurf(){
if(this.surfRight()==PROXY_ALLOK) if(this.surfRight()==PROXY_ALLOK) return true;
return true; return false;
else
return false;
} }
public long updateLastAccess(boolean incrementTimeUsed) { public long updateLastAccess(boolean incrementTimeUsed) {
return updateLastAccess(System.currentTimeMillis(), incrementTimeUsed); return updateLastAccess(System.currentTimeMillis(), incrementTimeUsed);

@ -78,17 +78,13 @@ public class htmlFilterContentTransformer extends htmlFilterAbstractTransformer
} }
private ArrayList bluelist = null; private ArrayList bluelist = null;
private boolean gettext = false;
public htmlFilterContentTransformer() { public htmlFilterContentTransformer() {
super(linkTags0, linkTags1); super(linkTags0, linkTags1);
} }
public void init(String initarg) { public void init(String initarg) {
if (initarg.equals("gettext")) { if (bluelist == null) {
// the initarg declares that the transformer applies a gettext-quotation on strings
gettext = true;
} else if (bluelist == null) {
// here, the initarg is used to load a list of bluelisted words // here, the initarg is used to load a list of bluelisted words
bluelist = new ArrayList(); bluelist = new ArrayList();
File f = new File(initarg); File f = new File(initarg);
@ -108,7 +104,7 @@ public class htmlFilterContentTransformer extends htmlFilterAbstractTransformer
} }
public boolean isIdentityTransformer() { public boolean isIdentityTransformer() {
return (bluelist.size() == 0) && (!gettext); return (bluelist.size() == 0);
} }
private static char[] genBlueLetters(int length) { private static char[] genBlueLetters(int length) {
@ -154,33 +150,6 @@ public class htmlFilterContentTransformer extends htmlFilterAbstractTransformer
return result; return result;
} }
public char[] transformText(char[] text) { public char[] transformText(char[] text) {
if (gettext) {
// serverCharBuffer sbb = new serverCharBuffer(text);
// //if (sbb.length() > 0) System.out.println(" TEXT: " + sbb.toString());
// serverCharBuffer[] sbbs = httpTemplate.splitQuotations(sbb);
// sbb = new serverCharBuffer();
// for (int i = 0; i < sbbs.length; i++) {
// if (sbbs[i].isWhitespace(true)) {
// sbb.append(sbbs[i]);
// } else if ((sbbs[i].byteAt(0) == httpTemplate.hash) ||
// (sbbs[i].startsWith(httpTemplate.dpdpa))) {
// // this is a template or a part of a template
// sbb.append(sbbs[i]);
// } else {
// // this is a text fragment, generate gettext quotation
// int ws = sbbs[i].whitespaceStart(true);
// int we = sbbs[i].whitespaceEnd(true);
// sbb.append(sbbs[i].getBytes(0, ws));
// sbb.append('_');
// sbb.append('(');
// sbb.append(sbbs[i].getBytes(ws, we));
// sbb.append(')');
// sbb.append(sbbs[i].getBytes(we));
// }
// }
// //if (sbb.length() > 0) System.out.println("GETTEXT: " + sbb.toString());
// return sbb.getChars();
}
if (bluelist != null) { if (bluelist != null) {
if (bluelistHit(text)) { if (bluelistHit(text)) {
// System.out.println("FILTERHIT: " + text); // System.out.println("FILTERHIT: " + text);

@ -512,7 +512,6 @@ public final class htmlFilterWriter extends Writer {
htmlFilterTransformer transformer = new htmlFilterContentTransformer(); htmlFilterTransformer transformer = new htmlFilterContentTransformer();
// TODO: this does not work at the moment // TODO: this does not work at the moment
System.exit(0); System.exit(0);
transformer.init("gettext");
Reader is = new FileReader(args[0]); Reader is = new FileReader(args[0]);
FileOutputStream fos = new FileOutputStream(new File(args[0] + ".out")); FileOutputStream fos = new FileOutputStream(new File(args[0] + ".out"));
Writer os = new htmlFilterWriter(fos, "UTF-8",scraper, transformer, false); Writer os = new htmlFilterWriter(fos, "UTF-8",scraper, transformer, false);

@ -45,12 +45,15 @@ package de.anomic.http;
import java.io.FilterOutputStream; import java.io.FilterOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream; import java.io.OutputStream;
public final class httpChunkedOutputStream extends FilterOutputStream import de.anomic.server.serverByteBuffer;
{ import de.anomic.server.serverCore;
private boolean finished = false; import de.anomic.server.serverFileUtils;
private static final byte[] crlf = {(byte)13,(byte)10};
public final class httpChunkedOutputStream extends FilterOutputStream {
private boolean finished = false;
public httpChunkedOutputStream(OutputStream out) { public httpChunkedOutputStream(OutputStream out) {
super(out); super(out);
@ -63,7 +66,9 @@ public final class httpChunkedOutputStream extends FilterOutputStream
public void finish() throws IOException { public void finish() throws IOException {
if (!this.finished) { if (!this.finished) {
this.out.write("0\r\n\r\n".getBytes()); this.out.write((byte) 48);
this.out.write(serverCore.crlf);
this.out.write(serverCore.crlf);
this.out.flush(); this.out.flush();
this.finished = true; this.finished = true;
} }
@ -74,9 +79,9 @@ public final class httpChunkedOutputStream extends FilterOutputStream
if (b.length == 0) return; if (b.length == 0) return;
this.out.write(Integer.toHexString(b.length).getBytes()); this.out.write(Integer.toHexString(b.length).getBytes());
this.out.write(crlf); this.out.write(serverCore.crlf);
this.out.write(b); this.out.write(b);
this.out.write(crlf); this.out.write(serverCore.crlf);
this.out.flush(); this.out.flush();
} }
@ -85,9 +90,32 @@ public final class httpChunkedOutputStream extends FilterOutputStream
if (len == 0) return; if (len == 0) return;
this.out.write(Integer.toHexString(len).getBytes()); this.out.write(Integer.toHexString(len).getBytes());
this.out.write(crlf); this.out.write(serverCore.crlf);
this.out.write(b, off, len); this.out.write(b, off, len);
this.out.write(crlf); this.out.write(serverCore.crlf);
this.out.flush();
}
public void write(serverByteBuffer b, int off, int len) throws IOException {
if (this.finished) throw new IOException("ChunkedOutputStream already finalized.");
if (len == 0) return;
this.out.write(Integer.toHexString(len).getBytes());
this.out.write(serverCore.crlf);
this.out.write(b.getBytes(off, off + len));
this.out.write(serverCore.crlf);
this.out.flush();
}
public void write(InputStream b) throws IOException {
if (this.finished) throw new IOException("ChunkedOutputStream already finalized.");
int len = b.available();
if (len == 0) return;
this.out.write(Integer.toHexString(len).getBytes());
this.out.write(serverCore.crlf);
serverFileUtils.copy(b, out, len);
this.out.write(serverCore.crlf);
this.out.flush(); this.out.flush();
} }
@ -95,9 +123,9 @@ public final class httpChunkedOutputStream extends FilterOutputStream
if (this.finished) throw new IOException("ChunkedOutputStream already finalized."); if (this.finished) throw new IOException("ChunkedOutputStream already finalized.");
this.out.write("1".getBytes()); this.out.write("1".getBytes());
this.out.write(crlf); this.out.write(serverCore.crlf);
this.out.write(b); this.out.write(b);
this.out.write(crlf); this.out.write(serverCore.crlf);
this.out.flush(); this.out.flush();
} }
} }

@ -491,6 +491,10 @@ public final class httpHeader extends TreeMap implements Map {
return null; return null;
} }
public String userAgent() {
return (String) get(USER_AGENT);
}
public long age() { public long age() {
Date lm = lastModified(); Date lm = lastModified();
Date sd = date(); Date sd = date();

@ -0,0 +1,76 @@
// httpSSI.java
// -----------------------------
// (C) 2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 26.06.2007 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.http;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import de.anomic.server.serverByteBuffer;
public class httpSSI {
public static void writeSSI(File referenceFile, serverByteBuffer in, httpChunkedOutputStream out) throws IOException {
writeSSI(referenceFile, in, 0, out);
}
public static void writeSSI(File referenceFile, serverByteBuffer in, int start, httpChunkedOutputStream out) throws IOException {
int p = in.indexOf("<!--#".getBytes(), start);
if (p == 0) {
int q = in.indexOf("-->".getBytes(), start + 10);
assert q >= 0;
parseSSI(referenceFile, in, start, q + 3 - start, out);
writeSSI(referenceFile, in, start + q + 3, out);
} else if (p > 0) {
int q = in.indexOf("-->".getBytes(), start + 10);
out.write(in, start, p - start);
parseSSI(referenceFile, in, start + p, q + 3 - start - p, out);
writeSSI(referenceFile, in, start + q + 3, out);
} else /* p < 0 */ {
out.write(in, start, in.length() - start);
}
}
private static void parseSSI(File referenceFile, serverByteBuffer in, int start, int length, httpChunkedOutputStream out) {
if (in.startsWith("<!--#include virtual=\"".getBytes(), start)) {
int q = in.indexOf("\"".getBytes(), start + 22);
if (q > 0) {
String path = in.toString(start + 22, q);
File loadFile = new File(referenceFile.getParentFile(), path);
try {
out.write(new FileInputStream(loadFile));
} catch (FileNotFoundException e) {
// do nothing
} catch (IOException e) {
// do nothing
}
}
}
}
}

@ -229,8 +229,8 @@ public final class httpTemplate {
return false; return false;
} }
public static byte[] writeTemplate(InputStream in, OutputStream out, Hashtable pattern, byte[] dflt) throws IOException { public static void writeTemplate(InputStream in, OutputStream out, Hashtable pattern, byte[] dflt) throws IOException {
return writeTemplate(in, out, pattern, dflt, new byte[0]); writeTemplate(in, out, pattern, dflt, new byte[0]);
} }
/** /**

@ -340,7 +340,7 @@ public final class httpd implements serverHandler {
return true; return true;
} }
private boolean handleYaCyHopAuthentication(httpHeader header) throws IOException { private boolean handleYaCyHopAuthentication(httpHeader header) {
// check if the user has allowed that his/her peer is used for hops // check if the user has allowed that his/her peer is used for hops
if (!this.allowYaCyHop) return false; if (!this.allowYaCyHop) return false;
@ -1364,7 +1364,7 @@ public final class httpd implements serverHandler {
if (!reqMethod.equals(httpHeader.METHOD_HEAD)){ if (!reqMethod.equals(httpHeader.METHOD_HEAD)){
if (!conProp.getProperty(httpHeader.CONNECTION_PROP_PERSISTENT,"close").equals("close")) { if (!conProp.getProperty(httpHeader.CONNECTION_PROP_PERSISTENT,"close").equals("close")) {
if (transferEnc == null && contentLength < 0) { if (transferEnc == null && contentLength < 0) {
throw new IllegalArgumentException("Message MUST contain a Content-Length or a non-identity transfer-coding heder field."); throw new IllegalArgumentException("Message MUST contain a Content-Length or a non-identity transfer-coding header field.");
} }
} }
if (transferEnc != null && contentLength >= 0) { if (transferEnc != null && contentLength >= 0) {

@ -77,13 +77,11 @@ package de.anomic.http;
import java.awt.Image; import java.awt.Image;
import java.awt.image.BufferedImage; import java.awt.image.BufferedImage;
import java.io.BufferedInputStream; import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream; import java.io.OutputStream;
import java.io.PushbackInputStream; import java.io.PushbackInputStream;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
@ -91,8 +89,6 @@ import java.lang.ref.SoftReference;
import java.lang.reflect.InvocationTargetException; import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method; import java.lang.reflect.Method;
import java.net.URLDecoder; import java.net.URLDecoder;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Date; import java.util.Date;
import java.util.Enumeration; import java.util.Enumeration;
import java.util.HashMap; import java.util.HashMap;
@ -134,9 +130,6 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
private static String[] defaultFiles = null; private static String[] defaultFiles = null;
private static File htDefaultPath = null; private static File htDefaultPath = null;
private static File htLocalePath = null; private static File htLocalePath = null;
private MessageDigest md5Digest = null;
private static final HashMap templateCache; private static final HashMap templateCache;
private static final HashMap templateMethodCache; private static final HashMap templateMethodCache;
@ -205,12 +198,6 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
//htLocaleSelection = switchboard.getConfig("htLocaleSelection","default"); //htLocaleSelection = switchboard.getConfig("htLocaleSelection","default");
} }
// initialise an message digest for Content-MD5 support ...
try {
this.md5Digest = MessageDigest.getInstance("MD5");
} catch (NoSuchAlgorithmException e) {
serverLog.logWarning("HTTPDFileHandler", "Content-MD5 support not availabel ...");
}
} }
public static final void initDefaultPath() { public static final void initDefaultPath() {
@ -585,7 +572,6 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
// if this file uses templates, then we use the template // if this file uses templates, then we use the template
// re-write - method to create an result // re-write - method to create an result
String mimeType = mimeTable.getProperty(targetExt,"text/html"); String mimeType = mimeTable.getProperty(targetExt,"text/html");
byte[] result;
boolean zipContent = requestHeader.acceptGzip() && httpd.shallTransportZipped("." + conProp.getProperty("EXT","")); boolean zipContent = requestHeader.acceptGzip() && httpd.shallTransportZipped("." + conProp.getProperty("EXT",""));
if (path.endsWith("html") || if (path.endsWith("html") ||
path.endsWith("xml") || path.endsWith("xml") ||
@ -675,148 +661,96 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
tp.putAll(templates); tp.putAll(templates);
// rewrite the file // rewrite the file
serverByteBuffer o = null;
InputStream fis = null; InputStream fis = null;
GZIPOutputStream zippedOut = null;
try { // read the file/template
// do fileCaching here byte[] templateContent = null;
byte[] templateContent = null; if (useTemplateCache) {
if (useTemplateCache) { long fileSize = targetFile.length();
long fileSize = targetFile.length(); if (fileSize <= 512 * 1024) {
if (fileSize <= 512*1024) { // read from cache
SoftReference ref = (SoftReference) templateCache.get(targetFile); SoftReference ref = (SoftReference) templateCache.get(targetFile);
if (ref != null) { if (ref != null) {
templateContent = (byte[]) ref.get(); templateContent = (byte[]) ref.get();
if (templateContent == null) if (templateContent == null) templateCache.remove(targetFile);
templateCache.remove(targetFile); }
}
if (templateContent == null) {
if (templateContent == null) { // loading the content of the template file into
// loading the content of the template file into a byte array // a byte array
templateContent = serverFileUtils.read(targetFile); templateContent = serverFileUtils.read(targetFile);
// storing the content into the cache // storing the content into the cache
ref = new SoftReference(templateContent); ref = new SoftReference(templateContent);
templateCache.put(targetFile,ref); templateCache.put(targetFile, ref);
if (this.theLogger.isLoggable(Level.FINEST)) if (this.theLogger.isLoggable(Level.FINEST))
this.theLogger.logFinest("Cache MISS for file " + targetFile); this.theLogger.logFinest("Cache MISS for file " + targetFile);
} else {
if (this.theLogger.isLoggable(Level.FINEST))
this.theLogger.logFinest("Cache HIT for file " + targetFile);
}
// creating an inputstream needed by the template rewrite function
fis = new ByteArrayInputStream(templateContent);
templateContent = null;
} else { } else {
fis = new BufferedInputStream(new FileInputStream(targetFile)); if (this.theLogger.isLoggable(Level.FINEST))
this.theLogger.logFinest("Cache HIT for file " + targetFile);
} }
// creating an inputstream needed by the template
// rewrite function
fis = new ByteArrayInputStream(templateContent);
templateContent = null;
} else { } else {
// read from file directly
fis = new BufferedInputStream(new FileInputStream(targetFile)); fis = new BufferedInputStream(new FileInputStream(targetFile));
} }
} else {
//use the page template fis = new BufferedInputStream(new FileInputStream(targetFile));
String supertemplate=""; }
if(tp.containsKey("SUPERTEMPLATE")){
supertemplate=(String) tp.get("SUPERTEMPLATE"); // write the array to the client
} // we can do that either in standard mode (whole thing completely) or in chunked mode
//if(sb.getConfig("usePageTemplate", "false").equals("true")){ // since yacy clients do not understand chunked mode, we use this only for communication with the administrator
if(!supertemplate.equals("")){ boolean yacyClient = requestHeader.userAgent().startsWith("yacy");
File pageFile=getOverlayedFile(supertemplate); //typically /env/page.html boolean chunked = !method.equals(httpHeader.METHOD_HEAD) && !yacyClient;
File pageClass=getOverlayedClass(supertemplate); if (chunked) {
if(pageFile != null && pageFile.exists()){ // send page in chunks and parse SSIs
//warning: o,tp and fis are reused serverByteBuffer o = new serverByteBuffer();
//this replaces the normal template function (below), and the // apply templates
//normal function is used for supertemplates. httpTemplate.writeTemplate(fis, o, tp, "-UNRESOLVED_PATTERN-".getBytes("UTF-8"));
httpd.sendRespondHeader(this.connectionProperties, out,
//search for header Data httpVersion, 200, null, mimeType, -1,
//XXX: This part may be slow :-( targetDate, null, tp.getOutgoingHeader(),
BufferedReader br=new BufferedReader(new InputStreamReader(fis)); null, "chunked", nocache);
String line; // send the content in chunked parts, see RFC 2616 section 3.6.1
boolean inheader=false; //byte[] result = o.toByteArray(); // this interrupts streaming (bad idea!)
StringBuffer header=new StringBuffer(); httpChunkedOutputStream chos = new httpChunkedOutputStream(out);
StringBuffer content=new StringBuffer(); httpSSI.writeSSI(targetFile, o, chos);
String content_s,header_s; //chos.write(result);
while((line=br.readLine())!=null){ chos.finish();
if(!inheader){ } else {
if(line.startsWith("<!--HEADER")){ // send page as whole thing, SSIs are not possible
inheader=true; String contentEncoding = (zipContent) ? "gzip" : null;
}else{ // apply templates
content.append(line); serverByteBuffer o = new serverByteBuffer();
}
}else{
if(line.endsWith("HEADER-->")){
inheader=false;
}else{
header.append(line);
}
}
}
//tp is from the servlet, fis is the content-only from the servlet
//this resolvs templates in both header and content
o = new serverByteBuffer();
fis=new ByteArrayInputStream((new String(content)).getBytes());
httpTemplate.writeTemplate(fis, o, tp, "-UNRESOLVED_PATTERN-".getBytes());
content_s=o.toString();
o = new serverByteBuffer();
fis=new ByteArrayInputStream(header.toString().getBytes());
httpTemplate.writeTemplate(fis, o, tp, "-UNRESOLVED_PATTERN-".getBytes());
header_s=o.toString();
//further processing of page.html (via page.class)?
if (pageClass != null && pageClass.exists()){
Object tmp = invokeServlet(pageClass, requestHeader, args);
if(tmp instanceof serverObjects){
tp=new servletProperties((serverObjects)tmp);
}else{
tp=(servletProperties)tmp;
}
}else
tp = new servletProperties();
tp.putASIS("header", header_s);
tp.putASIS("page", content_s);
fis=new BufferedInputStream(new FileInputStream(pageFile));
}
}
o = new serverByteBuffer();
if (zipContent) zippedOut = new GZIPOutputStream(o);
httpTemplate.writeTemplate(fis, (zipContent) ? (OutputStream)zippedOut: (OutputStream)o, tp, "-UNRESOLVED_PATTERN-".getBytes("UTF-8"));
if (zipContent) { if (zipContent) {
GZIPOutputStream zippedOut = new GZIPOutputStream(o);
httpTemplate.writeTemplate(fis, zippedOut, tp, "-UNRESOLVED_PATTERN-".getBytes("UTF-8"));
zippedOut.finish(); zippedOut.finish();
zippedOut.flush(); zippedOut.flush();
zippedOut.close(); zippedOut.close();
zippedOut = null; zippedOut = null;
} else {
httpTemplate.writeTemplate(fis, o, tp, "-UNRESOLVED_PATTERN-".getBytes("UTF-8"));
} }
if (method.equals(httpHeader.METHOD_HEAD)) {
result = o.toByteArray(); httpd.sendRespondHeader(this.connectionProperties, out,
httpVersion, 200, null, mimeType, o.length(),
if (this.md5Digest != null) { targetDate, null, tp.getOutgoingHeader(),
this.md5Digest.reset(); contentEncoding, null, nocache);
this.md5Digest.update(result); } else {
byte[] digest = this.md5Digest.digest(); byte[] result = o.toByteArray(); // this interrupts streaming (bad idea!)
StringBuffer digestString = new StringBuffer(); httpd.sendRespondHeader(this.connectionProperties, out,
for ( int i = 0; i < digest.length; i++ ) httpVersion, 200, null, mimeType, result.length,
digestString.append(Integer.toHexString( digest[i]&0xff)); targetDate, null, tp.getOutgoingHeader(),
contentEncoding, null, nocache);
} serverFileUtils.write(result, out);
} finally { }
if (zippedOut != null) try {zippedOut.close();} catch(Exception e) {}
if (o != null) try {o.close(); o = null;} catch(Exception e) {}
if (fis != null) try {fis.close(); fis=null;} catch(Exception e) {}
} }
// write the array to the client
long contentLength = result.length;
String contentEncoding = (zipContent)?"gzip":null;
httpd.sendRespondHeader(this.connectionProperties, out, httpVersion, 200, null, mimeType, contentLength, targetDate, null, tp.getOutgoingHeader(), contentEncoding, null, nocache);
if (! method.equals(httpHeader.METHOD_HEAD)) {
//Thread.sleep(200); // this solved the message problem (!!)
serverFileUtils.write(result, out);
}
} else { // no html } else { // no html
int statusCode = 200; int statusCode = 200;

@ -147,7 +147,8 @@ public class icapHeader extends TreeMap implements Map {
// to make the occurrence of multiple keys possible, we add them using a counter // to make the occurrence of multiple keys possible, we add them using a counter
public Object add(Object key, Object value) { public Object add(Object key, Object value) {
int c = keyCount((String) key); int c = keyCount((String) key);
if (c == 0) return put(key, value); else return put("*" + key + "-" + c, value); if (c == 0) return put(key, value);
return put("*" + key + "-" + c, value);
} }
public int keyCount(String key) { public int keyCount(String key) {
@ -160,7 +161,8 @@ public class icapHeader extends TreeMap implements Map {
// a convenience method to access the map with fail-over defaults // a convenience method to access the map with fail-over defaults
public Object get(Object key, Object dflt) { public Object get(Object key, Object dflt) {
Object result = get(key); Object result = get(key);
if (result == null) return dflt; else return result; if (result == null) return dflt;
return result;
} }
// return multiple results // return multiple results

@ -68,12 +68,7 @@ public class indexCollectionRI implements indexRI {
} }
public int size() { public int size() {
try { return collectionIndex.size();
return collectionIndex.size();
} catch (IOException e) {
e.printStackTrace();
return 0;
}
} }
public synchronized int indexSize(String wordHash) { public synchronized int indexSize(String wordHash) {
@ -196,11 +191,7 @@ public class indexCollectionRI implements indexRI {
} }
public synchronized void close() { public synchronized void close() {
try { collectionIndex.close();
collectionIndex.close();
} catch (IOException e) {
e.printStackTrace();
}
} }
} }

@ -27,7 +27,6 @@
package de.anomic.index; package de.anomic.index;
import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.text.ParseException; import java.text.ParseException;
import java.util.ArrayList; import java.util.ArrayList;
@ -285,7 +284,7 @@ public class indexURLEntry {
} }
} }
public kelondroRow.Entry toRowEntry() throws IOException { public kelondroRow.Entry toRowEntry() {
return this.entry; return this.entry;
} }

@ -32,7 +32,7 @@ public class kelondroBytesIntMap {
private kelondroRow rowdef; private kelondroRow rowdef;
private kelondroIndex index0, index1; private kelondroIndex index0, index1;
public kelondroBytesIntMap(kelondroIndex ki) throws IOException { public kelondroBytesIntMap(kelondroIndex ki) {
assert (ki.row().columns() == 2); // must be a key/index relation assert (ki.row().columns() == 2); // must be a key/index relation
assert (ki.row().width(1) == 4); // the value must be a b256-encoded int, 4 bytes long assert (ki.row().width(1) == 4); // the value must be a b256-encoded int, 4 bytes long
this.index0 = null; // not used this.index0 = null; // not used
@ -46,7 +46,7 @@ public class kelondroBytesIntMap {
this.index1 = null; // to show that this is the initialization phase this.index1 = null; // to show that this is the initialization phase
} }
public kelondroRow row() throws IOException { public kelondroRow row() {
return index0.row(); return index0.row();
} }

@ -264,7 +264,7 @@ public class kelondroCollectionIndex {
} }
} }
public int size() throws IOException { public int size() {
return index.size(); return index.size();
} }
@ -1000,7 +1000,7 @@ public class kelondroCollectionIndex {
} }
public synchronized void close() throws IOException { public synchronized void close() {
this.index.close(); this.index.close();
Iterator i = arrays.values().iterator(); Iterator i = arrays.values().iterator();
while (i.hasNext()) { while (i.hasNext()) {

@ -123,7 +123,7 @@ public class kelondroDyn {
return this.rowdef; return this.rowdef;
} }
public synchronized int sizeDyn() throws IOException { public synchronized int sizeDyn() {
//this.segmentCount = 0; //this.segmentCount = 0;
//Iterator i = keys(true); while (i.hasNext()) segmentCount++; //Iterator i = keys(true); while (i.hasNext()) segmentCount++;
//return segmentCount; //return segmentCount;

@ -68,7 +68,7 @@ public class kelondroDynTree {
private Hashtable buffer, cache; private Hashtable buffer, cache;
private long cycleBuffer; private long cycleBuffer;
public kelondroDynTree(File file, long preloadTime, int keylength, int nodesize, kelondroRow rowdef, char fillChar, boolean resetOnFail) throws IOException { public kelondroDynTree(File file, long preloadTime, int keylength, int nodesize, kelondroRow rowdef, char fillChar, boolean resetOnFail) {
// creates or opens a DynTree // creates or opens a DynTree
this.file = file; this.file = file;
this.preloadTime = preloadTime; this.preloadTime = preloadTime;

@ -46,14 +46,14 @@ public class kelondroFlexSplitTable implements kelondroIndex {
private File path; private File path;
private String tablename; private String tablename;
public kelondroFlexSplitTable(File path, String tablename, long preloadTime, kelondroRow rowdef, boolean resetOnFail) throws IOException { public kelondroFlexSplitTable(File path, String tablename, long preloadTime, kelondroRow rowdef, boolean resetOnFail) {
this.path = path; this.path = path;
this.tablename = tablename; this.tablename = tablename;
this.rowdef = rowdef; this.rowdef = rowdef;
init(preloadTime, resetOnFail); init(preloadTime, resetOnFail);
} }
public void init(long preloadTime, boolean resetOnFail) throws IOException { public void init(long preloadTime, boolean resetOnFail) {
// initialized tables map // initialized tables map
this.tables = new HashMap(); this.tables = new HashMap();

@ -423,12 +423,9 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
private final Map memoryStats() { private final Map memoryStats() {
// returns statistical data about this object // returns statistical data about this object
HashMap map = new HashMap(); HashMap map = new HashMap();
try { map.put("tableIndexChunkSize", (!RAMIndex) ? "0" : Integer.toString(index.row().objectsize));
map.put("tableIndexChunkSize", (!RAMIndex) ? "0" : Integer.toString(index.row().objectsize)); map.put("tableIndexCount", (!RAMIndex) ? "0" : Integer.toString(index.size()));
map.put("tableIndexCount", (!RAMIndex) ? "0" : Integer.toString(index.size())); map.put("tableIndexMem", (!RAMIndex) ? "0" : Integer.toString((int) (index.row().objectsize * index.size() * kelondroRowCollection.growfactor)));
map.put("tableIndexMem", (!RAMIndex) ? "0" : Integer.toString((int) (index.row().objectsize * index.size() * kelondroRowCollection.growfactor)));
} catch (IOException e) {
}
return map; return map;
} }

@ -24,7 +24,6 @@
package de.anomic.kelondro; package de.anomic.kelondro;
import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashSet; import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
@ -55,7 +54,7 @@ public class kelondroIntBytesMap {
return index1.memoryNeededForGrow(); return index1.memoryNeededForGrow();
} }
public kelondroRow row() throws IOException { public kelondroRow row() {
return index0.row(); return index0.row();
} }

@ -63,20 +63,20 @@ public class kelondroMapTable {
public void declareMaps( public void declareMaps(
String tablename, int keysize, kelondroOrder objectOrder, int nodesize, int cacheslots, String tablename, int keysize, kelondroOrder objectOrder, int nodesize, int cacheslots,
char fillChar, boolean resetOnFail) throws IOException { char fillChar, boolean resetOnFail) {
declareMaps(tablename, keysize, objectOrder, nodesize, cacheslots, null, null, null, fillChar, resetOnFail); declareMaps(tablename, keysize, objectOrder, nodesize, cacheslots, null, null, null, fillChar, resetOnFail);
} }
public void declareMaps( public void declareMaps(
String tablename, int keysize, kelondroOrder objectOrder, int nodesize, int cacheslots, String tablename, int keysize, kelondroOrder objectOrder, int nodesize, int cacheslots,
String[] sortfields, String[] longaccfields, String[] doubleaccfields, char fillChar, boolean resetOnFail) throws IOException { String[] sortfields, String[] longaccfields, String[] doubleaccfields, char fillChar, boolean resetOnFail) {
declareMaps(tablename, keysize, objectOrder, nodesize, cacheslots, sortfields, longaccfields, doubleaccfields, fillChar, 0, resetOnFail); declareMaps(tablename, keysize, objectOrder, nodesize, cacheslots, sortfields, longaccfields, doubleaccfields, fillChar, 0, resetOnFail);
} }
public void declareMaps( public void declareMaps(
String tablename, int keysize, kelondroOrder objectOrder, int nodesize, int cacheslots, String tablename, int keysize, kelondroOrder objectOrder, int nodesize, int cacheslots,
String[] sortfields, String[] longaccfields, String[] doubleaccfields, char fillChar, String[] sortfields, String[] longaccfields, String[] doubleaccfields, char fillChar,
long preloadTime, boolean resetOnFail) throws IOException { long preloadTime, boolean resetOnFail) {
if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared twice."); if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared twice.");
if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared already in other context."); if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared already in other context.");
File tablefile = new File(tablesPath, "table." + tablename + ".mdb"); File tablefile = new File(tablesPath, "table." + tablename + ".mdb");
@ -87,7 +87,7 @@ public class kelondroMapTable {
mTables.put(tablename, map); mTables.put(tablename, map);
} }
public void declareTree(String tablename, kelondroRow rowdef, long buffersize /*bytes*/, long preloadTime) throws IOException { public void declareTree(String tablename, kelondroRow rowdef, long buffersize /*bytes*/, long preloadTime) {
if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareTree: table '" + tablename + "' declared already in other context."); if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareTree: table '" + tablename + "' declared already in other context.");
if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareTree: table '" + tablename + "' declared twice."); if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareTree: table '" + tablename + "' declared twice.");
File tablefile = new File(tablesPath, "table." + tablename + ".tdb"); File tablefile = new File(tablesPath, "table." + tablename + ".tdb");
@ -110,7 +110,7 @@ public class kelondroMapTable {
tTables.put(tablename, tree); tTables.put(tablename, tree);
} }
public synchronized Map selectMap(String tablename, String key) throws IOException { public synchronized Map selectMap(String tablename, String key) {
kelondroMapObjects table = (kelondroMapObjects) mTables.get(tablename); kelondroMapObjects table = (kelondroMapObjects) mTables.get(tablename);
if (table == null) throw new RuntimeException("kelondroTables.selectMap: map table '" + tablename + "' does not exist."); if (table == null) throw new RuntimeException("kelondroTables.selectMap: map table '" + tablename + "' does not exist.");
if (key.length() > table.keySize()) key = key.substring(0, table.keySize()); if (key.length() > table.keySize()) key = key.substring(0, table.keySize());

@ -149,11 +149,7 @@ public class kelondroObjects {
} }
public synchronized int size() { public synchronized int size() {
try { return dyn.sizeDyn();
return dyn.sizeDyn();
} catch (IOException e) {
return 0;
}
} }
public void close() { public void close() {

@ -301,7 +301,7 @@ public class kelondroRowCollection {
addUnique(r, 0, r.length); addUnique(r, 0, r.length);
} }
public synchronized void addUniqueMultiple(List rows) throws IOException { public synchronized void addUniqueMultiple(List rows) {
assert this.sortBound == 0 : "sortBound = " + this.sortBound + ", chunkcount = " + this.chunkcount; assert this.sortBound == 0 : "sortBound = " + this.sortBound + ", chunkcount = " + this.chunkcount;
Iterator i = rows.iterator(); Iterator i = rows.iterator();
while (i.hasNext()) addUnique((kelondroRow.Entry) i.next()); while (i.hasNext()) addUnique((kelondroRow.Entry) i.next());

@ -138,7 +138,7 @@ public class kelondroSplittedTree implements kelondroIndex {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
public synchronized void addUnique(kelondroRow.Entry row, Date entryDate) throws IOException { public synchronized void addUnique(kelondroRow.Entry row, Date entryDate) {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }

@ -76,7 +76,7 @@ public class sevenzipParser extends AbstractParser implements Parser {
*/ */
private static final String[] LIBX_DEPENDENCIES = new String[] { "J7Zip-modified.jar" }; private static final String[] LIBX_DEPENDENCIES = new String[] { "J7Zip-modified.jar" };
public sevenzipParser() throws IOException { public sevenzipParser() {
super(LIBX_DEPENDENCIES); super(LIBX_DEPENDENCIES);
super.parserName = "7zip Archive Parser"; super.parserName = "7zip Archive Parser";
} }

@ -95,12 +95,7 @@ public final class plasmaCrawlLURL {
public plasmaCrawlLURL(File indexPath, long preloadTime) { public plasmaCrawlLURL(File indexPath, long preloadTime) {
super(); super();
try { urlIndexFile = new kelondroFlexSplitTable(new File(indexPath, "PUBLIC/TEXT"), "urls", preloadTime, indexURLEntry.rowdef, false);
urlIndexFile = new kelondroFlexSplitTable(new File(indexPath, "PUBLIC/TEXT"), "urls", preloadTime, indexURLEntry.rowdef, false);
} catch (IOException e) {
e.printStackTrace();
System.exit(-1);
}
// init result stacks // init result stacks
externResultStack = new LinkedList(); externResultStack = new LinkedList();

@ -143,13 +143,9 @@ public final class plasmaCrawlStacker {
this.log.logSevere("Unable to shutdown all remaining stackCrawl threads", e1); this.log.logSevere("Unable to shutdown all remaining stackCrawl threads", e1);
} }
try { this.log.logFine("Shutdown. Closing stackCrawl queue.");
this.log.logFine("Shutdown. Closing stackCrawl queue."); if (this.queue != null) this.queue.close();
if (this.queue != null) this.queue.close(); this.queue = null;
this.queue = null;
} catch (IOException e) {
this.log.logSevere("DB could not be closed properly.", e);
}
} }
public int getNumActiveWorker() { public int getNumActiveWorker() {
@ -540,7 +536,7 @@ public final class plasmaCrawlStacker {
} }
} }
public void close() throws IOException { public void close() {
// closing the db // closing the db
this.urlEntryCache.close(); this.urlEntryCache.close();

@ -1076,7 +1076,7 @@ public final class plasmaHTCache {
return this.resInfo; return this.resInfo;
} }
public boolean writeResourceInfo() throws IOException { public boolean writeResourceInfo() {
assert(this.nomalizedURLHash != null) : "URL Hash is null"; assert(this.nomalizedURLHash != null) : "URL Hash is null";
if (this.resInfo == null) return false; if (this.resInfo == null) return false;
try { try {

@ -1609,14 +1609,14 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
return cacheManager; return cacheManager;
} }
synchronized public void htEntryStoreEnqueued(plasmaHTCache.Entry entry) throws IOException { synchronized public void htEntryStoreEnqueued(plasmaHTCache.Entry entry) {
if (cacheManager.full()) if (cacheManager.full())
htEntryStoreProcess(entry); htEntryStoreProcess(entry);
else else
cacheManager.push(entry); cacheManager.push(entry);
} }
synchronized public boolean htEntryStoreProcess(plasmaHTCache.Entry entry) throws IOException { synchronized public boolean htEntryStoreProcess(plasmaHTCache.Entry entry) {
if (entry == null) return false; if (entry == null) return false;
@ -1721,11 +1721,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
public boolean htEntryStoreJob() { public boolean htEntryStoreJob() {
if (cacheManager.empty()) return false; if (cacheManager.empty()) return false;
try { return htEntryStoreProcess(cacheManager.pop());
return htEntryStoreProcess(cacheManager.pop());
} catch (IOException e) {
return false;
}
} }
public int htEntrySize() { public int htEntrySize() {

@ -155,7 +155,7 @@ public class plasmaSwitchboardQueue {
} }
} }
public void clear() throws IOException { public void clear() {
sbQueueStack = kelondroStack.reset(sbQueueStack); sbQueueStack = kelondroStack.reset(sbQueueStack);
} }

@ -59,7 +59,7 @@ public class plasmaWordConnotation {
private static final int nodesize = 4048; private static final int nodesize = 4048;
private kelondroDynTree refDB; private kelondroDynTree refDB;
public plasmaWordConnotation(File refDBfile, long preloadTime, char fillChar) throws IOException { public plasmaWordConnotation(File refDBfile, long preloadTime, char fillChar) {
refDB = new kelondroDynTree(refDBfile, preloadTime, wordlength, nodesize, new kelondroRow("byte[] word-" + wordlength + ", Cardinal count-" + countlength, kelondroNaturalOrder.naturalOrder, 0), fillChar, true); refDB = new kelondroDynTree(refDBfile, preloadTime, wordlength, nodesize, new kelondroRow("byte[] word-" + wordlength + ", Cardinal count-" + countlength, kelondroNaturalOrder.naturalOrder, 0), fillChar, true);
} }

@ -277,9 +277,13 @@ public final class serverByteBuffer extends OutputStream {
} }
public boolean startsWith(byte[] bs) { public boolean startsWith(byte[] bs) {
if (length < bs.length) return false; return startsWith(bs, 0);
}
public boolean startsWith(byte[] bs, int start) {
if (length - start < bs.length) return false;
for (int i = 0; i < bs.length; i++) { for (int i = 0; i < bs.length; i++) {
if (buffer[offset + i] != bs[i]) return false; if (buffer[offset + i + start] != bs[i]) return false;
} }
return true; return true;
} }

@ -126,6 +126,7 @@ public final class yacyClient {
obj.put("mytime", yacyCore.universalDateShortString(new Date())); obj.put("mytime", yacyCore.universalDateShortString(new Date()));
obj.put("myUTC", System.currentTimeMillis()); obj.put("myUTC", System.currentTimeMillis());
obj.put("seed", yacyCore.seedDB.mySeed.genSeedStr(key)); obj.put("seed", yacyCore.seedDB.mySeed.genSeedStr(key));
obj.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
// sending request // sending request
result = nxTools.table( result = nxTools.table(
@ -272,7 +273,8 @@ public final class yacyClient {
"&youare=" + target.hash + "&youare=" + target.hash +
"&key=" + key + "&key=" + key +
"&object=seed" + "&object=seed" +
"&env=" + seedHash "&env=" + seedHash +
"&network.unit.name=" + plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT)
), ),
target.getHexHash() + ".yacyh", target.getHexHash() + ".yacyh",
8000, 8000,
@ -309,7 +311,8 @@ public final class yacyClient {
"&key=" + "&key=" +
"&object=rwicount" + "&object=rwicount" +
"&env=" + wordHash + "&env=" + wordHash +
"&ttl=0" "&ttl=0" +
"&network.unit.name=" + plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT)
), ),
target.getHexHash() + ".yacyh", target.getHexHash() + ".yacyh",
10000, 10000,
@ -345,7 +348,8 @@ public final class yacyClient {
"&key=" + "&key=" +
"&object=lurlcount" + "&object=lurlcount" +
"&env=" + "&env=" +
"&ttl=0"; "&ttl=0" +
"&network.unit.name=" + plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT);
// seinding request // seinding request
try { try {
@ -452,6 +456,7 @@ public final class yacyClient {
obj.put("profile", crypt.simpleEncode(rankingProfile.toExternalString())); obj.put("profile", crypt.simpleEncode(rankingProfile.toExternalString()));
obj.put("constraint", constraint.exportB64()); obj.put("constraint", constraint.exportB64());
obj.put("mytime", yacyCore.universalDateShortString(new Date())); obj.put("mytime", yacyCore.universalDateShortString(new Date()));
obj.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
if (abstractCache != null) obj.put("abstracts", "auto"); if (abstractCache != null) obj.put("abstracts", "auto");
//yacyCore.log.logDebug("yacyClient.search url=" + url); //yacyCore.log.logDebug("yacyClient.search url=" + url);
@ -622,6 +627,7 @@ public final class yacyClient {
post.put("iam", yacyCore.seedDB.mySeed.hash); post.put("iam", yacyCore.seedDB.mySeed.hash);
post.put("youare", targetHash); post.put("youare", targetHash);
post.put("mytime", yacyCore.universalDateShortString(new Date())); post.put("mytime", yacyCore.universalDateShortString(new Date()));
post.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
// get target address // get target address
String address = targetAddress(targetHash); String address = targetAddress(targetHash);
@ -664,6 +670,7 @@ public final class yacyClient {
post.put("youare", targetHash); post.put("youare", targetHash);
post.put("subject", subject); post.put("subject", subject);
post.put("mytime", yacyCore.universalDateShortString(new Date())); post.put("mytime", yacyCore.universalDateShortString(new Date()));
post.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
try { try {
post.put("message", new String(message, "UTF-8")); post.put("message", new String(message, "UTF-8"));
} catch (UnsupportedEncodingException e) { } catch (UnsupportedEncodingException e) {
@ -722,6 +729,7 @@ public final class yacyClient {
post.put("filename", filename); post.put("filename", filename);
post.put("filesize", Long.toString(filesize)); post.put("filesize", Long.toString(filesize));
post.put("can-send-protocol", "http"); post.put("can-send-protocol", "http");
post.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
// send request // send request
try { try {
@ -763,6 +771,7 @@ public final class yacyClient {
post.put("filesize", Long.toString(file.length)); post.put("filesize", Long.toString(file.length));
post.put("md5", serverCodings.encodeMD5Hex(file)); post.put("md5", serverCodings.encodeMD5Hex(file));
post.put("access", access); post.put("access", access);
post.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
HashMap files = new HashMap(); HashMap files = new HashMap();
files.put("filename", file); files.put("filename", file);
@ -845,7 +854,8 @@ public final class yacyClient {
} }
post.put("depth", "0"); post.put("depth", "0");
post.put("ttl", "0"); post.put("ttl", "0");
post.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
// determining target address // determining target address
final String address = targetSeed.getClusterAddress(); final String address = targetSeed.getClusterAddress();
if (address == null) { return null; } if (address == null) { return null; }
@ -928,7 +938,8 @@ public final class yacyClient {
"&result=" + result + "&result=" + result +
"&reason=" + reason + "&reason=" + reason +
"&wordh=" + wordhashes + "&wordh=" + wordhashes +
"&lurlEntry=" + ((entry == null) ? "" : crypt.simpleEncode(entry.toString(), key)) "&lurlEntry=" + ((entry == null) ? "" : crypt.simpleEncode(entry.toString(), key)) +
"&network.unit.name=" + plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT)
), ),
targetSeed.getHexHash() + ".yacyh", targetSeed.getHexHash() + ".yacyh",
60000, 60000,
@ -1055,7 +1066,7 @@ public final class yacyClient {
post.put("iam", yacyCore.seedDB.mySeed.hash); post.put("iam", yacyCore.seedDB.mySeed.hash);
post.put("youare", targetSeed.hash); post.put("youare", targetSeed.hash);
post.put("wordc", Integer.toString(indexes.length)); post.put("wordc", Integer.toString(indexes.length));
post.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
int indexcount = 0; int indexcount = 0;
final StringBuffer entrypost = new StringBuffer(indexes.length*73); final StringBuffer entrypost = new StringBuffer(indexes.length*73);
@ -1131,6 +1142,8 @@ public final class yacyClient {
post.put("key", key); post.put("key", key);
post.put("iam", yacyCore.seedDB.mySeed.hash); post.put("iam", yacyCore.seedDB.mySeed.hash);
post.put("youare", targetSeed.hash); post.put("youare", targetSeed.hash);
post.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
String resource = ""; String resource = "";
int urlc = 0; int urlc = 0;
int urlPayloadSize = 0; int urlPayloadSize = 0;
@ -1183,6 +1196,8 @@ public final class yacyClient {
final serverObjects post = new serverObjects(2); final serverObjects post = new serverObjects(2);
post.put("iam", yacyCore.seedDB.mySeed.hash); post.put("iam", yacyCore.seedDB.mySeed.hash);
post.put("youare", targetSeed.hash); post.put("youare", targetSeed.hash);
post.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
String address = targetSeed.getClusterAddress(); String address = targetSeed.getClusterAddress();
if (address == null) { address = "localhost:8080"; } if (address == null) { address = "localhost:8080"; }
try { try {
@ -1226,7 +1241,8 @@ public final class yacyClient {
"&myseed=" + yacyCore.seedDB.mySeed.genSeedStr(null) + "&myseed=" + yacyCore.seedDB.mySeed.genSeedStr(null) +
"&count=10" + "&count=10" +
"&resource=global" + "&resource=global" +
"&query=" + wordhashe), "&query=" + wordhashe +
"&network.unit.name=" + plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT)),
target.getHexHash() + ".yacyh", target.getHexHash() + ".yacyh",
5000, 5000,
null, null,

@ -204,12 +204,7 @@ public class yacyCore {
// create or init news database // create or init news database
long memNews_time = Long.parseLong(switchboard.getConfig("ramCacheNews_time", "1000")); long memNews_time = Long.parseLong(switchboard.getConfig("ramCacheNews_time", "1000"));
try { newsPool = new yacyNewsPool(yacyDBPath, memNews_time);
newsPool = new yacyNewsPool(yacyDBPath, memNews_time);
} catch (IOException e) {
e.printStackTrace();
System.exit(-1);
}
loadSeedUploadMethods(); loadSeedUploadMethods();

@ -64,13 +64,13 @@ public class yacyNewsDB {
private long preloadTime; private long preloadTime;
protected kelondroIndex news; protected kelondroIndex news;
public yacyNewsDB(File path, long preloadTime) throws IOException { public yacyNewsDB(File path, long preloadTime) {
this.path = path; this.path = path;
this.preloadTime = preloadTime; this.preloadTime = preloadTime;
this.news = new kelondroCache(kelondroTree.open(path, true, preloadTime, yacyNewsRecord.rowdef), true, false); this.news = new kelondroCache(kelondroTree.open(path, true, preloadTime, yacyNewsRecord.rowdef), true, false);
} }
private void resetDB() throws IOException { private void resetDB() {
try {close();} catch (Exception e) {} try {close();} catch (Exception e) {}
if (path.exists()) path.delete(); if (path.exists()) path.delete();
this.news = new kelondroCache(kelondroTree.open(path, true, preloadTime, yacyNewsRecord.rowdef), true, false); this.news = new kelondroCache(kelondroTree.open(path, true, preloadTime, yacyNewsRecord.rowdef), true, false);
@ -85,7 +85,7 @@ public class yacyNewsDB {
close(); close();
} }
public int size() throws IOException { public int size() {
return news.size(); return news.size();
} }
@ -148,7 +148,7 @@ public class yacyNewsDB {
); );
} }
protected final kelondroRow.Entry r2b(yacyNewsRecord r) throws IOException { protected final kelondroRow.Entry r2b(yacyNewsRecord r) {
try { try {
if (r == null) return null; if (r == null) return null;
String attributes = r.attributes().toString(); String attributes = r.attributes().toString();

@ -265,7 +265,7 @@ public class yacyNewsPool {
private int maxDistribution; private int maxDistribution;
public yacyNewsPool(File yacyDBPath, long preloadTime) throws IOException { public yacyNewsPool(File yacyDBPath, long preloadTime) {
newsDB = new yacyNewsDB(new File(yacyDBPath, "news1.db"), preloadTime); newsDB = new yacyNewsDB(new File(yacyDBPath, "news1.db"), preloadTime);
outgoingNews = new yacyNewsQueue(new File(yacyDBPath, "newsOut1.stack"), newsDB); outgoingNews = new yacyNewsQueue(new File(yacyDBPath, "newsOut1.stack"), newsDB);
publishedNews = new yacyNewsQueue(new File(yacyDBPath, "newsPublished1.stack"), newsDB); publishedNews = new yacyNewsQueue(new File(yacyDBPath, "newsPublished1.stack"), newsDB);
@ -283,12 +283,7 @@ public class yacyNewsPool {
} }
public int dbSize() { public int dbSize() {
try { return newsDB.size();
return newsDB.size();
} catch (IOException e) {
e.printStackTrace();
return 0;
}
} }
public void publishMyNews(yacyNewsRecord record) { public void publishMyNews(yacyNewsRecord record) {

@ -168,6 +168,9 @@ public class yacySeed {
/** zero-value */ /** zero-value */
public static final String ZERO = "0"; public static final String ZERO = "0";
public static final String DFLT_NETWORK_UNIT = "freeworld";
public static final String DFLT_NETWORK_GROUP = "";
// class variables // class variables
/** the peer-hash */ /** the peer-hash */
public String hash; public String hash;

@ -750,12 +750,6 @@ CRDist1Target = kaskelix.de:8080,yacy.dyndns.org:8000,suma-lab.de:8080
# Hash of the peer, you would like to store to the data your installation collected. # Hash of the peer, you would like to store to the data your installation collected.
storagePeerHash = storagePeerHash =
#use /env/page.html (and if it exists page.class) as super template,
# which includes the generated page in #[page]#.
#This option is for developers only, because it may be integrated by default, or not,
#but its not useful to mix configurations with and without this supertemplate.
usePageTemplate = false
# Search sequence settings # Search sequence settings
# collection: # collection:
# time = time to get a RWI out of RAM cache, assortments and WORDS files # time = time to get a RWI out of RAM cache, assortments and WORDS files

Loading…
Cancel
Save