Merge branch 'master' of ssh://git@gitorious.org/yacy/rc1.git

Conflicts:
	htroot/PerformanceMemory_p.java
pull/1/head
Michael Peter Christen 13 years ago
commit 701b9a28a0

@ -114,8 +114,9 @@
</dl> </dl>
</fieldset> </fieldset>
:: ::
You installed YaCy with a package manager. To update YaCy, use the package manager:<p> <p>You installed YaCy with a package manager. To update YaCy, use the package manager:</p>
Debian: apt-get update &amp;&amp; apt-get install yacy</p> <p>manual update:<br/>apt-get update &amp;&amp; apt-get install yacy</p>
<p>automatic update: add the following line to /etc/crontab<br/>0 6 * * * root apt-get update &amp;&amp; apt-get -y --force-yes install yacy</p>
:: ::
#(/candeploy)# #(/candeploy)#

@ -210,7 +210,6 @@ public class sharedBlacklist_p {
prop.put("page", "1"); //result page prop.put("page", "1"); //result page
prop.put("status", STATUS_ENTRIES_ADDED); //list of added Entries prop.put("status", STATUS_ENTRIES_ADDED); //list of added Entries
int count = 0;//couter of added entries
PrintWriter pw = null; PrintWriter pw = null;
try { try {
// open the blacklist file // open the blacklist file
@ -238,7 +237,6 @@ public class sharedBlacklist_p {
// append the item to the file // append the item to the file
pw.println(newItem); pw.println(newItem);
count++;
if (Switchboard.urlBlacklist != null) { if (Switchboard.urlBlacklist != null) {
final String supportedBlacklistTypesStr = Blacklist.BLACKLIST_TYPES_STRING; final String supportedBlacklistTypesStr = Blacklist.BLACKLIST_TYPES_STRING;
final String[] supportedBlacklistTypes = supportedBlacklistTypesStr.split(","); final String[] supportedBlacklistTypes = supportedBlacklistTypesStr.split(",");

@ -27,6 +27,7 @@
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants; import net.yacy.search.SwitchboardConstants;
import net.yacy.search.ranking.BlockRank;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
@ -37,6 +38,8 @@ public class yacyinteractive {
public static serverObjects respond(final RequestHeader header, serverObjects post, final serverSwitch env) { public static serverObjects respond(final RequestHeader header, serverObjects post, final serverSwitch env) {
final Switchboard sb = (Switchboard) env; final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
BlockRank.ensureLoaded();
prop.put("topmenu", sb.getConfigBool("publicTopmenu", true) ? 1 : 0); prop.put("topmenu", sb.getConfigBool("publicTopmenu", true) ? 1 : 0);
final String promoteSearchPageGreeting = final String promoteSearchPageGreeting =
(env.getConfigBool(SwitchboardConstants.GREETING_NETWORK_NAME, false)) ? (env.getConfigBool(SwitchboardConstants.GREETING_NETWORK_NAME, false)) ?

@ -78,6 +78,7 @@ import net.yacy.search.query.AccessTracker;
import net.yacy.search.query.QueryParams; import net.yacy.search.query.QueryParams;
import net.yacy.search.query.SearchEvent; import net.yacy.search.query.SearchEvent;
import net.yacy.search.query.SearchEventCache; import net.yacy.search.query.SearchEventCache;
import net.yacy.search.ranking.BlockRank;
import net.yacy.search.ranking.RankingProfile; import net.yacy.search.ranking.RankingProfile;
import de.anomic.data.DidYouMean; import de.anomic.data.DidYouMean;
import de.anomic.data.UserDB; import de.anomic.data.UserDB;
@ -96,6 +97,7 @@ public class yacysearch {
final Switchboard sb = (Switchboard) env; final Switchboard sb = (Switchboard) env;
sb.localSearchLastAccess = System.currentTimeMillis(); sb.localSearchLastAccess = System.currentTimeMillis();
BlockRank.ensureLoaded();
final boolean searchAllowed = final boolean searchAllowed =
sb.getConfigBool("publicSearchpage", true) || sb.verifyAuthentication(header); sb.getConfigBool("publicSearchpage", true) || sb.verifyAuthentication(header);

@ -31,6 +31,7 @@ import net.yacy.document.LibraryProvider;
import net.yacy.document.geolocalization.GeoLocation; import net.yacy.document.geolocalization.GeoLocation;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants; import net.yacy.search.SwitchboardConstants;
import net.yacy.search.ranking.BlockRank;
import de.anomic.server.serverCore; import de.anomic.server.serverCore;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
@ -40,6 +41,7 @@ public class yacysearch_location {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
final Switchboard sb = (Switchboard) env; final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
BlockRank.ensureLoaded();
prop.put("kml", 0); prop.put("kml", 0);

@ -143,6 +143,10 @@ public final class Cache {
} }
} }
public static void commit() {
fileDB.flushAll();
}
/** /**
* clear the cache * clear the cache
*/ */

@ -34,7 +34,6 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.servletProperties;
import net.yacy.cora.document.ASCII; import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.document.MultiProtocolURI;

@ -67,7 +67,6 @@ public final class CrawlStacker {
private final Log log = new Log("STACKCRAWL"); private final Log log = new Log("STACKCRAWL");
private final WorkflowProcessor<Request> fastQueue, slowQueue; private final WorkflowProcessor<Request> fastQueue, slowQueue;
private long dnsMiss;
private final CrawlQueues nextQueue; private final CrawlQueues nextQueue;
private final CrawlSwitchboard crawler; private final CrawlSwitchboard crawler;
private final Segment indexSegment; private final Segment indexSegment;
@ -89,8 +88,6 @@ public final class CrawlStacker {
this.crawler = cs; this.crawler = cs;
this.indexSegment = indexSegment; this.indexSegment = indexSegment;
this.peers = peers; this.peers = peers;
//this.dnsHit = 0;
this.dnsMiss = 0;
this.acceptLocalURLs = acceptLocalURLs; this.acceptLocalURLs = acceptLocalURLs;
this.acceptGlobalURLs = acceptGlobalURLs; this.acceptGlobalURLs = acceptGlobalURLs;
this.domainList = domainList; this.domainList = domainList;
@ -179,7 +176,6 @@ public final class CrawlStacker {
} else { } else {
try { try {
this.slowQueue.enQueue(entry); this.slowQueue.enQueue(entry);
this.dnsMiss++;
} catch (final InterruptedException e) { } catch (final InterruptedException e) {
Log.logException(e); Log.logException(e);
} }

@ -25,6 +25,7 @@
package net.yacy.cora.sorting; package net.yacy.cora.sorting;
import java.io.Serializable;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Comparator; import java.util.Comparator;
import java.util.Iterator; import java.util.Iterator;
@ -39,10 +40,11 @@ import java.util.concurrent.TimeUnit;
* If the queue gets larger that the given maxsize, then elements from the tail of the queue * If the queue gets larger that the given maxsize, then elements from the tail of the queue
* are drained (deleted). * are drained (deleted).
*/ */
public class WeakPriorityBlockingQueue<E> { public class WeakPriorityBlockingQueue<E> implements Serializable {
private static final long serialVersionUID = 4573442576760691887L;
private final TreeSet<Element<E>> queue; // object within the stack, ordered using a TreeSet private final TreeSet<Element<E>> queue; // object within the stack, ordered using a TreeSet
private final Semaphore enqueued; // semaphore for elements in the stack private final Semaphore enqueued; // semaphore for elements in the stack
private final ArrayList<Element<E>> drained; // objects that had been on the stack but had been removed private final ArrayList<Element<E>> drained; // objects that had been on the stack but had been removed
protected int maxsize; protected int maxsize;
@ -281,7 +283,7 @@ public class WeakPriorityBlockingQueue<E> {
return this.drained.iterator(); return this.drained.iterator();
} }
public interface Element<E> { public interface Element<E> extends Serializable {
public long getWeight(); public long getWeight();
public E getElement(); public E getElement();
public boolean equals(Element<E> o); public boolean equals(Element<E> o);
@ -291,9 +293,11 @@ public class WeakPriorityBlockingQueue<E> {
public String toString(); public String toString();
} }
protected abstract static class AbstractElement<E> implements Element<E> { protected abstract static class AbstractElement<E> implements Element<E>, Serializable {
public long weight; private static final long serialVersionUID = -7026597258248026566L;
public long weight;
public E element; public E element;
@Override @Override
@ -328,7 +332,9 @@ public class WeakPriorityBlockingQueue<E> {
*/ */
public static class NaturalElement<E> extends AbstractElement<E> implements Element<E>, Comparable<NaturalElement<E>>, Comparator<NaturalElement<E>> { public static class NaturalElement<E> extends AbstractElement<E> implements Element<E>, Comparable<NaturalElement<E>>, Comparator<NaturalElement<E>> {
public NaturalElement(final E element, final long weight) { private static final long serialVersionUID = 6816543012966928794L;
public NaturalElement(final E element, final long weight) {
this.element = element; this.element = element;
this.weight = weight; this.weight = weight;
} }
@ -359,7 +365,9 @@ public class WeakPriorityBlockingQueue<E> {
*/ */
public static class ReverseElement<E> extends AbstractElement<E> implements Element<E>, Comparable<ReverseElement<E>>, Comparator<ReverseElement<E>> { public static class ReverseElement<E> extends AbstractElement<E> implements Element<E>, Comparable<ReverseElement<E>>, Comparator<ReverseElement<E>> {
public ReverseElement(final E element, final long weight) { private static final long serialVersionUID = -8166724491837508921L;
public ReverseElement(final E element, final long weight) {
this.element = element; this.element = element;
this.weight = weight; this.weight = weight;
} }

@ -59,7 +59,6 @@ public class ImportDump {
FileUtils.copy(dump, baos); FileUtils.copy(dump, baos);
String s = UTF8.String(baos.toByteArray()); String s = UTF8.String(baos.toByteArray());
int batchSize = 0;
int p, q; int p, q;
String t; String t;
loop: while (s.length() > 0) { loop: while (s.length() > 0) {
@ -78,11 +77,9 @@ public class ImportDump {
s = s.substring(p); s = s.substring(p);
//if (batchSize + t.length() >= maxBatch) { //if (batchSize + t.length() >= maxBatch) {
statement.executeBatch(); statement.executeBatch();
batchSize = 0;
//} //}
System.out.println(t); System.out.println(t);
statement.addBatch(t); statement.addBatch(t);
batchSize += t.length();
} }
statement.executeBatch(); statement.executeBatch();
} catch (SQLException e) { } catch (SQLException e) {

@ -27,6 +27,7 @@ import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.Serializable;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
@ -51,7 +52,7 @@ import org.xml.sax.helpers.DefaultHandler;
import de.anomic.crawler.retrieval.Response; import de.anomic.crawler.retrieval.Response;
public class OAIListFriendsLoader { public class OAIListFriendsLoader implements Serializable {
private static final long serialVersionUID = -8705115274655024604L; private static final long serialVersionUID = -8705115274655024604L;
@ -117,7 +118,7 @@ public class OAIListFriendsLoader {
} }
// get a resumption token using a SAX xml parser from am input stream // get a resumption token using a SAX xml parser from am input stream
private static class Parser extends DefaultHandler { public static class Parser extends DefaultHandler {
// class variables // class variables
private final StringBuilder buffer; private final StringBuilder buffer;
@ -162,6 +163,10 @@ public class OAIListFriendsLoader {
</BaseURLs> </BaseURLs>
*/ */
public int getCounter() {
return this.recordCounter;
}
public void startElement(final String uri, final String name, final String tag, final Attributes atts) throws SAXException { public void startElement(final String uri, final String name, final String tag, final Attributes atts) throws SAXException {
if ("baseURL".equals(tag)) { if ("baseURL".equals(tag)) {
this.recordCounter++; this.recordCounter++;

@ -335,7 +335,7 @@ public class Compressor implements BLOB, Iterable<byte[]> {
} }
} }
private void flushAll() { public void flushAll() {
while (!this.buffer.isEmpty()) { while (!this.buffer.isEmpty()) {
if (!flushOne()) break; if (!flushOne()) break;
} }

@ -25,6 +25,7 @@
package net.yacy.kelondro.blob; package net.yacy.kelondro.blob;
import java.io.Serializable;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
@ -38,7 +39,7 @@ import net.yacy.kelondro.order.NaturalOrder;
/** /**
* a mapping from a column name to maps with the value of the columns to the primary keys where the entry exist in the table * a mapping from a column name to maps with the value of the columns to the primary keys where the entry exist in the table
*/ */
public class MapColumnIndex { public class MapColumnIndex implements Serializable {
private static final long serialVersionUID=-424741536889467566L; private static final long serialVersionUID=-424741536889467566L;

@ -44,12 +44,10 @@ import net.yacy.cora.order.ByteOrder;
public class ByteArray { public class ByteArray {
private final byte[] buffer; private final byte[] buffer;
private final int hash;
public ByteArray(final byte[] bb) { public ByteArray(final byte[] bb) {
this.buffer = bb; this.buffer = bb;
this.hash = 0;
} }
public int length() { public int length() {

@ -1892,7 +1892,7 @@ public final class Switchboard extends serverSwitch
} }
public int cleanupJobSize() { public int cleanupJobSize() {
int c = 0; int c = 1; // "es gibt immer was zu tun"
if ( (this.crawlQueues.delegatedURL.stackSize() > 1000) ) { if ( (this.crawlQueues.delegatedURL.stackSize() > 1000) ) {
c++; c++;
} }
@ -1909,6 +1909,9 @@ public final class Switchboard extends serverSwitch
public boolean cleanupJob() { public boolean cleanupJob() {
try { try {
// flush the document compressor cache
Cache.commit();
// clear caches if necessary // clear caches if necessary
if ( !MemoryControl.request(8000000L, false) ) { if ( !MemoryControl.request(8000000L, false) ) {
for ( final Segment indexSegment : this.indexSegments ) { for ( final Segment indexSegment : this.indexSegments ) {

@ -272,7 +272,6 @@ public final class RWIProcess extends Thread
|| pattern.equals("smb://.*") || pattern.equals("smb://.*")
|| pattern.equals("file://.*"); || pattern.equals("file://.*");
long remaining; long remaining;
int count = 0;
pollloop: while ( true ) { pollloop: while ( true ) {
remaining = timeout - System.currentTimeMillis(); remaining = timeout - System.currentTimeMillis();
if (remaining <= 0) { if (remaining <= 0) {
@ -288,8 +287,6 @@ public final class RWIProcess extends Thread
break pollloop; break pollloop;
} }
assert (iEntry.urlhash().length == index.row().primaryKeyLength); assert (iEntry.urlhash().length == index.row().primaryKeyLength);
//if (iEntry.urlHash().length() != index.row().primaryKeyLength) continue;
count++;
// increase flag counts // increase flag counts
for ( int j = 0; j < 32; j++ ) { for ( int j = 0; j < 32; j++ ) {

@ -461,7 +461,6 @@ public class SnippetProcess {
final boolean nav_topics = SnippetProcess.this.query.navigators.equals("all") || SnippetProcess.this.query.navigators.indexOf("topics",0) >= 0; final boolean nav_topics = SnippetProcess.this.query.navigators.equals("all") || SnippetProcess.this.query.navigators.indexOf("topics",0) >= 0;
try { try {
//System.out.println("DEPLOYED WORKER " + id + " FOR " + this.neededResults + " RESULTS, timeoutd = " + (this.timeout - System.currentTimeMillis())); //System.out.println("DEPLOYED WORKER " + id + " FOR " + this.neededResults + " RESULTS, timeoutd = " + (this.timeout - System.currentTimeMillis()));
int loops = 0;
while (this.shallrun && System.currentTimeMillis() < this.timeout) { while (this.shallrun && System.currentTimeMillis() < this.timeout) {
//Log.logInfo("SnippetProcess", "***** timeleft = " + (this.timeout - System.currentTimeMillis())); //Log.logInfo("SnippetProcess", "***** timeleft = " + (this.timeout - System.currentTimeMillis()));
this.lastLifeSign = System.currentTimeMillis(); this.lastLifeSign = System.currentTimeMillis();
@ -509,7 +508,6 @@ public class SnippetProcess {
} }
} }
loops++;
resultEntry = fetchSnippet(page, solrContent, this.cacheStrategy); // does not fetch snippets if snippetMode == 0 resultEntry = fetchSnippet(page, solrContent, this.cacheStrategy); // does not fetch snippets if snippetMode == 0
if (resultEntry == null) if (resultEntry == null)
{ {

@ -56,7 +56,8 @@ import net.yacy.search.index.Segment;
public class BlockRank { public class BlockRank {
public static BinSearch[] ybrTables = null; // block-rank tables public static BinSearch[] ybrTables = null; // block-rank tables
private static File rankingPath;
private static int count;
/** /**
* collect host index information from other peers. All peers in the seed database are asked * collect host index information from other peers. All peers in the seed database are asked
@ -214,6 +215,7 @@ public class BlockRank {
byte[] hosth = new byte[6]; byte[] hosth = new byte[6];
String hosths, hostn; String hosths, hostn;
HostStat hs; HostStat hs;
ensureLoaded();
for (int ybr = 0; ybr < ybrTables.length; ybr++) { for (int ybr = 0; ybr < ybrTables.length; ybr++) {
row: for (int i = 0; i < ybrTables[ybr].size(); i++) { row: for (int i = 0; i < ybrTables[ybr].size(); i++) {
hosth = ybrTables[ybr].get(i, hosth); hosth = ybrTables[ybr].get(i, hosth);
@ -240,11 +242,18 @@ public class BlockRank {
* @param rankingPath * @param rankingPath
* @param count * @param count
*/ */
public static void loadBlockRankTable(final File rankingPath, final int count) { public static void loadBlockRankTable(final File rankingPath0, final int count0) {
if (!rankingPath.exists()) return; // lazy initialization to save memory during startup phase
rankingPath = rankingPath0;
count = count0;
}
public static void ensureLoaded() {
if (ybrTables != null) return;
ybrTables = new BinSearch[count]; ybrTables = new BinSearch[count];
String ybrName; String ybrName;
File f; File f;
Log.logInfo("BlockRank", "loading block rank table from " + rankingPath.toString());
try { try {
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
ybrName = "YBR-4-" + Digest.encodeHex(i, 2) + ".idx"; ybrName = "YBR-4-" + Digest.encodeHex(i, 2) + ".idx";
@ -287,6 +296,7 @@ public class BlockRank {
* @return * @return
*/ */
public static int ranking(final byte[] hash) { public static int ranking(final byte[] hash) {
ensureLoaded();
return ranking(hash, ybrTables); return ranking(hash, ybrTables);
} }

Loading…
Cancel
Save