boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added patches to prevent yacy-protocol DoS settings

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6751 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent e820ed061a
commit 748abfcffa
7 changed files with 98 additions and 37 deletions
Show Stats Download Patch File Download Diff File

26
htroot/PerformanceQueues_p.java
Unescape View File

@ -202,8 +202,11 @@ public class PerformanceQueues_p {
if (idlesleep < 1000) idlesleep = 1000;
if (threadName.equals("10_httpd")) { idlesleep = 0; busysleep = 0; memprereq = 0; }
onTheFlyReconfiguration(sb, threadName, idlesleep, busysleep, memprereq);
} if (setProfile) {
sb.setThreadPerformance(threadName, idlesleep, busysleep, memprereq);
idlesleep = sb.getConfigLong(threadName + "_idlesleep", idlesleep);
busysleep = sb.getConfigLong(threadName + "_busysleep", busysleep);
}
if (setProfile) {
if (threadName.equals(SwitchboardConstants.PEER_PING)
|| threadName.equals(SwitchboardConstants.SEED_UPLOAD)
|| threadName.equals(SwitchboardConstants.CLEANUP)
@ -223,8 +226,7 @@ public class PerformanceQueues_p {
if (idlesleep < 1000) idlesleep = 1000;
if (threadName.equals("10_httpd")) { idlesleep = 0; busysleep = 0; memprereq = 0; }
//if (threadName.equals(plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL) && (busysleep < 50)) busysleep = 50;
onTheFlyReconfiguration(sb, threadName, idlesleep, busysleep, memprereq);
sb.setThreadPerformance(threadName, idlesleep, busysleep, memprereq);
}
}
prop.put("table_" + c + "_idlesleep", idlesleep);
@ -368,22 +370,6 @@ public class PerformanceQueues_p {
// return rewrite values for templates
return prop;
}
/**
* @param switchboard
* @param threadName
* @param idlesleep
* @param busysleep
* @param memprereq
*/
private static void onTheFlyReconfiguration(final Switchboard switchboard, final String threadName, final long idlesleep,
final long busysleep, final long memprereq) {
// on-the-fly re-configuration
switchboard.setThreadPerformance(threadName, idlesleep, busysleep, memprereq);
switchboard.setConfig(threadName + "_idlesleep", idlesleep);
switchboard.setConfig(threadName + "_busysleep", busysleep);
switchboard.setConfig(threadName + "_memprereq", memprereq);
}
private static String d(final String a, final String b) {
return (a == null) ? b : a;

2
source/de/anomic/data/bookmarksDB.java
Unescape View File

@ -123,7 +123,7 @@ public class bookmarksDB {
// autoReCrawl
Switchboard sb = Switchboard.getSwitchboard();
this.autoReCrawl = new InstantBusyThread(this, "autoReCrawl", null, null);
this.autoReCrawl = new InstantBusyThread(this, "autoReCrawl", null, null, Long.MIN_VALUE, Long.MAX_VALUE, Long.MIN_VALUE, Long.MAX_VALUE);
long sleepTime = Long.parseLong(sb.getConfig("autoReCrawl_idlesleep" , SLEEP_TIME));
sb.deployThread("autoReCrawl", "autoReCrawl Scheduler", "simple scheduler for automatic re-crawls of bookmarked urls", null, autoReCrawl, 120000,
sleepTime, sleepTime, Long.parseLong(sb.getConfig("autoReCrawl_memprereq" , "-1"))

64
source/de/anomic/search/Switchboard.java
Unescape View File

@ -585,21 +585,69 @@ public final class Switchboard extends serverSwitch {
MemoryControl.gc(10000, "plasmaSwitchboard, help for profiler"); // help for profiler - thq
deployThread(SwitchboardConstants.CLEANUP, "Cleanup", "simple cleaning process for monitoring information", null,
new InstantBusyThread(this, SwitchboardConstants.CLEANUP_METHOD_START, SwitchboardConstants.CLEANUP_METHOD_JOBCOUNT, SwitchboardConstants.CLEANUP_METHOD_FREEMEM), 600000); // all 5 Minutes, wait 10 minutes until first run
new InstantBusyThread(
this,
SwitchboardConstants.CLEANUP_METHOD_START,
SwitchboardConstants.CLEANUP_METHOD_JOBCOUNT,
SwitchboardConstants.CLEANUP_METHOD_FREEMEM,
60000, Long.MAX_VALUE, 10000, Long.MAX_VALUE),
600000); // all 5 Minutes, wait 10 minutes until first run
deployThread(SwitchboardConstants.SURROGATES, "Surrogates", "A thread that polls the SURROGATES path and puts all Documents in one surroagte file into the indexing queue.", null,
new InstantBusyThread(this, SwitchboardConstants.SURROGATES_METHOD_START, SwitchboardConstants.SURROGATES_METHOD_JOBCOUNT, SwitchboardConstants.SURROGATES_METHOD_FREEMEM), 10000);
new InstantBusyThread(
this,
SwitchboardConstants.SURROGATES_METHOD_START,
SwitchboardConstants.SURROGATES_METHOD_JOBCOUNT,
SwitchboardConstants.SURROGATES_METHOD_FREEMEM,
20000, Long.MAX_VALUE, 0, Long.MAX_VALUE),
10000);
deployThread(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL, "Remote Crawl Job", "thread that performes a single crawl/indexing step triggered by a remote peer", null,
new InstantBusyThread(crawlQueues, SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_START, SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_JOBCOUNT, SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_FREEMEM), 30000);
new InstantBusyThread(
crawlQueues,
SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_START,
SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_JOBCOUNT,
SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_FREEMEM,
0, Long.MAX_VALUE, 0, Long.MAX_VALUE),
30000);
deployThread(SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER, "Remote Crawl URL Loader", "thread that loads remote crawl lists from other peers", "",
new InstantBusyThread(crawlQueues, SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_START, SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_JOBCOUNT, SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_FREEMEM), 30000); // error here?
new InstantBusyThread(
crawlQueues,
SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_START,
SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_JOBCOUNT,
SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_FREEMEM,
10000, Long.MAX_VALUE, 10000, Long.MAX_VALUE),
30000); // error here?
deployThread(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL, "Local Crawl", "thread that performes a single crawl step from the local crawl queue", "/IndexCreateWWWLocalQueue_p.html",
new InstantBusyThread(crawlQueues, SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_START, SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_JOBCOUNT, SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_FREEMEM), 10000);
new InstantBusyThread(
crawlQueues,
SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_START,
SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_JOBCOUNT,
SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_FREEMEM,
0, Long.MAX_VALUE, 0, Long.MAX_VALUE),
10000);
deployThread(SwitchboardConstants.SEED_UPLOAD, "Seed-List Upload", "task that a principal peer performes to generate and upload a seed-list to a ftp account", null,
new InstantBusyThread(yc, SwitchboardConstants.SEED_UPLOAD_METHOD_START, SwitchboardConstants.SEED_UPLOAD_METHOD_JOBCOUNT, SwitchboardConstants.SEED_UPLOAD_METHOD_FREEMEM), 180000);
new InstantBusyThread(
yc,
SwitchboardConstants.SEED_UPLOAD_METHOD_START,
SwitchboardConstants.SEED_UPLOAD_METHOD_JOBCOUNT,
SwitchboardConstants.SEED_UPLOAD_METHOD_FREEMEM,
600000, Long.MAX_VALUE, 300000, Long.MAX_VALUE),
180000);
deployThread(SwitchboardConstants.PEER_PING, "YaCy Core", "this is the p2p-control and peer-ping task", null,
new InstantBusyThread(yc, SwitchboardConstants.PEER_PING_METHOD_START, SwitchboardConstants.PEER_PING_METHOD_JOBCOUNT, SwitchboardConstants.PEER_PING_METHOD_FREEMEM), 2000);
new InstantBusyThread(
yc,
SwitchboardConstants.PEER_PING_METHOD_START,
SwitchboardConstants.PEER_PING_METHOD_JOBCOUNT,
SwitchboardConstants.PEER_PING_METHOD_FREEMEM,
60000, Long.MAX_VALUE, 60000, Long.MAX_VALUE),
2000);
deployThread(SwitchboardConstants.INDEX_DIST, "DHT Distribution", "selection, transfer and deletion of index entries that are not searched on your peer, but on others", null,
new InstantBusyThread(this, SwitchboardConstants.INDEX_DIST_METHOD_START, SwitchboardConstants.INDEX_DIST_METHOD_JOBCOUNT, SwitchboardConstants.INDEX_DIST_METHOD_FREEMEM), 5000,
new InstantBusyThread(
this,
SwitchboardConstants.INDEX_DIST_METHOD_START,
SwitchboardConstants.INDEX_DIST_METHOD_JOBCOUNT,
SwitchboardConstants.INDEX_DIST_METHOD_FREEMEM,
10000, Long.MAX_VALUE, 1000, Long.MAX_VALUE),
5000,
Long.parseLong(getConfig(SwitchboardConstants.INDEX_DIST_IDLESLEEP , "5000")),
Long.parseLong(getConfig(SwitchboardConstants.INDEX_DIST_BUSYSLEEP , "0")),
Long.parseLong(getConfig(SwitchboardConstants.INDEX_DIST_MEMPREREQ , "1000000")));

1
source/de/anomic/server/serverCore.java
Unescape View File

@ -181,6 +181,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
final serverSwitch switchboard,
final int commandMaxLength
) {
super(Long.MIN_VALUE, Long.MAX_VALUE, Long.MIN_VALUE, Long.MAX_VALUE);
this.timeout = timeout;
this.commandMaxLength = commandMaxLength;

5
source/de/anomic/server/serverSwitch.java
Unescape View File

@ -385,8 +385,9 @@ public class serverSwitch {
public void setThreadPerformance(final String threadName, final long idleMillis, final long busyMillis, final long memprereqBytes) {
final BusyThread thread = workerThreads.get(threadName);
if (thread != null) {
thread.setIdleSleep(idleMillis);
thread.setBusySleep(busyMillis);
setConfig(threadName + "_idlesleep", thread.setIdleSleep(idleMillis));
setConfig(threadName + "_busysleep", thread.setBusySleep(busyMillis));
setConfig(threadName + "_memprereq", memprereqBytes);
thread.setMemPreReqisite(memprereqBytes);
}
}

22
source/net/yacy/kelondro/workflow/AbstractBusyThread.java
Unescape View File

@ -40,6 +40,20 @@ public abstract class AbstractBusyThread extends AbstractThread implements BusyT
private boolean intermissionObedient = true;
private final Object syncObject = new Object();
private long maxIdleSleep = Long.MAX_VALUE, minIdleSleep = Long.MIN_VALUE;
private long maxBusySleep = Long.MAX_VALUE, minBusySleep = Long.MIN_VALUE;
public AbstractBusyThread(
long minIdleSleep,
long maxIdleSleep,
long minBusySleep,
long maxBusySleep) {
this.minIdleSleep = minIdleSleep;
this.maxIdleSleep = maxIdleSleep;
this.minBusySleep = minBusySleep;
this.maxBusySleep = maxBusySleep;
}
protected final void announceMoreSleepTime(final long millis) {
this.idletime += millis;
}
@ -51,14 +65,14 @@ public abstract class AbstractBusyThread extends AbstractThread implements BusyT
public final long setIdleSleep(final long milliseconds) {
// sets a sleep time for pauses between two jobs
idlePause = milliseconds;
return milliseconds;
idlePause = Math.min(this.maxIdleSleep, Math.max(this.minIdleSleep, milliseconds));
return idlePause;
}
public final long setBusySleep(final long milliseconds) {
// sets a sleep time for pauses between two jobs
busyPause = milliseconds;
return milliseconds;
busyPause = Math.min(this.maxBusySleep, Math.max(this.minBusySleep, milliseconds));
return busyPause;
}
public void setMemPreReqisite(final long freeBytes) {

15
source/net/yacy/kelondro/workflow/InstantBusyThread.java
Unescape View File

@ -38,7 +38,17 @@ public final class InstantBusyThread extends AbstractBusyThread implements BusyT
public static int instantThreadCounter = 0;
public static final TreeMap<Long, String> jobs = new TreeMap<Long, String>();
public InstantBusyThread(final Object env, final String jobExec, final String jobCount, final String freemem) {
public InstantBusyThread(
final Object env,
final String jobExec,
final String jobCount,
final String freemem,
long minIdleSleep,
long maxIdleSleep,
long minBusySleep,
long maxBusySleep) {
super(minIdleSleep, maxIdleSleep, minBusySleep, maxBusySleep);
// jobExec is the name of a method of the object 'env' that executes the one-step-run
// jobCount is the name of a method that returns the size of the job
// freemem is the name of a method that tries to free memory and returns void
@ -146,7 +156,8 @@ public final class InstantBusyThread extends AbstractBusyThread implements BusyT
public static BusyThread oneTimeJob(final Object env, final String jobExec, final Log log, final long startupDelay) {
// start the job and execute it once as background process
final BusyThread thread = new InstantBusyThread(env, jobExec, null, null);
final BusyThread thread = new InstantBusyThread(
env, jobExec, null, null, Long.MIN_VALUE, Long.MAX_VALUE, Long.MIN_VALUE, Long.MAX_VALUE);
thread.setStartupSleep(startupDelay);
thread.setIdleSleep(-1);
thread.setBusySleep(-1);

Write Preview
Loading…
Cancel
Save