boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- cleanup, removed unused imports

Browse Source 
- added crawling queue sizes to /api/status_p.xml, syntax same as in queues_p.html
- fixed a bug in queue enumeration that caused a out of bounds exception

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6842 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent 3aad50d38e
commit 90c3e5d6f6
25 changed files with 55 additions and 100 deletions
Show Stats Download Patch File Download Diff File

18
htroot/Crawler_p.html
Unescape View File

@ -22,18 +22,6 @@
<th>&nbsp;</th>
<th>Max</th>
</tr>
<tr class="TableCellLight">
<td align="left">Indexing</td>
<td align="right"><span id="indexingqueuesize">#[indexingSize]#</span></td>
<td>&nbsp;</td>
<td align="right"><span id="indexingqueuemax">#[indexingMax]#</span></td>
</tr>
<tr class="TableCellLight">
<td align="left">Loader</td>
<td align="right"><span id="loaderqueuesize">#[loaderSize]#</span></td>
<td>&nbsp;</td>
<td align="right"><span id="loaderqueuemax">#[loaderMax]#</span></td>
</tr>
<tr class="TableCellLight">
<td align="left">Local Crawler</td>
<td align="right"><span id="localcrawlerqueuesize">#[localCrawlSize]#</span></td>
@ -64,6 +52,12 @@
</td>
<td align="right">unlimited</td>
</tr>
<tr class="TableCellLight">
<td align="left">Loader</td>
<td align="right"><span id="loaderqueuesize">#[loaderSize]#</span></td>
<td>&nbsp;</td>
<td align="right"><span id="loaderqueuemax">#[loaderMax]#</span></td>
</tr>
</tbody>
</table>

2
htroot/Crawler_p.java
Unescape View File

@ -74,8 +74,6 @@ public class Crawler_p {
// inital values for AJAX Elements (without JavaScript)
final serverObjects prop = new serverObjects();
prop.put("rejected", 0);
prop.put("indexingSize", 0);
prop.put("indexingMax", 0);
prop.put("urlpublictextSize", 0);
prop.put("rwipublictextSize", 0);
prop.put("list", "0");

17
htroot/RSSLoader_p.java
Unescape View File

@ -27,7 +27,6 @@
import java.io.ByteArrayInputStream;
import java.net.MalformedURLException;
import java.util.Map;
import net.yacy.document.Document;
import net.yacy.document.ParserException;
@ -36,8 +35,6 @@ import net.yacy.kelondro.data.meta.DigestURI;
import de.anomic.crawler.retrieval.Response;
import de.anomic.http.server.RequestHeader;
import de.anomic.search.Segment;
import de.anomic.search.Segments;
import de.anomic.search.Switchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
@ -53,18 +50,6 @@ public class RSSLoader_p {
return prop;
}
// get segment
Segment indexSegment = null;
if (post.containsKey("segment")) {
String segmentName = post.get("segment");
if (sb.indexSegments.segmentExist(segmentName)) {
indexSegment = sb.indexSegments.segment(segmentName);
}
} else {
// take default segment
indexSegment = sb.indexSegments.segment(Segments.Process.PUBLIC);
}
DigestURI url = null;
final String urlString = post.get("url", "");
@ -103,7 +88,7 @@ public class RSSLoader_p {
}
// get the links out of the rss
Map<DigestURI, String> map = doc.getAnchors();
//Map<DigestURI, String> map = doc.getAnchors();
// put the urls into crawler using the proxy profile

19
htroot/api/queues_p.java
Unescape View File

@ -31,11 +31,10 @@ public class queues_p {
//wikiCode wikiTransformer = new wikiCode(switchboard);
final serverObjects prop = new serverObjects();
Segment segment = null;
if (post == null || !post.containsKey("html")) {
prop.setLocalized(false);
if (post.containsKey("segment") && sb.verifyAuthentication(header, false)) {
segment = sb.indexSegments.segment(post.get("segment"));
}
boolean html = post != null && post.containsKey("html");
prop.setLocalized(html);
if (post != null && post.containsKey("segment") && sb.verifyAuthentication(header, false)) {
segment = sb.indexSegments.segment(post.get("segment"));
}
if (segment == null) segment = sb.indexSegments.segment(Segments.Process.PUBLIC);
prop.put("rejected", "0");
@ -48,8 +47,8 @@ public class queues_p {
prop.putNum("rwipublictextSize", segment.termIndex().sizesMax());
// loader queue
prop.put("loaderSize", Integer.toString(sb.crawlQueues.workerSize()));
prop.put("loaderMax", sb.getConfigLong(SwitchboardConstants.CRAWLER_THREADS_ACTIVE_MAX, 10));
prop.putNum("loaderSize", sb.crawlQueues.workerSize());
prop.putNum("loaderMax", sb.getConfigLong(SwitchboardConstants.CRAWLER_THREADS_ACTIVE_MAX, 10));
if (sb.crawlQueues.workerSize() == 0) {
prop.put("list-loader", "0");
} else {
@ -68,18 +67,18 @@ public class queues_p {
}
//local crawl queue
prop.putNum("localCrawlSize", Integer.toString(sb.getThread(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL).getJobCount()));
prop.putNum("localCrawlSize", sb.getThread(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL).getJobCount());
prop.put("localCrawlState", sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL) ? STATE_PAUSED : STATE_RUNNING);
int stackSize = sb.crawlQueues.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE);
addNTable(sb, prop, "list-local", sb.crawlQueues.noticeURL.top(NoticedURL.STACK_TYPE_CORE, Math.min(10, stackSize)));
//global crawl queue
prop.putNum("limitCrawlSize", Integer.toString(sb.crawlQueues.limitCrawlJobSize()));
prop.putNum("limitCrawlSize", sb.crawlQueues.limitCrawlJobSize());
prop.put("limitCrawlState", STATE_RUNNING);
stackSize = sb.crawlQueues.noticeURL.stackSize(NoticedURL.STACK_TYPE_LIMIT);
//global crawl queue
prop.putNum("remoteCrawlSize", Integer.toString(sb.getThread(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL).getJobCount()));
prop.putNum("remoteCrawlSize", sb.getThread(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL).getJobCount());
prop.put("remoteCrawlState", sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL) ? STATE_PAUSED : STATE_RUNNING);
stackSize = sb.crawlQueues.noticeURL.stackSize(NoticedURL.STACK_TYPE_LIMIT);

22
htroot/api/status_p.java
Unescape View File

@ -20,11 +20,10 @@ public class status_p {
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
Segment segment = null;
if (post == null || !post.containsKey("html")) {
prop.setLocalized(false);
if (post.containsKey("segment") && sb.verifyAuthentication(header, false)) {
segment = sb.indexSegments.segment(post.get("segment"));
}
boolean html = post != null && post.containsKey("html");
prop.setLocalized(html);
if (post != null && post.containsKey("segment") && sb.verifyAuthentication(header, false)) {
segment = sb.indexSegments.segment(post.get("segment"));
}
if (segment == null) segment = sb.indexSegments.segment(Segments.Process.PUBLIC);
@ -33,9 +32,16 @@ public class status_p {
final int cacheMaxSize = (int) sb.getConfigLong(SwitchboardConstants.WORDCACHE_MAX_COUNT, 10000);
prop.putNum("ppm", sb.currentPPM());
prop.putNum("qpm", sb.peers.mySeed().getQPM());
prop.put("wordCacheSize", Integer.toString(segment.termIndex().getBufferSize()));
prop.put("wordCacheMaxSize", Integer.toString(cacheMaxSize));
//
prop.putNum("wordCacheSize", segment.termIndex().getBufferSize());
prop.putNum("wordCacheMaxSize", cacheMaxSize);
// crawl queues
prop.putNum("localCrawlSize", sb.getThread(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL).getJobCount());
prop.putNum("limitCrawlSize", sb.crawlQueues.limitCrawlJobSize());
prop.putNum("remoteCrawlSize", sb.getThread(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL).getJobCount());
prop.putNum("loaderSize", sb.crawlQueues.workerSize());
prop.putNum("loaderMax", sb.getConfigLong(SwitchboardConstants.CRAWLER_THREADS_ACTIVE_MAX, 10));
// memory usage and system attributes
prop.putNum("freeMemory", MemoryControl.free());
prop.putNum("totalMemory", MemoryControl.total());

14
htroot/api/status_p.xml
Unescape View File

@ -3,6 +3,20 @@
<ppm>#[ppm]#</ppm>
<wordCacheSize>#[wordCacheSize]#</wordCacheSize>
<wordCacheMaxSize>#[wordCacheMaxSize]#</wordCacheMaxSize>
<loaderqueue>
<size>#[loaderSize]#</size>
<max>#[loaderMax]#</max>
</loaderqueue>
<localcrawlerqueue>
<size>#[localCrawlSize]#</size>
</localcrawlerqueue>
<limitcrawlerqueue>
<size>#[limitCrawlSize]#</size>
</limitcrawlerqueue>
<remotecrawlerqueue>
<size>#[remoteCrawlSize]#</size>
</remotecrawlerqueue>
<memory>
<free>#[freeMemory]#</free>

1
htroot/yacy/search.java
Unescape View File

@ -29,7 +29,6 @@
// if the shell's current path is htroot/yacy
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

4
source/de/anomic/crawler/AbstractImporter.java
Unescape View File

@ -1,12 +1,9 @@
package de.anomic.crawler;
import net.yacy.kelondro.logging.Log;
public abstract class AbstractImporter extends Thread implements Importer {
private int jobID = -1;
private String jobType;
private Log log;
private boolean stopped = false;
private boolean paused = false;
private long globalStart = System.currentTimeMillis();
@ -19,7 +16,6 @@ public abstract class AbstractImporter extends Thread implements Importer {
this.jobType = theJobType;
// initializing the logger and setting a more verbose thread name
this.log = new Log("IMPORT_" + this.jobType + "_" + this.jobID);
this.setName("IMPORT_" + this.jobType + "_" + this.jobID);
}

1
source/de/anomic/crawler/Balancer.java
Unescape View File

@ -34,7 +34,6 @@ import java.util.concurrent.ConcurrentLinkedQueue;
import net.yacy.kelondro.index.BufferedObjectIndex;
import net.yacy.kelondro.index.HandleSet;
import net.yacy.kelondro.index.ObjectIndex;
import net.yacy.kelondro.index.Row;
import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log;

5
source/de/anomic/crawler/CrawlQueues.java
Unescape View File

@ -181,7 +181,10 @@ public class CrawlQueues {
synchronized (workers) {
final Request[] e = new Request[workers.size()];
int i = 0;
for (final crawlWorker w: workers.values()) e[i++] = w.request;
for (final crawlWorker w: workers.values()) {
if (i >= e.length) break;
e[i++] = w.request;
}
return e;
}
}

1
source/de/anomic/data/translator.java
Unescape View File

@ -41,7 +41,6 @@ import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.util.HashMap;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

1
source/de/anomic/data/userDB.java
Unescape View File

@ -49,7 +49,6 @@ import de.anomic.http.server.RequestHeader;
public final class userDB {
private static final int USERNAME_MAX_LENGTH = 128;
private static final int USERNAME_MIN_LENGTH = 4;
private MapHeap userTable;

3
source/de/anomic/search/ResultFetcher.java
Unescape View File

@ -26,12 +26,9 @@
package de.anomic.search;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import net.yacy.document.Condenser;
import net.yacy.kelondro.data.meta.DigestURI;

3
source/de/anomic/search/TextSnippet.java
Unescape View File

@ -25,7 +25,6 @@
package de.anomic.search;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Iterator;
@ -40,7 +39,6 @@ import net.yacy.document.parser.html.CharacterCoding;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.data.meta.URIMetadataRow;
import net.yacy.kelondro.data.word.Word;
import net.yacy.kelondro.data.word.WordReferenceVars;
import net.yacy.kelondro.index.ARC;
import net.yacy.kelondro.index.ConcurrentARC;
import net.yacy.kelondro.index.HandleSet;
@ -48,7 +46,6 @@ import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.util.ByteArray;
import net.yacy.kelondro.util.SetTools;
import net.yacy.repository.LoaderDispatcher;
import de.anomic.crawler.retrieval.Response;

4
source/net/yacy/dbtest.java
Unescape View File

@ -5,16 +5,13 @@ package net.yacy;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Random;
import javax.imageio.ImageIO;
import net.yacy.kelondro.index.ObjectIndex;
import net.yacy.kelondro.index.ObjectIndexCache;
import net.yacy.kelondro.index.Row;
import net.yacy.kelondro.index.RowSet;
import net.yacy.kelondro.index.RowSpaceExceededException;
@ -26,7 +23,6 @@ import net.yacy.kelondro.table.SQLTable;
import net.yacy.kelondro.table.SplitTable;
import net.yacy.kelondro.table.Table;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.kelondro.workflow.InstantBusyThread;
import net.yacy.visualization.ChartPlotter;

2
source/net/yacy/document/parser/images/genericImageParser.java
Unescape View File

@ -222,7 +222,7 @@ public class genericImageParser extends AbstractParser implements Idiom {
public static ImageInfo parseJavaImage(
final DigestURI location,
final BufferedImage image) throws ParserException {
final BufferedImage image) {
ImageInfo ii = new ImageInfo(location);
ii.image = image;

24
source/net/yacy/kelondro/blob/Stacks.java
Unescape View File

@ -68,36 +68,22 @@ public class Stacks {
}
private class StackInstance {
String name;
private final File location;
Stack stack;
public String name;
public Stack stack;
public StackInstance(final File location) throws IOException {
String filename = location.getName();
assert filename.startsWith(stacksPrefix + "_");
assert filename.endsWith(".bstack");
this.name = filename.substring(stacksPrefix.length() + 1, filename.length() - 7);
this.location = location;
this.stack = new Stack(location);
}
public StackInstance(final String stack) throws IOException {
this.location = new File(stacksLocation, stacksPrefix + "_" + stack + ".bstack");
this.name = stack;
this.stack = new Stack(location);
}
public String name() {
return this.name;
this.stack = new Stack(new File(stacksLocation, stacksPrefix + "_" + stack + ".bstack"));
}
public Stack stack() {
return this.stack;
}
public File location() {
return this.location;
}
}
private Stack getStack(final String stack) {
@ -107,13 +93,13 @@ public class Stacks {
try {
si = new StackInstance(stack);
this.stacks.put(stack, si);
return si.stack();
return si.stack;
} catch (IOException e) {
Log.logException(e);
return null;
}
}
return si.stack();
return si.stack;
}
/**

3
source/net/yacy/kelondro/data/meta/DigestURI.java
Unescape View File

@ -35,7 +35,6 @@ import java.io.Serializable;
import java.net.MalformedURLException;
import java.net.UnknownHostException;
import java.text.Collator;
import java.util.Iterator;
import java.util.Locale;
import java.util.TreeSet;
import java.util.regex.Matcher;
@ -936,7 +935,7 @@ public class DigestURI implements Serializable {
return hosthash6("http", host, 80);
}
private static String[] testTLDs = new String[] { "com", "net", "org", "uk", "fr", "de", "es", "it" };
//private static String[] testTLDs = new String[] { "com", "net", "org", "uk", "fr", "de", "es", "it" };
public static final int domLengthEstimation(final byte[] urlHashBytes) {
// generates an estimation of the original domain length

1
source/net/yacy/kelondro/index/RowCollection.java
Unescape View File

@ -33,7 +33,6 @@ import java.util.Random;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.SynchronousQueue;

2
source/net/yacy/kelondro/order/StackIterator.java
Unescape View File

@ -22,9 +22,7 @@
package net.yacy.kelondro.order;
import java.util.Collection;
import java.util.ConcurrentModificationException;
import java.util.Iterator;
public class StackIterator<E> implements CloneableIterator<E> {

1
source/net/yacy/kelondro/rwi/AbstractIndex.java
Unescape View File

@ -28,7 +28,6 @@
package net.yacy.kelondro.rwi;
import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.TreeMap;
import java.util.TreeSet;

1
source/net/yacy/kelondro/rwi/Index.java
Unescape View File

@ -29,7 +29,6 @@
package net.yacy.kelondro.rwi;
import java.io.IOException;
import java.util.HashMap;
import java.util.TreeMap;
import java.util.TreeSet;

4
source/net/yacy/kelondro/rwi/IndexCell.java
Unescape View File

@ -28,23 +28,19 @@ package net.yacy.kelondro.rwi;
import java.io.File;
import java.io.IOException;
import java.util.Set;
import java.util.concurrent.Semaphore;
import de.anomic.yacy.graphics.ProfilingGraph;
import net.yacy.kelondro.data.meta.URIMetadataRow;
import net.yacy.kelondro.index.ARC;
import net.yacy.kelondro.index.HandleSet;
import net.yacy.kelondro.index.Row;
import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.index.SimpleARC;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.ByteOrder;
import net.yacy.kelondro.order.CloneableIterator;
import net.yacy.kelondro.order.MergeIterator;
import net.yacy.kelondro.order.Order;
import net.yacy.kelondro.util.ByteArray;
import net.yacy.kelondro.util.EventTracker;
import net.yacy.kelondro.util.MemoryControl;

1
source/net/yacy/kelondro/rwi/ReferenceContainer.java
Unescape View File

@ -31,7 +31,6 @@ import java.util.Collection;
import java.util.ConcurrentModificationException;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import net.yacy.kelondro.index.HandleSet;

1
source/net/yacy/kelondro/rwi/ReferenceContainerCache.java
Unescape View File

@ -33,7 +33,6 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import net.yacy.kelondro.blob.HeapWriter;

Write Preview
Loading…
Cancel
Save