boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- simplified watchCrawler code

Browse Source 
- changed display of watchCrawler slightly

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2594 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent 74c3e7cf29
commit 5015e780c2
4 changed files with 64 additions and 195 deletions
Show Stats Download Patch File Download Diff File

38
htroot/WatchCrawler.html
Unescape View File

@ -17,12 +17,9 @@
wordcaches: <span id="wordcache">&nbsp;</span>
</p>
<p>
Loader:
<br />
Size: <span id="loaderqueuesize">&nbsp;&nbsp;&nbsp;</span><br />
Max: <span id="loaderqueuemax">&nbsp;&nbsp;&nbsp;</span><br />
Indexing Queue: size=<span id="indexingqueuesize">&nbsp;&nbsp;&nbsp;</span>; max=<span id="indexingqueuemax">&nbsp;&nbsp;&nbsp;</span>
</p>
<table border="0" cellpadding="2" cellspacing="1" id="loaderTable">
<table border="0" cellpadding="2" cellspacing="1" id="indexingTable">
<tbody>
<tr class="TableHeader">
<th>Initiator</th>
@ -30,15 +27,15 @@
<th>Modified Date</th>
<th>Anchor Name</th>
<th>URL</th>
<th>Size</th>
<th>Delete</th>
</tr>
</tbody>
</table>
<p>
Indexing Queue:<br />
Size: <span id="indexingqueuesize">&nbsp;&nbsp;&nbsp;</span><br />
Max: <span id="indexingqueuemax">&nbsp;&nbsp;&nbsp;</span><br />
Loader: size=<span id="loaderqueuesize">&nbsp;&nbsp;&nbsp;</span>; max=<span id="loaderqueuemax">&nbsp;&nbsp;&nbsp;</span>
</p>
<table border="0" cellpadding="2" cellspacing="1" id="indexingTable">
<table border="0" cellpadding="2" cellspacing="1" id="loaderTable">
<tbody>
<tr class="TableHeader">
<th>Initiator</th>
@ -52,10 +49,9 @@
</tbody>
</table>
<p>
Local Crawler<br />
Size: <span id="localcrawlerqueuesize">&nbsp;&nbsp;&nbsp;</span><br />
Local Crawler: size=<span id="localcrawlerqueuesize">&nbsp;&nbsp;&nbsp;</span>
</p>
<table border="0" cellpadding="2" cellspacing="1" id="localCrawlerTable" >
<table border="0" cellpadding="2" cellspacing="1" id="localCrawlerTable">
<tbody>
<tr class="TableHeader">
<th>Initiator</th>
@ -63,21 +59,25 @@
<th>Modified Date</th>
<th>Anchor Name</th>
<th>URL</th>
<th>Size</th>
<th>Delete</th>
</tr>
</tbody>
</table>
<p>
Remote Crawler<br />
Size: <span id="remotecrawlerqueuesize">&nbsp;&nbsp;&nbsp;</span><br />
Remote Crawler: size=<span id="remotecrawlerqueuesize">&nbsp;&nbsp;&nbsp;</span>
</p>
<table border="0" cellpadding="2" cellspacing="1" id="remoteCrawlerTable">
<tbody>
<tr class="TableHeader">
<th>Profile</th>
<th>Depth</th>
<th>Modified Date</th>
<th>Anchor Name</th>
<th>URL</th>
<th width="30">Initiator</th>
<th width="10">Depth</th>
<th width="60">Modified Date</th>
<th width="100">Anchor Name</th>
<th width="300">URL</th>
<th width="20">Size</th>
<th width="30">Delete</th>
</tr>
</tbody>
</table>

146
htroot/js/WatchCrawler.js
Unescape View File

@ -56,8 +56,6 @@ function requestQueues(){
}
function handleStatus(){
if(statusRPC.readyState != 4){
return;
@ -100,7 +98,6 @@ function handleStatus(){
}
}
function handleQueues(){
if(queuesRPC.readyState != 4){
return;
@ -109,14 +106,14 @@ function handleQueues(){
xml=getFirstChild(queuesResponse);
if(queuesResponse != null){
indexingqueue=getFirstChild(xml, "indexingqueue");
createIndexingTable(indexingqueue);
createIndexingTable(indexingqueue, "indexingTable");
indexingqueue_size=getValue(getFirstChild(indexingqueue, "size"));
indexingqueue_max=getValue(getFirstChild(indexingqueue, "max"));
document.getElementById("indexingqueuesize").firstChild.nodeValue=indexingqueue_size;
document.getElementById("indexingqueuemax").firstChild.nodeValue=indexingqueue_max;
loaderqueue=getFirstChild(xml, "loaderqueue");
createLoaderTable(getFirstChild(xml, "loaderqueue"));
createIndexingTable(loaderqueue, "loaderTable");
loaderqueue_size=getValue(getFirstChild(loaderqueue, "size"));
loaderqueue_max=getValue(getFirstChild(loaderqueue, "max"));
document.getElementById("loaderqueuesize").firstChild.nodeValue=loaderqueue_size;
@ -125,42 +122,18 @@ function handleQueues(){
localcrawlerqueue=getFirstChild(xml, "localcrawlerqueue");
localcrawlerqueue_size=getValue(getFirstChild(localcrawlerqueue, "size"));
document.getElementById("localcrawlerqueuesize").firstChild.nodeValue=localcrawlerqueue_size;
createLocalCrawlerTable(localcrawlerqueue);
createIndexingTable(localcrawlerqueue, "localCrawlerTable");
remotecrawlerqueue=getFirstChild(xml, "remotecrawlerqueue");
createRemoteCrawlerTable(remotecrawlerqueue);
createIndexingTable(remotecrawlerqueue, "remoteCrawlerTable");
remotecrawlerqueue_size=getValue(getFirstChild(remotecrawlerqueue, "size"));
document.getElementById("remotecrawlerqueuesize").firstChild.nodeValue=remotecrawlerqueue_size;
createRemoteCrawlerTable(remotecrawlerqueue);
}
}
function createLoaderTable(loaderqueue){
entries=loaderqueue.getElementsByTagName("entry");
loaderTable=document.getElementById("loaderTable");
clearTable(loaderTable, 1);
dark=false;
for(i=0;i<entries.length;i++){
initiator=getValue(getFirstChild(entries[i], "initiator"));
depth=getValue(getFirstChild(entries[i], "depth"));
modified=getValue(getFirstChild(entries[i], "modified"));
anchor=getValue(getFirstChild(entries[i], "anchor"));
url=getValue(getFirstChild(entries[i], "url"));
row=createLoaderRow(initiator, depth, modified, anchor, url);
//create row
if(dark){
row.setAttribute("class", "TableCellDark");
}else{
row.setAttribute("class", "TableCellLight");
}
getFirstChild(loaderTable, "tbody").appendChild(row);
dark=!dark;
}
}
function createIndexingTable(indexingqueue){
indexingTable=document.getElementById("indexingTable");
function createIndexingTable(indexingqueue, tablename){
indexingTable=document.getElementById(tablename);
entries=indexingqueue.getElementsByTagName("entry");
clearTable(indexingTable, 1);
@ -177,8 +150,11 @@ function createIndexingTable(indexingqueue){
if(getValue(getFirstChild(entries[i], "inProcess"))=="true"){
inProcess=true;
}
row=createIndexingRow(initiator, depth, modified, anchor, url, size, hash);
if (tablename=="indexingTable")
deletebutton=createLinkCol("IndexCreateIndexingQueue_p.html?deleteEntry="+hash, DELETE_STRING);
else
deletebutton=createCol("");
row=createIndexingRow(initiator, depth, modified, anchor, url, size, deletebutton);
//create row
if(inProcess){
@ -191,108 +167,22 @@ function createIndexingTable(indexingqueue){
getFirstChild(indexingTable, "tbody").appendChild(row);
dark=!dark;
}
}
function createLocalCrawlerTable(localcrawlerqueue){
localCrawlerTable=document.getElementById("localCrawlerTable");
entries=localcrawlerqueue.getElementsByTagName("entry");
clearTable(localCrawlerTable, 1);
dark=false;
for(i=0;i<entries.length;i++){
initiator=getValue(getFirstChild(entries[i], "initiator"));
depth=getValue(getFirstChild(entries[i], "depth"));
modified=getValue(getFirstChild(entries[i], "modified"));
anchor=getValue(getFirstChild(entries[i], "anchor"));
url=getValue(getFirstChild(entries[i], "url"));
//hash=getValue(getFirstChild(entries[i], "hash"));
inProcess=false;
if(getValue(getFirstChild(entries[i], "inProcess"))=="true"){
inProcess=true;
}
row=createLocalCrawlerRow(initiator, depth, modified, anchor, url);
//create row
if(inProcess){
row.setAttribute("class", "TableCellActive");
}else if(dark){
row.setAttribute("class", "TableCellDark");
}else{
row.setAttribute("class", "TableCellLight");
}
getFirstChild(localCrawlerTable, "tbody").appendChild(row);
dark=!dark;
}
}
function createRemoteCrawlerTable(remotecrawlerqueue){
remoteCrawlerTable=document.getElementById("remoteCrawlerTable");
entries=remotecrawlerqueue.getElementsByTagName("entry");
clearTable(remoteCrawlerTable, 1);
dark=false;
for(i=0;i<entries.length;i++){
profile=getValue(getFirstChild(entries[i], "profile"));
depth=getValue(getFirstChild(entries[i], "depth"));
modified=getValue(getFirstChild(entries[i], "modified"));
anchor=getValue(getFirstChild(entries[i], "anchor"));
url=getValue(getFirstChild(entries[i], "url"));
//hash=getValue(getFirstChild(entries[i], "hash"));
inProcess=false;
if(getValue(getFirstChild(entries[i], "inProcess"))=="true"){
inProcess=true;
}
row=createRemoteCrawlerRow(profile, depth, modified, anchor, url);
//create row
if(inProcess){
row.setAttribute("class", "TableCellActive");
}else if(dark){
row.setAttribute("class", "TableCellDark");
}else{
row.setAttribute("class", "TableCellLight");
}
getFirstChild(remoteCrawlerTable, "tbody").appendChild(row);
while(i++ <= 10) {
row=createIndexingRow("", "", "", "", "", "", createCol(""));
if(dark) {row.setAttribute("class", "TableCellDark");} else {row.setAttribute("class", "TableCellLight");}
getFirstChild(indexingTable, "tbody").appendChild(row);
dark=!dark;
}
}
function createIndexingRow(initiator, depth, modified, anchor, url, size, hash){
function createIndexingRow(initiator, depth, modified, anchor, url, size, deletebutton){
row=document.createElement("tr");
row.setAttribute("height", 10);
row.appendChild(createCol(initiator));
row.appendChild(createCol(depth));
row.appendChild(createCol(modified));
row.appendChild(createCol(anchor));
row.appendChild(createLinkCol(url, url));
row.appendChild(createCol(size));
row.appendChild(createLinkCol("IndexCreateIndexingQueue_p.html?deleteEntry="+hash, DELETE_STRING));
return row;
}
function createLoaderRow(initiator, depth, modified, anchor, url){
row=document.createElement("tr");
row.appendChild(createCol(initiator));
row.appendChild(createCol(depth));
row.appendChild(createCol(modified));
row.appendChild(createCol(anchor));
row.appendChild(createLinkCol(url, url));
row.appendChild(deletebutton);
return row;
}
function createLocalCrawlerRow(initiator, depth, modified, anchor, url){
row=document.createElement("tr");
row.appendChild(createCol(initiator));
row.appendChild(createCol(depth));
row.appendChild(createCol(modified));
row.appendChild(createCol(anchor));
row.appendChild(createLinkCol(url, url));
return row;
}
function createRemoteCrawlerRow(profile, depth, modified, anchor, url){
row=document.createElement("tr");
row.appendChild(createCol(profile));
row.appendChild(createCol(depth));
row.appendChild(createCol(modified));
row.appendChild(createCol(anchor));
row.appendChild(createLinkCol(url, url));
return row;
}

72
htroot/xml/queues_p.java
Unescape View File

@ -56,7 +56,6 @@ import de.anomic.data.wikiCode;
import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaCrawlLoaderMessage;
import de.anomic.plasma.plasmaCrawlNURL;
import de.anomic.plasma.plasmaCrawlProfile;
import de.anomic.plasma.crawler.http.CrawlWorker;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.plasma.plasmaSwitchboardQueue;
@ -164,67 +163,46 @@ public class queues_p {
}
//local crawl queue
prop.put("localCrawlSize", Integer.toString(switchboard.getThread("50_localcrawl").getJobCount()));
plasmaCrawlNURL.Entry urle;
String profileHandle;
plasmaCrawlProfile.entry profileEntry;
int i;
int showNum=0;
int size=10;
prop.put("localCrawlSize", Integer.toString(switchboard.getThread("50_localcrawl").getJobCount()));
int stackSize = switchboard.urlPool.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE);
plasmaCrawlNURL.Entry[] crawlerList = switchboard.urlPool.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_CORE, (int) (size * 1.20));
for (i = 0; (i < crawlerList.length) && (showNum < size); i++) {
urle = crawlerList[i];
if ((urle != null)&&(urle.url()!=null)) {
initiator = yacyCore.seedDB.getConnected(urle.initiator());
profileHandle = urle.profileHandle();
profileEntry = (profileHandle == null) ? null : switchboard.profiles.getEntry(profileHandle);
prop.put("list-local_"+showNum+"_initiator", ((initiator == null) ? "proxy" : initiator.getName()) );
prop.putNoHTML("list-local_"+showNum+"_profile", ((profileEntry == null) ? "unknown" : profileEntry.name()));
prop.put("list-local_"+showNum+"_depth", urle.depth());
prop.put("list-local_"+showNum+"_modified", daydate(urle.loaddate()) );
prop.putNoHTML("list-local_"+showNum+"_anchor", urle.name());
prop.putNoHTML("list-local_"+showNum+"_url", urle.url().toString());
prop.put("list-local_"+showNum+"_hash", urle.hash());
showNum++;
} else {
stackSize--;
}
}
prop.put("list-local", showNum);
addNTable(prop, "list-local", switchboard.urlPool.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_CORE, Math.min(10, stackSize)));
//global crawl queue
prop.put("remoteCrawlSize", Integer.toString(switchboard.getThread("61_globalcrawltrigger").getJobCount()));
//prop.put("remoteCrawlSize", Integer.toString(switchboard.getThread("62_remotetriggeredcrawl").getJobCount()));
size=10;
stackSize = switchboard.urlPool.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT);
if (stackSize == 0) {
prop.put("list-remote", 0);
} else {
crawlerList = switchboard.urlPool.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_LIMIT, size);
showNum = 0;
for (i = 0; (i < crawlerList.length) && (showNum < size); i++) {
urle = crawlerList[i];
if ((urle != null)&&(urle.url()!=null)) {
initiator = yacyCore.seedDB.getConnected(urle.initiator());
profileHandle = urle.profileHandle();
profileEntry = (profileHandle == null) ? null : switchboard.profiles.getEntry(profileHandle);
prop.putNoHTML("list-remote_"+i+"_profile", ((profileEntry == null) ? "unknown" : profileEntry.name()));
prop.put("list-remote_"+i+"_depth", urle.depth());
prop.put("list-remote_"+i+"_modified", daydate(urle.loaddate()) );
prop.putNoHTML("list-remote_"+i+"_anchor", urle.name());
prop.putNoHTML("list-remote_"+i+"_url", urle.url().toString());
showNum++;
}
}
prop.put("list-remote", showNum);
addNTable(prop, "list-remote", switchboard.urlPool.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_LIMIT, Math.min(10, stackSize)));
}
// return rewrite properties
return prop;
}
public static final void addNTable(serverObjects prop, String tableName, plasmaCrawlNURL.Entry[] crawlerList) {
int showNum = 0;
plasmaCrawlNURL.Entry urle;
yacySeed initiator;
for (int i = 0; i < crawlerList.length; i++) {
urle = crawlerList[i];
if ((urle != null) && (urle.url() != null)) {
initiator = yacyCore.seedDB.getConnected(urle.initiator());
prop.put(tableName + "_" + showNum + "_initiator", ((initiator == null) ? "proxy" : initiator.getName()));
prop.put(tableName + "_" + showNum + "_depth", urle.depth());
prop.put(tableName + "_" + showNum + "_modified", daydate(urle.loaddate()));
prop.putNoHTML(tableName + "_" + showNum + "_anchor", urle.name());
prop.putNoHTML(tableName + "_" + showNum + "_url", urle.url().toString());
prop.put(tableName + "_" + showNum + "_hash", urle.hash());
showNum++;
}
}
prop.put(tableName, showNum);
}
}

3
htroot/xml/queues_p.xml
Unescape View File

@ -44,11 +44,12 @@
<size>#[remoteCrawlSize]#</size>
#{list-remote}#
<entry>
<profile>#[profile]#</profile>
<initiator>#[initiator]#</initiator>
<depth>#[depth]#</depth>
<modified>#[modified]#</modified>
<anchor>#[anchor]#</anchor>
<url>#[url]#</url>
<hash>#[hash]#</hash>
<inProcess>#(inProcess)#false::true#(/inProcess)#</inProcess>
</entry>
#{/list-remote}#

Write Preview
Loading…
Cancel
Save