boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

* fixed problem with watch crawler

Browse Source 
* added new column to network table (remote crawl urls):
  the new value for provided URLs will be used for new remote crawl method


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4061 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent e332b844b2
commit d9472b6a3a
5 changed files with 42 additions and 29 deletions
Show Stats Download Patch File Download Diff File

20
htroot/Network.html
Unescape View File

@ -59,8 +59,9 @@
<td><strong>UTC</strong><br />Offset<br />&nbsp;</td> <td><strong>UTC</strong><br />Offset<br />&nbsp;</td>
<td><strong>Location</strong><br /><br />&nbsp;</td> <td><strong>Location</strong><br /><br />&nbsp;</td>
<td><strong>Uptime</strong><br />&nbsp;&nbsp;<a href="/Network.html?page=#[page]#&amp;sort=Uptime&amp;order=up">&lt;</a>&nbsp;<a href="/Network.html?page=#[page]#&amp;sort=Uptime&amp;order=down">&gt;</a></td> <td><strong>Uptime</strong><br />&nbsp;&nbsp;<a href="/Network.html?page=#[page]#&amp;sort=Uptime&amp;order=up">&lt;</a>&nbsp;<a href="/Network.html?page=#[page]#&amp;sort=Uptime&amp;order=down">&gt;</a></td>
<td><strong>#Links</strong><br />&nbsp;&nbsp;<a href="/Network.html?page=#[page]#&amp;sort=LCount&amp;order=up">&lt;</a>&nbsp;<a href="/Network.html?page=#[page]#&amp;sort=LCount&amp;order=down">&gt;</a></td> <td><strong>Links</strong><br />&nbsp;&nbsp;<a href="/Network.html?page=#[page]#&amp;sort=LCount&amp;order=up">&lt;</a>&nbsp;<a href="/Network.html?page=#[page]#&amp;sort=LCount&amp;order=down">&gt;</a></td>
<td><strong>#RWIs</strong><br />&nbsp;&nbsp;<a href="/Network.html?page=#[page]#&amp;sort=ICount&amp;order=up">&lt;</a>&nbsp;<a href="/Network.html?page=#[page]#&amp;sort=ICount&amp;order=down">&gt;</a></td> <td><strong>RWIs</strong><br />&nbsp;&nbsp;<a href="/Network.html?page=#[page]#&amp;sort=ICount&amp;order=up">&lt;</a>&nbsp;<a href="/Network.html?page=#[page]#&amp;sort=ICount&amp;order=down">&gt;</a></td>
<td><strong>URLs for<br />Remote<br />Crawl</strong><br />&nbsp;</td>
<td><strong>Sent<br />Words</strong><br />&nbsp;</td> <td><strong>Sent<br />Words</strong><br />&nbsp;</td>
<td><strong>Sent<br />URLs</strong><br />&nbsp;</td> <td><strong>Sent<br />URLs</strong><br />&nbsp;</td>
<td><strong>Received<br />Words</strong><br />&nbsp;</td> <td><strong>Received<br />Words</strong><br />&nbsp;</td>
@ -97,8 +98,9 @@
<td align="right">#[utc]#</td> <td align="right">#[utc]#</td>
<td align="right">#[location]#</td> <td align="right">#[location]#</td>
<td align="right">#[uptime]#</td> <td align="right">#[uptime]#</td>
<td align="right">#[links]#</td> <td align="right">#[LCount]#</td>
<td align="right">#[words]#</td> <td align="right">#[ICount]#</td>
<td align="right">#[RCount]#</td>
<td align="right">#[sI]#</td> <td align="right">#[sI]#</td>
<td align="right">#[sU]#</td> <td align="right">#[sU]#</td>
<td align="right">#[rI]#</td> <td align="right">#[rI]#</td>
@ -238,8 +240,9 @@
<td>Version</td> <td>Version</td>
<td>UTC</td> <td>UTC</td>
<td>Uptime</td> <td>Uptime</td>
<td>#Links</td> <td>Links</td>
<td>#RWIs</td> <td>RWIs</td>
<td>URLs for<br />Remote Crawl</td>
<td>Sent<br />Words</td> <td>Sent<br />Words</td>
<td>Sent<br />URLs</td> <td>Sent<br />URLs</td>
<td>Received<br />Words</td> <td>Received<br />Words</td>
@ -257,8 +260,9 @@
<td align="right">#[my-version]#</td> <td align="right">#[my-version]#</td>
<td align="right">#[my-utc]#</td> <td align="right">#[my-utc]#</td>
<td align="right">#[my-uptime]#</td> <td align="right">#[my-uptime]#</td>
<td align="right">#[my-links]#</td> <td align="right">#[my-LCount]#</td>
<td align="right">#[my-words]#</td> <td align="right">#[my-ICount]#</td>
<td align="right">#[my-RCount]#</td>
<td align="right">#[my-sI]#</td> <td align="right">#[my-sI]#</td>
<td align="right">#[my-sU]#</td> <td align="right">#[my-sU]#</td>
<td align="right">#[my-rI]#</td> <td align="right">#[my-rI]#</td>

34
htroot/Network.java
Unescape View File

@ -110,12 +110,14 @@ public class Network {
// update seed info // update seed info
yacyCore.peerActions.updateMySeed(); yacyCore.peerActions.updateMySeed();
long links; long LCount;
long words; long ICount;
long RCount;
try { try {
links = Long.parseLong(seed.get(yacySeed.LCOUNT, "0")); LCount = Long.parseLong(seed.get(yacySeed.LCOUNT, "0"));
words = Long.parseLong(seed.get(yacySeed.ICOUNT, "0")); ICount = Long.parseLong(seed.get(yacySeed.ICOUNT, "0"));
} catch (Exception e) {links = 0; words = 0;} RCount = Long.parseLong(seed.get(yacySeed.RCOUNT, "0"));
} catch (Exception e) {LCount = 0; ICount = 0; RCount = 0;}
// my-info // my-info
prop.put("table_my-name", seed.get(yacySeed.NAME, "-") ); prop.put("table_my-name", seed.get(yacySeed.NAME, "-") );
@ -124,16 +126,16 @@ public class Network {
prop.put("table_my-info", 0); prop.put("table_my-info", 0);
} else if(yacyCore.seedDB.mySeed.isJunior()) { } else if(yacyCore.seedDB.mySeed.isJunior()) {
prop.put("table_my-info", 1); prop.put("table_my-info", 1);
accPotLinks += links; accPotLinks += LCount;
accPotWords += words; accPotWords += ICount;
} else if(yacyCore.seedDB.mySeed.isSenior()) { } else if(yacyCore.seedDB.mySeed.isSenior()) {
prop.put("table_my-info", 2); prop.put("table_my-info", 2);
accActLinks += links; accActLinks += LCount;
accActWords += words; accActWords += ICount;
} else if(yacyCore.seedDB.mySeed.isPrincipal()) { } else if(yacyCore.seedDB.mySeed.isPrincipal()) {
prop.put("table_my-info", 3); prop.put("table_my-info", 3);
accActLinks += links; accActLinks += LCount;
accActWords += words; accActWords += ICount;
} }
prop.put("table_my-acceptcrawl", seed.getFlagAcceptRemoteCrawl() ? 1 : 0); prop.put("table_my-acceptcrawl", seed.getFlagAcceptRemoteCrawl() ? 1 : 0);
prop.put("table_my-dhtreceive", seed.getFlagAcceptRemoteIndex() ? 1 : 0); prop.put("table_my-dhtreceive", seed.getFlagAcceptRemoteIndex() ? 1 : 0);
@ -145,8 +147,9 @@ public class Network {
prop.put("table_my-version", seed.get(yacySeed.VERSION, "-")); prop.put("table_my-version", seed.get(yacySeed.VERSION, "-"));
prop.put("table_my-utc", seed.get(yacySeed.UTC, "-")); prop.put("table_my-utc", seed.get(yacySeed.UTC, "-"));
prop.put("table_my-uptime", serverDate.intervalToString(60000 * Long.parseLong(seed.get(yacySeed.UPTIME, "")))); prop.put("table_my-uptime", serverDate.intervalToString(60000 * Long.parseLong(seed.get(yacySeed.UPTIME, ""))));
prop.put("table_my-links", groupDigits(links)); prop.put("table_my-LCount", groupDigits(LCount));
prop.put("table_my-words", groupDigits(words)); prop.put("table_my-ICount", groupDigits(ICount));
prop.put("table_my-RCount", groupDigits(RCount));
prop.put("table_my-sI", groupDigits(seed.get(yacySeed.INDEX_OUT, "0"))); prop.put("table_my-sI", groupDigits(seed.get(yacySeed.INDEX_OUT, "0")));
prop.put("table_my-sU", groupDigits(seed.get(yacySeed.URL_OUT, "0"))); prop.put("table_my-sU", groupDigits(seed.get(yacySeed.URL_OUT, "0")));
prop.put("table_my-rI", groupDigits(seed.get(yacySeed.INDEX_IN, "0"))); prop.put("table_my-rI", groupDigits(seed.get(yacySeed.INDEX_IN, "0")));
@ -470,8 +473,9 @@ public class Network {
prop.put(STR_TABLE_LIST + conCount + "_lastSeen", /*seed.getLastSeenString() + " " +*/ lastseen); prop.put(STR_TABLE_LIST + conCount + "_lastSeen", /*seed.getLastSeenString() + " " +*/ lastseen);
prop.put(STR_TABLE_LIST + conCount + "_utc", seed.get(yacySeed.UTC, "-")); prop.put(STR_TABLE_LIST + conCount + "_utc", seed.get(yacySeed.UTC, "-"));
prop.put(STR_TABLE_LIST + conCount + "_uptime", serverDate.intervalToString(60000 * Long.parseLong(seed.get(yacySeed.UPTIME, "0")))); prop.put(STR_TABLE_LIST + conCount + "_uptime", serverDate.intervalToString(60000 * Long.parseLong(seed.get(yacySeed.UPTIME, "0"))));
prop.put(STR_TABLE_LIST + conCount + "_links", groupDigits(seed.get(yacySeed.LCOUNT, "0"))); prop.put(STR_TABLE_LIST + conCount + "_LCount", groupDigits(seed.get(yacySeed.LCOUNT, "0")));
prop.put(STR_TABLE_LIST + conCount + "_words", groupDigits(seed.get(yacySeed.ICOUNT, "0"))); prop.put(STR_TABLE_LIST + conCount + "_ICount", groupDigits(seed.get(yacySeed.ICOUNT, "0")));
prop.put(STR_TABLE_LIST + conCount + "_RCount", groupDigits(seed.get(yacySeed.RCOUNT, "0")));
prop.put(STR_TABLE_LIST + conCount + "_sI", groupDigits(seed.get(yacySeed.INDEX_OUT, "0"))); prop.put(STR_TABLE_LIST + conCount + "_sI", groupDigits(seed.get(yacySeed.INDEX_OUT, "0")));
prop.put(STR_TABLE_LIST + conCount + "_sU", groupDigits(seed.get(yacySeed.URL_OUT, "0"))); prop.put(STR_TABLE_LIST + conCount + "_sU", groupDigits(seed.get(yacySeed.URL_OUT, "0")));
prop.put(STR_TABLE_LIST + conCount + "_rI", groupDigits(seed.get(yacySeed.INDEX_IN, "0"))); prop.put(STR_TABLE_LIST + conCount + "_rI", groupDigits(seed.get(yacySeed.INDEX_IN, "0")));

10
htroot/Network.xml
Unescape View File

@ -9,8 +9,9 @@
<ppm>#[ppm]#</ppm> <ppm>#[ppm]#</ppm>
<qph>#[qph]#</qph> <qph>#[qph]#</qph>
<uptime>#[uptime]#</uptime> <uptime>#[uptime]#</uptime>
<links>#[links]#</links> <links>#[LCount]#</links>
<words>#[words]#</words> <words>#[ICount]#</words>
<rurls>#[RCount]#</rurls>
<lastseen>#[lastSeen]#</lastseen> <lastseen>#[lastSeen]#</lastseen>
<sendWords>#[sI]#</sendWords> <sendWords>#[sI]#</sendWords>
<receivedWords>#[rI]#</receivedWords> <receivedWords>#[rI]#</receivedWords>
@ -60,8 +61,9 @@
<version>#[my-version]#</version> <version>#[my-version]#</version>
<utc>#[my-utc]#</utc> <utc>#[my-utc]#</utc>
<uptime>#[my-uptime]#</uptime> <uptime>#[my-uptime]#</uptime>
<links>#[my-links]#</links> <links>#[my-LCount]#</links>
<words>#[my-words]#</words> <words>#[my-ICount]#</words>
<words>#[my-RCount]#</words>
<acceptcrawl>#[my-acceptcrawl]#</acceptcrawl> <acceptcrawl>#[my-acceptcrawl]#</acceptcrawl>
<acceptindex>#[my-dhtreceive]#</acceptindex> <acceptindex>#[my-dhtreceive]#</acceptindex>
<acceptranking>#[my-rankingreceive]#</acceptranking> <acceptranking>#[my-rankingreceive]#</acceptranking>

3
htroot/xml/queues_p.xml
Unescape View File

@ -1,4 +1,5 @@
<?xml version="1.0"?> <?xml version="1.0"?>
<queues>
<dbsize> <dbsize>
<urlpublictext>#[urlpublictextSize]#</urlpublictext> <urlpublictext>#[urlpublictextSize]#</urlpublictext>
<rwipublictext>#[rwipublictextSize]#</rwipublictext> <rwipublictext>#[rwipublictextSize]#</rwipublictext>
@ -64,4 +65,4 @@
</entry> </entry>
#{/list-remote}# #{/list-remote}#
</remotecrawlerqueue> </remotecrawlerqueue>
</xml> </queues>

4
source/de/anomic/plasma/plasmaCrawlBalancer.java
Unescape View File

@ -467,7 +467,9 @@ public class plasmaCrawlBalancer {
while ((urlRAMStack.size() <= dist) && (urlFileStack.size() > 0)) { while ((urlRAMStack.size() <= dist) && (urlFileStack.size() > 0)) {
// flush some entries from disc to ram stack // flush some entries from disc to ram stack
try { try {
urlRAMStack.add(new String(urlFileStack.pop().getColBytes(0))); kelondroRow.Entry t = urlFileStack.pop();
if (t == null) break;
urlRAMStack.add(new String(t.getColBytes(0)));
} catch (IOException e) { } catch (IOException e) {
break; break;
} }

Write Preview
Loading…
Cancel
Save