boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- some refactoring for internal RSSFeed (protocol of all actions as seen on status page)

Browse Source 
- added dht-out to internal RSSFeed (you can see now messages about distributed indexes on status page)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6948 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent 5a4684f21f
commit 1557e0f2d0
13 changed files with 51 additions and 41 deletions
Show Stats Download Patch File Download Diff File

10
htroot/Network.html
Unescape View File

@ -14,27 +14,27 @@
imageanimindex = 0;
imageloadindex = 0;
imagecycles = 0;
imagearray = new Array(4);
imagearray = new Array(6);
setTimeout("initanimation()", 100);
setTimeout("doanimation()", 1000);
function initanimation() {
handle = new Date().getTime();
for (j = 0; j < 4; j++) initanimationphase(j, handle);
for (j = 0; j < 6; j++) initanimationphase(j, handle);
}
function initanimationphase(phase, handle) {
imagearray[phase] = new Image(768, 576);
angle = phase * 90;
angle = phase * 60;
imagearray[phase].src = imagestub + angle + "&handle=" + handle;
}
function doanimation() {
document.getElementById("NetworkPicture").src = imagearray[imageanimindex].src;
imageanimindex++;
if (imageanimindex == 4) { imageanimindex = 0; }
if (imageanimindex == 6) { imageanimindex = 0; }
imagecycles++;
if (imagecycles == 25) {
initanimationphase(imageloadindex, new Date().getTime());
imageloadindex++;
if (imageloadindex == 4) { imageloadindex = 0; }
if (imageloadindex == 6) { imageloadindex = 0; }
imagecycles = 0;
}
setTimeout("doanimation()", 100);

2
htroot/Status.html
Unescape View File

@ -42,7 +42,7 @@
<div class="welcome">
<div style="margin:auto;text-align:center;">
<h2><img src="/env/grafics/kaskelix.png" width="128" height="64" alt="YaCy Websearch" style="margin-right:5px; vertical-align:middle;" />Welcome to YaCy!</h2>
<iframe src="rssTerminal.html?set=PEERNEWS,REMOTESEARCH,LOCALSEARCH,REMOTEINDEXING,LOCALINDEXING,INDEXRECEIVE&amp;width=600px&amp;height=180px&amp;maxlines=20&amp;maxwidth=120"
<iframe src="rssTerminal.html?set=PEERNEWS,REMOTESEARCH,LOCALSEARCH,REMOTEINDEXING,LOCALINDEXING,DHTRECEIVE,DHTSEND&amp;width=600px&amp;height=180px&amp;maxlines=20&amp;maxwidth=120"
style="width:600px;height:180px;margin:0px;border:1px solid black;" scrolling="no" name="newsframe"></iframe><br />
<img src="PerformanceGraph.png" id="graph" alt="PerformanceGraph" width="660" height="240"/><br />
<img src="Banner.png?textcolor=000000&amp;bgcolor=e7effc&amp;bordercolor=5090d0" alt="banner" width="468" height="60"/>

12
htroot/api/feed.java
Unescape View File

@ -33,16 +33,16 @@ public class feed {
int messageMaxCount = Math.min(post.getInt("count", 100), 1000);
RSSFeed feed;
channelIteration: for (int channelIndex = 0; channelIndex < channels.length; channelIndex++) {
channelIteration: for (String channel: channels) {
// prevent that unauthorized access to this servlet get results from private data
if ((!authorized) && (RSSFeed.privateChannels.contains(channels[channelIndex]))) continue channelIteration; // allow only public channels if not authorized
if ((!authorized) && (RSSFeed.privateChannels.contains(channel))) continue channelIteration; // allow only public channels if not authorized
if (channels[channelIndex].equals("TEST")) {
if (channel.equals("TEST")) {
// for interface testing return at least one single result
prop.putXML("channel_title", "YaCy News Testchannel");
prop.putXML("channel_description", "");
prop.put("channel_pubDate", (new Date()).toString());
prop.putXML("item_" + messageCount + "_title", channels[channelIndex] + ": " + "YaCy Test Entry " + (new Date()).toString());
prop.putXML("item_" + messageCount + "_title", channel + ": " + "YaCy Test Entry " + (new Date()).toString());
prop.putXML("item_" + messageCount + "_description", "abcdefg");
prop.putXML("item_" + messageCount + "_link", "http://yacy.net");
prop.put("item_" + messageCount + "_pubDate", (new Date()).toString());
@ -53,7 +53,7 @@ public class feed {
}
// read the channel
feed = RSSFeed.channels(channels[channelIndex]);
feed = RSSFeed.channels(channel);
if (feed == null || feed.isEmpty()) continue channelIteration;
RSSMessage message = feed.getChannel();
@ -67,7 +67,7 @@ public class feed {
if (message == null) continue;
// create RSS entry
prop.putXML("item_" + messageCount + "_title", channels[channelIndex] + ": " + message.getTitle());
prop.putXML("item_" + messageCount + "_title", channel + ": " + message.getTitle());
prop.putXML("item_" + messageCount + "_description", message.getDescription());
prop.putXML("item_" + messageCount + "_link", message.getLink());
prop.put("item_" + messageCount + "_pubDate", message.getPubDate());

2
htroot/terminal_p.html
Unescape View File

@ -54,7 +54,7 @@ function init() {
<div class="navouter">
<div class="navheader">Event Terminal</div>
<div class="navcontent">
<iframe src="/rssTerminal.html?set=PEERNEWS,REMOTESEARCH,LOCALSEARCH,REMOTEINDEXING,LOCALINDEXING,INDEXRECEIVE&amp;width=430px&amp;height=200px&amp;maxlines=22&amp;maxwidth=95&amp;color=fff&amp;background=000"
<iframe src="/rssTerminal.html?set=PEERNEWS,REMOTESEARCH,LOCALSEARCH,REMOTEINDEXING,LOCALINDEXING,DHTRECEIVE,DHTSEND&amp;width=430px&amp;height=200px&amp;maxlines=22&amp;maxwidth=95&amp;color=fff&amp;background=000"
style="width:430px;height:200px;margin:0;border-width:0;" scrolling="no" name="newsframe"></iframe>
</div>
</div>

2
htroot/yacy/search.java
Unescape View File

@ -277,7 +277,7 @@ public final class search {
rankingProfile
);
yacyCore.log.logInfo("INIT HASH SEARCH (query-" + abstracts + "): " + QueryParams.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.displayResults() + " links");
RSSFeed.channels(RSSFeed.REMOTESEARCH).addMessage(new RSSMessage("Remote Search Request from " + ((remoteSeed == null) ? "unknown" : remoteSeed.getName()), QueryParams.anonymizedQueryHashes(theQuery.queryHashes), ""));
RSSFeed.channels(RSSFeed.YaCyChannel.REMOTESEARCH).addMessage(new RSSMessage("Remote Search Request from " + ((remoteSeed == null) ? "unknown" : remoteSeed.getName()), QueryParams.anonymizedQueryHashes(theQuery.queryHashes), ""));
// make event
theSearch = SearchEventCache.getEvent(theQuery, sb.peers, sb.crawlResults, null, true, sb.loader);

2
htroot/yacy/transferRWI.java
Unescape View File

@ -219,7 +219,7 @@ public final class transferRWI {
String lastHash = wordhashes.get(wordhashes.size() - 1);
final long avdist = (FlatWordPartitionScheme.std.dhtDistance(firstHash.getBytes(), null, sb.peers.mySeed()) + FlatWordPartitionScheme.std.dhtDistance(lastHash.getBytes(), null, sb.peers.mySeed())) / 2;
sb.getLog().logInfo("Received " + received + " Entries " + wordc + " Words [" + firstHash + " .. " + lastHash + "]/" + avdist + " from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + "/" + receivedURL + " URLs, blocked " + blocked + " RWIs");
RSSFeed.channels(RSSFeed.INDEXRECEIVE).addMessage(new RSSMessage("Received " + received + " RWIs [" + firstHash + " .. " + lastHash + "]/" + avdist + " from " + otherPeerName + ", requesting " + unknownURL.size() + " URLs, blocked " + blocked, "", ""));
RSSFeed.channels(RSSFeed.YaCyChannel.DHTRECEIVE).addMessage(new RSSMessage("Received " + received + " RWIs [" + firstHash + " .. " + lastHash + "]/" + avdist + " from " + otherPeerName + ", requesting " + unknownURL.size() + " URLs, blocked " + blocked, "", ""));
}
result = "ok";

2
htroot/yacy/transferURL.java
Unescape View File

@ -157,7 +157,7 @@ public final class transferURL {
final int more = sb.indexSegments.urlMetadata(Segments.Process.DHTIN).size() - sizeBefore;
doublevalues = Integer.toString(received - more);
yacyCore.log.logInfo("Received " + received + " URLs from peer " + otherPeerName + " in " + (System.currentTimeMillis() - start) + " ms, blocked " + blocked + " URLs");
RSSFeed.channels(RSSFeed.INDEXRECEIVE).addMessage(new RSSMessage("Received " + received + " URLs from peer " + otherPeerName + ", blocked " + blocked, "", ""));
RSSFeed.channels(RSSFeed.YaCyChannel.DHTRECEIVE).addMessage(new RSSMessage("Received " + received + " URLs from peer " + otherPeerName + ", blocked " + blocked, "", ""));
if ((received - more) > 0) yacyCore.log.logSevere("Received " + doublevalues + " double URLs from peer " + otherPeerName);
result = "ok";
}

2
htroot/yacysearch.java
Unescape View File

@ -498,7 +498,7 @@ public class yacysearch {
// log
Log.logInfo("LOCAL_SEARCH", "INIT WORD SEARCH: " + theQuery.queryString + ":" + QueryParams.hashSet2hashString(theQuery.queryHashes) + " - " + theQuery.neededResults() + " links to be computed, " + theQuery.displayResults() + " lines to be displayed");
RSSFeed.channels(RSSFeed.LOCALSEARCH).addMessage(new RSSMessage("Local Search Request", theQuery.queryString, ""));
RSSFeed.channels(RSSFeed.YaCyChannel.LOCALSEARCH).addMessage(new RSSMessage("Local Search Request", theQuery.queryString, ""));
final long timestamp = System.currentTimeMillis();
// create a new search event

2
source/de/anomic/search/Switchboard.java
Unescape View File

@ -1855,7 +1855,7 @@ public final class Switchboard extends serverSwitch {
document,
condenser,
searchEvent);
RSSFeed.channels(Base64Order.enhancedCoder.equal(queueEntry.initiator(), peers.mySeed().hash.getBytes()) ? RSSFeed.LOCALINDEXING : RSSFeed.REMOTEINDEXING).addMessage(new RSSMessage("Indexed web page", dc_title, queueEntry.url().toNormalform(true, false)));
RSSFeed.channels(Base64Order.enhancedCoder.equal(queueEntry.initiator(), peers.mySeed().hash.getBytes()) ? RSSFeed.YaCyChannel.LOCALINDEXING : RSSFeed.YaCyChannel.REMOTEINDEXING).addMessage(new RSSMessage("Indexed web page", dc_title, queueEntry.url().toNormalform(true, false)));
} catch (final IOException e) {
if (this.log.isFine()) log.logFine("Not Indexed Resource '" + queueEntry.url().toNormalform(false, true) + "': process case=" + processCase);
addURLtoErrorDB(queueEntry.url(), (referrerURL == null) ? null : referrerURL.hash(), queueEntry.initiator(), dc_title, "error storing url: " + e.getMessage());

8
source/de/anomic/yacy/yacyClient.java
Unescape View File

@ -57,6 +57,7 @@ import java.util.TreeMap;
import java.util.regex.Pattern;
import net.yacy.cora.document.RSSFeed;
import net.yacy.cora.document.RSSMessage;
import net.yacy.cora.document.RSSReader;
import net.yacy.cora.protocol.HttpConnector;
import net.yacy.cora.services.Search;
@ -873,6 +874,8 @@ public final class yacyClient {
if (uhss == null) {
return "no unknownURL tag in response";
}
RSSFeed.channels(RSSFeed.YaCyChannel.DHTSEND).addMessage(new RSSMessage("Send " + indexes.size() + " RWIs to " + targetSeed.getName(), "", ""));
uhss = uhss.trim();
if (uhss.length() == 0 || uhss.equals(",")) { return null; } // all url's known, we are ready here
@ -902,9 +905,10 @@ public final class yacyClient {
return "no result from transferURL";
}
if (!(result.equals("ok"))) {
if (!result.equals("ok")) {
return result;
}
}
RSSFeed.channels(RSSFeed.YaCyChannel.DHTSEND).addMessage(new RSSMessage("Send " + uhs.length + " URLs to peer " + targetSeed.getName(), "", ""));
return null;
} finally {

2
source/de/anomic/yacy/yacyCore.java
Unescape View File

@ -98,7 +98,7 @@ public class yacyCore {
sb.setConfig("yacyStatus", "");
// create a peer news channel
final RSSFeed peernews = RSSFeed.channels(RSSFeed.PEERNEWS);
final RSSFeed peernews = RSSFeed.channels(RSSFeed.YaCyChannel.PEERNEWS);
peernews.addMessage(new RSSMessage("YaCy started", "", ""));
// ensure that correct IP is used

6
source/de/anomic/yacy/yacyPeerActions.java
Unescape View File

@ -197,7 +197,7 @@ public class yacyPeerActions {
if (res) {
// perform all actions if peer is effective new
this.processPeerArrival(peer);
RSSFeed.channels(RSSFeed.PEERNEWS).addMessage(new RSSMessage(peer.getName() + " joined the network", "", ""));
RSSFeed.channels(RSSFeed.YaCyChannel.PEERNEWS).addMessage(new RSSMessage(peer.getName() + " joined the network", "", ""));
}
return res;
}
@ -211,7 +211,7 @@ public class yacyPeerActions {
peer.put("dct", Long.toString(System.currentTimeMillis()));
seedDB.addDisconnected(peer); // update info
}
RSSFeed.channels(RSSFeed.PEERNEWS).addMessage(new RSSMessage(peer.getName() + " left the network", "", ""));
RSSFeed.channels(RSSFeed.YaCyChannel.PEERNEWS).addMessage(new RSSMessage(peer.getName() + " left the network", "", ""));
}
public void peerPing(final yacySeed peer) {
@ -220,7 +220,7 @@ public class yacyPeerActions {
seedDB.addPotential(peer);
// perform all actions
processPeerArrival(peer);
RSSFeed.channels(RSSFeed.PEERNEWS).addMessage(new RSSMessage(peer.getName() + " sent me a ping", "", ""));
RSSFeed.channels(RSSFeed.YaCyChannel.PEERNEWS).addMessage(new RSSMessage(peer.getName() + " sent me a ping", "", ""));
}
private void processPeerArrival(final yacySeed peer) {

40
source/net/yacy/cora/document/RSSFeed.java
Unescape View File

@ -27,14 +27,16 @@ import java.util.concurrent.ConcurrentLinkedQueue;
public class RSSFeed implements Iterable<Hit> {
// static channel names of feeds
public static final String TEST = "TEST";
public static final String PEERNEWS = "PEERNEWS";
public static final String REMOTESEARCH = "REMOTESEARCH";
public static final String LOCALSEARCH = "LOCALSEARCH";
public static final String REMOTEINDEXING = "REMOTEINDEXING";
public static final String LOCALINDEXING = "LOCALINDEXING";
public static final String INDEXRECEIVE = "INDEXRECEIVE";
public static enum YaCyChannel {
TEST,
PEERNEWS,
REMOTESEARCH,
LOCALSEARCH,
REMOTEINDEXING,
LOCALINDEXING,
DHTRECEIVE,
DHTSEND;
}
// test:
// http://localhost:8080/xml/feed.rss?set=PEERNEWS,REMOTESEARCH,LOCALSEARCH,REMOTEINDEXING,LOCALINDEXING
@ -43,10 +45,10 @@ public class RSSFeed implements Iterable<Hit> {
* the following private channels are declared to prevent that an access to the feed servlet
* gets results from news channels that are not for the public
*/
public static final HashSet<String> privateChannels = new HashSet<String>();
public static final HashSet<YaCyChannel> privateChannels = new HashSet<YaCyChannel>();
static {
privateChannels.add(LOCALSEARCH);
privateChannels.add(LOCALINDEXING);
privateChannels.add(YaCyChannel.LOCALSEARCH);
privateChannels.add(YaCyChannel.LOCALINDEXING);
}
@ -152,16 +154,20 @@ public class RSSFeed implements Iterable<Hit> {
/**
* the following static channels object is used to organize a storage array for RSS feeds
*/
private static final ConcurrentHashMap<String, RSSFeed> channels = new ConcurrentHashMap<String, RSSFeed>();
private static final ConcurrentHashMap<YaCyChannel, RSSFeed> channels = new ConcurrentHashMap<YaCyChannel, RSSFeed>();
public static RSSFeed channels(final String channelName) {
final ConcurrentHashMap<String, RSSFeed> channelss = channels;
RSSFeed feed = channelss.get(channelName);
for (YaCyChannel channel: YaCyChannel.values()) {
if (channel.name().equals(channelName)) return channels(channel);
}
return null;
}
public static RSSFeed channels(final YaCyChannel channel) {
RSSFeed feed = channels.get(channel);
if (feed != null) return feed;
feed = new RSSFeed();
feed.setChannel(new RSSMessage(channelName, "", ""));
feed.setChannel(new RSSMessage(channel.name(), "", ""));
feed.maxsize = 100;
channels.put(channelName, feed);
channels.put(channel, feed);
return feed;
}
}

Write Preview
Loading…
Cancel
Save