boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added network definition for different YaCy networks

Browse Source 
(needs much more work)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3919 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 18 years ago
parent 1d41ebf489
commit 0e57a8062b
17 changed files with 255 additions and 103 deletions
Show Stats Download Patch File Download Diff File

34
doc/index.html
Unescape View File

@ -101,31 +101,35 @@ globalheader();
<td width="20">&nbsp;</td> <td width="20">&nbsp;</td>
<td width="50%"> <td width="50%">
<b>YaCy is a peer-to-peer application<br>for web search.</b><br><br> <h3>Peer-to-Peer Web Search Engine</h3>
<ul> Anonymous and uncensored search in a distributed and community-driven network of search-engine peers.
<li>This is your web search engine for freedom of information: <br><br><br>
search requests are anonymous, independend and uncensored</li>
<li>No central server, no storage of user behaviour</li>
</ul>
</td>
<h3>Personal Search Appliance</h3>
Create your own search portal with up to 10 million pages in your private web index.
<td width="40">&nbsp;</td> <td width="40">&nbsp;</td>
<td width="50%"> <td width="20%">
<b>YaCy can be used as your own,<br>private search engine</b><br><br> <table>
<ul> <tr><td>We are supported by</td></tr>
<li>Set up a search portal</li> <tr><td><a href="http://suma-ev.de/"><img src="http://suma-ev.de/images/suma-logo-small.jpg" border="0" alt="SuMa-eV"></a></td></tr>
<li>Use YaCy to provide a search function for your own web pages</li> <tr><td><a href="http://suma-ev.de/">Gemeinn&uuml;tziger Verein zur F&ouml;rderung der Suchmaschinen-Technologie und des freien Wissenszugangs</a></td></tr>
<li>Host web pages with the built-in http-server</li> </table><br><br>
</ul> <table>
<tr><td>We collaborate with</td></tr>
<tr><td><a href="http://liebel.fzk.de/"><img src="http://www.kit.edu/fzk/resources/kit/img/kit_logo_large.gif" border="0" alt="Karlsruhe Institute of Technology"></a></td></tr>
<tr><td><a href="http://liebel.fzk.de/">Liebel-Lab at Forschungszentrum Karlsruhe</a></td></tr>
</table><br>
</td> </td>
<td width="20">&nbsp;</td> <td width="20">&nbsp;</td>
</tr></table> </tr></table>
<p>YaCy is GPL licensed, free software. <p>YaCy is GPL licensed, free software.
<p><i>F&uuml;r eine deutsche Dokumentation sehen sie bitte <a href="http://www.yacy-websuche.de/">hier</a></p> <p><i>F&uuml;r eine deutsche Dokumentation sehen sie bitte <a href="http://yacy-websuche.de/wiki/index.php/De:Start">hier</a></p>
<!-- ----- HERE ENDS CONTENT PART ----- --> <!-- ----- HERE ENDS CONTENT PART ----- -->

10
htroot/Network.html
Unescape View File

@ -14,20 +14,20 @@
<div class="SubMenu"> <div class="SubMenu">
<h3>YaCy Network</h3> <h3>YaCy Network</h3>
<ul class="SubMenu"> <ul class="SubMenu">
<li><a href="/Network.html" class="MenuItemLink">Network&nbsp;Overview</a></li> <li><a href="/Network.html" class="MenuItemLink">Network Overview</a></li>
<li><a href="/Network.html?page=1" class="MenuItemLink">Active&nbsp;Peers</a></li> <li><a href="/Network.html?page=1" class="MenuItemLink">Active&nbsp;Peers</a></li>
<li><a href="/Network.html?page=2" class="MenuItemLink">Passive&nbsp;Peers</a></li> <li><a href="/Network.html?page=2" class="MenuItemLink">Passive&nbsp;Peers</a></li>
<li><a href="/Network.html?page=3" class="MenuItemLink">Potential&nbsp;Peers</a></li> <li><a href="/Network.html?page=3" class="MenuItemLink">Potential&nbsp;Peers</a></li>
</ul> </ul>
</div> </div>
#(page)# #(page)#
<h2>Network Overview</h2> <h2>Overview of '#[networkName]#' Network</h2>
:: ::
<h2>Active Peers</h2> <h2>Active Peers in '#[networkName]#' Network</h2>
:: ::
<h2>Passive Peers</h2> <h2>Passive Peers in '#[networkName]#' Network</h2>
:: ::
<h2>Potential Peers</h2> <h2>Potential Peers in '#[networkName]#' Network</h2>
:: ::
<h2>Manually contacting Peer</h2> <h2>Manually contacting Peer</h2>
#(/page)# #(/page)#

2
htroot/Network.java
Unescape View File

@ -78,6 +78,8 @@ public class Network {
// return variable that accumulates replacements // return variable that accumulates replacements
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
prop.put("page_networkTitle", sb.getConfig("network.unit.description", "unspecified"));
prop.put("page_networkName", sb.getConfig("network.unit.name", "unspecified"));
final boolean overview = (post == null) || (post.get("page", "0").equals("0")); final boolean overview = (post == null) || (post.get("page", "0").equals("0"));
final String mySeedType = yacyCore.seedDB.mySeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN); final String mySeedType = yacyCore.seedDB.mySeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN);

3
htroot/NetworkPicture.java
Unescape View File

@ -43,7 +43,6 @@
// the intact and unchanged copyright notice. // the intact and unchanged copyright notice.
// Contributions and changes to the program code must be marked as such. // Contributions and changes to the program code must be marked as such.
import de.anomic.http.httpHeader; import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaGrafics; import de.anomic.plasma.plasmaGrafics;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
@ -79,7 +78,7 @@ public class NetworkPicture {
if (passiveLimit > 1000000) passiveLimit = 1000000; if (passiveLimit > 1000000) passiveLimit = 1000000;
if (potentialLimit > 1000000) potentialLimit = 1000000; if (potentialLimit > 1000000) potentialLimit = 1000000;
if (maxCount > 1000) maxCount = 1000; if (maxCount > 1000) maxCount = 1000;
return plasmaGrafics.getNetworkPicture(10000, width, height, passiveLimit, potentialLimit, maxCount, corona); return plasmaGrafics.getNetworkPicture(10000, width, height, passiveLimit, potentialLimit, maxCount, corona, env.getConfig("network.unit.name", "unspecified"), env.getConfig("network.unit.description", "unspecified"));
} }
} }

4
htroot/Status.java
Unescape View File

@ -119,10 +119,6 @@ public class Status {
} }
} }
/*
versionProbe=http://www.anomic.de/AnomicHTTPProxy/release.txt
superseedFile=superseed.txt
*/
// update seed info // update seed info
yacyCore.peerActions.updateMySeed(); yacyCore.peerActions.updateMySeed();

27
source/de/anomic/http/httpc.java
Unescape View File

@ -1373,6 +1373,33 @@ do upload
return a; return a;
} }
public static Map loadHashMap(URL url, httpRemoteProxyConfig proxy) {
try {
// should we use the proxy?
boolean useProxy = (proxy != null) &&
(proxy.useProxy()) &&
(proxy.useProxy4Yacy());
// sending request
final HashMap result = nxTools.table(
httpc.wget(
url,
url.getHost(),
8000,
null,
null,
(useProxy) ? proxy : null,
null
)
, "UTF-8");
if (result == null) return new HashMap();
return result;
} catch (Exception e) {
return new HashMap();
}
}
public static httpHeader whead( public static httpHeader whead(
URL url, URL url,
String vhost, String vhost,

1
source/de/anomic/plasma/plasmaCrawlLURL.java
Unescape View File

@ -187,6 +187,7 @@ public final class plasmaCrawlLURL {
oldEntry = null; oldEntry = null;
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace();
oldEntry = null; oldEntry = null;
} }
if ((oldEntry != null) && (entry.isOlder(oldEntry))) { if ((oldEntry != null) && (entry.isOlder(oldEntry))) {

27
source/de/anomic/plasma/plasmaGrafics.java
Unescape View File

@ -127,7 +127,7 @@ public class plasmaGrafics {
if (primarySearches == null) return null; // this was a local search and there are no threads if (primarySearches == null) return null; // this was a local search and there are no threads
// get a copy of a recent network picture // get a copy of a recent network picture
ymageMatrix eventPicture = getNetworkPicture(120000); ymageMatrix eventPicture = getNetworkPicture(120000, plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", "unspecified"), plasmaSwitchboard.getSwitchboard().getConfig("network.unit.description", "unspecified"));
if (eventPicture instanceof ymageMatrix) eventPicture = (ymageMatrix) eventPicture; //new ymageMatrix((ymageMatrix) eventPicture); if (eventPicture instanceof ymageMatrix) eventPicture = (ymageMatrix) eventPicture; //new ymageMatrix((ymageMatrix) eventPicture);
// TODO: fix cloning of ymageMatrix pictures // TODO: fix cloning of ymageMatrix pictures
@ -172,18 +172,18 @@ public class plasmaGrafics {
return eventPicture; return eventPicture;
} }
public static ymageMatrix getNetworkPicture(long maxAge) { public static ymageMatrix getNetworkPicture(long maxAge, String networkName, String networkTitle) {
return getNetworkPicture(maxAge, 640, 480, 300, 300, 1000, true); return getNetworkPicture(maxAge, 640, 480, 300, 300, 1000, true, networkName, networkTitle);
} }
public static ymageMatrix getNetworkPicture(long maxAge, int width, int height, int passiveLimit, int potentialLimit, int maxCount, boolean corona) { public static ymageMatrix getNetworkPicture(long maxAge, int width, int height, int passiveLimit, int potentialLimit, int maxCount, boolean corona, String networkName, String networkTitle) {
if ((networkPicture == null) || ((System.currentTimeMillis() - networkPictureDate) > maxAge)) { if ((networkPicture == null) || ((System.currentTimeMillis() - networkPictureDate) > maxAge)) {
drawNetworkPicture(width, height, passiveLimit, potentialLimit, maxCount, corona); drawNetworkPicture(width, height, passiveLimit, potentialLimit, maxCount, corona, networkName, networkTitle);
} }
return networkPicture; return networkPicture;
} }
private static void drawNetworkPicture(int width, int height, int passiveLimit, int potentialLimit, int maxCount, boolean corona) { private static void drawNetworkPicture(int width, int height, int passiveLimit, int potentialLimit, int maxCount, boolean corona, String networkName, String networkTitle) {
int innerradius = Math.min(width, height) / 5; int innerradius = Math.min(width, height) / 5;
int outerradius = innerradius + innerradius * yacyCore.seedDB.sizeConnected() / 100; int outerradius = innerradius + innerradius * yacyCore.seedDB.sizeConnected() / 100;
@ -196,7 +196,7 @@ public class plasmaGrafics {
// draw network circle // draw network circle
networkPicture.setColor(COL_DHTCIRCLE); networkPicture.setColor(COL_DHTCIRCLE);
networkPicture.arc(width / 2, height / 2, innerradius - 20, innerradius + 20, 0, 360); networkPicture.arc(width / 2, height / 2 + 20, innerradius - 20, innerradius + 20, 0, 360);
//System.out.println("Seed Maximum distance is " + yacySeed.maxDHTDistance); //System.out.println("Seed Maximum distance is " + yacySeed.maxDHTDistance);
//System.out.println("Seed Minimum distance is " + yacySeed.minDHTNumber); //System.out.println("Seed Minimum distance is " + yacySeed.minDHTNumber);
@ -212,7 +212,7 @@ public class plasmaGrafics {
while (e.hasMoreElements() && count < maxCount) { while (e.hasMoreElements() && count < maxCount) {
seed = (yacySeed) e.nextElement(); seed = (yacySeed) e.nextElement();
if (seed != null) { if (seed != null) {
drawNetworkPicturePeer(networkPicture, width / 2, height / 2, innerradius, outerradius, seed, COL_ACTIVE_DOT, COL_ACTIVE_LINE, COL_ACTIVE_TEXT, corona); drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, seed, COL_ACTIVE_DOT, COL_ACTIVE_LINE, COL_ACTIVE_TEXT, corona);
count++; count++;
} }
} }
@ -226,7 +226,7 @@ public class plasmaGrafics {
if (seed != null) { if (seed != null) {
lastseen = Math.abs((System.currentTimeMillis() - seed.getLastSeenUTC()) / 1000 / 60); lastseen = Math.abs((System.currentTimeMillis() - seed.getLastSeenUTC()) / 1000 / 60);
if (lastseen > passiveLimit) break; // we have enough, this list is sorted so we don't miss anything if (lastseen > passiveLimit) break; // we have enough, this list is sorted so we don't miss anything
drawNetworkPicturePeer(networkPicture, width / 2, height / 2, innerradius, outerradius, seed, COL_PASSIVE_DOT, COL_PASSIVE_LINE, COL_PASSIVE_TEXT, corona); drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, seed, COL_PASSIVE_DOT, COL_PASSIVE_LINE, COL_PASSIVE_TEXT, corona);
count++; count++;
} }
} }
@ -240,21 +240,22 @@ public class plasmaGrafics {
if (seed != null) { if (seed != null) {
lastseen = Math.abs((System.currentTimeMillis() - seed.getLastSeenUTC()) / 1000 / 60); lastseen = Math.abs((System.currentTimeMillis() - seed.getLastSeenUTC()) / 1000 / 60);
if (lastseen > potentialLimit) break; // we have enough, this list is sorted so we don't miss anything if (lastseen > potentialLimit) break; // we have enough, this list is sorted so we don't miss anything
drawNetworkPicturePeer(networkPicture, width / 2, height / 2, innerradius, outerradius, seed, COL_POTENTIAL_DOT, COL_POTENTIAL_LINE, COL_POTENTIAL_TEXT, corona); drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, seed, COL_POTENTIAL_DOT, COL_POTENTIAL_LINE, COL_POTENTIAL_TEXT, corona);
count++; count++;
} }
} }
totalCount += count; totalCount += count;
// draw my own peer // draw my own peer
drawNetworkPicturePeer(networkPicture, width / 2, height / 2, innerradius, outerradius, yacyCore.seedDB.mySeed, COL_WE_DOT, COL_WE_LINE, COL_WE_TEXT, corona); drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, yacyCore.seedDB.mySeed, COL_WE_DOT, COL_WE_LINE, COL_WE_TEXT, corona);
// draw description // draw description
networkPicture.setColor(COL_HEADLINE); networkPicture.setColor(COL_HEADLINE);
networkPicture.setMode(ymageMatrix.MODE_SUB); networkPicture.setMode(ymageMatrix.MODE_SUB);
ymageToolPrint.print(networkPicture, 2, 8, 0, "THE YACY NETWORK", -1); ymageToolPrint.print(networkPicture, 2, 8, 0, "YACY NETWORK '" + networkName.toUpperCase() + "'", -1);
ymageToolPrint.print(networkPicture, 2, 16, 0, "DRAWING OF " + totalCount + " SELECTED PEERS", -1); ymageToolPrint.print(networkPicture, 2, 16, 0, networkTitle.toUpperCase(), -1);
ymageToolPrint.print(networkPicture, width - 2, 8, 0, "SNAPSHOT FROM " + new Date().toString().toUpperCase(), 1); ymageToolPrint.print(networkPicture, width - 2, 8, 0, "SNAPSHOT FROM " + new Date().toString().toUpperCase(), 1);
ymageToolPrint.print(networkPicture, width - 2, 16, 0, "DRAWING OF " + totalCount + " SELECTED PEERS", 1);
// set timestamp // set timestamp
networkPictureDate = System.currentTimeMillis(); networkPictureDate = System.currentTimeMillis();

46
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -893,9 +893,47 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
// set loglevel and log // set loglevel and log
setLog(new serverLog("PLASMA")); setLog(new serverLog("PLASMA"));
if (applyPro) this.log.logInfo("This is the pro-version of YaCy"); if (applyPro) this.log.logInfo("This is the pro-version of YaCy");
// remote proxy configuration
this.remoteProxyConfig = httpRemoteProxyConfig.init(this);
this.log.logConfig("Remote proxy configuration:\n" + this.remoteProxyConfig.toString());
// load network configuration into settings
String networkUnitDefinition = getConfig("network.unit.definition", "yacy.network.unit");
String networkGroupDefinition = getConfig("network.group.definition", "yacy.network.group");
// include additional network definition properties into our settings
// note that these properties cannot be set in the application because they are
// _always_ overwritten each time with the default values. This is done so on purpose.
// the network definition should be made either consistent for all peers,
// or independently using a bootstrap URL
Map initProps;
if (networkUnitDefinition.startsWith("http://")) {
try {
this.setConfig(httpc.loadHashMap(new URL(networkUnitDefinition), remoteProxyConfig));
} catch (MalformedURLException e) {
}
} else {
File networkUnitDefinitionFile = new File(rootPath, networkUnitDefinition);
if (networkUnitDefinitionFile.exists()) {
initProps = serverFileUtils.loadHashMap(networkUnitDefinitionFile);
this.setConfig(initProps);
}
}
if (networkGroupDefinition.startsWith("http://")) {
try {
this.setConfig(httpc.loadHashMap(new URL(networkGroupDefinition), remoteProxyConfig));
} catch (MalformedURLException e) {
}
} else {
File networkGroupDefinitionFile = new File(rootPath, networkGroupDefinition);
if (networkGroupDefinitionFile.exists()) {
initProps = serverFileUtils.loadHashMap(networkGroupDefinitionFile);
this.setConfig(initProps);
}
}
// load values from configs // load values from configs
this.plasmaPath = new File(rootPath, getConfig(DBPATH, DBPATH_DEFAULT)); this.plasmaPath = new File(rootPath, getConfig(DBPATH, DBPATH_DEFAULT));
this.log.logConfig("Plasma DB Path: " + this.plasmaPath.toString()); this.log.logConfig("Plasma DB Path: " + this.plasmaPath.toString());
@ -913,12 +951,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
this.workPath = new File(rootPath, getConfig(WORK_PATH, WORK_PATH_DEFAULT)); this.workPath = new File(rootPath, getConfig(WORK_PATH, WORK_PATH_DEFAULT));
this.log.logConfig("Work Path: " + this.workPath.toString()); this.log.logConfig("Work Path: " + this.workPath.toString());
/* ============================================================================
* Remote Proxy configuration
* ============================================================================ */
this.remoteProxyConfig = httpRemoteProxyConfig.init(this);
this.log.logConfig("Remote proxy configuration:\n" + this.remoteProxyConfig.toString());
// set up local robots.txt // set up local robots.txt
this.robotstxtConfig = httpdRobotsTxtConfig.init(this); this.robotstxtConfig = httpdRobotsTxtConfig.init(this);

14
source/de/anomic/server/serverAbstractSwitch.java
Unescape View File

@ -84,7 +84,6 @@ public abstract class serverAbstractSwitch implements serverSwitch {
// predefine init's // predefine init's
Map initProps; Map initProps;
configRemoved = new HashMap();
if (initFile.exists()) if (initFile.exists())
initProps = serverFileUtils.loadHashMap(initFile); initProps = serverFileUtils.loadHashMap(initFile);
else else
@ -117,9 +116,9 @@ public abstract class serverAbstractSwitch implements serverSwitch {
else else
configProps = new HashMap(); configProps = new HashMap();
// remove all values from config that do not appear in init
configRemoved = new HashMap();
synchronized (configProps) { synchronized (configProps) {
// remove all values from config that do not appear in init
// (out-dated settings)
i = configProps.keySet().iterator(); i = configProps.keySet().iterator();
String key; String key;
while (i.hasNext()) { while (i.hasNext()) {
@ -212,6 +211,15 @@ public abstract class serverAbstractSwitch implements serverSwitch {
return accessTracker.keySet().iterator(); return accessTracker.keySet().iterator();
} }
public void setConfig(Map otherConfigs) {
Iterator i = otherConfigs.entrySet().iterator();
Map.Entry entry;
while (i.hasNext()) {
entry = (Map.Entry) i.next();
setConfig((String) entry.getKey(), (String) entry.getValue());
}
}
public void setConfig(String key, boolean value) { public void setConfig(String key, boolean value) {
setConfig(key, (value) ? "true" : "false"); setConfig(key, (value) ? "true" : "false");
} }

1
source/de/anomic/server/serverSwitch.java
Unescape View File

@ -96,6 +96,7 @@ public interface serverSwitch {
public Iterator /*of serverThread-Names (String)*/ threadNames(); public Iterator /*of serverThread-Names (String)*/ threadNames();
// the switchboard can be used to set and read properties // the switchboard can be used to set and read properties
public void setConfig(Map otherConfigs);
public void setConfig(String key, long value); public void setConfig(String key, long value);
public void setConfig(String key, String value); public void setConfig(String key, String value);
public String getConfig(String key, String dflt); public String getConfig(String key, String dflt);

26
source/de/anomic/yacy/yacyClient.java
Unescape View File

@ -259,9 +259,9 @@ public final class yacyClient {
final String key = crypt.randomSalt(); final String key = crypt.randomSalt();
try { try {
// should we use the proxy? // should we use the proxy?
boolean useProxy = (yacyCore.seedDB.sb.remoteProxyConfig != null) && boolean useProxy = (plasmaSwitchboard.getSwitchboard().remoteProxyConfig != null) &&
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) && (plasmaSwitchboard.getSwitchboard().remoteProxyConfig.useProxy()) &&
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy()); (plasmaSwitchboard.getSwitchboard().remoteProxyConfig.useProxy4Yacy());
// sending request // sending request
final HashMap result = nxTools.table( final HashMap result = nxTools.table(
@ -278,7 +278,7 @@ public final class yacyClient {
8000, 8000,
null, null,
null, null,
(useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null, (useProxy)? plasmaSwitchboard.getSwitchboard().remoteProxyConfig:null,
null null
) )
, "UTF-8"); , "UTF-8");
@ -526,18 +526,28 @@ public final class yacyClient {
assert (urlEntry.hash().length() == 12) : "urlEntry.hash() = " + urlEntry.hash(); assert (urlEntry.hash().length() == 12) : "urlEntry.hash() = " + urlEntry.hash();
if (urlEntry.hash().length() != 12) continue; // bad url hash if (urlEntry.hash().length() != 12) continue; // bad url hash
indexURLEntry.Components comp = urlEntry.comp(); indexURLEntry.Components comp = urlEntry.comp();
if (blacklist.isListed(plasmaURLPattern.BLACKLIST_SEARCH, comp.url())) continue; // block with backlist if (blacklist.isListed(plasmaURLPattern.BLACKLIST_SEARCH, comp.url())) {
urlManager.store(urlEntry); yacyCore.log.logInfo("remote search (client): filtered blacklisted url " + comp.url() + " from peer " + targetPeer.getName());
urlManager.stack(urlEntry, yacyCore.seedDB.mySeed.hash, targetPeer.hash, 2); continue; // block with backlist
}
// save the url entry // save the url entry
indexRWIEntry entry; indexRWIEntry entry;
if (urlEntry.word() == null) { if (urlEntry.word() == null) {
yacyCore.log.logWarning("DEBUG-SEARCH: no word attached from peer " + targetPeer.getName() + ", version " + targetPeer.getVersion()); yacyCore.log.logWarning("remote search (client): no word attached from peer " + targetPeer.getName() + ", version " + targetPeer.getVersion());
continue; // no word attached continue; // no word attached
} }
// the search-result-url transports all the attributes of word indexes // the search-result-url transports all the attributes of word indexes
entry = urlEntry.word(); entry = urlEntry.word();
if (!(entry.urlHash().equals(urlEntry.hash()))) {
yacyCore.log.logInfo("remote search (client): url-hash " + urlEntry.hash() + " does not belong to word-attached-hash " + entry.urlHash() + "; url = " + comp.url() + " from peer " + targetPeer.getName());
continue; // spammed
}
// passed all checks, store url
urlManager.store(urlEntry);
urlManager.stack(urlEntry, yacyCore.seedDB.mySeed.hash, targetPeer.hash, 2);
if (urlEntry.snippet() != null) { if (urlEntry.snippet() != null) {
// we don't store the snippets along the url entry, because they are search-specific. // we don't store the snippets along the url entry, because they are search-specific.

4
source/de/anomic/yacy/yacyCore.java
Unescape View File

@ -215,8 +215,8 @@ public class yacyCore {
// deploy peer actions // deploy peer actions
peerActions = new yacyPeerActions(seedDB, switchboard, peerActions = new yacyPeerActions(seedDB, switchboard,
new File(sb.getRootPath(), sb.getConfig("superseedFile", "superseed.txt")), new File(sb.getRootPath(), sb.getConfig("network.unit.bootstrap.file", "superseed.txt")),
switchboard.getConfig("superseedLocation", "http://www.yacy.net/yacy/superseed.txt")); switchboard.getConfig("network.unit.bootstrap.url", "http://www.yacy.net/yacy/superseed.txt"));
dhtAgent = new yacyDHTAction(seedDB); dhtAgent = new yacyDHTAction(seedDB);
peerActions.deploy(dhtAgent); peerActions.deploy(dhtAgent);
peerActions.deploy(new yacyNewsAction(newsPool)); peerActions.deploy(new yacyNewsAction(newsPool));

3
source/yacy.java
Unescape View File

@ -253,8 +253,7 @@ public final class yacy {
f = new File(homePath, "DATA/yacy.running"); f = new File(homePath, "DATA/yacy.running");
if (f.exists()) { // another instance running? VM crash? User will have to care about this if (f.exists()) { // another instance running? VM crash? User will have to care about this
serverLog.logSevere("STARTUP", "the file " + f + " exists, this usually means that another instance of YaCy is using this DATA-folder."); serverLog.logSevere("STARTUP", "WARNING: the file " + f + " exists, this usually means that a YaCy instance is still running");
serverLog.logSevere("STARTUP", "please make sure that DATA can be used exclusively by one YaCy. Quitting...");
f.delete(); f.delete();
} }
f.createNewFile(); f.createNewFile();

47
yacy.init
Unescape View File

@ -64,43 +64,14 @@ pkcs12ImportPwd =
# value is in milliseconds, default is one hour # value is in milliseconds, default is one hour
server.maxTrackingTime = 3600000 server.maxTrackingTime = 3600000
# peer-to-peer construction for distributed search # Network Definition
# we have several stages: # There can be separate YaCy networks, and managed sub-groups of the general network.
# 1st: a file within every distribution that has a list of URLs: # The essentials of the network definition are attached in separate property files.
# -> this is the superseed file # The property here can also be a url where the definition can be loaded.
# 2nd: the files that can be retrieved by the superseeds' URLs # In case of privately managed networks, this configuration must be changed BEFORE it is released
# are called seed list-files. # to the members of the separated network peers.
# -> the seed list-files contain IP/port combinations of running network.unit.definition = yacy.network.unit
# AnomicHTTPProxies network.group.definition = yacy.network.group
# 3rd: the peers that are targeted within the seed files are called superpeers
# 4th: the superpeers hold and share a list of all client/search/crawl peers
#
# some superpeers should be able to create again seed list-files.
# These superpeers must upload their IP or their list of peer-IP's to a
# ftp location to create the seed list-file.
# Everyone who do so should mail his/her new seed location to mc<at>anomic.de
# The seed list-file location will then be included in the superseed file.
# This superseed file is available then at two localtions:
# - it is included in every distribution and
# - updated through a specific URL-location
# we see the file name and the URL of the superseed here:
superseedFile=superseed.txt
superseedLocation=http://www.yacy.net/superseed.txt
# network definition
# we distiguish local and global networks. Each network type can have different user groups
# groups can be uncontrolled, moderated or controlled
# Each group has a group master, but in case the group is uncontrolled the only task of the
# group master is to provide a bootstrap file which contains peer addresses of peers that
# are member of the group. All attributes are given in a single network description string
# of the form:
# network = <group-name>:<network-name>:<network-domain>:{'uncontrolled'|'moderated'|'controlled'}:<bootstrap-uri>
# in case that the network is the uncontrolled global dht-managed community for everybody,
# the details defaults to:
# network = all:world:global:uncontrolled:http://yacy.net/
# the network-uri must have a sub-path yacy/seed.txt containing a list of urls pointing to the
# peer-address of peers within the group of that network
# several network definition strings can be listed
# clusters within a network: # clusters within a network:
# every network can have an unlimited number of clusters. Clusters may be also completely # every network can have an unlimited number of clusters. Clusters may be also completely
@ -716,7 +687,7 @@ useYacyReferer = true
useYacyReferer__pro = false useYacyReferer__pro = false
# allow only 443(https-port) for https-proxy? # allow only 443(https-port) for https-proxy?
# if you want to tunnel other protokols, set to false # if you want to tunnel other protocols, set to false
secureHttps = true secureHttps = true
# specifies if the httpdFileHandler should cache # specifies if the httpdFileHandler should cache

29
yacy.network.group
Unescape View File

@ -0,0 +1,29 @@
# YaCy Network Group Definition
# -----------------------------
# This is an addition to the yacy.network.unit configuration file.
# This file is adressed by the network.group.definition property in yacy.init
# The purpose of a group within a network is that some parts of a network may be managed independently,
# while the content of the network stays private for the whole network, mostly for a special purpose.
# This file needs to be configured if someone wants to participate with several peers to the network,
# but wants to manage the group of own peers with a single master peer.
# One example application would be the use of computing-teams
# Defintion of property domains:
# network.group.name = <any word, name of network>
# network.group.description = <any string, just informal; appears in network graphic>
# network.group.administration.control = 'uncontrolled'|'moderated'|'controlled'
# network.group.administration.manager = <a .yacy or .yacyh - domain, naming the manager peer>
# network.group.administration.request.authentification.method = 'salted-magic'
# network.group.administration.request.authentification.essentials = <any string, a secret password>
# -----------------------------------------------------------------#
# this is a work in progress. disabled properties are not yet used #
# -----------------------------------------------------------------#
# properties for group access administrator-authentification:
#network.group.name = everybody
#network.group.description = Our Peer-Group
#network.group.administration.control = uncontrolled
#network.group.administration.manager =
#network.group.administration.request.authentification.method = salted-magic
#network.group.administration.request.authentification.essentials =

72
yacy.network.unit
Unescape View File

@ -0,0 +1,72 @@
# YaCy Network Unit Definition
# ----------------------------
# There is not only a single global YaCy network, but there can be also separated networks
# This file describes the properties of the network, how it is bootstraped, if it is moderated
# and so on. It is also possible to describe authentification methods for in-protocol communication
# To distinguish between index areas where the shared web index is for a intranet or the public internet,
# we distinguish local and global networks. Each network type can have different user groups.
# Groups can be uncontrolled, moderated or controlled. The whole network may also have a global master.
# This file is adressed by the network.unit.definition property in yacy.init. If necessary this
# property can also address a URL where a network definition file can be hosted, so that a network
# can easily be moderated from a single central. In case of the public network of all independent peers
# for global indexing, the network definition is attached to the release and defines no network master or
# any other supervision.
# peer-to-peer construction using the bootstrap properties:
# 1st: a file within every distribution that has a list of URLs,
# or a location with a file containing a list of URLs:
# -> these are the superseed files (there are only two of these files)
# 2nd: the files that can be retrieved by the superseeds' URLs
# are called seed list-files. (there can be many of those)
# -> the seed list-files contain b.o. IP/port combinations of running YaCy peers
# -> seed-list files are written by principal peers
# Control Modes for Peer Groups:
# we have three different types of moderation:
# uncontrolled: only the peer's administrator may use protected pages of the YaCy interface
# moderated : the unit/group moderator may access some protected pages i.e. Crawl Start
# controlled : the unit/group moderator may access all pages of the peers within the same unit/group
# Defintion of property domains:
# network.unit.name = <any word, name of network; appears in network graphic>
# network.unit.description = <any string, just informal; appears in network graphic>
# network.unit.domain = 'global'|'local'
# network.unit.dhtredundancy = <integer number, 0 means no DHT enabled>
# network.unit.bootstrap.file = <a file in the application directory containing urls to seedlists>
# network.unit.bootstrap.url = <a url to a bootstrap.file containing urls to seedlists>
# network.unit.protocol.control = 'uncontrolled'|'moderated'|'controlled'
# network.unit.protocol.request.authentification.method = 'salted-magic'
# network.unit.protocol.request.authentification.essentials = <any string, a secret password>
# network.unit.protocol.response.authentification.method = 'challenge-pwresponse'
# network.unit.protocol.response.authentification.essentials = <a challenge string>':'<a secret response-string>
# network.unit.administration.control = 'uncontrolled'|'moderated'|'controlled'
# network.unit.administration.manager = <a .yacy or .yacyh - domain, naming the manager peer>
# network.unit.administration.request.authentification.method = 'salted-magic'
# network.unit.administration.request.authentification.essentials = <any string, a secret password>
# -----------------------------------------------------------------#
# this is a work in progress. disabled properties are not yet used #
# -----------------------------------------------------------------#
# general network definition
network.unit.name = freeworld
network.unit.description = Public YaCy Community
#network.unit.domain = global
network.unit.dhtredundancy.junior = 1
network.unit.dhtredundancy.senior = 3
network.unit.bootstrap.file = superseed.txt
network.unit.bootstrap.url = http://www.yacy.net/superseed.txt
# properties for in-protocol response authentification:
#network.unit.protocol.control = uncontrolled
#network.unit.protocol.request.authentification.method = salted-magic
#network.unit.protocol.request.authentification.essentials =
#network.unit.protocol.response.authentification.method = challenge-pwresponse
#network.unit.protocol.response.authentification.essentials =
# properties for unit access administrator-authentification:
#network.unit.administration.control = uncontrolled
#network.unit.administration.manager =
#network.unit.administration.request.authentification.method = salted-magic
#network.unit.administration.request.authentification.essentials =
Write Preview
Loading…
Cancel
Save