boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fix for wrong index initialization after network switch

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5203 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent e50934bbd9
commit 6e7d113eac
15 changed files with 37 additions and 24 deletions
Show Stats Download Patch File Download Diff File

2
build.properties
Unescape View File

@ -3,7 +3,7 @@ javacSource=1.5
javacTarget=1.5
# Release Configuration
releaseVersion=0.602
releaseVersion=0.603
stdReleaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
embReleaseFile=yacy_emb_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
proReleaseFile=yacy_pro_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz

3
htroot/Banner.java
Unescape View File

@ -34,6 +34,7 @@ import javax.imageio.ImageIO;
import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaGrafics;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.plasma.plasmaSwitchboardConstants;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacySeed;
@ -65,7 +66,7 @@ public class Banner {
int myppm = 0;
double myqph = 0;
String type = "";
final String network = env.getConfig("network.unit.name", "unspecified").toUpperCase();
final String network = env.getConfig(plasmaSwitchboardConstants.NETWORK_NAME, "unspecified").toUpperCase();
final int peers = sb.webIndex.seedDB.sizeConnected() + 1; // the '+ 1': the own peer is not included in sizeConnected()
long nlinks = sb.webIndex.seedDB.countActiveURL();
long nwords = sb.webIndex.seedDB.countActiveRWI();

4
htroot/ConfigBasic.java
Unescape View File

@ -134,7 +134,7 @@ public class ConfigBasic {
}
// set a use case
String networkName = sb.getConfig("network.unit.name", "");
String networkName = sb.getConfig(plasmaSwitchboardConstants.NETWORK_NAME, "");
if (post != null && post.containsKey("usecase")) {
if (post.get("usecase", "").equals("freeworld") && !networkName.equals("freeworld")) {
// switch to freeworld network
@ -159,7 +159,7 @@ public class ConfigBasic {
}
}
networkName = sb.getConfig("network.unit.name", "");
networkName = sb.getConfig(plasmaSwitchboardConstants.NETWORK_NAME, "");
if (networkName.equals("freeworld")) {
prop.put("setUseCase", 1);
prop.put("setUseCase_freeworldChecked", 1);

2
htroot/ConfigNetwork_p.java
Unescape View File

@ -204,7 +204,7 @@ public class ConfigNetwork_p {
// set network configuration
prop.putHTML("network.unit.definition", sb.getConfig("network.unit.definition", ""));
prop.putHTML("network.unit.name", sb.getConfig("network.unit.name", ""));
prop.putHTML("network.unit.name", sb.getConfig(plasmaSwitchboardConstants.NETWORK_NAME, ""));
prop.putHTML("network.unit.description", sb.getConfig("network.unit.description", ""));
prop.putHTML("network.unit.domain", sb.getConfig("network.unit.domain", ""));
prop.putHTML("network.unit.dht", sb.getConfig("network.unit.dht", ""));

2
htroot/CrawlStart_p.java
Unescape View File

@ -39,7 +39,7 @@ public class CrawlStart_p {
// define visible variables
String a = sb.webIndex.seedDB.mySeed().getPublicAddress();
prop.put("starturl", (sb.getConfig("network.unit.name", "").equals("intranet")) ? "http://" + ((a == null) ? "localhost:" + sb.getConfig("port", "8080") : a) + "/repository/" : "http://");
prop.put("starturl", (sb.getConfig(plasmaSwitchboardConstants.NETWORK_NAME, "").equals("intranet")) ? "http://" + ((a == null) ? "localhost:" + sb.getConfig("port", "8080") : a) + "/repository/" : "http://");
prop.put("proxyPrefetchDepth", env.getConfig("proxyPrefetchDepth", "0"));
prop.put("crawlingDepth", env.getConfig("crawlingDepth", "0"));
prop.put("crawlingFilter", env.getConfig("crawlingFilter", "0"));

3
htroot/Network.java
Unescape View File

@ -39,6 +39,7 @@ import de.anomic.crawler.HTTPLoader;
import de.anomic.http.HttpClient;
import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.plasma.plasmaSwitchboardConstants;
import de.anomic.server.serverCodings;
import de.anomic.server.serverDate;
import de.anomic.server.serverObjects;
@ -61,7 +62,7 @@ public class Network {
final serverObjects prop = new serverObjects();
prop.setLocalized(!(requestHeader.get("PATH")).endsWith(".xml"));
prop.putHTML("page_networkTitle", sb.getConfig("network.unit.description", "unspecified"));
prop.putHTML("page_networkName", sb.getConfig("network.unit.name", "unspecified"));
prop.putHTML("page_networkName", sb.getConfig(plasmaSwitchboardConstants.NETWORK_NAME, "unspecified"));
final boolean overview = (post == null) || (post.get("page", "0").equals("0"));
final String mySeedType = sb.webIndex.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN);

3
htroot/NetworkPicture.java
Unescape View File

@ -27,6 +27,7 @@
import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaGrafics;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.plasma.plasmaSwitchboardConstants;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.ymage.ymageMatrix;
@ -63,7 +64,7 @@ public class NetworkPicture {
if (passiveLimit > 1000000) passiveLimit = 1000000;
if (potentialLimit > 1000000) potentialLimit = 1000000;
if (maxCount > 1000) maxCount = 1000;
return plasmaGrafics.getNetworkPicture(sb.webIndex.seedDB, 10000, width, height, passiveLimit, potentialLimit, maxCount, corona, env.getConfig("network.unit.name", "unspecified"), env.getConfig("network.unit.description", "unspecified"), bgcolor);
return plasmaGrafics.getNetworkPicture(sb.webIndex.seedDB, 10000, width, height, passiveLimit, potentialLimit, maxCount, corona, env.getConfig(plasmaSwitchboardConstants.NETWORK_NAME, "unspecified"), env.getConfig("network.unit.description", "unspecified"), bgcolor);
}
}

4
htroot/Status.java
Unescape View File

@ -205,10 +205,10 @@ public class Status {
}
}
final String peerStatus = ((sb.webIndex.seedDB.mySeed() == null) ? yacySeed.PEERTYPE_VIRGIN : sb.webIndex.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN));
if (peerStatus.equals(yacySeed.PEERTYPE_VIRGIN) && sb.getConfig("network.unit.name", "").equals("freeworld")) {
if (peerStatus.equals(yacySeed.PEERTYPE_VIRGIN) && sb.getConfig(plasmaSwitchboardConstants.NETWORK_NAME, "").equals("freeworld")) {
prop.put(PEERSTATUS, "0");
prop.put("urgentStatusVirgin", "1");
} else if (peerStatus.equals(yacySeed.PEERTYPE_JUNIOR) && sb.getConfig("network.unit.name", "").equals("freeworld")) {
} else if (peerStatus.equals(yacySeed.PEERTYPE_JUNIOR) && sb.getConfig(plasmaSwitchboardConstants.NETWORK_NAME, "").equals("freeworld")) {
prop.put(PEERSTATUS, "1");
prop.put("warningStatusJunior", "1");
} else if (peerStatus.equals(yacySeed.PEERTYPE_SENIOR)) {

5
htroot/yacysearch.java
Unescape View File

@ -139,9 +139,10 @@ public class yacysearch {
}
// SEARCH
//final boolean indexDistributeGranted = sb.getConfig(plasmaSwitchboard.INDEX_DIST_ALLOW, "true").equals("true");
//final boolean indexReceiveGranted = sb.getConfig("allowReceiveIndex", "true").equals("true");
final boolean indexReceiveGranted = sb.getConfigBool(plasmaSwitchboardConstants.INDEX_RECEIVE_ALLOW, true);
global = global && indexReceiveGranted; // if the user does not want indexes from remote peers, it cannot be a global search
//final boolean offline = yacyCore.seedDB.mySeed().isVirgin();
final boolean clustersearch = sb.isRobinsonMode() &&
(sb.getConfig("cluster.mode", "").equals("privatecluster") ||
sb.getConfig("cluster.mode", "").equals("publiccluster"));

2
source/de/anomic/plasma/plasmaGrafics.java
Unescape View File

@ -114,7 +114,7 @@ public class plasmaGrafics {
if (primarySearches == null) return null; // this was a local search and there are no threads
// get a copy of a recent network picture
final ymageMatrix eventPicture = getNetworkPicture(seedDB, 120000, plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", "unspecified"), plasmaSwitchboard.getSwitchboard().getConfig("network.unit.description", "unspecified"), COL_BACKGROUND);
final ymageMatrix eventPicture = getNetworkPicture(seedDB, 120000, plasmaSwitchboard.getSwitchboard().getConfig(plasmaSwitchboardConstants.NETWORK_NAME, "unspecified"), plasmaSwitchboard.getSwitchboard().getConfig("network.unit.description", "unspecified"), COL_BACKGROUND);
//if (eventPicture instanceof ymageMatrix) eventPicture = (ymageMatrix) eventPicture; //new ymageMatrix((ymageMatrix) eventPicture);
// TODO: fix cloning of ymageMatrix pictures

12
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -300,9 +300,12 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
setRemotecrawlPPM(Math.max(1, (int) getConfigLong("network.unit.remotecrawl.speed", 60)));
}
// load the network definition
overwriteNetworkDefinition();
// start indexing management
log.logConfig("Starting Indexing Management");
final String networkName = getConfig("network.unit.name", "");
final String networkName = getConfig(plasmaSwitchboardConstants.NETWORK_NAME, "");
webIndex = new plasmaWordIndex(networkName, log, indexPrimaryPath, indexSecondaryPath, wordCacheMaxCount);
crawlResults = new ResultURLs();
@ -312,9 +315,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
serverInstantBusyThread.oneTimeJob(this, "loadSeedLists", yacyCore.log, 0);
final long startedSeedListAquisition = System.currentTimeMillis();
// load the network definition
overwriteNetworkDefinition();
// set up local robots.txt
this.robotstxtConfig = httpdRobotsTxtConfig.init(this);
@ -728,7 +728,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
setConfig("crawlResponse", "false");
setConfig(plasmaSwitchboardConstants.INDEX_DIST_ALLOW, false);
setConfig(plasmaSwitchboardConstants.INDEX_RECEIVE_ALLOW, false);
webIndex.seedDB.mySeed().setFlagAcceptRemoteIndex(false);
}
// in freeworld network set full p2p mode
@ -737,7 +736,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
setConfig("crawlResponse", "true");
setConfig(plasmaSwitchboardConstants.INDEX_DIST_ALLOW, true);
setConfig(plasmaSwitchboardConstants.INDEX_RECEIVE_ALLOW, true);
webIndex.seedDB.mySeed().setFlagAcceptRemoteIndex(true);
}
}
@ -762,7 +760,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
final File indexPrimaryPath = getConfigPath(plasmaSwitchboardConstants.INDEX_PRIMARY_PATH, plasmaSwitchboardConstants.INDEX_PATH_DEFAULT);
final File indexSecondaryPath = (getConfig(plasmaSwitchboardConstants.INDEX_SECONDARY_PATH, "").length() == 0) ? indexPrimaryPath : new File(getConfig(plasmaSwitchboardConstants.INDEX_SECONDARY_PATH, ""));
final int wordCacheMaxCount = (int) getConfigLong(plasmaSwitchboardConstants.WORDCACHE_MAX_COUNT, 20000);
this.webIndex = new plasmaWordIndex(getConfig("network.unit.name", ""), getLog(), indexPrimaryPath, indexSecondaryPath, wordCacheMaxCount);
this.webIndex = new plasmaWordIndex(getConfig(plasmaSwitchboardConstants.NETWORK_NAME, ""), getLog(), indexPrimaryPath, indexSecondaryPath, wordCacheMaxCount);
}
// start up crawl jobs
continueCrawlJob(plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL);

5
source/de/anomic/plasma/plasmaSwitchboardConstants.java
Unescape View File

@ -531,4 +531,9 @@ public final class plasmaSwitchboardConstants {
*/
public static final String STR_REMOTECRAWLTRIGGER = "REMOTECRAWLTRIGGER: REMOTE CRAWL TO PEER ";
/**
* network properties
*
*/
public static final String NETWORK_NAME = "network.unit.name";
}

4
source/de/anomic/plasma/plasmaWordIndex.java
Unescape View File

@ -116,6 +116,10 @@ public final class plasmaWordIndex implements indexRI {
public yacyPeerActions peerActions;
public plasmaWordIndex(final String networkName, final serverLog log, final File indexPrimaryRoot, final File indexSecondaryRoot, final int entityCacheMaxSize) {
if (networkName == null || networkName.length() == 0) {
log.logSevere("no network name given - shutting down");
System.exit(0);
}
this.log = log;
this.primaryRoot = new File(indexPrimaryRoot, networkName);
this.secondaryRoot = new File(indexSecondaryRoot, networkName);

3
source/de/anomic/yacy/yacyClient.java
Unescape View File

@ -79,6 +79,7 @@ import de.anomic.plasma.plasmaSearchRankingProcess;
import de.anomic.plasma.plasmaSearchRankingProfile;
import de.anomic.plasma.plasmaSnippetCache;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.plasma.plasmaSwitchboardConstants;
import de.anomic.plasma.plasmaWordIndex;
import de.anomic.server.serverByteBuffer;
import de.anomic.server.serverCodings;
@ -1083,7 +1084,7 @@ public final class yacyClient {
"&count=10" +
"&resource=global" +
"&query=" + wordhashe +
"&network.unit.name=" + plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT),
"&network.unit.name=" + plasmaSwitchboard.getSwitchboard().getConfig(plasmaSwitchboardConstants.NETWORK_NAME, yacySeed.DFLT_NETWORK_UNIT),
reqHeader, 10000, target.getHexHash() + ".yacyh");
final HashMap<String, String> result = nxTools.table(content, "UTF-8");
System.out.println("Result=" + result.toString());

7
source/de/anomic/yacy/yacyNetwork.java
Unescape View File

@ -35,6 +35,7 @@ import org.apache.commons.httpclient.methods.multipart.Part;
import de.anomic.http.DefaultCharsetStringPart;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.plasma.plasmaSwitchboardConstants;
import de.anomic.server.serverCodings;
import de.anomic.server.serverDate;
import de.anomic.server.serverObjects;
@ -46,8 +47,8 @@ public class yacyNetwork {
if ((post == null) || (env == null)) return false;
// identify network
final String unitName = post.get("network.unit.name", yacySeed.DFLT_NETWORK_UNIT); // the network unit
if (!unitName.equals(env.getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT))) {
final String unitName = post.get(plasmaSwitchboardConstants.NETWORK_NAME, yacySeed.DFLT_NETWORK_UNIT); // the network unit
if (!unitName.equals(env.getConfig(plasmaSwitchboardConstants.NETWORK_NAME, yacySeed.DFLT_NETWORK_UNIT))) {
return false;
}
@ -86,7 +87,7 @@ public class yacyNetwork {
post.add(new DefaultCharsetStringPart("myUTC", Long.toString(System.currentTimeMillis())));
// network identification
post.add(new DefaultCharsetStringPart("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT)));
post.add(new DefaultCharsetStringPart(plasmaSwitchboardConstants.NETWORK_NAME, plasmaSwitchboard.getSwitchboard().getConfig(plasmaSwitchboardConstants.NETWORK_NAME, yacySeed.DFLT_NETWORK_UNIT)));
// authentification essentials
final String authentificationControl = sb.getConfig("network.unit.protocol.control", "uncontrolled");

Write Preview
Loading…
Cancel
Save