boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

bugfixes

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@204 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 20 years ago
parent 650ca3955a
commit e89ded9e41
7 changed files with 41 additions and 17 deletions
Show Stats Download Patch File Download Diff File

4
build.properties
Unescape View File

@ -3,9 +3,9 @@ javacSource=1.4
javacTarget=1.4 javacTarget=1.4
# Release Configuration # Release Configuration
releaseVersion=0.375 releaseVersion=0.376
releaseFile=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz releaseFile=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
releaseDir=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr} releaseDir=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}
releaseNr=$Revision$ releaseNr=$Revision$
# defining some file/directory access rights # defining some file/directory access rights

17
htroot/Network.java
Unescape View File

@ -167,9 +167,18 @@ public class Network {
boolean complete = post.containsKey("ip"); boolean complete = post.containsKey("ip");
Enumeration e = null; Enumeration e = null;
switch (page) { switch (page) {
case 1 : e = yacyCore.seedDB.seedsSortedConnected(post.get("order", "down").equals("up"), post.get("sort", "LCount")); break; case 1 :
case 2 : e = yacyCore.seedDB.seedsSortedDisconnected(post.get("order", "up").equals("up"), post.get("sort", "LastSeen")); break; e = yacyCore.seedDB.seedsSortedConnected(post.get("order", "down").equals("up"), post.get("sort", "LCount"));
case 3 : e = yacyCore.seedDB.seedsSortedPotential(post.get("order", "up").equals("up"), post.get("sort", "LastSeen")); break; prop.put("table_total", yacyCore.seedDB.sizeConnected());
break;
case 2 :
e = yacyCore.seedDB.seedsSortedDisconnected(post.get("order", "up").equals("up"), post.get("sort", "LastSeen"));
prop.put("table_total", yacyCore.seedDB.sizeDisconnected());
break;
case 3 :
e = yacyCore.seedDB.seedsSortedPotential(post.get("order", "up").equals("up"), post.get("sort", "LastSeen"));
prop.put("table_total", yacyCore.seedDB.sizePotential());
break;
} }
while ((e.hasMoreElements()) && (conCount < maxCount)) { while ((e.hasMoreElements()) && (conCount < maxCount)) {
seed = (yacySeed) e.nextElement(); seed = (yacySeed) e.nextElement();
@ -228,7 +237,7 @@ public class Network {
prop.put("table_list", conCount); prop.put("table_list", conCount);
prop.put("table", 1); prop.put("table", 1);
prop.put("table_num", conCount); prop.put("table_num", conCount);
prop.put("table_total", (maxCount > conCount) ? conCount : maxCount); //prop.put("table_total", (maxCount > conCount) ? conCount : maxCount);
prop.put("table_complete", ((complete)? 1 : 0) ); prop.put("table_complete", ((complete)? 1 : 0) );
} }
} }

2
htroot/index.java
Unescape View File

@ -114,7 +114,7 @@ public class index {
// process search words // process search words
String querystring = (String) post.get("search", ""); String querystring = (String) post.get("search", "");
try { sb.facilityDB.update("zeitgeist", querystring, post); } catch (IOException e) {} try { sb.facilityDB.update("zeitgeist", querystring, post); } catch (Exception e) {}
TreeSet query = cleanQuery(querystring); TreeSet query = cleanQuery(querystring);
// filter out stopwords // filter out stopwords
TreeSet filtered = kelondroMSetTools.joinConstructive(query, plasmaSwitchboard.stopwords); TreeSet filtered = kelondroMSetTools.joinConstructive(query, plasmaSwitchboard.stopwords);

9
htroot/yacy/crawlReceipt.java
Unescape View File

@ -45,6 +45,7 @@
import de.anomic.http.httpHeader; import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaCrawlNURL; import de.anomic.plasma.plasmaCrawlNURL;
import de.anomic.plasma.plasmaCrawlLURL;
import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.plasma.plasmaURL; import de.anomic.plasma.plasmaURL;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
@ -113,6 +114,14 @@ public class crawlReceipt {
switchboard.loadedURL.newEntry(propStr, true, youare, iam, 1); switchboard.loadedURL.newEntry(propStr, true, youare, iam, 1);
switchboard.noticeURL.remove(urlhash); switchboard.noticeURL.remove(urlhash);
// write log
plasmaCrawlLURL.entry entry = switchboard.loadedURL.getEntry(urlhash);
if (entry == null) {
switchboard.getLog().logError("RECEIVED wrong RECEIPT for hash " + urlhash + " from peer " + iam);
} else {
switchboard.getLog().logInfo("RECEIVED RECEIPT for url " + entry.url().toString());
}
// ready for more // ready for more
prop.put("delay", "10"); prop.put("delay", "10");
} else { } else {

5
source/de/anomic/kelondro/kelondroMap.java
Unescape View File

@ -125,7 +125,10 @@ public class kelondroMap {
if (accfields != null) for (int i = 0; i < accfields.length; i++) accMap.put(accfields[i], accumulator[i]); if (accfields != null) for (int i = 0; i < accfields.length; i++) accMap.put(accfields[i], accumulator[i]);
} }
public int keySize() {
return dyn.columnSize(0);
}
public synchronized void set(String key, Map newMap) throws IOException { public synchronized void set(String key, Map newMap) throws IOException {
// update elementCount // update elementCount
if ((sortfields != null) || (accfields != null)) { if ((sortfields != null) || (accfields != null)) {

3
source/de/anomic/kelondro/kelondroTables.java
Unescape View File

@ -101,6 +101,7 @@ public class kelondroTables {
public synchronized void update(String tablename, String key, Map map) throws IOException { public synchronized void update(String tablename, String key, Map map) throws IOException {
kelondroMap table = (kelondroMap) mTables.get(tablename); kelondroMap table = (kelondroMap) mTables.get(tablename);
if (table == null) throw new RuntimeException("kelondroTables.update: map table '" + tablename + "' does not exist."); if (table == null) throw new RuntimeException("kelondroTables.update: map table '" + tablename + "' does not exist.");
if (key.length() > table.keySize()) key = key.substring(0, table.keySize());
table.set(key, map); table.set(key, map);
mTables.put(tablename, table); mTables.put(tablename, table);
} }
@ -122,6 +123,7 @@ public class kelondroTables {
public synchronized Map selectMap(String tablename, String key) throws IOException { public synchronized Map selectMap(String tablename, String key) throws IOException {
kelondroMap table = (kelondroMap) mTables.get(tablename); kelondroMap table = (kelondroMap) mTables.get(tablename);
if (table == null) throw new RuntimeException("kelondroTables.selectMap: map table '" + tablename + "' does not exist."); if (table == null) throw new RuntimeException("kelondroTables.selectMap: map table '" + tablename + "' does not exist.");
if (key.length() > table.keySize()) key = key.substring(0, table.keySize());
return table.get(key); return table.get(key);
} }
@ -171,6 +173,7 @@ public class kelondroTables {
public synchronized void delete(String tablename, String key) throws IOException { public synchronized void delete(String tablename, String key) throws IOException {
kelondroMap table = (kelondroMap) mTables.get(tablename); kelondroMap table = (kelondroMap) mTables.get(tablename);
if (key.length() > table.keySize()) key = key.substring(0, table.keySize());
if (table != null) {table.remove(key); mTables.put(tablename, table); return;} if (table != null) {table.remove(key); mTables.put(tablename, table); return;}
kelondroTree Tree = (kelondroTree) tTables.get(tablename); kelondroTree Tree = (kelondroTree) tTables.get(tablename);

18
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -645,7 +645,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
} }
processLocalCrawling(urlEntry, profile); processLocalCrawling(urlEntry, profile);
return false; return true;
} }
public int remoteTriggeredCrawlJobSize() { public int remoteTriggeredCrawlJobSize() {
@ -969,7 +969,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
if (u == null) return plasmaURL.dummyHash; else return u.toString(); if (u == null) return plasmaURL.dummyHash; else return u.toString();
} }
/*
private void processCrawlingX(plasmaCrawlNURL.entry urlEntry, String initiator) { private void processCrawlingX(plasmaCrawlNURL.entry urlEntry, String initiator) {
if (urlEntry.url() == null) return; if (urlEntry.url() == null) return;
String profileHandle = urlEntry.profileHandle(); String profileHandle = urlEntry.profileHandle();
@ -984,11 +984,11 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
", permission=" + ((yacyCore.seedDB == null) ? "undefined" : (((yacyCore.seedDB.mySeed.isSenior()) || (yacyCore.seedDB.mySeed.isPrincipal())) ? "true" : "false"))); ", permission=" + ((yacyCore.seedDB == null) ? "undefined" : (((yacyCore.seedDB.mySeed.isSenior()) || (yacyCore.seedDB.mySeed.isPrincipal())) ? "true" : "false")));
boolean tryRemote = boolean tryRemote =
(profile.remoteIndexing()) /* granted */ && (profile.remoteIndexing()) &&
(urlEntry.depth() == profile.generalDepth()) /* leaf node */ && (urlEntry.depth() == profile.generalDepth()) &&
(urlEntry.initiator() != null) && (!(urlEntry.initiator().equals(plasmaURL.dummyHash))) /* not proxy */ && (urlEntry.initiator() != null) && (!(urlEntry.initiator().equals(plasmaURL.dummyHash))) &&
((yacyCore.seedDB.mySeed.isSenior()) || ((yacyCore.seedDB.mySeed.isSenior()) ||
(yacyCore.seedDB.mySeed.isPrincipal())) /* qualified */; (yacyCore.seedDB.mySeed.isPrincipal())) ;
if (tryRemote) { if (tryRemote) {
boolean success = processRemoteCrawlTrigger(urlEntry); boolean success = processRemoteCrawlTrigger(urlEntry);
@ -997,16 +997,16 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
processLocalCrawling(urlEntry, profile); processLocalCrawling(urlEntry, profile);
} }
} }
*/
private boolean processLocalCrawling(plasmaCrawlNURL.entry urlEntry, plasmaCrawlProfile.entry profile) { private boolean processLocalCrawling(plasmaCrawlNURL.entry urlEntry, plasmaCrawlProfile.entry profile) {
// work off one Crawl stack entry // work off one Crawl stack entry
if ((urlEntry == null) && (urlEntry.url() == null)) { if ((urlEntry == null) && (urlEntry.url() == null)) {
log.logInfo("LOCALCRAWL[" + noticeURL.coreStackSize() + ", " + noticeURL.remoteStackSize() + "]: urlEntry=null"); log.logInfo("LOCALCRAWL[" + noticeURL.coreStackSize() + ", " + noticeURL.limitStackSize() + ", " + noticeURL.overhangStackSize() + ", " + noticeURL.remoteStackSize() + "]: urlEntry=null");
return false; return false;
} }
cacheLoader.loadParallel(urlEntry.url(), urlEntry.referrerHash(), urlEntry.initiator(), urlEntry.depth(), profile); cacheLoader.loadParallel(urlEntry.url(), urlEntry.referrerHash(), urlEntry.initiator(), urlEntry.depth(), profile);
log.logInfo("LOCALCRAWL[" + noticeURL.coreStackSize() + ", " + noticeURL.remoteStackSize() + "]: enqueued for load " + urlEntry.url()); log.logInfo("LOCALCRAWL[" + noticeURL.coreStackSize() + ", " + noticeURL.limitStackSize() + ", " + noticeURL.overhangStackSize() + ", " + noticeURL.remoteStackSize() + "]: enqueued for load " + urlEntry.url());
return true; return true;
} }

Write Preview
Loading…
Cancel
Save