boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- one try to fix the httpd problem

Browse Source 
- fix for handling of collection index that appears when removing elements
- added another navigation method (stub, not working yet)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4543 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent 7cc4ff05c9
commit 9eddc1506b
5 changed files with 79 additions and 12 deletions
Show Stats Download Patch File Download Diff File

29
htroot/yacy/user/sidebar_navigation.java
Unescape View File

@ -25,6 +25,7 @@
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
@ -36,6 +37,7 @@ import de.anomic.plasma.plasmaSearchQuery;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyURL;
public class sidebar_navigation {
@ -119,15 +121,34 @@ public class sidebar_navigation {
prop.put("navigation_topwords_words_" + hintcount + "_resource", theQuery.searchdom());
prop.put("navigation_topwords_words_" + hintcount + "_zonecode", theQuery.zonecode);
}
prop.put("navigation_topwords_words", hintcount);
if (hintcount++ > MAX_TOPWORDS) {
break;
}
hintcount++;
if (hintcount >= MAX_TOPWORDS) break;
}
prop.put("navigation_topwords_words", hintcount);
prop.put("navigation_topwords", "1");
}
}
// compose language zone drill-down
int c = 0;
final Iterator<Map.Entry<String, Integer>> iter = theSearch.getRankingResult().getZoneStatistics().entrySet().iterator();
Map.Entry<String, Integer> entry;
while (iter.hasNext()) {
entry = iter.next();
if ((theQuery == null) || (theQuery.queryString == null)) break;
prop.putHTML("navigation_languagezone_zones_" + c + "_zone", entry.getKey() + " (" + entry.getValue() + ")");
prop.putHTML("navigation_languagezone_zones_" + c + "_search", theQuery.queryString.replace(' ', '+'));
prop.put("navigation_languagezone_zones_" + c + "_count", theQuery.displayResults());
prop.put("navigation_languagezone_zones_" + c + "_offset", "0");
prop.put("navigation_languagezone_zones_" + c + "_contentdom", theQuery.contentdom());
prop.put("navigation_languagezone_zones_" + c + "_resource", theQuery.searchdom());
prop.put("navigation_languagezone_zones_" + c + "_zonecode", yacyURL.zone2map.get(entry.getKey()).intValue());
prop.put("navigation_languagezone_zones", c);
c++;
}
prop.put("navigation_languagezone", (c > 2) ? "1" : "0");
// compose page navigation
StringBuffer resnav = new StringBuffer();
int thispage = offset / theQuery.displayResults();

16
source/de/anomic/http/httpdFileHandler.java
Unescape View File

@ -851,7 +851,17 @@ public final class httpdFileHandler {
if (chunkedOut != null) {
chunkedOut.finish();
}
}
// flush all
try {newOut.flush();}catch (Exception e) {}
// wait a little time until everything closes so that clients can read from the streams/sockets
if ((contentLength >= 0) && ((String)requestHeader.get(httpHeader.CONNECTION, "close")).indexOf("keep-alive") == -1) {
// in case that the client knows the size in advance (contentLength present) the waiting will have no effect on the interface performance
// but if the client waits on a connection interruption this will slow down.
try {Thread.sleep(2000);} catch (InterruptedException e) {} // FIXME: is this necessary?
}
}
// check mime type again using the result array: these are 'magics'
// if (serverByteBuffer.equals(result, 1, "PNG".getBytes())) mimeType = mimeTable.getProperty("png","text/html");
@ -921,10 +931,6 @@ public final class httpdFileHandler {
} finally {
try {out.flush();}catch (Exception e) {}
if (((String)requestHeader.get(httpHeader.CONNECTION, "close")).indexOf("keep-alive") == -1) {
// wait a little time until everything closes so that clients can read from the streams/sockets
try {Thread.sleep(50);} catch (InterruptedException e) {} // FIXME: is this necessary?
}
}
}

2
source/de/anomic/kelondro/kelondroCollectionIndex.java
Unescape View File

@ -920,7 +920,7 @@ public class kelondroCollectionIndex {
array_remove(
oldPartitionNumber, serialNumber, this.payloadrow.objectsize,
oldrownumber);
index.remove(key, true);
index.remove(key, false);
return removed;
}

11
source/de/anomic/plasma/plasmaSearchRankingProcess.java
Unescape View File

@ -68,6 +68,7 @@ public final class plasmaSearchRankingProcess {
private TreeSet<String> misses; // contains url-hashes that could not been found in the LURL-DB
private plasmaWordIndex wordIndex;
private HashMap<String, indexContainer>[] localSearchContainerMaps;
private int[] domZones;
public plasmaSearchRankingProcess(plasmaWordIndex wordIndex, plasmaSearchQuery query, int maxentries, int concurrency) {
// we collect the urlhashes and construct a list with urlEntry objects
@ -90,6 +91,8 @@ public final class plasmaSearchRankingProcess {
this.wordIndex = wordIndex;
this.flagcount = new int[32];
for (int i = 0; i < 32; i++) {this.flagcount[i] = 0;}
this.domZones = new int[8];
for (int i = 0; i < 8; i++) {this.domZones[i] = 0;}
}
public long ranking(indexRWIVarEntry word) {
@ -176,6 +179,10 @@ public final class plasmaSearchRankingProcess {
continue;
}
// count domZones
System.out.println("DEBUG domDomain dom=" + wordIndex.loadedURL.load(iEntry.urlHash, iEntry, 0).comp().url().getHost() + ", zone=" + yacyURL.domDomain(iEntry.urlHash()));
this.domZones[yacyURL.domDomain(iEntry.urlHash())]++;
// insert
if ((maxentries < 0) || (stack.size() < maxentries)) {
// in case that we don't have enough yet, accept any new entry
@ -327,6 +334,10 @@ public final class plasmaSearchRankingProcess {
return this.local_resourceSize;
}
public Map<String, Integer> getZoneStatistics() {
return yacyURL.zoneStatistics(this.domZones);
}
public indexRWIEntry remove(String urlHash) {
kelondroSortStack<indexRWIVarEntry>.stackElement se = stack.remove(urlHash.hashCode());
if (se == null) return null;

33
source/de/anomic/yacy/yacyURL.java
Unescape View File

@ -31,6 +31,7 @@ import java.io.File;
import java.net.MalformedURLException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -382,7 +383,7 @@ public class yacyURL {
public static final int language_domain_africa_zone = 128 + 32; //{5, 7};
public static final int language_domain_any_zone = 255;
public static final String[] regions = {"europe", "english", "spanish", "asia", "middleeast", "africa"};
public static final HashMap<String, Integer> zone2map = new HashMap<String, Integer>();
static {
// create a dummy hash
@ -398,6 +399,14 @@ public class yacyURL {
insertTLDProps(TLD_Africa, 5); // africa
insertTLDProps(TLD_Generic, 6); // anything else, mixed languages, mainly english
// the id=7 is used to flag local addresses
zone2map.put("europe", language_domain_europe_zone);
zone2map.put("english", language_domain_english_zone);
zone2map.put("spanish", language_domain_spanish_zone);
zone2map.put("asia", language_domain_asia_zone);
zone2map.put("middleeast", language_domain_middleeast_zone);
zone2map.put("africa", language_domain_africa_zone);
zone2map.put("any", language_domain_any_zone);
}
// class variables
@ -1093,7 +1102,7 @@ public class yacyURL {
// returns the ID of the domain of the domain
assert (urlHash != null);
assert (urlHash.length() == 12) : "urlhash = " + urlHash;
return (kelondroBase64Order.enhancedCoder.decodeByte(urlHash.charAt(11)) & 12) >> 2;
return (kelondroBase64Order.enhancedCoder.decodeByte(urlHash.charAt(11)) & 28) >> 2;
}
public static boolean isLocalDomain(String urlhash) {
@ -1123,6 +1132,26 @@ public class yacyURL {
return language;
}
public static Map<String, Integer> zoneStatistics(int[] domAccumulators) {
assert domAccumulators.length == 8;
HashMap<String, Integer> zoneCounter = new HashMap<String, Integer>();
Iterator<Map.Entry<String, Integer>> j;
Map.Entry<String, Integer> entry;
for (int i = 0; i < 8; i++) {
j = zone2map.entrySet().iterator();
while (j.hasNext()) {
entry = j.next();
if ((i & entry.getValue().intValue()) != 0) {
if (zoneCounter.containsKey(entry.getKey())) {
zoneCounter.put(entry.getKey(), zoneCounter.get(entry.getKey()) + domAccumulators[i]);
} else {
zoneCounter.put(entry.getKey(), domAccumulators[i]);
}
}
}
}
return zoneCounter;
}
public static void main(String[] args) {
String[][] test = new String[][]{

Write Preview
Loading…
Cancel
Save