Merge branch 'master' of ssh://git@gitorious.org/yacy/rc1.git

pull/1/head
Michael Peter Christen 13 years ago
commit cf79b6cee3

@ -41,9 +41,9 @@
<classpathentry kind="lib" path="lib/jcl-over-slf4j-1.6.1.jar"/>
<classpathentry kind="lib" path="lib/wstx-asl-3.2.7.jar"/>
<classpathentry kind="lib" path="lib/commons-codec-1.6.jar"/>
<classpathentry kind="lib" path="lib/httpcore-4.1.4.jar"/>
<classpathentry kind="lib" path="lib/httpclient-4.1.3.jar"/>
<classpathentry kind="lib" path="lib/httpmime-4.1.3.jar"/>
<classpathentry kind="lib" path="lib/httpcore-4.2.jar"/>
<classpathentry kind="lib" path="lib/httpclient-4.2.jar"/>
<classpathentry kind="lib" path="lib/httpmime-4.2.jar"/>
<classpathentry kind="lib" path="lib/commons-io-2.1.jar"/>
<classpathentry kind="lib" path="lib/apache-solr-solrj-3.6.0.jar" sourcepath="/solrj/src"/>
<classpathentry kind="lib" path="lib/commons-compress-1.4.1.jar"/>

@ -50,9 +50,9 @@
<string>$JAVAROOT/lib/commons-logging-1.1.1.jar</string>
<string>$JAVAROOT/lib/fontbox-1.6.0.jar</string>
<string>$JAVAROOT/lib/geronimo-stax-api_1.0_spec-1.0.1.jar</string>
<string>$JAVAROOT/lib/httpclient-4.1.3.jar</string>
<string>$JAVAROOT/lib/httpcore-4.1.4.jar</string>
<string>$JAVAROOT/lib/httpmime-4.1.3.jar</string>
<string>$JAVAROOT/lib/httpclient-4.2.jar</string>
<string>$JAVAROOT/lib/httpcore-4.2.jar</string>
<string>$JAVAROOT/lib/httpmime-4.2.jar</string>
<string>$JAVAROOT/lib/icu4j-core.jar</string>
<string>$JAVAROOT/lib/J7Zip-modified.jar</string>
<string>$JAVAROOT/lib/jakarta-oro-2.0.8.jar</string>

@ -60,11 +60,14 @@ SetCompressor /SOLID LZMA
; http://www.java.com/de/download/manual.jsp BundleId +1 / +2
; User-Agent to see the 64bit link: Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Win64; x64; Trident/4.0)
; at least we need Java 6
!define JRE_VERSION6 "1.6"
; jre-6u31-windows-i586.exe
!define JRE_32 "http://javadl.sun.com/webapps/download/AutoDL?BundleId=61044"
; jre-6u31-windows-x64.exe
!define JRE_64 "http://javadl.sun.com/webapps/download/AutoDL?BundleId=60338"
; download link Oracle Java 7 Update 4
; 32 bit
!define JRE_32 "http://javadl.sun.com/webapps/download/AutoDL?BundleId=63691"
; 64 bit
!define JRE_64 "http://javadl.sun.com/webapps/download/AutoDL?BundleId=63692"
;!define JRE_32 "http://yacy.berlios.de/download.php?what=jre&version=32&yacyrevnr=@REPL_REVISION_NR@"
;!define JRE_64 "http://yacy.berlios.de/download.php?what=jre&version=64&yacyrevnr=@REPL_REVISION_NR@"

@ -168,9 +168,9 @@
<pathelement location="${lib}/commons-logging-1.1.1.jar" />
<pathelement location="${lib}/fontbox-1.6.0.jar" />
<pathelement location="${lib}/geronimo-stax-api_1.0_spec-1.0.1.jar" />
<pathelement location="${lib}/httpclient-4.1.3.jar" />
<pathelement location="${lib}/httpcore-4.1.4.jar" />
<pathelement location="${lib}/httpmime-4.1.3.jar" />
<pathelement location="${lib}/httpclient-4.2.jar" />
<pathelement location="${lib}/httpcore-4.2.jar" />
<pathelement location="${lib}/httpmime-4.2.jar" />
<pathelement location="${lib}/icu4j-core.jar" />
<pathelement location="${lib}/J7Zip-modified.jar" />
<pathelement location="${lib}/jakarta-oro-2.0.8.jar" />

@ -110,7 +110,8 @@ public class PerformanceMemory_p {
prop.putNum("EcoList_" + c + "_tableSize", mapx.get(Table.StatKeys.tableSize));
assert mapx.get(Table.StatKeys.tableKeyMem) != null : mapx;
mem = Long.parseLong(mapx.get(Table.StatKeys.tableKeyMem));
String v = mapx.get(Table.StatKeys.tableKeyMem);
mem = v == null ? 0 : Long.parseLong(v);
totalmem += mem;
prop.put("EcoList_" + c + "_tableKeyMem", Formatter.bytesToString(mem));
prop.put("EcoList_" + c + "_tableKeyChunkSize", mapx.get(Table.StatKeys.tableKeyChunkSize));

@ -255,7 +255,7 @@ public final class search {
indexSegment,
rankingProfile,
header.get(RequestHeader.USER_AGENT, ""),
false
false, 0.0d, 0.0d, 0.0d
);
Network.log.logInfo("INIT HASH SEARCH (abstracts only): " + QueryParams.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.itemsPerPage() + " links");
@ -318,7 +318,7 @@ public final class search {
sb.indexSegments.segment(Segments.Process.PUBLIC),
rankingProfile,
header.get(RequestHeader.USER_AGENT, ""),
false
false, 0.0d, 0.0d, 0.0d
);
Network.log.logInfo("INIT HASH SEARCH (query-" + abstracts + "): " + QueryParams.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.itemsPerPage() + " links");
EventChannel.channels(EventChannel.REMOTESEARCH).addMessage(new RSSMessage("Remote Search Request from " + ((remoteSeed == null) ? "unknown" : remoteSeed.getName()), QueryParams.anonymizedQueryHashes(theQuery.queryHashes), ""));

@ -485,6 +485,29 @@ public class yacysearch {
}
}
int radius = 0;
double lon = 0.0d, lat = 0.0d, rad = 0.0d;
if ((radius = querystring.indexOf("/radius/")) >= 0) {
int ve = querystring.indexOf(' ', radius + 8);
String geo = "";
if (ve < 0) {
geo = querystring.substring(radius);
querystring = querystring.substring(0, radius).trim();
} else {
geo = querystring.substring(radius, ve);
querystring = querystring.substring(0, radius) + querystring.substring(ve);
}
geo = geo.substring(8);
String[] sp = geo.split("/");
if (sp.length == 3) try {
lat = Double.parseDouble(sp[0]);
lon = Double.parseDouble(sp[1]);
rad = Double.parseDouble(sp[2]);
} catch (NumberFormatException e) {
lon = 0.0d; lat = 0.0d; rad = 0.0d;
}
}
String tenant = null;
if ( post.containsKey("tenant") ) {
tenant = post.get("tenant");
@ -757,7 +780,8 @@ public class yacysearch {
header.get(RequestHeader.USER_AGENT, ""),
sb.getConfigBool(SwitchboardConstants.SEARCH_VERIFY_DELETE, false)
&& sb.getConfigBool(SwitchboardConstants.NETWORK_SEARCHVERIFY, false)
&& sb.peers.mySeed().getFlagAcceptRemoteIndex());
&& sb.peers.mySeed().getFlagAcceptRemoteIndex(),
lat, lon, rad);
EventTracker.delete(EventTracker.EClass.SEARCH);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(
theQuery.id(true),

@ -6,72 +6,144 @@
<script type="text/javascript" src="http://openlayers.org/api/OpenLayers.js"></script>
<script type="text/javascript" src="http://www.openstreetmap.org/openlayers/OpenStreetMap.js"></script>
<script type="text/javascript">
function adaptHeight() {
document.getElementById('map').style.height = (document.documentElement.clientHeight - 130) + "px";
}
window.onresize = adaptHeight;
</script>
<script type="text/javascript">
// possible values for dom: location,metatag,alltext,title,publisher,creator,subject
var map;
var searchLayer_md = null;
var searchLayer_co = null;
var searchLayer_md0 = null; // two layers each for double-buffering during search
var searchLayer_md1 = null;
var searchLayer_co0 = null;
var searchLayer_co1 = null;
var path_mdsearch = 'yacysearch_location.rss?dom=alltext&query=';
var path_losearch = 'yacysearch_location.rss?dom=location&query=';
var marker_md = new OpenLayers.Icon("/env/grafics/marker_red.png", new OpenLayers.Size(11,16));
var marker_co = new OpenLayers.Icon("/env/grafics/star_yellow.png", new OpenLayers.Size(25,25));
// possible values for dom: location,metatag,alltext,title,publisher,creator,subject
var shallZoom = true; // flag if the map shall be zoomed to the search result
var shallReload = false; // flag if the search shall be repeated upon next resize of the window
var poisfound = 0;
var kmNormal = 100; // ca. 1km grid for radius
var meterNormal = 100000; // ca. 1m grid for location
function init() {
map = new OpenLayers.Map('map', {
maxResolution:'auto',
controls:[
new OpenLayers.Control.Navigation(),
new OpenLayers.Control.PanZoomBar(),
new OpenLayers.Control.ZoomBox(),
new OpenLayers.Control.LayerSwitcher(),
new OpenLayers.Control.Attribution()]
});
// automatically adapt height of map upon window resize
function adaptHeight() {
document.getElementById('map').style.height = (document.documentElement.clientHeight - 130) + "px";
}
window.onresize = adaptHeight;
//layerMaplint = new OpenLayers.Layer.OSM.Maplint("Maplint");
layerWMS = new OpenLayers.Layer.WMS( "OpenLayers WMS", "http://labs.metacarta.com/wms/vmap0", {layers: 'basic'} );
layerMapnik = new OpenLayers.Layer.OSM.Mapnik("Mapnik");
layerCycleMap = new OpenLayers.Layer.OSM.CycleMap("CycleMap");
map.addLayer(layerMapnik);
map.addLayer(layerCycleMap);
//map.addLayer(layerMaplint);
map.addLayer(layerWMS);
map.addControl(new OpenLayers.Control.LayerSwitcher());
map.setCenter(new OpenLayers.LonLat(9,48) // Center of the map
// initialize the map layers
function init() {
map = new OpenLayers.Map('map', {
maxResolution:'auto',
controls:[
new OpenLayers.Control.Navigation(),
new OpenLayers.Control.PanZoomBar(),
new OpenLayers.Control.ZoomBox(),
new OpenLayers.Control.LayerSwitcher(),
new OpenLayers.Control.Attribution()]
});
//layerMaplint = new OpenLayers.Layer.OSM.Maplint("Maplint");
layerWMS = new OpenLayers.Layer.WMS( "OpenLayers WMS", "http://labs.metacarta.com/wms/vmap0", {layers: 'basic'} );
layerMapnik = new OpenLayers.Layer.OSM.Mapnik("Mapnik");
layerCycleMap = new OpenLayers.Layer.OSM.CycleMap("CycleMap");
map.addLayer(layerMapnik);
map.addLayer(layerCycleMap);
//map.addLayer(layerMaplint);
map.addLayer(layerWMS);
map.addControl(new OpenLayers.Control.LayerSwitcher());
map.setCenter(new OpenLayers.LonLat(9,48) // Center of the map
.transform(
new OpenLayers.Projection("EPSG:4326"), // transform from WGS 1984
new OpenLayers.Projection("EPSG:900913") // to Spherical Mercator Projection
), 5 // Zoom level
);
map.events.register( "zoomend", map, zoomChanged );
map.events.register( "movestart", map, zoomChanged );
//#(initsearch)#::
var query = '#[query]#'.replace(' ', '+');
//#(/initsearch)#
#(initsearch)#::
var query = '#[query]#'.replace(' ', '+');
searchLayer_co = new OpenLayers.Layer.GeoRSS('GeoRSS', path_losearch + query, {'icon':marker_co});
map.addLayer(searchLayer_co);
searchLayer_md = new OpenLayers.Layer.GeoRSS('GeoRSS', path_mdsearch + query, {'icon':marker_md});
map.addLayer(searchLayer_md);
#(/initsearch)#
window.setInterval("refresh()", 1000);
}
function zoomChanged () {
shallZoom = false; // no automatic zoom if user hs zoomed
refreshsearch();
}
// called if a search is submitted
function search() {
var query = document.getElementById('query').value.replace(' ', '+');
if (searchLayer_md != null) searchLayer_md.destroy();
if (searchLayer_co != null) searchLayer_co.destroy();
var center = map.getCenter().transform(new OpenLayers.Projection("EPSG:900913"), new OpenLayers.Projection("EPSG:4326"));
searchLayer_co = new OpenLayers.Layer.GeoRSS('GeoRSS', path_losearch + query + '&lon=' + center.lon + '&lat=' + center.lat + '&zoom=' + map.getZoom(), {'icon':marker_co});
map.addLayer(searchLayer_co);
searchLayer_md = new OpenLayers.Layer.GeoRSS('GeoRSS', path_mdsearch + query, {'icon':marker_md});
map.addLayer(searchLayer_md);
//map.zoomToExtent(searchLayer_co.getExtent());
//map.panTo(searchLayer_co.getExtent().getCenterLonLat());
//map.setCenter(searchLayer_co.getExtent().getLonLatFromViewPortPx, 5);
//map.setCenter(searchLayer_co.getExtent().getCenterLonLat(), map.getZoomForExtent(searchLayer_co.getExtent(), true));
//document.getElementById('apilink').setAttribute('href', 'yacysearch_location.rss?query=' + query);
refreshsearch();
}
// refresh is called if the map is zoomed or paned to get a new search result for the viewport
function refreshsearch() {
var query = document.getElementById('query').value.replace(' ', '+');
if (searchLayer_md0 != null) searchLayer_md0.destroy();
if (searchLayer_co0 != null) searchLayer_co0.destroy();
var center = map.getCenter().transform(new OpenLayers.Projection("EPSG:900913"), new OpenLayers.Projection("EPSG:4326"));
var extend = map.getExtent().transform(new OpenLayers.Projection("EPSG:900913"), new OpenLayers.Projection("EPSG:4326"));
var radius = Math.sqrt(extend.getWidth() * extend.getWidth() + extend.getHeight() * extend.getHeight()) / 2;
radius = Math.floor(radius * kmNormal + 1) / kmNormal;
var lon = Math.floor(center.lon * meterNormal) / meterNormal;
var lat = Math.floor(center.lat * meterNormal) / meterNormal;
searchLayer_co0 = new OpenLayers.Layer.GeoRSS('Communities', path_losearch + query + '&lon=' + lon + '&lat=' + lat + '&r=' + radius + '&z=' + map.getZoom(), {'icon':marker_co});
searchLayer_md0 = new OpenLayers.Layer.GeoRSS('Point Of Interest', path_mdsearch + query + '&lon=' + lon + '&lat=' + lat + '&r=' + radius + '&z=' + map.getZoom(), {'icon':marker_md});
map.addLayer(searchLayer_co0);
map.addLayer(searchLayer_md0);
//document.getElementById('apilink').setAttribute('href', 'yacysearch_location.rss?query=' + query);
poisfound = 0;
document.getElementById('resultline').innerHTML = "";
}
function refresh() {
// check double-buffer state
if (searchLayer_md0 != null) {
// switch the double-buffer
if (searchLayer_md1 != null) searchLayer_md1.destroy();
if (searchLayer_co1 != null) searchLayer_co1.destroy();
searchLayer_md1 = searchLayer_md0;
searchLayer_co1 = searchLayer_co0;
searchLayer_md0 = null;
searchLayer_co0 = null;
}
// check if we need to do something
if (searchLayer_co1 == null || searchLayer_md1 == null) {
return;
}
var cocount = searchLayer_co1.markers.length;
var mdcount = searchLayer_md1.markers.length;
//if (mdcount + cocount <= poisfound) {
// return;
//}
// update map and result line
poisfound = mdcount + cocount;
document.getElementById('resultline').innerHTML = poisfound + " POIs found in visible map";
// check if any of the markers is visible on the screen.
// this is only necessary if shallZoom = false since this would set shallZoom = true
var bounds = searchLayer_md1.getDataExtent();
/*
if (!shallZoom) {
// check all markers
//if (!bounds.intersectsBounds(map.getExtent())) shallZoom = true;
var anyInScreen = false;
var mapbounds = map.getExtent();
//alert(searchLayer_md1.markers.length + " markers");
for (var marker in searchLayer_md1.markers) {
if (marker.lonlat != null && mapbounds.containsLonLat(marker.lonlat, {})) {anyInScreen = true; break;}
}
if (!anyInScreen) shallZoom = true;
}
*/
// zoom to layer bounds
if (shallZoom) {
if (bounds == null) {return;} //bounds = searchLayer_co1.getDataExtent();
map.panTo(bounds.getCenterLonLat());
map.zoomTo(map.getZoomForExtent(bounds));
}
}
</script>
@ -98,9 +170,10 @@ To see a list of all APIs, please visit the <a href="http://www.yacy-websuche.de
<h2>#[promoteSearchPageGreeting]#</h2>
<div class="yacylogo"><a href="#[promoteSearchPageGreeting.homepage]#" class="yacylogo"><img src="#[promoteSearchPageGreeting.smallImage]#" alt="yacysearch" /></a></div>
<fieldset class="yacys">
<input type="text" value="#(initsearch)#::#[query]##(/initsearch)#" id="query" size="42" />
<input type="text" value="#(initsearch)#::#[query]##(/initsearch)#" id="query" size="42" onFocus="this.select()" />
<input type="submit" onclick="search(); return false;" value="search" onsubmit="search(); return false;" />
</fieldset>
<div id="resultline"></div>
</form><br/>
<div id="map" style="clear:both; width:100%; height:720px"></div>
<script type="text/javascript">

@ -37,8 +37,6 @@ import de.anomic.server.serverSwitch;
public class yacysearch_location {
private static final String space = " ";
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
@ -62,10 +60,13 @@ public class yacysearch_location {
final boolean search_subject = alltext || post.get("dom", "").indexOf("subject",0) >= 0;
final long maximumTime = post.getLong("maximumTime", 5000);
final int maximumRecords = post.getInt("maximumRecords", 3000);
final double lon = post.getDouble("lon", 0.0d);
final double lat = post.getDouble("lat", 0.0d);
final double radius = post.getDouble("r", 0.0d);
//i.e. http://localhost:8090/yacysearch_location.kml?query=berlin&maximumTime=2000&maximumRecords=100
int placemarkCounter = 0;
if (search_query) {
if (query.length() > 0 && search_query) {
final Set<Location> locations = LibraryProvider.geoLoc.find(query, true);
for (final String qp: query.split(" ")) {
locations.addAll(LibraryProvider.geoLoc.find(qp, true));
@ -87,11 +88,11 @@ public class yacysearch_location {
}
}
if (metatag || search_title || search_publisher || search_creator || search_subject) try {
if (query.length() > 0 && (metatag || search_title || search_publisher || search_creator || search_subject)) try {
// get a queue of search results
final String rssSearchServiceURL = "http://127.0.0.1:" + sb.getConfig("port", "8090") + "/yacysearch.rss";
final BlockingQueue<RSSMessage> results = new LinkedBlockingQueue<RSSMessage>();
SRURSSConnector.searchSRURSS(results, rssSearchServiceURL, query, maximumTime, Integer.MAX_VALUE, null, false, null);
SRURSSConnector.searchSRURSS(results, rssSearchServiceURL, lon == 0.0d && lat == 0.0d ? query : query + " /radius/" + lat + "/" + lon + "/" + radius, maximumTime, Integer.MAX_VALUE, null, false, null);
// take the results and compute some locations
RSSMessage message;

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

@ -173,4 +173,10 @@
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
END OF TERMS AND CONDITIONS
This project contains annotations derived from JCIP-ANNOTATIONS
Copyright (c) 2005 Brian Goetz and Tim Peierls.
See http://www.jcip.net and the Creative Commons Attribution License
(http://creativecommons.org/licenses/by/2.5)

@ -77,7 +77,7 @@
<compilation-unit>
<package-root>source</package-root>
<package-root>htroot</package-root>
<classpath mode="compile">lib/activation.jar:lib/apache-mime4j-0.6.jar:lib/apache-solr-solrj-3.6.0.jar:lib/bcmail-jdk15-145.jar:lib/bcprov-jdk15-145.jar:lib/bzip2.jar:lib/commons-codec-1.6.jar:lib/commons-fileupload-1.2.2.jar:lib/commons-httpclient-3.1.jar:lib/commons-io-2.1.jar:lib/commons-jxpath-1.3.jar:lib/commons-logging-1.1.1.jar:lib/fontbox-1.6.0.jar:lib/geronimo-stax-api_1.0_spec-1.0.1.jar:lib/httpclient-4.1.3.jar:lib/httpcore-4.1.4.jar:lib/httpmime-4.1.3.jar:lib/icu4j-core.jar:lib/J7Zip-modified.jar:lib/jakarta-oro-2.0.8.jar:lib/jcifs-1.3.15.jar:lib/jcl-over-slf4j-1.6.1.jar:lib/jempbox-1.6.0.jar:lib/jsch-0.1.42.jar:lib/json-simple-1.1.jar:lib/log4j-1.2.16.jar:lib/metadata-extractor-2.4.0-beta-1.jar:lib/mysql-connector-java-5.1.12-bin.jar:lib/pdfbox-1.6.0.jar:lib/poi-3.6-20091214.jar:lib/poi-scratchpad-3.6-20091214.jar:lib/servlet-api.jar:lib/slf4j-api-1.6.1.jar:lib/slf4j-jdk14-1.6.1.jar:lib/webcat-0.1-swf.jar:lib/wstx-asl-3.2.7.jar:lib/xercesImpl.jar:lib/xml-apis.jar:htroot/processing/domaingraph/applet/domaingraph.jar</classpath>
<classpath mode="compile">lib/activation.jar:lib/apache-mime4j-0.6.jar:lib/apache-solr-solrj-3.6.0.jar:lib/bcmail-jdk15-145.jar:lib/bcprov-jdk15-145.jar:lib/bzip2.jar:lib/commons-codec-1.6.jar:lib/commons-fileupload-1.2.2.jar:lib/commons-httpclient-3.1.jar:lib/commons-io-2.1.jar:lib/commons-jxpath-1.3.jar:lib/commons-logging-1.1.1.jar:lib/fontbox-1.6.0.jar:lib/geronimo-stax-api_1.0_spec-1.0.1.jar:lib/httpclient-4.2.jar:lib/httpcore-4.2.jar:lib/httpmime-4.2.jar:lib/icu4j-core.jar:lib/J7Zip-modified.jar:lib/jakarta-oro-2.0.8.jar:lib/jcifs-1.3.15.jar:lib/jcl-over-slf4j-1.6.1.jar:lib/jempbox-1.6.0.jar:lib/jsch-0.1.42.jar:lib/json-simple-1.1.jar:lib/log4j-1.2.16.jar:lib/metadata-extractor-2.4.0-beta-1.jar:lib/mysql-connector-java-5.1.12-bin.jar:lib/pdfbox-1.6.0.jar:lib/poi-3.6-20091214.jar:lib/poi-scratchpad-3.6-20091214.jar:lib/servlet-api.jar:lib/slf4j-api-1.6.1.jar:lib/slf4j-jdk14-1.6.1.jar:lib/webcat-0.1-swf.jar:lib/wstx-asl-3.2.7.jar:lib/xercesImpl.jar:lib/xml-apis.jar:htroot/processing/domaingraph/applet/domaingraph.jar</classpath>
<source-level>1.6</source-level>
</compilation-unit>
</java-data>

@ -222,7 +222,7 @@ public final class ResultURLs {
public static void main(final String[] args) {
try {
final DigestURI url = new DigestURI("http", "www.yacy.net", 80, "/");
final URIMetadataRow urlRef = new URIMetadataRow(url, "YaCy Homepage", "", "", "", 0.0f, 0.0f, new Date(), new Date(), new Date(), "", new byte[] {}, 123, 42, '?', new Bitfield(), UTF8.getBytes("de"), 0, 0, 0, 0, 0, 0);
final URIMetadataRow urlRef = new URIMetadataRow(url, "YaCy Homepage", "", "", "", 0.0d, 0.0d, new Date(), new Date(), new Date(), "", new byte[] {}, 123, 42, '?', new Bitfield(), UTF8.getBytes("de"), 0, 0, 0, 0, 0, 0);
final EventOrigin stackNo = EventOrigin.LOCAL_CRAWLING;
System.out.println("valid test:\n=======");
// add

@ -579,6 +579,7 @@ public final class HTTPDFileHandler {
e.getTargetException().getMessage() +
"; java.awt.graphicsenv='" + System.getProperty("java.awt.graphicsenv","") + "'");
Log.logException(e);
Log.logException(e.getCause());
Log.logException(e.getTargetException());
targetClass = null;
}
@ -978,12 +979,9 @@ public final class HTTPDFileHandler {
if (e.getCause() instanceof InterruptedException) {
throw new InterruptedException(e.getCause().getMessage());
}
theLogger.logSevere("INTERNAL ERROR: " + e.toString() + ":" +
e.getMessage() +
" target exception at " + targetClass + ": " +
e.getTargetException().toString() + ":" +
e.getTargetException().getMessage(),e);
Log.logException(e);
Log.logException(e.getCause());
Log.logException(e.getTargetException());
targetClass = null;
throw e;
}

@ -298,15 +298,18 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
: new InetSocketAddress(bindIP, bindPort);
}
@Override
public void open() {
this.log.logConfig("* server started on " + Domains.myPublicLocalIP() + ":" + this.extendedPort);
}
@Override
public void freemem() {
// FIXME: can we something here to flush memory? Idea: Reduce the size of some of our various caches.
}
// class body
@Override
public boolean job() throws Exception {
try {
// prepare for new connection
@ -407,6 +410,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
}
}
@Override
public synchronized void close() {
// consuming the isInterrupted Flag. Otherwise we could not properly close the session pool
Thread.interrupted();
@ -456,6 +460,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
return l;
}
@Override
public int getJobCount() {
final Thread[] threadList = new Thread[sessionThreadGroup.activeCount()];
serverCore.sessionThreadGroup.enumerate(threadList, false);
@ -527,6 +532,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
sessionCounter++;
}
@Override
public int hashCode() {
// return a hash code so it is possible to store objects of httpc objects in a HashSet
return this.hashIndex;
@ -619,6 +625,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
*
* @see java.lang.Thread#run()
*/
@Override
public void run() {
this.runningsession = true;
@ -804,6 +811,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
} catch (final InvocationTargetException e) {
serverCore.this.log.logSevere("command execution, target exception " + e.getMessage() + " for client " + this.userAddress.getHostAddress(), e);
// we extract a target exception
writeLine(this.commandObj.error(e.getCause()));
writeLine(this.commandObj.error(e.getTargetException()));
break;
} catch (final NoSuchMethodException e) {

@ -183,6 +183,10 @@ public class serverObjects extends HashMap<String, String> implements Cloneable
return (null == this.put(key, Float.toString(value))) ? Float.NaN : value;
}
public double put(final String key, final double value) {
return (null == this.put(key, Double.toString(value))) ? Double.NaN : value;
}
/**
* same as {@link #put(String, double)} but for integer types
* @return Returns 0 for the error case.
@ -359,6 +363,16 @@ public class serverObjects extends HashMap<String, String> implements Cloneable
}
}
public double getDouble(final String key, final double dflt) {
final String s = removeByteOrderMark(super.get(key));
if (s == null) return dflt;
try {
return Double.parseDouble(s);
} catch (final NumberFormatException e) {
return dflt;
}
}
public boolean getBoolean(final String key, final boolean dflt) {
String s = removeByteOrderMark(super.get(key));
if (s == null) return dflt;

@ -25,6 +25,7 @@
package net.yacy.cora.document;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
@ -1967,8 +1968,8 @@ public class MultiProtocolURI implements Serializable, Comparable<MultiProtocolU
}
public InputStream getInputStream(final String userAgent, final int timeout) throws IOException {
if (isFile()) return new FileInputStream(getFSFile());
if (isSMB()) return new SmbFileInputStream(getSmbFile());
if (isFile()) return new BufferedInputStream(new FileInputStream(getFSFile()));
if (isSMB()) return new BufferedInputStream(new SmbFileInputStream(getSmbFile()));
if (isFTP()) {
final FTPClient client = new FTPClient();
client.open(this.host, this.port < 0 ? 21 : this.port);

@ -28,10 +28,10 @@ import java.text.ParseException;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.date.ISO8601Formatter;
@ -81,12 +81,14 @@ public class RSSMessage implements Hit, Comparable<RSSMessage>, Comparator<RSSMe
return this.keys;
}
@Override
public String toString() {
return this.keys.iterator().next();
}
}
private static String artificialGuidPrefix = "c0_";
private static String calculatedGuidPrefix = "c1_";
public static final RSSMessage POISON = new RSSMessage("", "", "");
public static final HashSet<String> tags = new HashSet<String>();
@ -99,16 +101,16 @@ public class RSSMessage implements Hit, Comparable<RSSMessage>, Comparator<RSSMe
private final Map<String, String> map;
public RSSMessage(final String title, final String description, final String link) {
this.map = new ConcurrentHashMap<String, String>();
this.map = new HashMap<String, String>();
this.map.put("title", title);
this.map.put("description", description);
this.map.put("link", link);
this.map.put("pubDate", ISO8601Formatter.FORMATTER.format());
this.map.put("guid", artificialGuidPrefix + Integer.toHexString((title + description + link).hashCode()));
}
public RSSMessage(final String title, final String description, final DigestURI link) {
this.map = new ConcurrentHashMap<String, String>();
this.map = new HashMap<String, String>();
this.map.put("title", title);
this.map.put("description", description);
this.map.put("link", link.toNormalform(true, false));
@ -117,22 +119,19 @@ public class RSSMessage implements Hit, Comparable<RSSMessage>, Comparator<RSSMe
}
public RSSMessage() {
this.map = new ConcurrentHashMap<String, String>();
this.map = new HashMap<String, String>();
}
public void setValue(final String name, final String value) {
this.map.put(name, value);
// if possible generate a guid if not existent so far
if ((name.equals("title") || name.equals("description") || name.equals("link")) &&
(!this.map.containsKey("guid") || this.map.get("guid").startsWith(artificialGuidPrefix))) {
this.map.put("guid", artificialGuidPrefix + Integer.toHexString((getTitle() + getDescription() + getLink()).hashCode()));
}
}
@Override
public String getTitle() {
return Token.title.valueFrom(this.map, "");
}
@Override
public String getLink() {
return Token.link.valueFrom(this.map, "");
}
@ -158,22 +157,27 @@ public class RSSMessage implements Hit, Comparable<RSSMessage>, Comparator<RSSMe
return o1.compareTo(o2);
}
@Override
public String getDescription() {
return Token.description.valueFrom(this.map, "");
}
@Override
public String getAuthor() {
return Token.author.valueFrom(this.map, "");
}
@Override
public String getCopyright() {
return Token.copyright.valueFrom(this.map, "");
}
@Override
public String getCategory() {
return Token.category.valueFrom(this.map, "");
}
@Override
public String[] getSubject() {
final String subject = Token.subject.valueFrom(this.map, "");
if (subject.indexOf(',') >= 0) return subject.split(",");
@ -181,14 +185,17 @@ public class RSSMessage implements Hit, Comparable<RSSMessage>, Comparator<RSSMe
return subject.split(" ");
}
@Override
public String getReferrer() {
return Token.referrer.valueFrom(this.map, "");
}
@Override
public String getLanguage() {
return Token.language.valueFrom(this.map, "");
}
@Override
public Date getPubDate() {
final String dateString = Token.pubDate.valueFrom(this.map, "");
Date date;
@ -204,18 +211,27 @@ public class RSSMessage implements Hit, Comparable<RSSMessage>, Comparator<RSSMe
return date;
}
@Override
public String getGuid() {
return Token.guid.valueFrom(this.map, "");
String guid = Token.guid.valueFrom(this.map, "");
if ((guid.length() == 0 || guid.startsWith(artificialGuidPrefix)) &&
(this.map.containsKey("title") || this.map.containsKey("description") || this.map.containsKey("link"))) {
guid = calculatedGuidPrefix + Integer.toHexString(getTitle().hashCode() + getDescription().hashCode() + getLink().hashCode());
this.map.put("guid", guid);
}
return guid;
}
public String getTTL() {
return Token.ttl.valueFrom(this.map, "");
}
@Override
public String getDocs() {
return Token.docs.valueFrom(this.map, "");
}
@Override
public long getSize() {
final String size = Token.size.valueFrom(this.map, "-1");
return (size == null || size.length() == 0) ? -1 : Long.parseLong(size);
@ -227,10 +243,12 @@ public class RSSMessage implements Hit, Comparable<RSSMessage>, Comparator<RSSMe
return sb.toString();
}
@Override
public float getLon() {
return Float.parseFloat(Token.lon.valueFrom(this.map, "0.0"));
}
@Override
public float getLat() {
return Float.parseFloat(Token.lat.valueFrom(this.map, "0.0"));
}
@ -240,18 +258,22 @@ public class RSSMessage implements Hit, Comparable<RSSMessage>, Comparator<RSSMe
return this.map.toString();
}
@Override
public void setAuthor(final String author) {
setValue("author", author);
}
@Override
public void setCategory(final String category) {
setValue("category", category);
}
@Override
public void setCopyright(final String copyright) {
setValue("copyright", copyright);
}
@Override
public void setSubject(final String[] tags) {
final StringBuilder sb = new StringBuilder(tags.length * 10);
for (final String tag: tags) sb.append(tag).append(',');
@ -259,38 +281,47 @@ public class RSSMessage implements Hit, Comparable<RSSMessage>, Comparator<RSSMe
setValue("subject", sb.toString());
}
@Override
public void setDescription(final String description) {
setValue("description", description);
}
@Override
public void setDocs(final String docs) {
setValue("docs", docs);
}
@Override
public void setGuid(final String guid) {
setValue("guid", guid);
}
@Override
public void setLanguage(final String language) {
setValue("language", language);
}
@Override
public void setLink(final String link) {
setValue("link", link);
}
@Override
public void setPubDate(final Date pubdate) {
setValue("pubDate", ISO8601Formatter.FORMATTER.format(pubdate));
}
@Override
public void setReferrer(final String referrer) {
setValue("referrer", referrer);
}
@Override
public void setSize(final long size) {
setValue("size", Long.toString(size));
}
@Override
public void setTitle(final String title) {
setValue("title", title);
}

@ -20,6 +20,7 @@
package net.yacy.cora.document;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
@ -57,9 +58,10 @@ public class RSSReader extends DefaultHandler {
this.type = Type.none;
}
public RSSReader(final int maxsize, final InputStream stream, final Type type) throws IOException {
public RSSReader(final int maxsize, InputStream stream, final Type type) throws IOException {
this(maxsize);
this.type = type;
if (!(stream instanceof ByteArrayInputStream) && !(stream instanceof BufferedInputStream)) stream = new BufferedInputStream(stream);
final SAXParserFactory factory = SAXParserFactory.newInstance();
try {
final SAXParser saxParser = factory.newSAXParser();

@ -50,6 +50,8 @@ import net.yacy.cora.protocol.ConnectionInfo;
import net.yacy.cora.protocol.HeaderFramework;
import org.apache.http.Header;
import org.apache.http.HeaderElement;
import org.apache.http.HeaderElementIterator;
import org.apache.http.HttpEntity;
import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpHost;
@ -67,6 +69,7 @@ import org.apache.http.client.params.CookiePolicy;
import org.apache.http.client.params.HttpClientParams;
import org.apache.http.client.protocol.ClientContext;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.conn.ConnectionKeepAliveStrategy;
import org.apache.http.conn.params.ConnRouteParams;
import org.apache.http.conn.routing.HttpRoute;
import org.apache.http.conn.scheme.PlainSocketFactory;
@ -77,10 +80,10 @@ import org.apache.http.entity.InputStreamEntity;
import org.apache.http.entity.mime.MultipartEntity;
import org.apache.http.entity.mime.content.ContentBody;
import org.apache.http.entity.mime.content.StringBody;
import org.apache.http.impl.client.AbstractHttpClient;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
import org.apache.http.impl.conn.PoolingClientConnectionManager;
import org.apache.http.message.BasicHeaderElementIterator;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpParams;
@ -139,7 +142,7 @@ public class HTTPClient {
schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
schemeRegistry.register(new Scheme("https", 443, getSSLSocketFactory()));
final ThreadSafeClientConnManager clientConnectionManager = new ThreadSafeClientConnManager(schemeRegistry);
final PoolingClientConnectionManager clientConnectionManager = new PoolingClientConnectionManager(schemeRegistry);
// Create and initialize HTTP parameters
final HttpParams httpParams = new BasicHttpParams();
@ -154,7 +157,7 @@ public class HTTPClient {
clientConnectionManager.setDefaultMaxPerRoute(2);
// Increase max connections for localhost
final HttpHost localhost = new HttpHost("localhost");
clientConnectionManager.setMaxForRoute(new HttpRoute(localhost), maxcon);
clientConnectionManager.setMaxPerRoute(new HttpRoute(localhost), maxcon);
/**
* HTTP protocol settings
*/
@ -188,10 +191,12 @@ public class HTTPClient {
httpClient = new DefaultHttpClient(clientConnectionManager, httpParams);
// disable the cookiestore, cause this may cause segfaults and is not needed
((DefaultHttpClient) httpClient).setCookieStore(null);
// add cutom keep alive strategy
addCustomKeepAliveStrategy((DefaultHttpClient) httpClient);
// ask for gzip
((AbstractHttpClient) httpClient).addRequestInterceptor(new GzipRequestInterceptor());
((DefaultHttpClient) httpClient).addRequestInterceptor(new GzipRequestInterceptor());
// uncompress gzip
((AbstractHttpClient) httpClient).addResponseInterceptor(new GzipResponseInterceptor());
((DefaultHttpClient) httpClient).addResponseInterceptor(new GzipResponseInterceptor());
if (idledConnectionEvictor == null) {
idledConnectionEvictor = new IdledConnectionEvictor(clientConnectionManager);
@ -233,7 +238,7 @@ public class HTTPClient {
*/
public static void setMaxRouteHost(final String host) {
final HttpHost mHost = new HttpHost(host);
((ThreadSafeClientConnManager) httpClient.getConnectionManager()).setMaxForRoute(new HttpRoute(mHost), 50);
((PoolingClientConnectionManager) httpClient.getConnectionManager()).setMaxPerRoute(new HttpRoute(mHost), 50);
}
/**
@ -655,7 +660,7 @@ public class HTTPClient {
if (ProxySettings.use)
ConnRouteParams.setDefaultProxy(httpParams, ProxySettings.getProxyHost());
// TODO find a better way for this
ProxySettings.setProxyCreds((AbstractHttpClient) httpClient);
ProxySettings.setProxyCreds((DefaultHttpClient) httpClient);
}
private void storeConnectionInfo(final HttpUriRequest httpUriRequest) {
@ -703,6 +708,38 @@ public class HTTPClient {
final SSLSocketFactory sslSF = new SSLSocketFactory(sslContext, SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
return sslSF;
}
/**
* If the Keep-Alive header is not present in the response,
* HttpClient assumes the connection can be kept alive indefinitely.
* Here we limit this to 5 seconds.
*
* @param defaultHttpClient
*/
private static void addCustomKeepAliveStrategy(final DefaultHttpClient defaultHttpClient) {
defaultHttpClient.setKeepAliveStrategy(new ConnectionKeepAliveStrategy() {
public long getKeepAliveDuration(HttpResponse response, HttpContext context) {
// Honor 'keep-alive' header
String param, value;
HeaderElement element;
HeaderElementIterator it = new BasicHeaderElementIterator(
response.headerIterator(HTTP.CONN_KEEP_ALIVE));
while (it.hasNext()) {
element = it.nextElement();
param = element.getName();
value = element.getValue();
if (value != null && param.equalsIgnoreCase("timeout")) {
try {
return Long.parseLong(value) * 1000;
} catch(final NumberFormatException e) {
}
}
}
// Keep alive for 5 seconds only
return 5 * 1000;
}
});
}
/**
* testing

@ -64,6 +64,10 @@ public class SolrDoc extends SolrInputDocument {
this.setField(key.getSolrFieldName(), value);
}
public final void addSolr(final SolrField key, final double value) {
this.setField(key.getSolrFieldName(), value);
}
public final void addSolr(final SolrField key, final boolean value) {
this.setField(key.getSolrFieldName(), value);
}

@ -87,7 +87,7 @@ public class Document {
private boolean resorted;
private final Set<String> languages;
private final boolean indexingDenied;
private final float lon, lat;
private final double lon, lat;
private final Object parserObject; // the source object that was used to create the Document
public Document(final MultiProtocolURI location, final String mimeType, final String charset,
@ -95,7 +95,7 @@ public class Document {
final Set<String> languages,
final String[] keywords, final String title, final String author, final String publisher,
final String[] sections, final String abstrct,
final float lon, final float lat,
final double lon, final double lat,
final Object text,
final Map<MultiProtocolURI, Properties> anchors,
final Map<MultiProtocolURI, String> rss,
@ -400,11 +400,11 @@ dc_rights
return this.emaillinks;
}
public float lon() {
public double lon() {
return this.lon;
}
public float lat() {
public double lat() {
return this.lat;
}
@ -743,7 +743,7 @@ dc_rights
final Map<MultiProtocolURI, Properties> anchors = new HashMap<MultiProtocolURI, Properties>();
final Map<MultiProtocolURI, String> rss = new HashMap<MultiProtocolURI, String>();
final Map<MultiProtocolURI, ImageEntry> images = new HashMap<MultiProtocolURI, ImageEntry>();
float lon = 0.0f, lat = 0.0f;
double lon = 0.0d, lat = 0.0d;
for (final Document doc: docs) {
@ -784,7 +784,7 @@ dc_rights
anchors.putAll(doc.getAnchors());
rss.putAll(doc.getRSS());
ContentScraper.addAllImages(images, doc.getImages());
if (doc.lon() != 0.0f && doc.lat() != 0.0f) { lon = doc.lon(); lat = doc.lat(); }
if (doc.lon() != 0.0d && doc.lat() != 0.0d) { lon = doc.lon(); lat = doc.lat(); }
}
return new Document(
location,

@ -7,7 +7,7 @@
// $LastChangedBy$
//
// LICENSE
//
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
@ -43,9 +43,9 @@ import net.yacy.kelondro.logging.Log;
public class DCEntry extends TreeMap<String, String> {
private static final long serialVersionUID = -2050291583515701559L;
// use a collator to relax when distinguishing between lowercase und uppercase letters
private static final Collator insensitiveCollator = Collator.getInstance(Locale.US);
static {
@ -53,19 +53,19 @@ public class DCEntry extends TreeMap<String, String> {
insensitiveCollator.setDecomposition(Collator.NO_DECOMPOSITION);
}
public static final DCEntry poison = new DCEntry();
public DCEntry() {
super((Collator) insensitiveCollator.clone());
}
public DCEntry(
DigestURI url,
Date date,
String title,
String author,
String body,
float lat,
float lon
double lat,
double lon
) {
super((Collator) insensitiveCollator.clone());
this.put("dc:identifier", url.toNormalform(true, false));
@ -73,10 +73,10 @@ public class DCEntry extends TreeMap<String, String> {
this.put("dc:title", title);
this.put("dc:creator", author);
this.put("dc:description", body);
this.put("geo:lat", Float.toString(lat));
this.put("geo:long", Float.toString(lon));
this.put("geo:lat", Double.toString(lat));
this.put("geo:long", Double.toString(lon));
}
/*
DC according to rfc 5013
@ -108,7 +108,7 @@ public class DCEntry extends TreeMap<String, String> {
return new Date();
}
}
public DigestURI getIdentifier(boolean useRelationAsAlternative) {
String u = this.get("url");
if (u == null) u = this.get("dc:identifier");
@ -130,7 +130,7 @@ public class DCEntry extends TreeMap<String, String> {
return null;
}
}
public DigestURI getRelation() {
String u = this.get("dc:relation");
if (u == null) return null;
@ -146,7 +146,7 @@ public class DCEntry extends TreeMap<String, String> {
return null;
}
}
private String bestU(String[] urls) {
for (String uu: urls) {
if (uu.startsWith("http://") && (uu.endsWith(".html") || uu.endsWith(".htm") || uu.endsWith(".pdf") || uu.endsWith(".doc") || uu.endsWith(".rss") || uu.endsWith(".xml"))) return uu;
@ -164,7 +164,7 @@ public class DCEntry extends TreeMap<String, String> {
}
return urls[0];
}
//modified by copperdust; Ukraine, 2012
public String getLanguage() {//final language computation
String l = this.get("dc:language");//from document metainfo
@ -172,31 +172,31 @@ public class DCEntry extends TreeMap<String, String> {
if (l == null) return this.get("language");//from TLD
return l;
}
public String getType() {
String t = this.get("dc:type");
if (t == null) return "";
return t;
}
public String getFormat() {
String t = this.get("dc:format");
if (t == null) return "";
return t;
}
public String getSource() {
String t = this.get("dc:source");
if (t == null) return "";
return t;
}
public String getRights() {
String t = this.get("dc:rights");
if (t == null) return "";
return t;
}
public String getTitle() {
String t = this.get("title");
if (t == null) t = this.get("dc:title");
@ -204,14 +204,14 @@ public class DCEntry extends TreeMap<String, String> {
if (t == null) return "";
return t;
}
public String getPublisher() {
String t = this.get("dc:publisher");
t = stripCDATA(t);
if (t == null) return "";
return t;
}
public String getCreator() {
String t = this.get("author");
if (t == null) t = this.get("dc:creator");
@ -219,7 +219,7 @@ public class DCEntry extends TreeMap<String, String> {
if (t == null) return "";
return t;
}
public String getDescription() {
String t = this.get("body");
if (t == null) t = this.get("dc:description");
@ -227,31 +227,31 @@ public class DCEntry extends TreeMap<String, String> {
if (t == null) return "";
return t;
}
public String[] getSubject() {
String t = this.get("categories");
if (t == null) this.get("dc:subject");
if (t == null) t = this.get("dc:subject");
t = stripCDATA(t);
if (t == null) return new String[]{};
return t.split(";");
}
public float getLon() {
public double getLon() {
String t = this.get("geo:long");
if (t == null) this.get("geo:lon");
if (t == null) t = this.get("geo:lon");
t = stripCDATA(t);
if (t == null) return 0.0f;
return Float.parseFloat(t);
if (t == null) return 0.0d;
return Double.parseDouble(t);
}
public float getLat() {
public double getLat() {
String t = this.get("geo:lat");
if (t == null) this.get("geo:lat");
if (t == null) t = this.get("geo:lat");
t = stripCDATA(t);
if (t == null) return 0.0f;
return Float.parseFloat(t);
if (t == null) return 0.0d;
return Double.parseDouble(t);
}
private String stripCDATA(String s) {
if (s == null) return null;
s = s.trim();
@ -259,11 +259,11 @@ public class DCEntry extends TreeMap<String, String> {
if (s.endsWith("]]")) s = s.substring(0, s.length() - 2);
return s;
}
public Document document() {
HashSet<String> languages = new HashSet<String>();
languages.add(getLanguage());
return new Document(
getIdentifier(true),
"text/html",
@ -283,7 +283,7 @@ public class DCEntry extends TreeMap<String, String> {
null,
false);
}
public void writeXML(OutputStreamWriter os) throws IOException {
Document doc = document();
if (doc != null) {

@ -214,7 +214,7 @@ public class PhpBB3Dao implements Dao {
String text = xmlCleaner(rs.getString("post_text"));
String user = getUser(rs.getInt("poster_id"));
Date date = new Date(rs.getLong("post_time") * 1000L);
return new DCEntry(url, date, subject, user, text, 0.0f, 0.0f);
return new DCEntry(url, date, subject, user, text, 0.0d, 0.0d);
}
public static String xmlCleaner(String s) {

@ -130,7 +130,7 @@ public class ContentScraper extends AbstractScraper implements Scraper {
private final List<String> li;
private final CharBuffer content;
private final EventListenerList htmlFilterEventListeners;
private float lon, lat;
private double lon, lat;
private MultiProtocolURI canonical;
@ -803,7 +803,7 @@ public class ContentScraper extends AbstractScraper implements Scraper {
// <meta NAME="ICBM" CONTENT="38.90551492, 1.454004505" />
// <meta NAME="geo.position" CONTENT="38.90551492;1.454004505" />
public float getLon() {
public double getLon() {
if (this.lon != 0.0f) return this.lon;
String s = this.metas.get("ICBM"); // InterContinental Ballistic Missile (abbrev. supposed to be a joke: http://www.jargon.net/jargonfile/i/ICBMaddress.html), see http://geourl.org/add.html#icbm
if (s != null) {
@ -811,8 +811,8 @@ public class ContentScraper extends AbstractScraper implements Scraper {
if (p < 0) p = s.indexOf(',');
if (p < 0) p = s.indexOf(' ');
if (p > 0) {
this.lat = Float.parseFloat(s.substring(0, p).trim());
this.lon = Float.parseFloat(s.substring(p + 1).trim());
this.lat = Double.parseDouble(s.substring(0, p).trim());
this.lon = Double.parseDouble(s.substring(p + 1).trim());
}
}
if (this.lon != 0.0f) return this.lon;
@ -822,15 +822,15 @@ public class ContentScraper extends AbstractScraper implements Scraper {
if (p < 0) p = s.indexOf(',');
if (p < 0) p = s.indexOf(' ');
if (p > 0) {
this.lat = Float.parseFloat(s.substring(0, p).trim());
this.lon = Float.parseFloat(s.substring(p + 1).trim());
this.lat = Double.parseDouble(s.substring(0, p).trim());
this.lon = Double.parseDouble(s.substring(p + 1).trim());
}
}
return this.lon;
}
public float getLat() {
if (this.lat != 0.0f) return this.lat;
public double getLat() {
if (this.lat != 0.0d) return this.lat;
getLon(); // parse with getLon() method which creates also the lat value
return this.lat;
}

@ -133,9 +133,14 @@ public class CitationReference implements Reference, Serializable {
return false;
}
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
@Override
public int hashCode() {
return ByteArray.hashCode(this.urlhash());
if (this.hashCache == Integer.MIN_VALUE) {
this.hashCache = ByteArray.hashCode(this.urlhash());
}
return this.hashCache;
}
@Override

@ -9,7 +9,7 @@
// $LastChangedBy$
//
// LICENSE
//
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
@ -52,8 +52,8 @@ public final class ImageReferenceRow extends AbstractReference implements /*Imag
* object for termination of concurrent blocking queue processing
*/
public static final ImageReferenceRow poison = new ImageReferenceRow((Row.Entry) null);
public static final Row urlEntryRow = new Row(new Column[]{
new Column("h", Column.celltype_string, Column.encoder_bytes, Word.commonHashLength, "urlhash"),
new Column("f", Column.celltype_cardinal, Column.encoder_b256, 4, "created"),
@ -65,7 +65,7 @@ public final class ImageReferenceRow extends AbstractReference implements /*Imag
new Column("i", Column.celltype_cardinal, Column.encoder_b256, 2, "height"), // pixels
new Column("i", Column.celltype_cardinal, Column.encoder_b256, 2, "iso"), // iso number
new Column("i", Column.celltype_cardinal, Column.encoder_b256, 2, "verschlusszeit"), // the x in 1/x
new Column("i", Column.celltype_cardinal, Column.encoder_b256, 2, "blende"),
new Column("i", Column.celltype_cardinal, Column.encoder_b256, 2, "blende"),
new Column("i", Column.celltype_cardinal, Column.encoder_b256, 4, "distance"),
new Column("o", Column.celltype_cardinal, Column.encoder_b256, 4, "author-id"), // author, creator, operator, camera-number
new Column("o", Column.celltype_cardinal, Column.encoder_b256, 4, "group-id"), // may be also a crawl start identifier
@ -82,7 +82,7 @@ public final class ImageReferenceRow extends AbstractReference implements /*Imag
Base64Order.enhancedCoder
);
// available chars: b,e,j,q
// static properties
private static final int col_urlhash = 0; // h 12 the url hash b64-encoded
private static final int col_lastModified = 1; // a 2 last-modified time of the document where word appears
@ -101,15 +101,15 @@ public final class ImageReferenceRow extends AbstractReference implements /*Imag
private static final int col_posofphrase = 17; // o 1 number of the phrase where word appears
private static final int col_reserve1 = 18; // i 1 reserve1
private static final int col_reserve2 = 19; // k 1 reserve2
// ideas for the classification bytes
// 0 : content-type (person-portrait, persons-group, landscape, buildings, technical, artistical)
// 1 : content-situation (a categorization of the type, like: person/standing, building/factory, artistical/cubistic)
// 2 : content-category (a classification that is taken from the text environment by text analysis)
// 3 :
// 3 :
private final Row.Entry entry;
public ImageReferenceRow(final byte[] urlHash,
final int urlLength, // byte-length of complete URL
final int urlComps, // number of path components
@ -147,7 +147,7 @@ public final class ImageReferenceRow extends AbstractReference implements /*Imag
this.entry.setCol(col_reserve1, 0);
this.entry.setCol(col_reserve2, 0);
}
public ImageReferenceRow(final byte[] urlHash,
final int urlLength, // byte-length of complete URL
final int urlComps, // number of path components
@ -174,44 +174,47 @@ public final class ImageReferenceRow extends AbstractReference implements /*Imag
this.entry.setCol(col_reserve1, 0);
this.entry.setCol(col_reserve2, 0);
}
public ImageReferenceRow(final String urlHash, final String code) {
// the code is the external form of the row minus the leading urlHash entry
this.entry = urlEntryRow.newEntry(UTF8.getBytes((urlHash + code)));
}
public ImageReferenceRow(final String external) {
this.entry = urlEntryRow.newEntry(external, true);
}
public ImageReferenceRow(final byte[] row) {
this.entry = urlEntryRow.newEntry(row);
}
public ImageReferenceRow(final byte[] row, final int offset, final boolean clone) {
this.entry = urlEntryRow.newEntry(row, offset, clone);
}
public ImageReferenceRow(final Row.Entry rentry) {
// FIXME: see if cloning is necessary
this.entry = rentry;
}
@Override
public ImageReferenceRow clone() {
final byte[] b = new byte[urlEntryRow.objectsize];
System.arraycopy(entry.bytes(), 0, b, 0, urlEntryRow.objectsize);
System.arraycopy(this.entry.bytes(), 0, b, 0, urlEntryRow.objectsize);
return new ImageReferenceRow(b);
}
@Override
public String toPropertyForm() {
return entry.toPropertyForm('=', true, true, false, false);
return this.entry.toPropertyForm('=', true, true, false, false);
}
@Override
public Entry toKelondroEntry() {
return this.entry;
}
@Override
public byte[] urlhash() {
return this.entry.getColBytes(col_urlhash, true);
}
@ -220,10 +223,11 @@ public final class ImageReferenceRow extends AbstractReference implements /*Imag
return (int) this.entry.getColLong(col_lastModified); // this is the time in MicoDateDays format
}
@Override
public long lastModified() {
return MicroDate.reverseMicroDateDays((int) this.entry.getColLong(col_lastModified));
}
public long freshUntil() {
return MicroDate.reverseMicroDateDays((int) this.entry.getColLong(col_freshUntil));
}
@ -232,6 +236,7 @@ public final class ImageReferenceRow extends AbstractReference implements /*Imag
return (int) this.entry.getColLong(col_hitcount);
}
@Override
public Collection<Integer> positions() {
return new ArrayList<Integer>(0);
}
@ -253,38 +258,45 @@ public final class ImageReferenceRow extends AbstractReference implements /*Imag
public char getType() {
return (char) this.entry.getColByte(col_doctype);
}
public int urllength() {
return (int) this.entry.getColLong(col_urlLength);
}
public int urlcomps() {
return (int) this.entry.getColLong(col_urlComps);
}
public Bitfield flags() {
return new Bitfield(this.entry.getColBytes(col_flags, true));
}
@Override
public String toString() {
return toPropertyForm();
}
@Override
public boolean isOlder(final Reference other) {
if (other == null) return false;
if (this.lastModified() < other.lastModified()) return true;
return false;
}
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
@Override
public int hashCode() {
return ByteArray.hashCode(this.urlhash());
if (this.hashCache == Integer.MIN_VALUE) {
this.hashCache = ByteArray.hashCode(this.urlhash());
}
return this.hashCache;
}
@Override
public void join(Reference oe) {
throw new UnsupportedOperationException("");
}
}

@ -381,9 +381,14 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere
this.wordsintext = this.wordsintext + oe.wordsintext();
}
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
@Override
public int hashCode() {
return ByteArray.hashCode(this.urlHash);
if (this.hashCache == Integer.MIN_VALUE) {
this.hashCache = ByteArray.hashCode(this.urlHash);
}
return this.hashCache;
}
public void addPosition(int position) {

@ -160,12 +160,14 @@ public class DigestURI extends MultiProtocolURI implements Serializable {
this.hash = null;
}
/* (non-Javadoc)
* @see java.lang.Object#hashCode()
*/
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
@Override
public int hashCode() {
return ByteArray.hashCode(hash());
if (this.hashCache == Integer.MIN_VALUE) {
this.hashCache = ByteArray.hashCode(hash());
}
return this.hashCache;
}
public static final int flagTypeID(final String hash) {

@ -9,12 +9,12 @@
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
@ -31,17 +31,17 @@ import net.yacy.kelondro.rwi.Reference;
public interface URIMetadata extends URIReference {
public String dc_title();
public String dc_creator();
public String dc_publisher();
public String dc_subject();
public float lat();
public float lon();
public double lat();
public double lon();
public long ranking();
public Date loaddate();
@ -71,13 +71,13 @@ public interface URIMetadata extends URIReference {
public int lvideo();
public int lapp();
public String snippet();
public Reference word();
public boolean isOlder(final URIMetadata other);
public String toString(final String snippet);
}

@ -123,7 +123,7 @@ public class URIMetadataRow implements URIMetadata {
final String dc_creator,
final String dc_subject,
final String dc_publisher,
final float lon, final float lat, // decimal degrees as in WGS84; if unknown both values may be 0.0f;
final double lon, final double lat, // decimal degrees as in WGS84; if unknown both values may be 0.0d;
final Date mod,
final Date load,
final Date fresh,
@ -194,8 +194,8 @@ public class URIMetadataRow implements URIMetadata {
final String dc_creator,
final String dc_subject,
final String dc_publisher,
final float lat,
final float lon) {
final double lat,
final double lon) {
final CharBuffer s = new CharBuffer(3600, 360);
s.append(url.toNormalform(false, true)).appendLF();
s.append(dc_title).appendLF();
@ -205,7 +205,7 @@ public class URIMetadataRow implements URIMetadata {
s.appendLF();
if (dc_publisher.length() > 80) s.append(dc_publisher, 0, 80); else s.append(dc_publisher);
s.appendLF();
if (lon == 0.0f && lat == 0.0f) s.appendLF(); else s.append(Float.toString(lat)).append(',').append(Float.toString(lon)).appendLF();
if (lon == 0.0f && lat == 0.0f) s.appendLF(); else s.append(Double.toString(lat)).append(',').append(Double.toString(lon)).appendLF();
return UTF8.getBytes(s.toString());
}
@ -418,11 +418,11 @@ public class URIMetadataRow implements URIMetadata {
return this.metadata().dc_subject();
}
public float lat() {
public double lat() {
return this.metadata().lat();
}
public float lon() {
public double lon() {
return this.metadata().lon();
}
@ -638,15 +638,15 @@ public class URIMetadataRow implements URIMetadata {
public String dc_creator() { return this.dc_creator; }
public String dc_publisher() { return this.dc_publisher; }
public String dc_subject() { return this.dc_subject; }
public float lat() {
if (this.latlon == null || this.latlon.length() == 0) return 0.0f;
public double lat() {
if (this.latlon == null || this.latlon.length() == 0) return 0.0d;
final int p = this.latlon.indexOf(',');
return p < 0 ? 0.0f : Float.parseFloat(this.latlon.substring(0, p));
return p < 0 ? 0.0f : Double.parseDouble(this.latlon.substring(0, p));
}
public float lon() {
if (this.latlon == null || this.latlon.length() == 0) return 0.0f;
public double lon() {
if (this.latlon == null || this.latlon.length() == 0) return 0.0d;
final int p = this.latlon.indexOf(',');
return p < 0 ? 0.0f : Float.parseFloat(this.latlon.substring(p + 1));
return p < 0 ? 0.0f : Double.parseDouble(this.latlon.substring(p + 1));
}
}

@ -215,6 +215,7 @@ public final class WordReferenceRow extends AbstractReference implements WordRef
this.out = new LinkedBlockingQueue<WordReferenceRow>();
for (int i = 0; i < concurrency; i++) {
this.worker[i] = new Thread() {
@Override
public void run() {
String s;
try {
@ -290,22 +291,27 @@ public final class WordReferenceRow extends AbstractReference implements WordRef
return new WordReferenceRow(b);
}
@Override
public String toPropertyForm() {
return this.entry.toPropertyForm('=', true, true, false, false);
}
@Override
public Entry toKelondroEntry() {
return this.entry;
}
@Override
public byte[] urlhash() {
return this.entry.getColBytes(col_urlhash, true);
}
@Override
public int virtualAge() {
return (int) this.entry.getColLong(col_lastModified); // this is the time in MicoDateDays format
}
@Override
public long lastModified() {
return MicroDate.reverseMicroDateDays((int) this.entry.getColLong(col_lastModified));
}
@ -314,10 +320,12 @@ public final class WordReferenceRow extends AbstractReference implements WordRef
return MicroDate.reverseMicroDateDays((int) this.entry.getColLong(col_freshUntil));
}
@Override
public int hitcount() {
return (0xff & this.entry.getColByte(col_hitcount));
}
@Override
public Collection<Integer> positions() {
return new ArrayList<Integer>(0);
}
@ -327,54 +335,67 @@ public final class WordReferenceRow extends AbstractReference implements WordRef
return (int) this.entry.getColLong(col_posintext);
}
@Override
public int posinphrase() {
return (0xff & this.entry.getColByte(col_posinphrase));
}
@Override
public int posofphrase() {
return (0xff & this.entry.getColByte(col_posofphrase));
}
@Override
public int wordsintext() {
return (int) this.entry.getColLong(col_wordsInText);
}
@Override
public int phrasesintext() {
return (int) this.entry.getColLong(col_phrasesInText);
}
@Override
public byte[] getLanguage() {
return this.entry.getColBytes(col_language, true);
}
@Override
public char getType() {
return (char) this.entry.getColByte(col_doctype);
}
@Override
public int wordsintitle() {
return (0xff & this.entry.getColByte(col_wordsInTitle));
}
@Override
public int llocal() {
return (0xff & this.entry.getColByte(col_llocal));
}
@Override
public int lother() {
return (0xff & this.entry.getColByte(col_lother));
}
@Override
public int urllength() {
return (0xff & this.entry.getColByte(col_urlLength));
}
@Override
public int urlcomps() {
return (0xff & this.entry.getColByte(col_urlComps));
}
@Override
public Bitfield flags() {
return new Bitfield(this.entry.getColBytes(col_flags, false));
}
@Override
public double termFrequency() {
return (((double) hitcount()) / ((double) (wordsintext() + wordsintitle() + 1)));
}
@ -393,11 +414,17 @@ public final class WordReferenceRow extends AbstractReference implements WordRef
return Base64Order.enhancedCoder.equal(urlhash(), other.urlhash());
}
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
@Override
public int hashCode() {
return ByteArray.hashCode(urlhash());
if (this.hashCache == Integer.MIN_VALUE) {
this.hashCache = ByteArray.hashCode(urlhash());
}
return this.hashCache;
}
@Override
public void join(final Reference oe) {
throw new UnsupportedOperationException("");

@ -31,7 +31,6 @@ import java.util.Comparator;
import java.util.Queue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.Semaphore;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.UTF8;
@ -394,9 +393,14 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
return Base64Order.enhancedCoder.equal(this.urlHash, other.urlHash);
}
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
@Override
public int hashCode() {
return ByteArray.hashCode(this.urlHash);
if (this.hashCache == Integer.MIN_VALUE) {
this.hashCache = ByteArray.hashCode(this.urlHash);
}
return this.hashCache;
}
@Override
@ -419,7 +423,7 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
* @return a blocking queue filled with WordReferenceVars that is still filled when the object is returned
*/
public static BlockingQueue<WordReferenceVars> transform(final ReferenceContainer<WordReference> container) {
public static BlockingQueue<WordReferenceVars> transform(final ReferenceContainer<WordReference> container, final long maxtime) {
final LinkedBlockingQueue<WordReferenceVars> vars = new LinkedBlockingQueue<WordReferenceVars>();
if (container.size() <= 100) {
// transform without concurrency to omit thread creation overhead
@ -431,7 +435,7 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
} catch (final InterruptedException e) {}
return vars;
}
final Thread distributor = new TransformDistributor(container, vars);
final Thread distributor = new TransformDistributor(container, vars, maxtime);
distributor.start();
// return the resulting queue while the processing queues are still working
@ -442,32 +446,37 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
ReferenceContainer<WordReference> container;
BlockingQueue<WordReferenceVars> out;
long maxtime;
public TransformDistributor(final ReferenceContainer<WordReference> container, final BlockingQueue<WordReferenceVars> out) {
public TransformDistributor(final ReferenceContainer<WordReference> container, final BlockingQueue<WordReferenceVars> out, final long maxtime) {
this.container = container;
this.out = out;
this.maxtime = maxtime;
}
@Override
public void run() {
// start the transformation threads
final int cores0 = Math.min(cores, this.container.size() / 100) + 1;
final Semaphore termination = new Semaphore(cores0);
final TransformWorker[] worker = new TransformWorker[cores0];
for (int i = 0; i < cores0; i++) {
worker[i] = new TransformWorker(this.out, termination);
worker[i] = new TransformWorker(this.out, this.maxtime);
worker[i].start();
}
long timeout = System.currentTimeMillis() + this.maxtime;
// fill the queue
int p = this.container.size();
while (p > 0) {
p--;
worker[p % cores0].add(this.container.get(p, false));
if (p % 100 == 0 && System.currentTimeMillis() > timeout) break;
}
// insert poison to stop the queues
for (int i = 0; i < cores0; i++) worker[i].add(WordReferenceRow.poisonRowEntry);
for (int i = 0; i < cores0; i++) {
worker[i].add(WordReferenceRow.poisonRowEntry);
}
}
}
@ -475,12 +484,12 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
BlockingQueue<Row.Entry> in;
BlockingQueue<WordReferenceVars> out;
Semaphore termination;
long maxtime;
public TransformWorker(final BlockingQueue<WordReferenceVars> out, final Semaphore termination) {
public TransformWorker(final BlockingQueue<WordReferenceVars> out, final long maxtime) {
this.in = new LinkedBlockingQueue<Row.Entry>();
this.out = out;
this.termination = termination;
this.maxtime = maxtime;
}
public void add(final Row.Entry entry) {
@ -493,15 +502,13 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
@Override
public void run() {
Row.Entry entry;
long timeout = System.currentTimeMillis() + this.maxtime;
try {
while ((entry = this.in.take()) != WordReferenceRow.poisonRowEntry) this.out.put(new WordReferenceVars(new WordReferenceRow(entry)));
while ((entry = this.in.take()) != WordReferenceRow.poisonRowEntry) {
this.out.put(new WordReferenceVars(new WordReferenceRow(entry)));
if (System.currentTimeMillis() > timeout) break;
}
} catch (final InterruptedException e) {}
// insert poison to signal the termination to next queue
try {
this.termination.acquire();
if (this.termination.availablePermits() == 0) this.out.put(WordReferenceVars.poison);
} catch (final InterruptedException e) {}
}
}

@ -157,14 +157,14 @@ public final class Row implements Serializable {
public final Entry newEntry(final byte[] rowinstance) {
if (rowinstance == null) return null;
assert (this.objectOrder.wellformed(rowinstance, 0, this.primaryKeyLength)) : "row not well-formed: rowinstance[0] = " + UTF8.String(rowinstance, 0, this.primaryKeyLength) + " / " + NaturalOrder.arrayList(rowinstance, 0, this.primaryKeyLength);
return new Entry(rowinstance, false);
return new Entry(rowinstance, 0, false);
}
public final Entry newEntry(final Entry oldrow, final int fromColumn) {
if (oldrow == null) return null;
assert (oldrow.getColBytes(0, false)[0] != 0);
assert (this.objectOrder.wellformed(oldrow.getPrimaryKeyBytes(), 0, this.primaryKeyLength));
return new Entry(oldrow, fromColumn, false);
return new Entry(oldrow.rowinstance, oldrow.offset + oldrow.colstart(fromColumn), false);
}
public final Entry newEntry(final byte[] rowinstance, final int start, final boolean clone) {
@ -251,14 +251,6 @@ public final class Row implements Serializable {
this.offset = 0;
}
public Entry(final byte[] newrow, final boolean forceclone) {
this(newrow, 0, forceclone);
}
public Entry(final Entry oldrow, final int fromColumn, final boolean forceclone) {
this(oldrow.rowinstance, oldrow.offset + oldrow.colstart(fromColumn), forceclone);
}
public Entry(final byte[] newrow, final int start, final boolean forceclone) {
if (forceclone || newrow.length - start < Row.this.objectsize) {
this.rowinstance = new byte[Row.this.objectsize];
@ -643,7 +635,7 @@ public final class Row implements Serializable {
private final int index;
public EntryIndex(final byte[] row, final int i) {
super(row, false);
super(row, 0, false);
this.index = i;
}
public int index() {

@ -76,6 +76,7 @@ public final class Log {
}
public final void logSevere(final String message, final Throwable thrown) {
if (thrown == null) return;
enQueueLog(this.theLogger, Level.SEVERE, message, thrown);
}
@ -88,6 +89,7 @@ public final class Log {
}
public final void logWarning(final String message, final Throwable thrown) {
if (thrown == null) return;
enQueueLog(this.theLogger, Level.WARNING, message, thrown);
}
@ -100,6 +102,7 @@ public final class Log {
}
public final void logConfig(final String message, final Throwable thrown) {
if (thrown == null) return;
enQueueLog(this.theLogger, Level.CONFIG, message, thrown);
}
@ -112,6 +115,7 @@ public final class Log {
}
public final void logInfo(final String message, final Throwable thrown) {
if (thrown == null) return;
enQueueLog(this.theLogger, Level.INFO, message, thrown);
}
@ -124,6 +128,7 @@ public final class Log {
}
public final void logFine(final String message, final Throwable thrown) {
if (thrown == null) return;
enQueueLog(this.theLogger, Level.FINE, message, thrown);
}
@ -136,6 +141,7 @@ public final class Log {
}
public final void logFiner(final String message, final Throwable thrown) {
if (thrown == null) return;
enQueueLog(this.theLogger, Level.FINER, message, thrown);
}
@ -148,6 +154,7 @@ public final class Log {
}
public final void logFinest(final String message, final Throwable thrown) {
if (thrown == null) return;
enQueueLog(this.theLogger, Level.FINEST, message, thrown);
}
@ -165,6 +172,7 @@ public final class Log {
enQueueLog(appName, Level.SEVERE, message);
}
public final static void logSevere(final String appName, final String message, final Throwable thrown) {
if (thrown == null) return;
enQueueLog(appName, Level.SEVERE, message, thrown);
}
@ -172,9 +180,11 @@ public final class Log {
enQueueLog(appName, Level.WARNING, message);
}
public final static void logException(final Throwable thrown) {
if (thrown == null) return;
enQueueLog("StackTrace", Level.WARNING, thrown.getMessage(), thrown);
}
public final static void logWarning(final String appName, final String message, final Throwable thrown) {
if (thrown == null) return;
enQueueLog(appName, Level.WARNING, message, thrown);
}
@ -182,6 +192,7 @@ public final class Log {
enQueueLog(appName, Level.CONFIG, message);
}
public final static void logConfig(final String appName, final String message, final Throwable thrown) {
if (thrown == null) return;
enQueueLog(appName, Level.CONFIG, message, thrown);
}
@ -189,6 +200,7 @@ public final class Log {
enQueueLog(appName, Level.INFO, message);
}
public final static void logInfo(final String appName, final String message, final Throwable thrown) {
if (thrown == null) return;
enQueueLog(appName, Level.INFO, message, thrown);
}
@ -196,6 +208,7 @@ public final class Log {
enQueueLog(appName, Level.FINE, message);
}
public final static void logFine(final String appName, final String message, final Throwable thrown) {
if (thrown == null) return;
enQueueLog(appName, Level.FINE, message, thrown);
}
public final static boolean isFine(final String appName) {
@ -206,6 +219,7 @@ public final class Log {
enQueueLog(appName, Level.FINER, message);
}
public final static void logFiner(final String appName, final String message, final Throwable thrown) {
if (thrown == null) return;
enQueueLog(appName, Level.FINER, message, thrown);
}
@ -213,6 +227,7 @@ public final class Log {
enQueueLog(appName, Level.FINEST, message);
}
public final static void logFinest(final String appName, final String message, final Throwable thrown) {
if (thrown == null) return;
enQueueLog(appName, Level.FINEST, message, thrown);
}
public final static boolean isFinest(final String appName) {
@ -220,6 +235,7 @@ public final class Log {
}
private final static void enQueueLog(final Logger logger, final Level level, final String message, final Throwable thrown) {
if (thrown == null) return;
if (logRunnerThread == null || !logRunnerThread.isAlive()) {
logger.log(level, message, thrown);
} else {
@ -244,6 +260,7 @@ public final class Log {
}
private final static void enQueueLog(final String loggername, final Level level, final String message, final Throwable thrown) {
if (thrown == null) return;
if (logRunnerThread == null || !logRunnerThread.isAlive()) {
Logger.getLogger(loggername).log(level, message, thrown);
} else {
@ -385,12 +402,15 @@ public final class Log {
Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler(){
@Override
public void uncaughtException(final Thread t, final Throwable e) {
if (e == null) return;
final String msg = String.format("Thread %s: %s",t.getName(), e.getMessage());
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
final PrintStream ps = new PrintStream(baos);
e.printStackTrace(ps);
ps.close();
exceptionLog.logSevere(msg + "\n" + baos.toString(), e);
Log.logException(e);
Log.logException(e.getCause());
//System.err.print("Exception in thread \"" + t.getName() + "\" ");
//e.printStackTrace(System.err);
}

@ -35,14 +35,14 @@ import net.yacy.kelondro.logging.Log;
public class MergeIterator<E> implements CloneableIterator<E> {
private final Comparator<E> comp;
private final CloneableIterator<E> a;
private final CloneableIterator<E> b;
private E na, nb;
private final Method merger;
private final boolean up;
public MergeIterator(
final CloneableIterator<E> a,
final CloneableIterator<E> b,
@ -61,51 +61,54 @@ public class MergeIterator<E> implements CloneableIterator<E> {
nextb();
}
@Override
public MergeIterator<E> clone(final Object modifier) {
assert a != null;
assert b != null;
assert merger != null;
return new MergeIterator<E>(a.clone(modifier), b.clone(modifier), comp, merger, up);
assert this.a != null;
assert this.b != null;
assert this.merger != null;
return new MergeIterator<E>(this.a.clone(modifier), this.b.clone(modifier), this.comp, this.merger, this.up);
}
private void nexta() {
try {
if (a != null && a.hasNext()) na = a.next(); else na = null;
if (this.a != null && this.a.hasNext()) this.na = this.a.next(); else this.na = null;
} catch (final ConcurrentModificationException e) {
na = null;
this.na = null;
}
}
private void nextb() {
try {
if (b != null && b.hasNext()) nb = b.next(); else nb = null;
if (this.b != null && this.b.hasNext()) this.nb = this.b.next(); else this.nb = null;
} catch (final ConcurrentModificationException e) {
nb = null;
this.nb = null;
}
}
@Override
public boolean hasNext() {
return (na != null) || (nb != null);
return (this.na != null) || (this.nb != null);
}
@SuppressWarnings("unchecked")
@Override
@SuppressWarnings("unchecked")
public E next() {
E s;
if (na == null) {
s = nb;
if (this.na == null) {
s = this.nb;
nextb();
return s;
}
if (nb == null) {
s = na;
if (this.nb == null) {
s = this.na;
nexta();
return s;
}
// compare the Objects
final int c = comp.compare(na, nb);
final int c = this.comp.compare(this.na, this.nb);
if (c == 0) {
try {
//System.out.print("MERGE OF " + na.toString() + " AND " + nb.toString() + ": ");
s = (E) this.merger.invoke(null, new Object[]{na, nb});
s = (E) this.merger.invoke(null, new Object[]{this.na, this.nb});
//System.out.println("RESULT IS " + s.toString());
} catch (final IllegalArgumentException e) {
Log.logException(e);
@ -115,26 +118,28 @@ public class MergeIterator<E> implements CloneableIterator<E> {
s = null;
} catch (final InvocationTargetException e) {
Log.logException(e);
Log.logException(e.getCause());
s = null;
}
nexta();
nextb();
return s;
} else if ((up && c < 0) || (!up && c > 0)) {
s = na;
} else if ((this.up && c < 0) || (!this.up && c > 0)) {
s = this.na;
nexta();
return s;
} else {
s = nb;
s = this.nb;
nextb();
return s;
}
}
@Override
public void remove() {
throw new java.lang.UnsupportedOperationException("merge does not support remove");
}
public static <A> CloneableIterator<A> cascade(final Collection<CloneableIterator<A>> iterators, final Order<A> c, final Method merger, final boolean up) {
// this extends the ability to combine two iterators
// to the ability of combining a set of iterators
@ -142,7 +147,7 @@ public class MergeIterator<E> implements CloneableIterator<E> {
if (iterators.isEmpty()) return null;
return cascade(iterators.iterator(), c, merger, up);
}
private static <A> CloneableIterator<A> cascade(final Iterator<CloneableIterator<A>> iiterators, final Order<A> c, final Method merger, final boolean up) {
if (iiterators == null) return null;
if (!(iiterators.hasNext())) return null;
@ -151,7 +156,7 @@ public class MergeIterator<E> implements CloneableIterator<E> {
assert merger != null;
return new MergeIterator<A>(one, cascade(iiterators, c, merger, up), c, merger, up);
}
public static final Method simpleMerge;
static {
Method meth = null;
@ -166,9 +171,9 @@ public class MergeIterator<E> implements CloneableIterator<E> {
meth = null;
}
assert meth != null;
simpleMerge = meth;
simpleMerge = meth;
}
// do not remove the following method, it is not reference anywhere directly but indirectly using reflection
// please see initialization of simpleMerge above
public static Object mergeEqualByReplace(final Object a, final Object b) {

@ -7,7 +7,7 @@
// $LastChangedBy$
//
// LICENSE
//
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
@ -42,28 +42,28 @@ import net.yacy.cora.order.ByteOrder;
*/
public class ByteArray {
private final byte[] buffer;
private int hash;
private final int hash;
public ByteArray(final byte[] bb) {
this.buffer = bb;
this.hash = 0;
}
public int length() {
return buffer.length;
return this.buffer.length;
}
public byte[] asBytes() {
return this.buffer;
}
public byte readByte(final int pos) {
return buffer[pos];
return this.buffer[pos];
}
public static boolean startsWith(final byte[] buffer, final byte[] pattern) {
// compares two byte arrays: true, if pattern appears completely at offset position
if (buffer == null && pattern == null) return true;
@ -72,22 +72,25 @@ public class ByteArray {
for (int i = 0; i < pattern.length; i++) if (buffer[i] != pattern[i]) return false;
return true;
}
public int compareTo(final ByteArray b, final ByteOrder order) {
assert this.buffer.length == b.buffer.length;
return order.compare(this.buffer, b.buffer);
}
public int compareTo(final int aoffset, final int alength, final ByteArray b, final int boffset, final int blength, final ByteOrder order) {
assert alength == blength;
return order.compare(this.buffer, aoffset, b.buffer, boffset, blength);
}
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
@Override
public int hashCode() {
if (this.hash != 0) return this.hash;
this.hash = hashCode(this.buffer);
return this.hash;
if (this.hashCache == Integer.MIN_VALUE) {
this.hashCache = ByteArray.hashCode(this.buffer);
}
return this.hashCache;
}
/**
@ -100,18 +103,18 @@ public class ByteArray {
for (byte c: b) h = 31 * h + (c & 0xFF);
return h;
}
@Override
public boolean equals(Object other) {
ByteArray b = (ByteArray) other;
if (buffer == null && b == null) return true;
if (buffer == null || b == null) return false;
if (this.buffer == null && b == null) return true;
if (this.buffer == null || b == null) return false;
if (this.buffer.length != b.buffer.length) return false;
int l = this.buffer.length;
while (--l >= 0) if (this.buffer[l] != b.buffer[l]) return false;
return true;
}
public static long parseDecimal(final byte[] s) throws NumberFormatException {
if (s == null) throw new NumberFormatException("null");
@ -121,7 +124,7 @@ public class ByteArray {
long limit;
long multmin;
long digit;
if (max <= 0) throw new NumberFormatException(UTF8.String(s));
if (s[0] == '-') {
negative = true;

@ -84,10 +84,12 @@ public class InstantBlockingThread<J extends WorkflowJob> extends AbstractBlocki
}
}
@Override
public int getJobCount() {
return getManager().queueSize();
}
@Override
@SuppressWarnings("unchecked")
public J job(final J next) throws Exception {
J out = null;
@ -116,8 +118,9 @@ public class InstantBlockingThread<J extends WorkflowJob> extends AbstractBlocki
terminate(false);
} catch (final InvocationTargetException e) {
final String targetException = e.getTargetException().getMessage();
Log.logException(e.getTargetException());
Log.logException(e);
Log.logException(e.getCause());
Log.logException(e.getTargetException());
if ((targetException != null) &&
((targetException.indexOf("heap space",0) > 0) ||
(targetException.indexOf("NullPointerException",0) > 0))) {

@ -83,6 +83,7 @@ public final class InstantBusyThread extends AbstractBusyThread implements BusyT
this.handle = Long.valueOf(System.currentTimeMillis() + getName().hashCode());
}
@Override
public int getJobCount() {
if (this.jobCountMethod == null) return Integer.MAX_VALUE;
try {
@ -101,6 +102,7 @@ public final class InstantBusyThread extends AbstractBusyThread implements BusyT
}
}
@Override
public boolean job() throws Exception {
instantThreadCounter++;
//System.out.println("started job " + this.handle + ": " + this.getName());
@ -121,6 +123,7 @@ public final class InstantBusyThread extends AbstractBusyThread implements BusyT
} catch (final InvocationTargetException e) {
final String targetException = e.getTargetException().getMessage();
Log.logException(e);
Log.logException(e.getCause());
Log.logException(e.getTargetException());
Log.logSevere("BUSYTHREAD", "Runtime Error in serverInstantThread.job, thread '" + getName() + "': " + e.getMessage() + "; target exception: " + targetException, e.getTargetException());
} catch (final OutOfMemoryError e) {
@ -136,6 +139,7 @@ public final class InstantBusyThread extends AbstractBusyThread implements BusyT
return jobHasDoneSomething;
}
@Override
public void freemem() {
if (this.freememExecMethod == null) return;
try {

@ -263,7 +263,7 @@ public final class Protocol
if ( mytype == null ) {
mytype = "";
}
// set my own seed according to new information
// we overwrite our own IP number only
if ( serverCore.useStaticIP ) {
@ -786,7 +786,7 @@ public final class Protocol
// store remote result to local result container
// insert one container into the search result buffer
// one is enough, only the references are used, not the word
containerCache.add(container[0], false, target.getName() + "/" + target.hash, result.joincount, true);
containerCache.add(container[0], false, target.getName() + "/" + target.hash, result.joincount, true, 5000);
containerCache.addExpectedRemoteReferences(-count);
// insert the containers to the index

@ -741,7 +741,7 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
private boolean getFlag(final int flag) {
final String flags = get(Seed.FLAGS, Seed.FLAGSZERO);
return (new bitfield(UTF8.getBytes(flags))).get(flag);
return (new bitfield(ASCII.getBytes(flags))).get(flag);
}
private void setFlag(final int flag, final boolean value) {
@ -749,7 +749,7 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
if ( flags.length() != 4 ) {
flags = Seed.FLAGSZERO;
}
final bitfield f = new bitfield(UTF8.getBytes(flags));
final bitfield f = new bitfield(ASCII.getBytes(flags));
f.set(flag, value);
this.dna.put(Seed.FLAGS, UTF8.String(f.getBytes()));
}

@ -143,7 +143,7 @@ public final class yacyRelease extends yacyVersion {
// return a dev-release or a main-release
if ((latestdev != null) &&
((latestmain == null) || (latestdev.compareTo(latestmain) > 0)) &&
(!(Float.toString(latestdev.getReleaseNr()).matches(blacklist)))) {
(!(Double.toString(latestdev.getReleaseNr()).matches(blacklist)))) {
// consider a dev-release
if (latestdev.compareTo(thisVersion()) <= 0) {
Network.log.logInfo(
@ -155,7 +155,7 @@ public final class yacyRelease extends yacyVersion {
}
if (latestmain != null) {
// consider a main release
if ((Float.toString(latestmain.getReleaseNr()).matches(blacklist))) {
if ((Double.toString(latestmain.getReleaseNr()).matches(blacklist))) {
Network.log.logInfo(
"rulebasedUpdateInfo: latest dev " + (latestdev == null ? "null" : latestdev.getName()) +
" matches with blacklist '" + blacklist + "'");
@ -172,7 +172,7 @@ public final class yacyRelease extends yacyVersion {
}
if ((concept.equals("main")) && (latestmain != null)) {
// return a main-release
if ((Float.toString(latestmain.getReleaseNr()).matches(blacklist))) {
if ((Double.toString(latestmain.getReleaseNr()).matches(blacklist))) {
Network.log.logInfo(
"rulebasedUpdateInfo: latest main " + latestmain.getName() +
" matches with blacklist'" + blacklist + "'");

@ -21,7 +21,7 @@ public class yacyVersion implements Comparator<yacyVersion>, Comparable<yacyVers
* this value is overwritten when a peer with later version appears*/
public static double latestRelease = 0.1; //
private float releaseNr;
private double releaseNr;
private final String dateStamp;
private int svn;
private final boolean mainRelease;
@ -58,9 +58,9 @@ public class yacyVersion implements Comparator<yacyVersion>, Comparable<yacyVers
throw new RuntimeException("release file name '" + release + "' is not valid, 3 information parts expected");
}
try {
this.releaseNr = Float.parseFloat(comp[0]);
this.releaseNr = Double.parseDouble(comp[0]);
} catch (final NumberFormatException e) {
throw new RuntimeException("release file name '" + release + "' is not valid, '" + comp[0] + "' should be a float number");
throw new RuntimeException("release file name '" + release + "' is not valid, '" + comp[0] + "' should be a double number");
}
this.mainRelease = ((int) (getReleaseNr() * 100)) % 10 == 0 || (host != null && host.endsWith("yacy.net"));
//System.out.println("Release version " + this.releaseNr + " is " + ((this.mainRelease) ? "main" : "std"));
@ -111,7 +111,7 @@ public class yacyVersion implements Comparator<yacyVersion>, Comparable<yacyVers
*/
@Override
public int compare(final yacyVersion v0, final yacyVersion v1) {
int r = (Float.valueOf(v0.getReleaseNr())).compareTo(Float.valueOf(v1.getReleaseNr()));
int r = (Double.valueOf(v0.getReleaseGitNr())).compareTo(Double.valueOf(v1.getReleaseGitNr()));
if (r != 0) return r;
r = v0.getDateStamp().compareTo(v1.getDateStamp());
if (r != 0) return r;
@ -122,7 +122,7 @@ public class yacyVersion implements Comparator<yacyVersion>, Comparable<yacyVers
public boolean equals(final Object obj) {
if (obj instanceof yacyVersion) {
final yacyVersion v = (yacyVersion) obj;
return (getReleaseNr() == v.getReleaseNr()) && (getSvn() == v.getSvn()) && (getName().equals(v.getName()));
return (getReleaseGitNr() == v.getReleaseGitNr()) && (getSvn() == v.getSvn()) && (getName().equals(v.getName()));
}
return false;
}
@ -152,7 +152,7 @@ public class yacyVersion implements Comparator<yacyVersion>, Comparable<yacyVers
return new String[]{"dev", "0000"};
}
final String mainversion = (Float.parseFloat(matcher.group(1)) < 0.11 ? "dev" : matcher.group(1));
final String mainversion = (Double.parseDouble(matcher.group(1)) < 0.11 ? "dev" : matcher.group(1));
String revision = matcher.group(2);
for(int i=revision.length();i<5;++i) revision += "0";
return new String[]{mainversion, revision};
@ -201,13 +201,17 @@ public class yacyVersion implements Comparator<yacyVersion>, Comparable<yacyVers
}
/**
* release number as float (e. g. 7.04)
* release number as Double (e. g. 7.04)
* @return
*/
public float getReleaseNr() {
public double getReleaseNr() {
return this.releaseNr;
}
public double getReleaseGitNr() {
// combine release number with git number
return this.getReleaseNr() + ((getSvn()) / 10000000.0d);
}
public String getName() {
return this.name;

@ -264,7 +264,7 @@ public class Segment {
try {
container = ReferenceContainer.emptyContainer(Segment.wordReferenceFactory, wordhash, 1);
container.add(ientry);
rankingProcess.add(container, true, sourceName, -1, !i.hasNext());
rankingProcess.add(container, true, sourceName, -1, !i.hasNext(), 5000);
} catch (final RowSpaceExceededException e) {
continue;
}
@ -377,7 +377,7 @@ public class Segment {
document.dc_subject(' '), // tags
document.dc_publisher(), // publisher (may be important to get location data)
document.lon(), // decimal degrees as in WGS84;
document.lat(), // if unknown both values may be 0.0f;
document.lat(), // if unknown both values may be 0.0d;
modDate, // modification date
loadDate, // loaded date
new Date(loadDate.getTime() + Math.max(0, loadDate.getTime() - modDate.getTime()) / 2), // freshdate, computed with Proxy-TTL formula

@ -108,6 +108,10 @@ public class SolrConfiguration extends ConfigurationSet implements Serializable
if (isEmpty() || contains(key.name())) solrdoc.addSolr(key, value);
}
protected void addSolr(final SolrDoc solrdoc, final SolrField key, final double value) {
if (isEmpty() || contains(key.name())) solrdoc.addSolr(key, value);
}
protected void addSolr(final SolrDoc solrdoc, final SolrField key, final boolean value) {
if (isEmpty() || contains(key.name())) solrdoc.addSolr(key, value);
}

@ -44,7 +44,7 @@ public class AccessTracker {
public static final int minSize = 100;
public static final int maxSize = 1000;
public static final int maxAge = 10 * 60 * 1000;
public static final int maxAge = 24 * 60 * 60 * 1000;
public enum Location {local, remote}

@ -141,6 +141,7 @@ public final class QueryParams {
public boolean specialRights; // is true if the user has a special authorization and my use more database-extensive options
public final String userAgent;
public boolean filterfailurls;
public double lat, lon, radius;
public QueryParams(
final String queryString,
@ -198,6 +199,9 @@ public final class QueryParams {
this.userAgent = userAgent;
this.transmitcount = 0;
this.filterfailurls = false;
this.lat = 0.0d;
this.lon = 0.0d;
this.radius = 0.0d;
}
public QueryParams(
@ -224,7 +228,8 @@ public final class QueryParams {
final Segment indexSegment,
final RankingProfile ranking,
final String userAgent,
final boolean filterfailurls) {
final boolean filterfailurls,
final double lat, final double lon, final double radius) {
this.queryString = queryString;
this.queryHashes = queryHashes;
@ -271,8 +276,15 @@ public final class QueryParams {
this.userAgent = userAgent;
this.transmitcount = 0;
this.filterfailurls = filterfailurls;
// we normalize here the location and radius because that should cause a better caching
// and as surplus it will increase privacy
this.lat = Math.floor(lat * this.kmNormal) / this.kmNormal;
this.lon = Math.floor(lon * this.kmNormal) / this.kmNormal;
this.radius = Math.floor(radius * this.kmNormal + 1) / this.kmNormal;
}
double kmNormal = 100.d; // 100 =ca 40000.d / 360.d == 111.11 - if lat/lon is multiplied with this, rounded and diveded by this, the location is normalized to a 1km grid
public Segment getSegment() {
return this.indexSegment;
}
@ -511,6 +523,8 @@ public final class QueryParams {
context.append(asterisk);
context.append(this.modifier.s);
context.append(asterisk);
context.append(this.lat).append(asterisk).append(this.lon).append(asterisk).append(this.radius);
context.append(asterisk);
context.append(this.snippetCacheStrategy == null ? "null" : this.snippetCacheStrategy.name());
String result = context.toString();
if (anonymized) {

@ -83,7 +83,9 @@ public final class RWIProcess extends Thread
//private final int[] domZones;
private SortedMap<byte[], ReferenceContainer<WordReference>> localSearchInclusion;
private int remote_resourceSize, remote_indexCount, remote_peerCount;
private int remote_resourceSize;
private int remote_indexCount;
private int remote_peerCount;
private int local_indexCount;
private final AtomicInteger maxExpectedRemoteReferences, expectedRemoteReferences,
receivedRemoteReferences;
@ -204,7 +206,7 @@ public final class RWIProcess extends Thread
System.currentTimeMillis() - timer),
false);
if ( !index.isEmpty() ) {
add(index, true, "local index: " + this.query.getSegment().getLocation(), -1, true);
add(index, true, "local index: " + this.query.getSegment().getLocation(), -1, true, 10000);
}
} catch ( final Exception e ) {
Log.logException(e);
@ -218,7 +220,8 @@ public final class RWIProcess extends Thread
final boolean local,
final String resourceName,
final int fullResource,
final boolean finalizeAddAtEnd) {
final boolean finalizeAddAtEnd,
final long maxtime) {
// we collect the urlhashes and construct a list with urlEntry objects
// attention: if minEntries is too high, this method will not terminate within the maxTime
//Log.logInfo("RWIProcess", "added a container, size = " + index.size());
@ -239,7 +242,7 @@ public final class RWIProcess extends Thread
long timer = System.currentTimeMillis();
// normalize entries
final BlockingQueue<WordReferenceVars> decodedEntries = this.order.normalizeWith(index);
final BlockingQueue<WordReferenceVars> decodedEntries = this.order.normalizeWith(index, maxtime);
int is = index.size();
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(
this.query.id(true),
@ -626,6 +629,18 @@ public final class RWIProcess extends Thread
continue;
}
// check geo coordinates
double lat, lon;
if (this.query.radius > 0.0d && this.query.lat != 0.0d && this.query.lon != 0.0d && (lat = page.lat()) > 0.0d && (lon = page.lon()) > 0.0d) {
double latDelta = this.query.lat - lat;
double lonDelta = this.query.lon - lon;
double distance = Math.sqrt(latDelta * latDelta + lonDelta * lonDelta); // pythagoras
if (distance > this.query.radius) {
this.sortout++;
continue;
}
}
// check vocabulary constraint
final String tags = page.dc_subject();
final String[] taglist = tags == null || tags.length() == 0 ? new String[0] : SPACE_PATTERN.split(page.dc_subject());

@ -360,7 +360,7 @@ public class SnippetProcess {
Worker worker;
if (this.workerThreads == null) {
this.workerThreads = new Worker[deployCount];
synchronized(this.workerThreads) {
synchronized(this.workerThreads) {try {
for (int i = 0; i < this.workerThreads.length; i++) {
if (this.result.sizeAvailable() >= neededResults ||
(this.rankingProcess.feedingIsFinished() && this.rankingProcess.sizeQueue() == 0)) {
@ -376,7 +376,7 @@ public class SnippetProcess {
}
}
}
}
} catch (OutOfMemoryError e) {}}
} else {
// there are still worker threads running, but some may be dead.
// if we find dead workers, reanimate them

@ -65,11 +65,11 @@ public class ReferenceOrder {
this.language = language;
}
public BlockingQueue<WordReferenceVars> normalizeWith(final ReferenceContainer<WordReference> container) {
public BlockingQueue<WordReferenceVars> normalizeWith(final ReferenceContainer<WordReference> container, long maxtime) {
final LinkedBlockingQueue<WordReferenceVars> out = new LinkedBlockingQueue<WordReferenceVars>();
int threads = cores;
if (container.size() < 100) threads = 2;
final Thread distributor = new NormalizeDistributor(container, out, threads);
final Thread distributor = new NormalizeDistributor(container, out, threads, maxtime);
distributor.start();
try {
distributor.join(10); // let the distributor work for at least 10 milliseconds
@ -85,17 +85,19 @@ public class ReferenceOrder {
ReferenceContainer<WordReference> container;
LinkedBlockingQueue<WordReferenceVars> out;
private final int threads;
private final long maxtime;
public NormalizeDistributor(final ReferenceContainer<WordReference> container, final LinkedBlockingQueue<WordReferenceVars> out, final int threads) {
public NormalizeDistributor(final ReferenceContainer<WordReference> container, final LinkedBlockingQueue<WordReferenceVars> out, final int threads, final long maxtime) {
this.container = container;
this.out = out;
this.threads = threads;
this.maxtime = maxtime;
}
@Override
public void run() {
// transform the reference container into a stream of parsed entries
final BlockingQueue<WordReferenceVars> vars = WordReferenceVars.transform(this.container);
final BlockingQueue<WordReferenceVars> vars = WordReferenceVars.transform(this.container, this.maxtime);
// start the transformation threads
final Semaphore termination = new Semaphore(this.threads);
@ -148,6 +150,7 @@ public class ReferenceOrder {
}
}
@Override
public void run() {
try {
WordReferenceVars iEntry;

@ -100,9 +100,14 @@ public class MediaSnippet implements Comparable<MediaSnippet>, Comparator<MediaS
if ((this.attr == null) || (this.attr.length() == 0)) this.attr = "_";
}
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
@Override
public int hashCode() {
return ByteArray.hashCode(this.href.hash());
if (this.hashCache == Integer.MIN_VALUE) {
this.hashCache = ByteArray.hashCode(this.href.hash());
}
return this.hashCache;
}
@Override

@ -9,7 +9,7 @@
// $LastChangedBy$
//
// LICENSE
//
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
@ -48,7 +48,7 @@ import net.yacy.search.index.Segment;
public class ResultEntry implements Comparable<ResultEntry>, Comparator<ResultEntry> {
// payload objects
private final URIMetadataRow urlentry;
private String alternative_urlstring;
@ -56,10 +56,10 @@ public class ResultEntry implements Comparable<ResultEntry>, Comparator<ResultEn
private final TextSnippet textSnippet;
private final List<MediaSnippet> mediaSnippets;
private final Segment indexSegment;
// statistic objects
public long dbRetrievalTime, snippetComputationTime, ranking;
public ResultEntry(final URIMetadataRow urlentry,
final Segment indexSegment,
SeedDB peers,
@ -103,9 +103,13 @@ public class ResultEntry implements Comparable<ResultEntry>, Comparator<ResultEn
if ((p = this.alternative_urlname.indexOf('?')) > 0) this.alternative_urlname = this.alternative_urlname.substring(0, p);
}
}
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
@Override
public int hashCode() {
return ByteArray.hashCode(this.urlentry.hash());
if (this.hashCache == Integer.MIN_VALUE) {
this.hashCache = ByteArray.hashCode(this.urlentry.hash());
}
return this.hashCache;
}
@Override
public boolean equals(final Object obj) {
@ -178,10 +182,10 @@ public class ResultEntry implements Comparable<ResultEntry>, Comparator<ResultEn
public int lapp() {
return this.urlentry.lapp();
}
public float lat() {
public double lat() {
return this.urlentry.lat();
}
public float lon() {
public double lon() {
return this.urlentry.lon();
}
public WordReferenceVars word() {

@ -400,9 +400,14 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
return o1.compareTo(o2);
}
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
@Override
public int hashCode() {
return ByteArray.hashCode(this.urlhash);
if (this.hashCache == Integer.MIN_VALUE) {
this.hashCache = ByteArray.hashCode(this.urlhash);
}
return this.hashCache;
}
@Override

Loading…
Cancel
Save