Merge branch 'master' of ssh://git@gitorious.org/yacy/rc1.git

pull/1/head
Michael Peter Christen 13 years ago
commit cf79b6cee3

@ -41,9 +41,9 @@
<classpathentry kind="lib" path="lib/jcl-over-slf4j-1.6.1.jar"/>
<classpathentry kind="lib" path="lib/wstx-asl-3.2.7.jar"/>
<classpathentry kind="lib" path="lib/commons-codec-1.6.jar"/>
<classpathentry kind="lib" path="lib/httpcore-4.1.4.jar"/>
<classpathentry kind="lib" path="lib/httpclient-4.1.3.jar"/>
<classpathentry kind="lib" path="lib/httpmime-4.1.3.jar"/>
<classpathentry kind="lib" path="lib/httpcore-4.2.jar"/>
<classpathentry kind="lib" path="lib/httpclient-4.2.jar"/>
<classpathentry kind="lib" path="lib/httpmime-4.2.jar"/>
<classpathentry kind="lib" path="lib/commons-io-2.1.jar"/>
<classpathentry kind="lib" path="lib/apache-solr-solrj-3.6.0.jar" sourcepath="/solrj/src"/>
<classpathentry kind="lib" path="lib/commons-compress-1.4.1.jar"/>

@ -50,9 +50,9 @@
<string>$JAVAROOT/lib/commons-logging-1.1.1.jar</string>
<string>$JAVAROOT/lib/fontbox-1.6.0.jar</string>
<string>$JAVAROOT/lib/geronimo-stax-api_1.0_spec-1.0.1.jar</string>
<string>$JAVAROOT/lib/httpclient-4.1.3.jar</string>
<string>$JAVAROOT/lib/httpcore-4.1.4.jar</string>
<string>$JAVAROOT/lib/httpmime-4.1.3.jar</string>
<string>$JAVAROOT/lib/httpclient-4.2.jar</string>
<string>$JAVAROOT/lib/httpcore-4.2.jar</string>
<string>$JAVAROOT/lib/httpmime-4.2.jar</string>
<string>$JAVAROOT/lib/icu4j-core.jar</string>
<string>$JAVAROOT/lib/J7Zip-modified.jar</string>
<string>$JAVAROOT/lib/jakarta-oro-2.0.8.jar</string>

@ -60,11 +60,14 @@ SetCompressor /SOLID LZMA
; http://www.java.com/de/download/manual.jsp BundleId +1 / +2
; User-Agent to see the 64bit link: Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Win64; x64; Trident/4.0)
; at least we need Java 6
!define JRE_VERSION6 "1.6"
; jre-6u31-windows-i586.exe
!define JRE_32 "http://javadl.sun.com/webapps/download/AutoDL?BundleId=61044"
; jre-6u31-windows-x64.exe
!define JRE_64 "http://javadl.sun.com/webapps/download/AutoDL?BundleId=60338"
; download link Oracle Java 7 Update 4
; 32 bit
!define JRE_32 "http://javadl.sun.com/webapps/download/AutoDL?BundleId=63691"
; 64 bit
!define JRE_64 "http://javadl.sun.com/webapps/download/AutoDL?BundleId=63692"
;!define JRE_32 "http://yacy.berlios.de/download.php?what=jre&version=32&yacyrevnr=@REPL_REVISION_NR@"
;!define JRE_64 "http://yacy.berlios.de/download.php?what=jre&version=64&yacyrevnr=@REPL_REVISION_NR@"

@ -168,9 +168,9 @@
<pathelement location="${lib}/commons-logging-1.1.1.jar" />
<pathelement location="${lib}/fontbox-1.6.0.jar" />
<pathelement location="${lib}/geronimo-stax-api_1.0_spec-1.0.1.jar" />
<pathelement location="${lib}/httpclient-4.1.3.jar" />
<pathelement location="${lib}/httpcore-4.1.4.jar" />
<pathelement location="${lib}/httpmime-4.1.3.jar" />
<pathelement location="${lib}/httpclient-4.2.jar" />
<pathelement location="${lib}/httpcore-4.2.jar" />
<pathelement location="${lib}/httpmime-4.2.jar" />
<pathelement location="${lib}/icu4j-core.jar" />
<pathelement location="${lib}/J7Zip-modified.jar" />
<pathelement location="${lib}/jakarta-oro-2.0.8.jar" />

@ -110,7 +110,8 @@ public class PerformanceMemory_p {
prop.putNum("EcoList_" + c + "_tableSize", mapx.get(Table.StatKeys.tableSize));
assert mapx.get(Table.StatKeys.tableKeyMem) != null : mapx;
mem = Long.parseLong(mapx.get(Table.StatKeys.tableKeyMem));
String v = mapx.get(Table.StatKeys.tableKeyMem);
mem = v == null ? 0 : Long.parseLong(v);
totalmem += mem;
prop.put("EcoList_" + c + "_tableKeyMem", Formatter.bytesToString(mem));
prop.put("EcoList_" + c + "_tableKeyChunkSize", mapx.get(Table.StatKeys.tableKeyChunkSize));

@ -255,7 +255,7 @@ public final class search {
indexSegment,
rankingProfile,
header.get(RequestHeader.USER_AGENT, ""),
false
false, 0.0d, 0.0d, 0.0d
);
Network.log.logInfo("INIT HASH SEARCH (abstracts only): " + QueryParams.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.itemsPerPage() + " links");
@ -318,7 +318,7 @@ public final class search {
sb.indexSegments.segment(Segments.Process.PUBLIC),
rankingProfile,
header.get(RequestHeader.USER_AGENT, ""),
false
false, 0.0d, 0.0d, 0.0d
);
Network.log.logInfo("INIT HASH SEARCH (query-" + abstracts + "): " + QueryParams.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.itemsPerPage() + " links");
EventChannel.channels(EventChannel.REMOTESEARCH).addMessage(new RSSMessage("Remote Search Request from " + ((remoteSeed == null) ? "unknown" : remoteSeed.getName()), QueryParams.anonymizedQueryHashes(theQuery.queryHashes), ""));

@ -485,6 +485,29 @@ public class yacysearch {
}
}
int radius = 0;
double lon = 0.0d, lat = 0.0d, rad = 0.0d;
if ((radius = querystring.indexOf("/radius/")) >= 0) {
int ve = querystring.indexOf(' ', radius + 8);
String geo = "";
if (ve < 0) {
geo = querystring.substring(radius);
querystring = querystring.substring(0, radius).trim();
} else {
geo = querystring.substring(radius, ve);
querystring = querystring.substring(0, radius) + querystring.substring(ve);
}
geo = geo.substring(8);
String[] sp = geo.split("/");
if (sp.length == 3) try {
lat = Double.parseDouble(sp[0]);
lon = Double.parseDouble(sp[1]);
rad = Double.parseDouble(sp[2]);
} catch (NumberFormatException e) {
lon = 0.0d; lat = 0.0d; rad = 0.0d;
}
}
String tenant = null;
if ( post.containsKey("tenant") ) {
tenant = post.get("tenant");
@ -757,7 +780,8 @@ public class yacysearch {
header.get(RequestHeader.USER_AGENT, ""),
sb.getConfigBool(SwitchboardConstants.SEARCH_VERIFY_DELETE, false)
&& sb.getConfigBool(SwitchboardConstants.NETWORK_SEARCHVERIFY, false)
&& sb.peers.mySeed().getFlagAcceptRemoteIndex());
&& sb.peers.mySeed().getFlagAcceptRemoteIndex(),
lat, lon, rad);
EventTracker.delete(EventTracker.EClass.SEARCH);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(
theQuery.id(true),

@ -6,72 +6,144 @@
<script type="text/javascript" src="http://openlayers.org/api/OpenLayers.js"></script>
<script type="text/javascript" src="http://www.openstreetmap.org/openlayers/OpenStreetMap.js"></script>
<script type="text/javascript">
function adaptHeight() {
document.getElementById('map').style.height = (document.documentElement.clientHeight - 130) + "px";
}
window.onresize = adaptHeight;
</script>
<script type="text/javascript">
// possible values for dom: location,metatag,alltext,title,publisher,creator,subject
var map;
var searchLayer_md = null;
var searchLayer_co = null;
var searchLayer_md0 = null; // two layers each for double-buffering during search
var searchLayer_md1 = null;
var searchLayer_co0 = null;
var searchLayer_co1 = null;
var path_mdsearch = 'yacysearch_location.rss?dom=alltext&query=';
var path_losearch = 'yacysearch_location.rss?dom=location&query=';
var marker_md = new OpenLayers.Icon("/env/grafics/marker_red.png", new OpenLayers.Size(11,16));
var marker_co = new OpenLayers.Icon("/env/grafics/star_yellow.png", new OpenLayers.Size(25,25));
// possible values for dom: location,metatag,alltext,title,publisher,creator,subject
var shallZoom = true; // flag if the map shall be zoomed to the search result
var shallReload = false; // flag if the search shall be repeated upon next resize of the window
var poisfound = 0;
var kmNormal = 100; // ca. 1km grid for radius
var meterNormal = 100000; // ca. 1m grid for location
function init() {
map = new OpenLayers.Map('map', {
maxResolution:'auto',
controls:[
new OpenLayers.Control.Navigation(),
new OpenLayers.Control.PanZoomBar(),
new OpenLayers.Control.ZoomBox(),
new OpenLayers.Control.LayerSwitcher(),
new OpenLayers.Control.Attribution()]
});
// automatically adapt height of map upon window resize
function adaptHeight() {
document.getElementById('map').style.height = (document.documentElement.clientHeight - 130) + "px";
}
window.onresize = adaptHeight;
//layerMaplint = new OpenLayers.Layer.OSM.Maplint("Maplint");
layerWMS = new OpenLayers.Layer.WMS( "OpenLayers WMS", "http://labs.metacarta.com/wms/vmap0", {layers: 'basic'} );
layerMapnik = new OpenLayers.Layer.OSM.Mapnik("Mapnik");
layerCycleMap = new OpenLayers.Layer.OSM.CycleMap("CycleMap");
map.addLayer(layerMapnik);
map.addLayer(layerCycleMap);
//map.addLayer(layerMaplint);
map.addLayer(layerWMS);
// initialize the map layers
function init() {
map = new OpenLayers.Map('map', {
maxResolution:'auto',
controls:[
new OpenLayers.Control.Navigation(),
new OpenLayers.Control.PanZoomBar(),
new OpenLayers.Control.ZoomBox(),
new OpenLayers.Control.LayerSwitcher(),
new OpenLayers.Control.Attribution()]
});
//layerMaplint = new OpenLayers.Layer.OSM.Maplint("Maplint");
layerWMS = new OpenLayers.Layer.WMS( "OpenLayers WMS", "http://labs.metacarta.com/wms/vmap0", {layers: 'basic'} );
layerMapnik = new OpenLayers.Layer.OSM.Mapnik("Mapnik");
layerCycleMap = new OpenLayers.Layer.OSM.CycleMap("CycleMap");
map.addLayer(layerMapnik);
map.addLayer(layerCycleMap);
//map.addLayer(layerMaplint);
map.addLayer(layerWMS);
map.addControl(new OpenLayers.Control.LayerSwitcher());
map.setCenter(new OpenLayers.LonLat(9,48) // Center of the map
map.addControl(new OpenLayers.Control.LayerSwitcher());
map.setCenter(new OpenLayers.LonLat(9,48) // Center of the map
.transform(
new OpenLayers.Projection("EPSG:4326"), // transform from WGS 1984
new OpenLayers.Projection("EPSG:900913") // to Spherical Mercator Projection
), 5 // Zoom level
);
map.events.register( "zoomend", map, zoomChanged );
map.events.register( "movestart", map, zoomChanged );
//#(initsearch)#::
var query = '#[query]#'.replace(' ', '+');
//#(/initsearch)#
#(initsearch)#::
var query = '#[query]#'.replace(' ', '+');
searchLayer_co = new OpenLayers.Layer.GeoRSS('GeoRSS', path_losearch + query, {'icon':marker_co});
map.addLayer(searchLayer_co);
searchLayer_md = new OpenLayers.Layer.GeoRSS('GeoRSS', path_mdsearch + query, {'icon':marker_md});
map.addLayer(searchLayer_md);
#(/initsearch)#
window.setInterval("refresh()", 1000);
}
function zoomChanged () {
shallZoom = false; // no automatic zoom if user hs zoomed
refreshsearch();
}
// called if a search is submitted
function search() {
var query = document.getElementById('query').value.replace(' ', '+');
if (searchLayer_md != null) searchLayer_md.destroy();
if (searchLayer_co != null) searchLayer_co.destroy();
var center = map.getCenter().transform(new OpenLayers.Projection("EPSG:900913"), new OpenLayers.Projection("EPSG:4326"));
searchLayer_co = new OpenLayers.Layer.GeoRSS('GeoRSS', path_losearch + query + '&lon=' + center.lon + '&lat=' + center.lat + '&zoom=' + map.getZoom(), {'icon':marker_co});
map.addLayer(searchLayer_co);
searchLayer_md = new OpenLayers.Layer.GeoRSS('GeoRSS', path_mdsearch + query, {'icon':marker_md});
map.addLayer(searchLayer_md);
//map.zoomToExtent(searchLayer_co.getExtent());
//map.panTo(searchLayer_co.getExtent().getCenterLonLat());
//map.setCenter(searchLayer_co.getExtent().getLonLatFromViewPortPx, 5);
//map.setCenter(searchLayer_co.getExtent().getCenterLonLat(), map.getZoomForExtent(searchLayer_co.getExtent(), true));
//document.getElementById('apilink').setAttribute('href', 'yacysearch_location.rss?query=' + query);
refreshsearch();
}
// refresh is called if the map is zoomed or paned to get a new search result for the viewport
function refreshsearch() {
var query = document.getElementById('query').value.replace(' ', '+');
if (searchLayer_md0 != null) searchLayer_md0.destroy();
if (searchLayer_co0 != null) searchLayer_co0.destroy();
var center = map.getCenter().transform(new OpenLayers.Projection("EPSG:900913"), new OpenLayers.Projection("EPSG:4326"));
var extend = map.getExtent().transform(new OpenLayers.Projection("EPSG:900913"), new OpenLayers.Projection("EPSG:4326"));
var radius = Math.sqrt(extend.getWidth() * extend.getWidth() + extend.getHeight() * extend.getHeight()) / 2;
radius = Math.floor(radius * kmNormal + 1) / kmNormal;
var lon = Math.floor(center.lon * meterNormal) / meterNormal;
var lat = Math.floor(center.lat * meterNormal) / meterNormal;
searchLayer_co0 = new OpenLayers.Layer.GeoRSS('Communities', path_losearch + query + '&lon=' + lon + '&lat=' + lat + '&r=' + radius + '&z=' + map.getZoom(), {'icon':marker_co});
searchLayer_md0 = new OpenLayers.Layer.GeoRSS('Point Of Interest', path_mdsearch + query + '&lon=' + lon + '&lat=' + lat + '&r=' + radius + '&z=' + map.getZoom(), {'icon':marker_md});
map.addLayer(searchLayer_co0);
map.addLayer(searchLayer_md0);
//document.getElementById('apilink').setAttribute('href', 'yacysearch_location.rss?query=' + query);
poisfound = 0;
document.getElementById('resultline').innerHTML = "";
}
function refresh() {
// check double-buffer state
if (searchLayer_md0 != null) {
// switch the double-buffer
if (searchLayer_md1 != null) searchLayer_md1.destroy();
if (searchLayer_co1 != null) searchLayer_co1.destroy();
searchLayer_md1 = searchLayer_md0;
searchLayer_co1 = searchLayer_co0;
searchLayer_md0 = null;
searchLayer_co0 = null;
}
// check if we need to do something
if (searchLayer_co1 == null || searchLayer_md1 == null) {
return;
}
var cocount = searchLayer_co1.markers.length;
var mdcount = searchLayer_md1.markers.length;
//if (mdcount + cocount <= poisfound) {
// return;
//}
// update map and result line
poisfound = mdcount + cocount;
document.getElementById('resultline').innerHTML = poisfound + " POIs found in visible map";
// check if any of the markers is visible on the screen.
// this is only necessary if shallZoom = false since this would set shallZoom = true
var bounds = searchLayer_md1.getDataExtent();
/*
if (!shallZoom) {
// check all markers
//if (!bounds.intersectsBounds(map.getExtent())) shallZoom = true;
var anyInScreen = false;
var mapbounds = map.getExtent();
//alert(searchLayer_md1.markers.length + " markers");
for (var marker in searchLayer_md1.markers) {
if (marker.lonlat != null && mapbounds.containsLonLat(marker.lonlat, {})) {anyInScreen = true; break;}
}
if (!anyInScreen) shallZoom = true;
}
*/
// zoom to layer bounds
if (shallZoom) {
if (bounds == null) {return;} //bounds = searchLayer_co1.getDataExtent();
map.panTo(bounds.getCenterLonLat());
map.zoomTo(map.getZoomForExtent(bounds));
}
}
</script>
@ -98,9 +170,10 @@ To see a list of all APIs, please visit the <a href="http://www.yacy-websuche.de
<h2>#[promoteSearchPageGreeting]#</h2>
<div class="yacylogo"><a href="#[promoteSearchPageGreeting.homepage]#" class="yacylogo"><img src="#[promoteSearchPageGreeting.smallImage]#" alt="yacysearch" /></a></div>
<fieldset class="yacys">
<input type="text" value="#(initsearch)#::#[query]##(/initsearch)#" id="query" size="42" />
<input type="text" value="#(initsearch)#::#[query]##(/initsearch)#" id="query" size="42" onFocus="this.select()" />
<input type="submit" onclick="search(); return false;" value="search" onsubmit="search(); return false;" />
</fieldset>
<div id="resultline"></div>
</form><br/>
<div id="map" style="clear:both; width:100%; height:720px"></div>
<script type="text/javascript">

@ -37,8 +37,6 @@ import de.anomic.server.serverSwitch;
public class yacysearch_location {
private static final String space = " ";
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
@ -62,10 +60,13 @@ public class yacysearch_location {
final boolean search_subject = alltext || post.get("dom", "").indexOf("subject",0) >= 0;
final long maximumTime = post.getLong("maximumTime", 5000);
final int maximumRecords = post.getInt("maximumRecords", 3000);
final double lon = post.getDouble("lon", 0.0d);
final double lat = post.getDouble("lat", 0.0d);
final double radius = post.getDouble("r", 0.0d);
//i.e. http://localhost:8090/yacysearch_location.kml?query=berlin&maximumTime=2000&maximumRecords=100
int placemarkCounter = 0;
if (search_query) {
if (query.length() > 0 && search_query) {
final Set<Location> locations = LibraryProvider.geoLoc.find(query, true);
for (final String qp: query.split(" ")) {
locations.addAll(LibraryProvider.geoLoc.find(qp, true));
@ -87,11 +88,11 @@ public class yacysearch_location {
}
}
if (metatag || search_title || search_publisher || search_creator || search_subject) try {
if (query.length() > 0 && (metatag || search_title || search_publisher || search_creator || search_subject)) try {
// get a queue of search results
final String rssSearchServiceURL = "http://127.0.0.1:" + sb.getConfig("port", "8090") + "/yacysearch.rss";
final BlockingQueue<RSSMessage> results = new LinkedBlockingQueue<RSSMessage>();
SRURSSConnector.searchSRURSS(results, rssSearchServiceURL, query, maximumTime, Integer.MAX_VALUE, null, false, null);
SRURSSConnector.searchSRURSS(results, rssSearchServiceURL, lon == 0.0d && lat == 0.0d ? query : query + " /radius/" + lat + "/" + lon + "/" + radius, maximumTime, Integer.MAX_VALUE, null, false, null);
// take the results and compute some locations
RSSMessage message;

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

@ -174,3 +174,9 @@
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
This project contains annotations derived from JCIP-ANNOTATIONS
Copyright (c) 2005 Brian Goetz and Tim Peierls.
See http://www.jcip.net and the Creative Commons Attribution License
(http://creativecommons.org/licenses/by/2.5)

@ -77,7 +77,7 @@
<compilation-unit>
<package-root>source</package-root>
<package-root>htroot</package-root>
<classpath mode="compile">lib/activation.jar:lib/apache-mime4j-0.6.jar:lib/apache-solr-solrj-3.6.0.jar:lib/bcmail-jdk15-145.jar:lib/bcprov-jdk15-145.jar:lib/bzip2.jar:lib/commons-codec-1.6.jar:lib/commons-fileupload-1.2.2.jar:lib/commons-httpclient-3.1.jar:lib/commons-io-2.1.jar:lib/commons-jxpath-1.3.jar:lib/commons-logging-1.1.1.jar:lib/fontbox-1.6.0.jar:lib/geronimo-stax-api_1.0_spec-1.0.1.jar:lib/httpclient-4.1.3.jar:lib/httpcore-4.1.4.jar:lib/httpmime-4.1.3.jar:lib/icu4j-core.jar:lib/J7Zip-modified.jar:lib/jakarta-oro-2.0.8.jar:lib/jcifs-1.3.15.jar:lib/jcl-over-slf4j-1.6.1.jar:lib/jempbox-1.6.0.jar:lib/jsch-0.1.42.jar:lib/json-simple-1.1.jar:lib/log4j-1.2.16.jar:lib/metadata-extractor-2.4.0-beta-1.jar:lib/mysql-connector-java-5.1.12-bin.jar:lib/pdfbox-1.6.0.jar:lib/poi-3.6-20091214.jar:lib/poi-scratchpad-3.6-20091214.jar:lib/servlet-api.jar:lib/slf4j-api-1.6.1.jar:lib/slf4j-jdk14-1.6.1.jar:lib/webcat-0.1-swf.jar:lib/wstx-asl-3.2.7.jar:lib/xercesImpl.jar:lib/xml-apis.jar:htroot/processing/domaingraph/applet/domaingraph.jar</classpath>
<classpath mode="compile">lib/activation.jar:lib/apache-mime4j-0.6.jar:lib/apache-solr-solrj-3.6.0.jar:lib/bcmail-jdk15-145.jar:lib/bcprov-jdk15-145.jar:lib/bzip2.jar:lib/commons-codec-1.6.jar:lib/commons-fileupload-1.2.2.jar:lib/commons-httpclient-3.1.jar:lib/commons-io-2.1.jar:lib/commons-jxpath-1.3.jar:lib/commons-logging-1.1.1.jar:lib/fontbox-1.6.0.jar:lib/geronimo-stax-api_1.0_spec-1.0.1.jar:lib/httpclient-4.2.jar:lib/httpcore-4.2.jar:lib/httpmime-4.2.jar:lib/icu4j-core.jar:lib/J7Zip-modified.jar:lib/jakarta-oro-2.0.8.jar:lib/jcifs-1.3.15.jar:lib/jcl-over-slf4j-1.6.1.jar:lib/jempbox-1.6.0.jar:lib/jsch-0.1.42.jar:lib/json-simple-1.1.jar:lib/log4j-1.2.16.jar:lib/metadata-extractor-2.4.0-beta-1.jar:lib/mysql-connector-java-5.1.12-bin.jar:lib/pdfbox-1.6.0.jar:lib/poi-3.6-20091214.jar:lib/poi-scratchpad-3.6-20091214.jar:lib/servlet-api.jar:lib/slf4j-api-1.6.1.jar:lib/slf4j-jdk14-1.6.1.jar:lib/webcat-0.1-swf.jar:lib/wstx-asl-3.2.7.jar:lib/xercesImpl.jar:lib/xml-apis.jar:htroot/processing/domaingraph/applet/domaingraph.jar</classpath>
<source-level>1.6</source-level>
</compilation-unit>
</java-data>

@ -222,7 +222,7 @@ public final class ResultURLs {
public static void main(final String[] args) {
try {
final DigestURI url = new DigestURI("http", "www.yacy.net", 80, "/");
final URIMetadataRow urlRef = new URIMetadataRow(url, "YaCy Homepage", "", "", "", 0.0f, 0.0f, new Date(), new Date(), new Date(), "", new byte[] {}, 123, 42, '?', new Bitfield(), UTF8.getBytes("de"), 0, 0, 0, 0, 0, 0);
final URIMetadataRow urlRef = new URIMetadataRow(url, "YaCy Homepage", "", "", "", 0.0d, 0.0d, new Date(), new Date(), new Date(), "", new byte[] {}, 123, 42, '?', new Bitfield(), UTF8.getBytes("de"), 0, 0, 0, 0, 0, 0);
final EventOrigin stackNo = EventOrigin.LOCAL_CRAWLING;
System.out.println("valid test:\n=======");
// add

@ -579,6 +579,7 @@ public final class HTTPDFileHandler {
e.getTargetException().getMessage() +
"; java.awt.graphicsenv='" + System.getProperty("java.awt.graphicsenv","") + "'");
Log.logException(e);
Log.logException(e.getCause());
Log.logException(e.getTargetException());
targetClass = null;
}
@ -978,12 +979,9 @@ public final class HTTPDFileHandler {
if (e.getCause() instanceof InterruptedException) {
throw new InterruptedException(e.getCause().getMessage());
}
theLogger.logSevere("INTERNAL ERROR: " + e.toString() + ":" +
e.getMessage() +
" target exception at " + targetClass + ": " +
e.getTargetException().toString() + ":" +
e.getTargetException().getMessage(),e);
Log.logException(e);
Log.logException(e.getCause());
Log.logException(e.getTargetException());
targetClass = null;
throw e;
}

@ -298,15 +298,18 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
: new InetSocketAddress(bindIP, bindPort);
}
@Override
public void open() {
this.log.logConfig("* server started on " + Domains.myPublicLocalIP() + ":" + this.extendedPort);
}
@Override
public void freemem() {
// FIXME: can we something here to flush memory? Idea: Reduce the size of some of our various caches.
}
// class body
@Override
public boolean job() throws Exception {
try {
// prepare for new connection
@ -407,6 +410,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
}
}
@Override
public synchronized void close() {
// consuming the isInterrupted Flag. Otherwise we could not properly close the session pool
Thread.interrupted();
@ -456,6 +460,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
return l;
}
@Override
public int getJobCount() {
final Thread[] threadList = new Thread[sessionThreadGroup.activeCount()];
serverCore.sessionThreadGroup.enumerate(threadList, false);
@ -527,6 +532,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
sessionCounter++;
}
@Override
public int hashCode() {
// return a hash code so it is possible to store objects of httpc objects in a HashSet
return this.hashIndex;
@ -619,6 +625,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
*
* @see java.lang.Thread#run()
*/
@Override
public void run() {
this.runningsession = true;
@ -804,6 +811,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
} catch (final InvocationTargetException e) {
serverCore.this.log.logSevere("command execution, target exception " + e.getMessage() + " for client " + this.userAddress.getHostAddress(), e);
// we extract a target exception
writeLine(this.commandObj.error(e.getCause()));
writeLine(this.commandObj.error(e.getTargetException()));
break;
} catch (final NoSuchMethodException e) {

@ -183,6 +183,10 @@ public class serverObjects extends HashMap<String, String> implements Cloneable
return (null == this.put(key, Float.toString(value))) ? Float.NaN : value;
}
public double put(final String key, final double value) {
return (null == this.put(key, Double.toString(value))) ? Double.NaN : value;
}
/**
* same as {@link #put(String, double)} but for integer types
* @return Returns 0 for the error case.
@ -359,6 +363,16 @@ public class serverObjects extends HashMap<String, String> implements Cloneable
}
}
public double getDouble(final String key, final double dflt) {
final String s = removeByteOrderMark(super.get(key));
if (s == null) return dflt;
try {
return Double.parseDouble(s);
} catch (final NumberFormatException e) {
return dflt;
}
}
public boolean getBoolean(final String key, final boolean dflt) {
String s = removeByteOrderMark(super.get(key));
if (s == null) return dflt;

@ -25,6 +25,7 @@
package net.yacy.cora.document;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
@ -1967,8 +1968,8 @@ public class MultiProtocolURI implements Serializable, Comparable<MultiProtocolU
}
public InputStream getInputStream(final String userAgent, final int timeout) throws IOException {
if (isFile()) return new FileInputStream(getFSFile());
if (isSMB()) return new SmbFileInputStream(getSmbFile());
if (isFile()) return new BufferedInputStream(new FileInputStream(getFSFile()));
if (isSMB()) return new BufferedInputStream(new SmbFileInputStream(getSmbFile()));
if (isFTP()) {
final FTPClient client = new FTPClient();
client.open(this.host, this.port < 0 ? 21 : this.port);

@ -28,10 +28,10 @@ import java.text.ParseException;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.date.ISO8601Formatter;
@ -81,12 +81,14 @@ public class RSSMessage implements Hit, Comparable<RSSMessage>, Comparator<RSSMe
return this.keys;
}
@Override
public String toString() {
return this.keys.iterator().next();
}
}
private static String artificialGuidPrefix = "c0_";
private static String calculatedGuidPrefix = "c1_";
public static final RSSMessage POISON = new RSSMessage("", "", "");
public static final HashSet<String> tags = new HashSet<String>();
@ -99,7 +101,7 @@ public class RSSMessage implements Hit, Comparable<RSSMessage>, Comparator<RSSMe
private final Map<String, String> map;
public RSSMessage(final String title, final String description, final String link) {
this.map = new ConcurrentHashMap<String, String>();
this.map = new HashMap<String, String>();
this.map.put("title", title);
this.map.put("description", description);
this.map.put("link", link);
@ -108,7 +110,7 @@ public class RSSMessage implements Hit, Comparable<RSSMessage>, Comparator<RSSMe
}
public RSSMessage(final String title, final String description, final DigestURI link) {
this.map = new ConcurrentHashMap<String, String>();
this.map = new HashMap<String, String>();
this.map.put("title", title);
this.map.put("description", description);
this.map.put("link", link.toNormalform(true, false));
@ -117,22 +119,19 @@ public class RSSMessage implements Hit, Comparable<RSSMessage>, Comparator<RSSMe
}
public RSSMessage() {
this.map = new ConcurrentHashMap<String, String>();
this.map = new HashMap<String, String>();
}
public void setValue(final String name, final String value) {
this.map.put(name, value);
// if possible generate a guid if not existent so far
if ((name.equals("title") || name.equals("description") || name.equals("link")) &&
(!this.map.containsKey("guid") || this.map.get("guid").startsWith(artificialGuidPrefix))) {
this.map.put("guid", artificialGuidPrefix + Integer.toHexString((getTitle() + getDescription() + getLink()).hashCode()));
}
}
@Override
public String getTitle() {
return Token.title.valueFrom(this.map, "");
}
@Override
public String getLink() {
return Token.link.valueFrom(this.map, "");
}
@ -158,22 +157,27 @@ public class RSSMessage implements Hit, Comparable<RSSMessage>, Comparator<RSSMe
return o1.compareTo(o2);
}
@Override
public String getDescription() {
return Token.description.valueFrom(this.map, "");
}
@Override
public String getAuthor() {
return Token.author.valueFrom(this.map, "");
}
@Override
public String getCopyright() {
return Token.copyright.valueFrom(this.map, "");
}
@Override
public String getCategory() {
return Token.category.valueFrom(this.map, "");
}
@Override
public String[] getSubject() {
final String subject = Token.subject.valueFrom(this.map, "");
if (subject.indexOf(',') >= 0) return subject.split(",");
@ -181,14 +185,17 @@ public class RSSMessage implements Hit, Comparable<RSSMessage>, Comparator<RSSMe
return subject.split(" ");
}
@Override
public String getReferrer() {
return Token.referrer.valueFrom(this.map, "");
}
@Override
public String getLanguage() {
return Token.language.valueFrom(this.map, "");
}
@Override
public Date getPubDate() {
final String dateString = Token.pubDate.valueFrom(this.map, "");
Date date;
@ -204,18 +211,27 @@ public class RSSMessage implements Hit, Comparable<RSSMessage>, Comparator<RSSMe
return date;
}
@Override
public String getGuid() {
return Token.guid.valueFrom(this.map, "");
String guid = Token.guid.valueFrom(this.map, "");
if ((guid.length() == 0 || guid.startsWith(artificialGuidPrefix)) &&
(this.map.containsKey("title") || this.map.containsKey("description") || this.map.containsKey("link"))) {
guid = calculatedGuidPrefix + Integer.toHexString(getTitle().hashCode() + getDescription().hashCode() + getLink().hashCode());
this.map.put("guid", guid);
}
return guid;
}
public String getTTL() {
return Token.ttl.valueFrom(this.map, "");
}
@Override
public String getDocs() {
return Token.docs.valueFrom(this.map, "");
}
@Override
public long getSize() {
final String size = Token.size.valueFrom(this.map, "-1");
return (size == null || size.length() == 0) ? -1 : Long.parseLong(size);
@ -227,10 +243,12 @@ public class RSSMessage implements Hit, Comparable<RSSMessage>, Comparator<RSSMe
return sb.toString();
}
@Override
public float getLon() {
return Float.parseFloat(Token.lon.valueFrom(this.map, "0.0"));
}
@Override
public float getLat() {
return Float.parseFloat(Token.lat.valueFrom(this.map, "0.0"));
}
@ -240,18 +258,22 @@ public class RSSMessage implements Hit, Comparable<RSSMessage>, Comparator<RSSMe
return this.map.toString();
}
@Override
public void setAuthor(final String author) {
setValue("author", author);
}
@Override
public void setCategory(final String category) {
setValue("category", category);
}
@Override
public void setCopyright(final String copyright) {
setValue("copyright", copyright);
}
@Override
public void setSubject(final String[] tags) {
final StringBuilder sb = new StringBuilder(tags.length * 10);
for (final String tag: tags) sb.append(tag).append(',');
@ -259,38 +281,47 @@ public class RSSMessage implements Hit, Comparable<RSSMessage>, Comparator<RSSMe
setValue("subject", sb.toString());
}
@Override
public void setDescription(final String description) {
setValue("description", description);
}
@Override
public void setDocs(final String docs) {
setValue("docs", docs);
}
@Override
public void setGuid(final String guid) {
setValue("guid", guid);
}
@Override
public void setLanguage(final String language) {
setValue("language", language);
}
@Override
public void setLink(final String link) {
setValue("link", link);
}
@Override
public void setPubDate(final Date pubdate) {
setValue("pubDate", ISO8601Formatter.FORMATTER.format(pubdate));
}
@Override
public void setReferrer(final String referrer) {
setValue("referrer", referrer);
}
@Override
public void setSize(final long size) {
setValue("size", Long.toString(size));
}
@Override
public void setTitle(final String title) {
setValue("title", title);
}

@ -20,6 +20,7 @@
package net.yacy.cora.document;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
@ -57,9 +58,10 @@ public class RSSReader extends DefaultHandler {
this.type = Type.none;
}
public RSSReader(final int maxsize, final InputStream stream, final Type type) throws IOException {
public RSSReader(final int maxsize, InputStream stream, final Type type) throws IOException {
this(maxsize);
this.type = type;
if (!(stream instanceof ByteArrayInputStream) && !(stream instanceof BufferedInputStream)) stream = new BufferedInputStream(stream);
final SAXParserFactory factory = SAXParserFactory.newInstance();
try {
final SAXParser saxParser = factory.newSAXParser();

@ -50,6 +50,8 @@ import net.yacy.cora.protocol.ConnectionInfo;
import net.yacy.cora.protocol.HeaderFramework;
import org.apache.http.Header;
import org.apache.http.HeaderElement;
import org.apache.http.HeaderElementIterator;
import org.apache.http.HttpEntity;
import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpHost;
@ -67,6 +69,7 @@ import org.apache.http.client.params.CookiePolicy;
import org.apache.http.client.params.HttpClientParams;
import org.apache.http.client.protocol.ClientContext;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.conn.ConnectionKeepAliveStrategy;
import org.apache.http.conn.params.ConnRouteParams;
import org.apache.http.conn.routing.HttpRoute;
import org.apache.http.conn.scheme.PlainSocketFactory;
@ -77,10 +80,10 @@ import org.apache.http.entity.InputStreamEntity;
import org.apache.http.entity.mime.MultipartEntity;
import org.apache.http.entity.mime.content.ContentBody;
import org.apache.http.entity.mime.content.StringBody;
import org.apache.http.impl.client.AbstractHttpClient;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
import org.apache.http.impl.conn.PoolingClientConnectionManager;
import org.apache.http.message.BasicHeaderElementIterator;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpParams;
@ -139,7 +142,7 @@ public class HTTPClient {
schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
schemeRegistry.register(new Scheme("https", 443, getSSLSocketFactory()));
final ThreadSafeClientConnManager clientConnectionManager = new ThreadSafeClientConnManager(schemeRegistry);
final PoolingClientConnectionManager clientConnectionManager = new PoolingClientConnectionManager(schemeRegistry);
// Create and initialize HTTP parameters
final HttpParams httpParams = new BasicHttpParams();
@ -154,7 +157,7 @@ public class HTTPClient {
clientConnectionManager.setDefaultMaxPerRoute(2);
// Increase max connections for localhost
final HttpHost localhost = new HttpHost("localhost");
clientConnectionManager.setMaxForRoute(new HttpRoute(localhost), maxcon);
clientConnectionManager.setMaxPerRoute(new HttpRoute(localhost), maxcon);
/**
* HTTP protocol settings
*/
@ -188,10 +191,12 @@ public class HTTPClient {
httpClient = new DefaultHttpClient(clientConnectionManager, httpParams);
// disable the cookiestore, cause this may cause segfaults and is not needed
((DefaultHttpClient) httpClient).setCookieStore(null);
// add cutom keep alive strategy
addCustomKeepAliveStrategy((DefaultHttpClient) httpClient);
// ask for gzip
((AbstractHttpClient) httpClient).addRequestInterceptor(new GzipRequestInterceptor());
((DefaultHttpClient) httpClient).addRequestInterceptor(new GzipRequestInterceptor());
// uncompress gzip
((AbstractHttpClient) httpClient).addResponseInterceptor(new GzipResponseInterceptor());
((DefaultHttpClient) httpClient).addResponseInterceptor(new GzipResponseInterceptor());
if (idledConnectionEvictor == null) {
idledConnectionEvictor = new IdledConnectionEvictor(clientConnectionManager);
@ -233,7 +238,7 @@ public class HTTPClient {
*/
public static void setMaxRouteHost(final String host) {
final HttpHost mHost = new HttpHost(host);
((ThreadSafeClientConnManager) httpClient.getConnectionManager()).setMaxForRoute(new HttpRoute(mHost), 50);
((PoolingClientConnectionManager) httpClient.getConnectionManager()).setMaxPerRoute(new HttpRoute(mHost), 50);
}
/**
@ -655,7 +660,7 @@ public class HTTPClient {
if (ProxySettings.use)
ConnRouteParams.setDefaultProxy(httpParams, ProxySettings.getProxyHost());
// TODO find a better way for this
ProxySettings.setProxyCreds((AbstractHttpClient) httpClient);
ProxySettings.setProxyCreds((DefaultHttpClient) httpClient);
}
private void storeConnectionInfo(final HttpUriRequest httpUriRequest) {
@ -704,6 +709,38 @@ public class HTTPClient {
return sslSF;
}
/**
* If the Keep-Alive header is not present in the response,
* HttpClient assumes the connection can be kept alive indefinitely.
* Here we limit this to 5 seconds.
*
* @param defaultHttpClient
*/
private static void addCustomKeepAliveStrategy(final DefaultHttpClient defaultHttpClient) {
defaultHttpClient.setKeepAliveStrategy(new ConnectionKeepAliveStrategy() {
public long getKeepAliveDuration(HttpResponse response, HttpContext context) {
// Honor 'keep-alive' header
String param, value;
HeaderElement element;
HeaderElementIterator it = new BasicHeaderElementIterator(
response.headerIterator(HTTP.CONN_KEEP_ALIVE));
while (it.hasNext()) {
element = it.nextElement();
param = element.getName();
value = element.getValue();
if (value != null && param.equalsIgnoreCase("timeout")) {
try {
return Long.parseLong(value) * 1000;
} catch(final NumberFormatException e) {
}
}
}
// Keep alive for 5 seconds only
return 5 * 1000;
}
});
}
/**
* testing
*

@ -64,6 +64,10 @@ public class SolrDoc extends SolrInputDocument {
this.setField(key.getSolrFieldName(), value);
}
public final void addSolr(final SolrField key, final double value) {
this.setField(key.getSolrFieldName(), value);
}
public final void addSolr(final SolrField key, final boolean value) {
this.setField(key.getSolrFieldName(), value);
}

@ -87,7 +87,7 @@ public class Document {
private boolean resorted;
private final Set<String> languages;
private final boolean indexingDenied;
private final float lon, lat;
private final double lon, lat;
private final Object parserObject; // the source object that was used to create the Document
public Document(final MultiProtocolURI location, final String mimeType, final String charset,
@ -95,7 +95,7 @@ public class Document {
final Set<String> languages,
final String[] keywords, final String title, final String author, final String publisher,
final String[] sections, final String abstrct,
final float lon, final float lat,
final double lon, final double lat,
final Object text,
final Map<MultiProtocolURI, Properties> anchors,
final Map<MultiProtocolURI, String> rss,
@ -400,11 +400,11 @@ dc_rights
return this.emaillinks;
}
public float lon() {
public double lon() {
return this.lon;
}
public float lat() {
public double lat() {
return this.lat;
}
@ -743,7 +743,7 @@ dc_rights
final Map<MultiProtocolURI, Properties> anchors = new HashMap<MultiProtocolURI, Properties>();
final Map<MultiProtocolURI, String> rss = new HashMap<MultiProtocolURI, String>();
final Map<MultiProtocolURI, ImageEntry> images = new HashMap<MultiProtocolURI, ImageEntry>();
float lon = 0.0f, lat = 0.0f;
double lon = 0.0d, lat = 0.0d;
for (final Document doc: docs) {
@ -784,7 +784,7 @@ dc_rights
anchors.putAll(doc.getAnchors());
rss.putAll(doc.getRSS());
ContentScraper.addAllImages(images, doc.getImages());
if (doc.lon() != 0.0f && doc.lat() != 0.0f) { lon = doc.lon(); lat = doc.lat(); }
if (doc.lon() != 0.0d && doc.lat() != 0.0d) { lon = doc.lon(); lat = doc.lat(); }
}
return new Document(
location,

@ -64,8 +64,8 @@ public class DCEntry extends TreeMap<String, String> {
String title,
String author,
String body,
float lat,
float lon
double lat,
double lon
) {
super((Collator) insensitiveCollator.clone());
this.put("dc:identifier", url.toNormalform(true, false));
@ -73,8 +73,8 @@ public class DCEntry extends TreeMap<String, String> {
this.put("dc:title", title);
this.put("dc:creator", author);
this.put("dc:description", body);
this.put("geo:lat", Float.toString(lat));
this.put("geo:long", Float.toString(lon));
this.put("geo:lat", Double.toString(lat));
this.put("geo:long", Double.toString(lon));
}
/*
@ -230,26 +230,26 @@ public class DCEntry extends TreeMap<String, String> {
public String[] getSubject() {
String t = this.get("categories");
if (t == null) this.get("dc:subject");
if (t == null) t = this.get("dc:subject");
t = stripCDATA(t);
if (t == null) return new String[]{};
return t.split(";");
}
public float getLon() {
public double getLon() {
String t = this.get("geo:long");
if (t == null) this.get("geo:lon");
if (t == null) t = this.get("geo:lon");
t = stripCDATA(t);
if (t == null) return 0.0f;
return Float.parseFloat(t);
if (t == null) return 0.0d;
return Double.parseDouble(t);
}
public float getLat() {
public double getLat() {
String t = this.get("geo:lat");
if (t == null) this.get("geo:lat");
if (t == null) t = this.get("geo:lat");
t = stripCDATA(t);
if (t == null) return 0.0f;
return Float.parseFloat(t);
if (t == null) return 0.0d;
return Double.parseDouble(t);
}
private String stripCDATA(String s) {

@ -214,7 +214,7 @@ public class PhpBB3Dao implements Dao {
String text = xmlCleaner(rs.getString("post_text"));
String user = getUser(rs.getInt("poster_id"));
Date date = new Date(rs.getLong("post_time") * 1000L);
return new DCEntry(url, date, subject, user, text, 0.0f, 0.0f);
return new DCEntry(url, date, subject, user, text, 0.0d, 0.0d);
}
public static String xmlCleaner(String s) {

@ -130,7 +130,7 @@ public class ContentScraper extends AbstractScraper implements Scraper {
private final List<String> li;
private final CharBuffer content;
private final EventListenerList htmlFilterEventListeners;
private float lon, lat;
private double lon, lat;
private MultiProtocolURI canonical;
@ -803,7 +803,7 @@ public class ContentScraper extends AbstractScraper implements Scraper {
// <meta NAME="ICBM" CONTENT="38.90551492, 1.454004505" />
// <meta NAME="geo.position" CONTENT="38.90551492;1.454004505" />
public float getLon() {
public double getLon() {
if (this.lon != 0.0f) return this.lon;
String s = this.metas.get("ICBM"); // InterContinental Ballistic Missile (abbrev. supposed to be a joke: http://www.jargon.net/jargonfile/i/ICBMaddress.html), see http://geourl.org/add.html#icbm
if (s != null) {
@ -811,8 +811,8 @@ public class ContentScraper extends AbstractScraper implements Scraper {
if (p < 0) p = s.indexOf(',');
if (p < 0) p = s.indexOf(' ');
if (p > 0) {
this.lat = Float.parseFloat(s.substring(0, p).trim());
this.lon = Float.parseFloat(s.substring(p + 1).trim());
this.lat = Double.parseDouble(s.substring(0, p).trim());
this.lon = Double.parseDouble(s.substring(p + 1).trim());
}
}
if (this.lon != 0.0f) return this.lon;
@ -822,15 +822,15 @@ public class ContentScraper extends AbstractScraper implements Scraper {
if (p < 0) p = s.indexOf(',');
if (p < 0) p = s.indexOf(' ');
if (p > 0) {
this.lat = Float.parseFloat(s.substring(0, p).trim());
this.lon = Float.parseFloat(s.substring(p + 1).trim());
this.lat = Double.parseDouble(s.substring(0, p).trim());
this.lon = Double.parseDouble(s.substring(p + 1).trim());
}
}
return this.lon;
}
public float getLat() {
if (this.lat != 0.0f) return this.lat;
public double getLat() {
if (this.lat != 0.0d) return this.lat;
getLon(); // parse with getLon() method which creates also the lat value
return this.lat;
}

@ -133,9 +133,14 @@ public class CitationReference implements Reference, Serializable {
return false;
}
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
@Override
public int hashCode() {
return ByteArray.hashCode(this.urlhash());
if (this.hashCache == Integer.MIN_VALUE) {
this.hashCache = ByteArray.hashCode(this.urlhash());
}
return this.hashCache;
}
@Override

@ -200,18 +200,21 @@ public final class ImageReferenceRow extends AbstractReference implements /*Imag
@Override
public ImageReferenceRow clone() {
final byte[] b = new byte[urlEntryRow.objectsize];
System.arraycopy(entry.bytes(), 0, b, 0, urlEntryRow.objectsize);
System.arraycopy(this.entry.bytes(), 0, b, 0, urlEntryRow.objectsize);
return new ImageReferenceRow(b);
}
@Override
public String toPropertyForm() {
return entry.toPropertyForm('=', true, true, false, false);
return this.entry.toPropertyForm('=', true, true, false, false);
}
@Override
public Entry toKelondroEntry() {
return this.entry;
}
@Override
public byte[] urlhash() {
return this.entry.getColBytes(col_urlhash, true);
}
@ -220,6 +223,7 @@ public final class ImageReferenceRow extends AbstractReference implements /*Imag
return (int) this.entry.getColLong(col_lastModified); // this is the time in MicoDateDays format
}
@Override
public long lastModified() {
return MicroDate.reverseMicroDateDays((int) this.entry.getColLong(col_lastModified));
}
@ -232,6 +236,7 @@ public final class ImageReferenceRow extends AbstractReference implements /*Imag
return (int) this.entry.getColLong(col_hitcount);
}
@Override
public Collection<Integer> positions() {
return new ArrayList<Integer>(0);
}
@ -271,17 +276,24 @@ public final class ImageReferenceRow extends AbstractReference implements /*Imag
return toPropertyForm();
}
@Override
public boolean isOlder(final Reference other) {
if (other == null) return false;
if (this.lastModified() < other.lastModified()) return true;
return false;
}
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
@Override
public int hashCode() {
return ByteArray.hashCode(this.urlhash());
if (this.hashCache == Integer.MIN_VALUE) {
this.hashCache = ByteArray.hashCode(this.urlhash());
}
return this.hashCache;
}
@Override
public void join(Reference oe) {
throw new UnsupportedOperationException("");

@ -381,9 +381,14 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere
this.wordsintext = this.wordsintext + oe.wordsintext();
}
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
@Override
public int hashCode() {
return ByteArray.hashCode(this.urlHash);
if (this.hashCache == Integer.MIN_VALUE) {
this.hashCache = ByteArray.hashCode(this.urlHash);
}
return this.hashCache;
}
public void addPosition(int position) {

@ -160,12 +160,14 @@ public class DigestURI extends MultiProtocolURI implements Serializable {
this.hash = null;
}
/* (non-Javadoc)
* @see java.lang.Object#hashCode()
*/
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
@Override
public int hashCode() {
return ByteArray.hashCode(hash());
if (this.hashCache == Integer.MIN_VALUE) {
this.hashCache = ByteArray.hashCode(hash());
}
return this.hashCache;
}
public static final int flagTypeID(final String hash) {

@ -38,9 +38,9 @@ public interface URIMetadata extends URIReference {
public String dc_subject();
public float lat();
public double lat();
public float lon();
public double lon();
public long ranking();

@ -123,7 +123,7 @@ public class URIMetadataRow implements URIMetadata {
final String dc_creator,
final String dc_subject,
final String dc_publisher,
final float lon, final float lat, // decimal degrees as in WGS84; if unknown both values may be 0.0f;
final double lon, final double lat, // decimal degrees as in WGS84; if unknown both values may be 0.0d;
final Date mod,
final Date load,
final Date fresh,
@ -194,8 +194,8 @@ public class URIMetadataRow implements URIMetadata {
final String dc_creator,
final String dc_subject,
final String dc_publisher,
final float lat,
final float lon) {
final double lat,
final double lon) {
final CharBuffer s = new CharBuffer(3600, 360);
s.append(url.toNormalform(false, true)).appendLF();
s.append(dc_title).appendLF();
@ -205,7 +205,7 @@ public class URIMetadataRow implements URIMetadata {
s.appendLF();
if (dc_publisher.length() > 80) s.append(dc_publisher, 0, 80); else s.append(dc_publisher);
s.appendLF();
if (lon == 0.0f && lat == 0.0f) s.appendLF(); else s.append(Float.toString(lat)).append(',').append(Float.toString(lon)).appendLF();
if (lon == 0.0f && lat == 0.0f) s.appendLF(); else s.append(Double.toString(lat)).append(',').append(Double.toString(lon)).appendLF();
return UTF8.getBytes(s.toString());
}
@ -418,11 +418,11 @@ public class URIMetadataRow implements URIMetadata {
return this.metadata().dc_subject();
}
public float lat() {
public double lat() {
return this.metadata().lat();
}
public float lon() {
public double lon() {
return this.metadata().lon();
}
@ -638,15 +638,15 @@ public class URIMetadataRow implements URIMetadata {
public String dc_creator() { return this.dc_creator; }
public String dc_publisher() { return this.dc_publisher; }
public String dc_subject() { return this.dc_subject; }
public float lat() {
if (this.latlon == null || this.latlon.length() == 0) return 0.0f;
public double lat() {
if (this.latlon == null || this.latlon.length() == 0) return 0.0d;
final int p = this.latlon.indexOf(',');
return p < 0 ? 0.0f : Float.parseFloat(this.latlon.substring(0, p));
return p < 0 ? 0.0f : Double.parseDouble(this.latlon.substring(0, p));
}
public float lon() {
if (this.latlon == null || this.latlon.length() == 0) return 0.0f;
public double lon() {
if (this.latlon == null || this.latlon.length() == 0) return 0.0d;
final int p = this.latlon.indexOf(',');
return p < 0 ? 0.0f : Float.parseFloat(this.latlon.substring(p + 1));
return p < 0 ? 0.0f : Double.parseDouble(this.latlon.substring(p + 1));
}
}

@ -215,6 +215,7 @@ public final class WordReferenceRow extends AbstractReference implements WordRef
this.out = new LinkedBlockingQueue<WordReferenceRow>();
for (int i = 0; i < concurrency; i++) {
this.worker[i] = new Thread() {
@Override
public void run() {
String s;
try {
@ -290,22 +291,27 @@ public final class WordReferenceRow extends AbstractReference implements WordRef
return new WordReferenceRow(b);
}
@Override
public String toPropertyForm() {
return this.entry.toPropertyForm('=', true, true, false, false);
}
@Override
public Entry toKelondroEntry() {
return this.entry;
}
@Override
public byte[] urlhash() {
return this.entry.getColBytes(col_urlhash, true);
}
@Override
public int virtualAge() {
return (int) this.entry.getColLong(col_lastModified); // this is the time in MicoDateDays format
}
@Override
public long lastModified() {
return MicroDate.reverseMicroDateDays((int) this.entry.getColLong(col_lastModified));
}
@ -314,10 +320,12 @@ public final class WordReferenceRow extends AbstractReference implements WordRef
return MicroDate.reverseMicroDateDays((int) this.entry.getColLong(col_freshUntil));
}
@Override
public int hitcount() {
return (0xff & this.entry.getColByte(col_hitcount));
}
@Override
public Collection<Integer> positions() {
return new ArrayList<Integer>(0);
}
@ -327,54 +335,67 @@ public final class WordReferenceRow extends AbstractReference implements WordRef
return (int) this.entry.getColLong(col_posintext);
}
@Override
public int posinphrase() {
return (0xff & this.entry.getColByte(col_posinphrase));
}
@Override
public int posofphrase() {
return (0xff & this.entry.getColByte(col_posofphrase));
}
@Override
public int wordsintext() {
return (int) this.entry.getColLong(col_wordsInText);
}
@Override
public int phrasesintext() {
return (int) this.entry.getColLong(col_phrasesInText);
}
@Override
public byte[] getLanguage() {
return this.entry.getColBytes(col_language, true);
}
@Override
public char getType() {
return (char) this.entry.getColByte(col_doctype);
}
@Override
public int wordsintitle() {
return (0xff & this.entry.getColByte(col_wordsInTitle));
}
@Override
public int llocal() {
return (0xff & this.entry.getColByte(col_llocal));
}
@Override
public int lother() {
return (0xff & this.entry.getColByte(col_lother));
}
@Override
public int urllength() {
return (0xff & this.entry.getColByte(col_urlLength));
}
@Override
public int urlcomps() {
return (0xff & this.entry.getColByte(col_urlComps));
}
@Override
public Bitfield flags() {
return new Bitfield(this.entry.getColBytes(col_flags, false));
}
@Override
public double termFrequency() {
return (((double) hitcount()) / ((double) (wordsintext() + wordsintitle() + 1)));
}
@ -393,11 +414,17 @@ public final class WordReferenceRow extends AbstractReference implements WordRef
return Base64Order.enhancedCoder.equal(urlhash(), other.urlhash());
}
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
@Override
public int hashCode() {
return ByteArray.hashCode(urlhash());
if (this.hashCache == Integer.MIN_VALUE) {
this.hashCache = ByteArray.hashCode(urlhash());
}
return this.hashCache;
}
@Override
public void join(final Reference oe) {
throw new UnsupportedOperationException("");

@ -31,7 +31,6 @@ import java.util.Comparator;
import java.util.Queue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.Semaphore;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.UTF8;
@ -394,9 +393,14 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
return Base64Order.enhancedCoder.equal(this.urlHash, other.urlHash);
}
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
@Override
public int hashCode() {
return ByteArray.hashCode(this.urlHash);
if (this.hashCache == Integer.MIN_VALUE) {
this.hashCache = ByteArray.hashCode(this.urlHash);
}
return this.hashCache;
}
@Override
@ -419,7 +423,7 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
* @return a blocking queue filled with WordReferenceVars that is still filled when the object is returned
*/
public static BlockingQueue<WordReferenceVars> transform(final ReferenceContainer<WordReference> container) {
public static BlockingQueue<WordReferenceVars> transform(final ReferenceContainer<WordReference> container, final long maxtime) {
final LinkedBlockingQueue<WordReferenceVars> vars = new LinkedBlockingQueue<WordReferenceVars>();
if (container.size() <= 100) {
// transform without concurrency to omit thread creation overhead
@ -431,7 +435,7 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
} catch (final InterruptedException e) {}
return vars;
}
final Thread distributor = new TransformDistributor(container, vars);
final Thread distributor = new TransformDistributor(container, vars, maxtime);
distributor.start();
// return the resulting queue while the processing queues are still working
@ -442,32 +446,37 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
ReferenceContainer<WordReference> container;
BlockingQueue<WordReferenceVars> out;
long maxtime;
public TransformDistributor(final ReferenceContainer<WordReference> container, final BlockingQueue<WordReferenceVars> out) {
public TransformDistributor(final ReferenceContainer<WordReference> container, final BlockingQueue<WordReferenceVars> out, final long maxtime) {
this.container = container;
this.out = out;
this.maxtime = maxtime;
}
@Override
public void run() {
// start the transformation threads
final int cores0 = Math.min(cores, this.container.size() / 100) + 1;
final Semaphore termination = new Semaphore(cores0);
final TransformWorker[] worker = new TransformWorker[cores0];
for (int i = 0; i < cores0; i++) {
worker[i] = new TransformWorker(this.out, termination);
worker[i] = new TransformWorker(this.out, this.maxtime);
worker[i].start();
}
long timeout = System.currentTimeMillis() + this.maxtime;
// fill the queue
int p = this.container.size();
while (p > 0) {
p--;
worker[p % cores0].add(this.container.get(p, false));
if (p % 100 == 0 && System.currentTimeMillis() > timeout) break;
}
// insert poison to stop the queues
for (int i = 0; i < cores0; i++) worker[i].add(WordReferenceRow.poisonRowEntry);
for (int i = 0; i < cores0; i++) {
worker[i].add(WordReferenceRow.poisonRowEntry);
}
}
}
@ -475,12 +484,12 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
BlockingQueue<Row.Entry> in;
BlockingQueue<WordReferenceVars> out;
Semaphore termination;
long maxtime;
public TransformWorker(final BlockingQueue<WordReferenceVars> out, final Semaphore termination) {
public TransformWorker(final BlockingQueue<WordReferenceVars> out, final long maxtime) {
this.in = new LinkedBlockingQueue<Row.Entry>();
this.out = out;
this.termination = termination;
this.maxtime = maxtime;
}
public void add(final Row.Entry entry) {
@ -493,15 +502,13 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
@Override
public void run() {
Row.Entry entry;
long timeout = System.currentTimeMillis() + this.maxtime;
try {
while ((entry = this.in.take()) != WordReferenceRow.poisonRowEntry) this.out.put(new WordReferenceVars(new WordReferenceRow(entry)));
while ((entry = this.in.take()) != WordReferenceRow.poisonRowEntry) {
this.out.put(new WordReferenceVars(new WordReferenceRow(entry)));
if (System.currentTimeMillis() > timeout) break;
}
} catch (final InterruptedException e) {}
// insert poison to signal the termination to next queue
try {
this.termination.acquire();
if (this.termination.availablePermits() == 0) this.out.put(WordReferenceVars.poison);
} catch (final InterruptedException e) {}
}
}

@ -157,14 +157,14 @@ public final class Row implements Serializable {
public final Entry newEntry(final byte[] rowinstance) {
if (rowinstance == null) return null;
assert (this.objectOrder.wellformed(rowinstance, 0, this.primaryKeyLength)) : "row not well-formed: rowinstance[0] = " + UTF8.String(rowinstance, 0, this.primaryKeyLength) + " / " + NaturalOrder.arrayList(rowinstance, 0, this.primaryKeyLength);
return new Entry(rowinstance, false);
return new Entry(rowinstance, 0, false);
}
public final Entry newEntry(final Entry oldrow, final int fromColumn) {
if (oldrow == null) return null;
assert (oldrow.getColBytes(0, false)[0] != 0);
assert (this.objectOrder.wellformed(oldrow.getPrimaryKeyBytes(), 0, this.primaryKeyLength));
return new Entry(oldrow, fromColumn, false);
return new Entry(oldrow.rowinstance, oldrow.offset + oldrow.colstart(fromColumn), false);
}
public final Entry newEntry(final byte[] rowinstance, final int start, final boolean clone) {
@ -251,14 +251,6 @@ public final class Row implements Serializable {
this.offset = 0;
}
public Entry(final byte[] newrow, final boolean forceclone) {
this(newrow, 0, forceclone);
}
public Entry(final Entry oldrow, final int fromColumn, final boolean forceclone) {
this(oldrow.rowinstance, oldrow.offset + oldrow.colstart(fromColumn), forceclone);
}
public Entry(final byte[] newrow, final int start, final boolean forceclone) {
if (forceclone || newrow.length - start < Row.this.objectsize) {
this.rowinstance = new byte[Row.this.objectsize];
@ -643,7 +635,7 @@ public final class Row implements Serializable {
private final int index;
public EntryIndex(final byte[] row, final int i) {
super(row, false);
super(row, 0, false);
this.index = i;
}
public int index() {

@ -76,6 +76,7 @@ public final class Log {
}
public final void logSevere(final String message, final Throwable thrown) {
if (thrown == null) return;
enQueueLog(this.theLogger, Level.SEVERE, message, thrown);
}
@ -88,6 +89,7 @@ public final class Log {
}
public final void logWarning(final String message, final Throwable thrown) {
if (thrown == null) return;
enQueueLog(this.theLogger, Level.WARNING, message, thrown);
}
@ -100,6 +102,7 @@ public final class Log {
}
public final void logConfig(final String message, final Throwable thrown) {
if (thrown == null) return;
enQueueLog(this.theLogger, Level.CONFIG, message, thrown);
}
@ -112,6 +115,7 @@ public final class Log {
}
public final void logInfo(final String message, final Throwable thrown) {
if (thrown == null) return;
enQueueLog(this.theLogger, Level.INFO, message, thrown);
}
@ -124,6 +128,7 @@ public final class Log {
}
public final void logFine(final String message, final Throwable thrown) {
if (thrown == null) return;
enQueueLog(this.theLogger, Level.FINE, message, thrown);
}
@ -136,6 +141,7 @@ public final class Log {
}
public final void logFiner(final String message, final Throwable thrown) {
if (thrown == null) return;
enQueueLog(this.theLogger, Level.FINER, message, thrown);
}
@ -148,6 +154,7 @@ public final class Log {
}
public final void logFinest(final String message, final Throwable thrown) {
if (thrown == null) return;
enQueueLog(this.theLogger, Level.FINEST, message, thrown);
}
@ -165,6 +172,7 @@ public final class Log {
enQueueLog(appName, Level.SEVERE, message);
}
public final static void logSevere(final String appName, final String message, final Throwable thrown) {
if (thrown == null) return;
enQueueLog(appName, Level.SEVERE, message, thrown);
}
@ -172,9 +180,11 @@ public final class Log {
enQueueLog(appName, Level.WARNING, message);
}
public final static void logException(final Throwable thrown) {
if (thrown == null) return;
enQueueLog("StackTrace", Level.WARNING, thrown.getMessage(), thrown);
}
public final static void logWarning(final String appName, final String message, final Throwable thrown) {
if (thrown == null) return;
enQueueLog(appName, Level.WARNING, message, thrown);
}
@ -182,6 +192,7 @@ public final class Log {
enQueueLog(appName, Level.CONFIG, message);
}
public final static void logConfig(final String appName, final String message, final Throwable thrown) {
if (thrown == null) return;
enQueueLog(appName, Level.CONFIG, message, thrown);
}
@ -189,6 +200,7 @@ public final class Log {
enQueueLog(appName, Level.INFO, message);
}
public final static void logInfo(final String appName, final String message, final Throwable thrown) {
if (thrown == null) return;
enQueueLog(appName, Level.INFO, message, thrown);
}
@ -196,6 +208,7 @@ public final class Log {
enQueueLog(appName, Level.FINE, message);
}
public final static void logFine(final String appName, final String message, final Throwable thrown) {
if (thrown == null) return;
enQueueLog(appName, Level.FINE, message, thrown);
}
public final static boolean isFine(final String appName) {
@ -206,6 +219,7 @@ public final class Log {
enQueueLog(appName, Level.FINER, message);
}
public final static void logFiner(final String appName, final String message, final Throwable thrown) {
if (thrown == null) return;
enQueueLog(appName, Level.FINER, message, thrown);
}
@ -213,6 +227,7 @@ public final class Log {
enQueueLog(appName, Level.FINEST, message);
}
public final static void logFinest(final String appName, final String message, final Throwable thrown) {
if (thrown == null) return;
enQueueLog(appName, Level.FINEST, message, thrown);
}
public final static boolean isFinest(final String appName) {
@ -220,6 +235,7 @@ public final class Log {
}
private final static void enQueueLog(final Logger logger, final Level level, final String message, final Throwable thrown) {
if (thrown == null) return;
if (logRunnerThread == null || !logRunnerThread.isAlive()) {
logger.log(level, message, thrown);
} else {
@ -244,6 +260,7 @@ public final class Log {
}
private final static void enQueueLog(final String loggername, final Level level, final String message, final Throwable thrown) {
if (thrown == null) return;
if (logRunnerThread == null || !logRunnerThread.isAlive()) {
Logger.getLogger(loggername).log(level, message, thrown);
} else {
@ -385,12 +402,15 @@ public final class Log {
Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler(){
@Override
public void uncaughtException(final Thread t, final Throwable e) {
if (e == null) return;
final String msg = String.format("Thread %s: %s",t.getName(), e.getMessage());
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
final PrintStream ps = new PrintStream(baos);
e.printStackTrace(ps);
ps.close();
exceptionLog.logSevere(msg + "\n" + baos.toString(), e);
Log.logException(e);
Log.logException(e.getCause());
//System.err.print("Exception in thread \"" + t.getName() + "\" ");
//e.printStackTrace(System.err);
}

@ -61,51 +61,54 @@ public class MergeIterator<E> implements CloneableIterator<E> {
nextb();
}
@Override
public MergeIterator<E> clone(final Object modifier) {
assert a != null;
assert b != null;
assert merger != null;
return new MergeIterator<E>(a.clone(modifier), b.clone(modifier), comp, merger, up);
assert this.a != null;
assert this.b != null;
assert this.merger != null;
return new MergeIterator<E>(this.a.clone(modifier), this.b.clone(modifier), this.comp, this.merger, this.up);
}
private void nexta() {
try {
if (a != null && a.hasNext()) na = a.next(); else na = null;
if (this.a != null && this.a.hasNext()) this.na = this.a.next(); else this.na = null;
} catch (final ConcurrentModificationException e) {
na = null;
this.na = null;
}
}
private void nextb() {
try {
if (b != null && b.hasNext()) nb = b.next(); else nb = null;
if (this.b != null && this.b.hasNext()) this.nb = this.b.next(); else this.nb = null;
} catch (final ConcurrentModificationException e) {
nb = null;
this.nb = null;
}
}
@Override
public boolean hasNext() {
return (na != null) || (nb != null);
return (this.na != null) || (this.nb != null);
}
@SuppressWarnings("unchecked")
@Override
@SuppressWarnings("unchecked")
public E next() {
E s;
if (na == null) {
s = nb;
if (this.na == null) {
s = this.nb;
nextb();
return s;
}
if (nb == null) {
s = na;
if (this.nb == null) {
s = this.na;
nexta();
return s;
}
// compare the Objects
final int c = comp.compare(na, nb);
final int c = this.comp.compare(this.na, this.nb);
if (c == 0) {
try {
//System.out.print("MERGE OF " + na.toString() + " AND " + nb.toString() + ": ");
s = (E) this.merger.invoke(null, new Object[]{na, nb});
s = (E) this.merger.invoke(null, new Object[]{this.na, this.nb});
//System.out.println("RESULT IS " + s.toString());
} catch (final IllegalArgumentException e) {
Log.logException(e);
@ -115,22 +118,24 @@ public class MergeIterator<E> implements CloneableIterator<E> {
s = null;
} catch (final InvocationTargetException e) {
Log.logException(e);
Log.logException(e.getCause());
s = null;
}
nexta();
nextb();
return s;
} else if ((up && c < 0) || (!up && c > 0)) {
s = na;
} else if ((this.up && c < 0) || (!this.up && c > 0)) {
s = this.na;
nexta();
return s;
} else {
s = nb;
s = this.nb;
nextb();
return s;
}
}
@Override
public void remove() {
throw new java.lang.UnsupportedOperationException("merge does not support remove");
}

@ -44,7 +44,7 @@ import net.yacy.cora.order.ByteOrder;
public class ByteArray {
private final byte[] buffer;
private int hash;
private final int hash;
public ByteArray(final byte[] bb) {
@ -53,7 +53,7 @@ public class ByteArray {
}
public int length() {
return buffer.length;
return this.buffer.length;
}
public byte[] asBytes() {
@ -61,7 +61,7 @@ public class ByteArray {
}
public byte readByte(final int pos) {
return buffer[pos];
return this.buffer[pos];
}
public static boolean startsWith(final byte[] buffer, final byte[] pattern) {
@ -83,11 +83,14 @@ public class ByteArray {
return order.compare(this.buffer, aoffset, b.buffer, boffset, blength);
}
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
@Override
public int hashCode() {
if (this.hash != 0) return this.hash;
this.hash = hashCode(this.buffer);
return this.hash;
if (this.hashCache == Integer.MIN_VALUE) {
this.hashCache = ByteArray.hashCode(this.buffer);
}
return this.hashCache;
}
/**
@ -104,8 +107,8 @@ public class ByteArray {
@Override
public boolean equals(Object other) {
ByteArray b = (ByteArray) other;
if (buffer == null && b == null) return true;
if (buffer == null || b == null) return false;
if (this.buffer == null && b == null) return true;
if (this.buffer == null || b == null) return false;
if (this.buffer.length != b.buffer.length) return false;
int l = this.buffer.length;
while (--l >= 0) if (this.buffer[l] != b.buffer[l]) return false;

@ -84,10 +84,12 @@ public class InstantBlockingThread<J extends WorkflowJob> extends AbstractBlocki
}
}
@Override
public int getJobCount() {
return getManager().queueSize();
}
@Override
@SuppressWarnings("unchecked")
public J job(final J next) throws Exception {
J out = null;
@ -116,8 +118,9 @@ public class InstantBlockingThread<J extends WorkflowJob> extends AbstractBlocki
terminate(false);
} catch (final InvocationTargetException e) {
final String targetException = e.getTargetException().getMessage();
Log.logException(e.getTargetException());
Log.logException(e);
Log.logException(e.getCause());
Log.logException(e.getTargetException());
if ((targetException != null) &&
((targetException.indexOf("heap space",0) > 0) ||
(targetException.indexOf("NullPointerException",0) > 0))) {

@ -83,6 +83,7 @@ public final class InstantBusyThread extends AbstractBusyThread implements BusyT
this.handle = Long.valueOf(System.currentTimeMillis() + getName().hashCode());
}
@Override
public int getJobCount() {
if (this.jobCountMethod == null) return Integer.MAX_VALUE;
try {
@ -101,6 +102,7 @@ public final class InstantBusyThread extends AbstractBusyThread implements BusyT
}
}
@Override
public boolean job() throws Exception {
instantThreadCounter++;
//System.out.println("started job " + this.handle + ": " + this.getName());
@ -121,6 +123,7 @@ public final class InstantBusyThread extends AbstractBusyThread implements BusyT
} catch (final InvocationTargetException e) {
final String targetException = e.getTargetException().getMessage();
Log.logException(e);
Log.logException(e.getCause());
Log.logException(e.getTargetException());
Log.logSevere("BUSYTHREAD", "Runtime Error in serverInstantThread.job, thread '" + getName() + "': " + e.getMessage() + "; target exception: " + targetException, e.getTargetException());
} catch (final OutOfMemoryError e) {
@ -136,6 +139,7 @@ public final class InstantBusyThread extends AbstractBusyThread implements BusyT
return jobHasDoneSomething;
}
@Override
public void freemem() {
if (this.freememExecMethod == null) return;
try {

@ -786,7 +786,7 @@ public final class Protocol
// store remote result to local result container
// insert one container into the search result buffer
// one is enough, only the references are used, not the word
containerCache.add(container[0], false, target.getName() + "/" + target.hash, result.joincount, true);
containerCache.add(container[0], false, target.getName() + "/" + target.hash, result.joincount, true, 5000);
containerCache.addExpectedRemoteReferences(-count);
// insert the containers to the index

@ -741,7 +741,7 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
private boolean getFlag(final int flag) {
final String flags = get(Seed.FLAGS, Seed.FLAGSZERO);
return (new bitfield(UTF8.getBytes(flags))).get(flag);
return (new bitfield(ASCII.getBytes(flags))).get(flag);
}
private void setFlag(final int flag, final boolean value) {
@ -749,7 +749,7 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
if ( flags.length() != 4 ) {
flags = Seed.FLAGSZERO;
}
final bitfield f = new bitfield(UTF8.getBytes(flags));
final bitfield f = new bitfield(ASCII.getBytes(flags));
f.set(flag, value);
this.dna.put(Seed.FLAGS, UTF8.String(f.getBytes()));
}

@ -143,7 +143,7 @@ public final class yacyRelease extends yacyVersion {
// return a dev-release or a main-release
if ((latestdev != null) &&
((latestmain == null) || (latestdev.compareTo(latestmain) > 0)) &&
(!(Float.toString(latestdev.getReleaseNr()).matches(blacklist)))) {
(!(Double.toString(latestdev.getReleaseNr()).matches(blacklist)))) {
// consider a dev-release
if (latestdev.compareTo(thisVersion()) <= 0) {
Network.log.logInfo(
@ -155,7 +155,7 @@ public final class yacyRelease extends yacyVersion {
}
if (latestmain != null) {
// consider a main release
if ((Float.toString(latestmain.getReleaseNr()).matches(blacklist))) {
if ((Double.toString(latestmain.getReleaseNr()).matches(blacklist))) {
Network.log.logInfo(
"rulebasedUpdateInfo: latest dev " + (latestdev == null ? "null" : latestdev.getName()) +
" matches with blacklist '" + blacklist + "'");
@ -172,7 +172,7 @@ public final class yacyRelease extends yacyVersion {
}
if ((concept.equals("main")) && (latestmain != null)) {
// return a main-release
if ((Float.toString(latestmain.getReleaseNr()).matches(blacklist))) {
if ((Double.toString(latestmain.getReleaseNr()).matches(blacklist))) {
Network.log.logInfo(
"rulebasedUpdateInfo: latest main " + latestmain.getName() +
" matches with blacklist'" + blacklist + "'");

@ -21,7 +21,7 @@ public class yacyVersion implements Comparator<yacyVersion>, Comparable<yacyVers
* this value is overwritten when a peer with later version appears*/
public static double latestRelease = 0.1; //
private float releaseNr;
private double releaseNr;
private final String dateStamp;
private int svn;
private final boolean mainRelease;
@ -58,9 +58,9 @@ public class yacyVersion implements Comparator<yacyVersion>, Comparable<yacyVers
throw new RuntimeException("release file name '" + release + "' is not valid, 3 information parts expected");
}
try {
this.releaseNr = Float.parseFloat(comp[0]);
this.releaseNr = Double.parseDouble(comp[0]);
} catch (final NumberFormatException e) {
throw new RuntimeException("release file name '" + release + "' is not valid, '" + comp[0] + "' should be a float number");
throw new RuntimeException("release file name '" + release + "' is not valid, '" + comp[0] + "' should be a double number");
}
this.mainRelease = ((int) (getReleaseNr() * 100)) % 10 == 0 || (host != null && host.endsWith("yacy.net"));
//System.out.println("Release version " + this.releaseNr + " is " + ((this.mainRelease) ? "main" : "std"));
@ -111,7 +111,7 @@ public class yacyVersion implements Comparator<yacyVersion>, Comparable<yacyVers
*/
@Override
public int compare(final yacyVersion v0, final yacyVersion v1) {
int r = (Float.valueOf(v0.getReleaseNr())).compareTo(Float.valueOf(v1.getReleaseNr()));
int r = (Double.valueOf(v0.getReleaseGitNr())).compareTo(Double.valueOf(v1.getReleaseGitNr()));
if (r != 0) return r;
r = v0.getDateStamp().compareTo(v1.getDateStamp());
if (r != 0) return r;
@ -122,7 +122,7 @@ public class yacyVersion implements Comparator<yacyVersion>, Comparable<yacyVers
public boolean equals(final Object obj) {
if (obj instanceof yacyVersion) {
final yacyVersion v = (yacyVersion) obj;
return (getReleaseNr() == v.getReleaseNr()) && (getSvn() == v.getSvn()) && (getName().equals(v.getName()));
return (getReleaseGitNr() == v.getReleaseGitNr()) && (getSvn() == v.getSvn()) && (getName().equals(v.getName()));
}
return false;
}
@ -152,7 +152,7 @@ public class yacyVersion implements Comparator<yacyVersion>, Comparable<yacyVers
return new String[]{"dev", "0000"};
}
final String mainversion = (Float.parseFloat(matcher.group(1)) < 0.11 ? "dev" : matcher.group(1));
final String mainversion = (Double.parseDouble(matcher.group(1)) < 0.11 ? "dev" : matcher.group(1));
String revision = matcher.group(2);
for(int i=revision.length();i<5;++i) revision += "0";
return new String[]{mainversion, revision};
@ -201,13 +201,17 @@ public class yacyVersion implements Comparator<yacyVersion>, Comparable<yacyVers
}
/**
* release number as float (e. g. 7.04)
* release number as Double (e. g. 7.04)
* @return
*/
public float getReleaseNr() {
public double getReleaseNr() {
return this.releaseNr;
}
public double getReleaseGitNr() {
// combine release number with git number
return this.getReleaseNr() + ((getSvn()) / 10000000.0d);
}
public String getName() {
return this.name;

@ -264,7 +264,7 @@ public class Segment {
try {
container = ReferenceContainer.emptyContainer(Segment.wordReferenceFactory, wordhash, 1);
container.add(ientry);
rankingProcess.add(container, true, sourceName, -1, !i.hasNext());
rankingProcess.add(container, true, sourceName, -1, !i.hasNext(), 5000);
} catch (final RowSpaceExceededException e) {
continue;
}
@ -377,7 +377,7 @@ public class Segment {
document.dc_subject(' '), // tags
document.dc_publisher(), // publisher (may be important to get location data)
document.lon(), // decimal degrees as in WGS84;
document.lat(), // if unknown both values may be 0.0f;
document.lat(), // if unknown both values may be 0.0d;
modDate, // modification date
loadDate, // loaded date
new Date(loadDate.getTime() + Math.max(0, loadDate.getTime() - modDate.getTime()) / 2), // freshdate, computed with Proxy-TTL formula

@ -108,6 +108,10 @@ public class SolrConfiguration extends ConfigurationSet implements Serializable
if (isEmpty() || contains(key.name())) solrdoc.addSolr(key, value);
}
protected void addSolr(final SolrDoc solrdoc, final SolrField key, final double value) {
if (isEmpty() || contains(key.name())) solrdoc.addSolr(key, value);
}
protected void addSolr(final SolrDoc solrdoc, final SolrField key, final boolean value) {
if (isEmpty() || contains(key.name())) solrdoc.addSolr(key, value);
}

@ -44,7 +44,7 @@ public class AccessTracker {
public static final int minSize = 100;
public static final int maxSize = 1000;
public static final int maxAge = 10 * 60 * 1000;
public static final int maxAge = 24 * 60 * 60 * 1000;
public enum Location {local, remote}

@ -141,6 +141,7 @@ public final class QueryParams {
public boolean specialRights; // is true if the user has a special authorization and my use more database-extensive options
public final String userAgent;
public boolean filterfailurls;
public double lat, lon, radius;
public QueryParams(
final String queryString,
@ -198,6 +199,9 @@ public final class QueryParams {
this.userAgent = userAgent;
this.transmitcount = 0;
this.filterfailurls = false;
this.lat = 0.0d;
this.lon = 0.0d;
this.radius = 0.0d;
}
public QueryParams(
@ -224,7 +228,8 @@ public final class QueryParams {
final Segment indexSegment,
final RankingProfile ranking,
final String userAgent,
final boolean filterfailurls) {
final boolean filterfailurls,
final double lat, final double lon, final double radius) {
this.queryString = queryString;
this.queryHashes = queryHashes;
@ -271,8 +276,15 @@ public final class QueryParams {
this.userAgent = userAgent;
this.transmitcount = 0;
this.filterfailurls = filterfailurls;
// we normalize here the location and radius because that should cause a better caching
// and as surplus it will increase privacy
this.lat = Math.floor(lat * this.kmNormal) / this.kmNormal;
this.lon = Math.floor(lon * this.kmNormal) / this.kmNormal;
this.radius = Math.floor(radius * this.kmNormal + 1) / this.kmNormal;
}
double kmNormal = 100.d; // 100 =ca 40000.d / 360.d == 111.11 - if lat/lon is multiplied with this, rounded and diveded by this, the location is normalized to a 1km grid
public Segment getSegment() {
return this.indexSegment;
}
@ -511,6 +523,8 @@ public final class QueryParams {
context.append(asterisk);
context.append(this.modifier.s);
context.append(asterisk);
context.append(this.lat).append(asterisk).append(this.lon).append(asterisk).append(this.radius);
context.append(asterisk);
context.append(this.snippetCacheStrategy == null ? "null" : this.snippetCacheStrategy.name());
String result = context.toString();
if (anonymized) {

@ -83,7 +83,9 @@ public final class RWIProcess extends Thread
//private final int[] domZones;
private SortedMap<byte[], ReferenceContainer<WordReference>> localSearchInclusion;
private int remote_resourceSize, remote_indexCount, remote_peerCount;
private int remote_resourceSize;
private int remote_indexCount;
private int remote_peerCount;
private int local_indexCount;
private final AtomicInteger maxExpectedRemoteReferences, expectedRemoteReferences,
receivedRemoteReferences;
@ -204,7 +206,7 @@ public final class RWIProcess extends Thread
System.currentTimeMillis() - timer),
false);
if ( !index.isEmpty() ) {
add(index, true, "local index: " + this.query.getSegment().getLocation(), -1, true);
add(index, true, "local index: " + this.query.getSegment().getLocation(), -1, true, 10000);
}
} catch ( final Exception e ) {
Log.logException(e);
@ -218,7 +220,8 @@ public final class RWIProcess extends Thread
final boolean local,
final String resourceName,
final int fullResource,
final boolean finalizeAddAtEnd) {
final boolean finalizeAddAtEnd,
final long maxtime) {
// we collect the urlhashes and construct a list with urlEntry objects
// attention: if minEntries is too high, this method will not terminate within the maxTime
//Log.logInfo("RWIProcess", "added a container, size = " + index.size());
@ -239,7 +242,7 @@ public final class RWIProcess extends Thread
long timer = System.currentTimeMillis();
// normalize entries
final BlockingQueue<WordReferenceVars> decodedEntries = this.order.normalizeWith(index);
final BlockingQueue<WordReferenceVars> decodedEntries = this.order.normalizeWith(index, maxtime);
int is = index.size();
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(
this.query.id(true),
@ -626,6 +629,18 @@ public final class RWIProcess extends Thread
continue;
}
// check geo coordinates
double lat, lon;
if (this.query.radius > 0.0d && this.query.lat != 0.0d && this.query.lon != 0.0d && (lat = page.lat()) > 0.0d && (lon = page.lon()) > 0.0d) {
double latDelta = this.query.lat - lat;
double lonDelta = this.query.lon - lon;
double distance = Math.sqrt(latDelta * latDelta + lonDelta * lonDelta); // pythagoras
if (distance > this.query.radius) {
this.sortout++;
continue;
}
}
// check vocabulary constraint
final String tags = page.dc_subject();
final String[] taglist = tags == null || tags.length() == 0 ? new String[0] : SPACE_PATTERN.split(page.dc_subject());

@ -360,7 +360,7 @@ public class SnippetProcess {
Worker worker;
if (this.workerThreads == null) {
this.workerThreads = new Worker[deployCount];
synchronized(this.workerThreads) {
synchronized(this.workerThreads) {try {
for (int i = 0; i < this.workerThreads.length; i++) {
if (this.result.sizeAvailable() >= neededResults ||
(this.rankingProcess.feedingIsFinished() && this.rankingProcess.sizeQueue() == 0)) {
@ -376,7 +376,7 @@ public class SnippetProcess {
}
}
}
}
} catch (OutOfMemoryError e) {}}
} else {
// there are still worker threads running, but some may be dead.
// if we find dead workers, reanimate them

@ -65,11 +65,11 @@ public class ReferenceOrder {
this.language = language;
}
public BlockingQueue<WordReferenceVars> normalizeWith(final ReferenceContainer<WordReference> container) {
public BlockingQueue<WordReferenceVars> normalizeWith(final ReferenceContainer<WordReference> container, long maxtime) {
final LinkedBlockingQueue<WordReferenceVars> out = new LinkedBlockingQueue<WordReferenceVars>();
int threads = cores;
if (container.size() < 100) threads = 2;
final Thread distributor = new NormalizeDistributor(container, out, threads);
final Thread distributor = new NormalizeDistributor(container, out, threads, maxtime);
distributor.start();
try {
distributor.join(10); // let the distributor work for at least 10 milliseconds
@ -85,17 +85,19 @@ public class ReferenceOrder {
ReferenceContainer<WordReference> container;
LinkedBlockingQueue<WordReferenceVars> out;
private final int threads;
private final long maxtime;
public NormalizeDistributor(final ReferenceContainer<WordReference> container, final LinkedBlockingQueue<WordReferenceVars> out, final int threads) {
public NormalizeDistributor(final ReferenceContainer<WordReference> container, final LinkedBlockingQueue<WordReferenceVars> out, final int threads, final long maxtime) {
this.container = container;
this.out = out;
this.threads = threads;
this.maxtime = maxtime;
}
@Override
public void run() {
// transform the reference container into a stream of parsed entries
final BlockingQueue<WordReferenceVars> vars = WordReferenceVars.transform(this.container);
final BlockingQueue<WordReferenceVars> vars = WordReferenceVars.transform(this.container, this.maxtime);
// start the transformation threads
final Semaphore termination = new Semaphore(this.threads);
@ -148,6 +150,7 @@ public class ReferenceOrder {
}
}
@Override
public void run() {
try {
WordReferenceVars iEntry;

@ -100,9 +100,14 @@ public class MediaSnippet implements Comparable<MediaSnippet>, Comparator<MediaS
if ((this.attr == null) || (this.attr.length() == 0)) this.attr = "_";
}
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
@Override
public int hashCode() {
return ByteArray.hashCode(this.href.hash());
if (this.hashCache == Integer.MIN_VALUE) {
this.hashCache = ByteArray.hashCode(this.href.hash());
}
return this.hashCache;
}
@Override

@ -103,9 +103,13 @@ public class ResultEntry implements Comparable<ResultEntry>, Comparator<ResultEn
if ((p = this.alternative_urlname.indexOf('?')) > 0) this.alternative_urlname = this.alternative_urlname.substring(0, p);
}
}
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
@Override
public int hashCode() {
return ByteArray.hashCode(this.urlentry.hash());
if (this.hashCache == Integer.MIN_VALUE) {
this.hashCache = ByteArray.hashCode(this.urlentry.hash());
}
return this.hashCache;
}
@Override
public boolean equals(final Object obj) {
@ -178,10 +182,10 @@ public class ResultEntry implements Comparable<ResultEntry>, Comparator<ResultEn
public int lapp() {
return this.urlentry.lapp();
}
public float lat() {
public double lat() {
return this.urlentry.lat();
}
public float lon() {
public double lon() {
return this.urlentry.lon();
}
public WordReferenceVars word() {

@ -400,9 +400,14 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
return o1.compareTo(o2);
}
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
@Override
public int hashCode() {
return ByteArray.hashCode(this.urlhash);
if (this.hashCache == Integer.MIN_VALUE) {
this.hashCache = ByteArray.hashCode(this.urlhash);
}
return this.hashCache;
}
@Override

Loading…
Cancel
Save