boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- merged files

Browse Source 
- changed behaviour of delete button in vocabulary edit
- fixed size numbe in vocabulary listing
pull/1/head
Michael Peter Christen 13 years ago
parent dd88d0ace2
commit 963f92ed9a
36 changed files with 428 additions and 478 deletions
Show Stats Download Patch File Download Diff File

2
defaults/yacy.init
Unescape View File

@ -691,7 +691,7 @@ crawlPause.localsearch=50
crawlPause.remotesearch=10
# Some configuration values for the crawler
crawler.clientTimeout=9000
crawler.clientTimeout=30000
# http crawler specific settings; size in bytes
crawler.http.accept=text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8

7
htroot/DemoServlet.java
Unescape View File

@ -1,11 +1,6 @@
import java.util.Iterator;
import net.yacy.yacy;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.interaction.Interaction;
import net.yacy.search.Switchboard;
import de.anomic.data.BookmarkHelper;
import de.anomic.data.UserDB;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
@ -17,8 +12,6 @@ public final class DemoServlet {
// return variable that accumulates replacements
final serverObjects prop = new serverObjects();
final Switchboard sb = Switchboard.getSwitchboard();
prop.put("temperature", "-10°C");
// Display currently logged on user

1
htroot/DemoServletInteraction.java
Unescape View File

@ -1,5 +1,4 @@
import net.yacy.cora.protocol.RequestHeader;
import de.anomic.data.BookmarkHelper;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;

2
htroot/DemoServletRDF.java
Unescape View File

@ -1,6 +1,4 @@
import net.yacy.yacy;
import net.yacy.cora.protocol.RequestHeader;
import de.anomic.data.BookmarkHelper;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;

12
htroot/DictionaryLoader_p.java
Unescape View File

@ -24,8 +24,8 @@ import java.net.MalformedURLException;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.services.federated.yacy.CacheStrategy;
import net.yacy.document.LibraryProvider;
import net.yacy.document.geolocalization.GeonamesLocation;
import net.yacy.document.geolocalization.OpenGeoDBLocation;
import net.yacy.document.geolocation.GeonamesLocation;
import net.yacy.document.geolocation.OpenGeoDBLocation;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.FileUtils;
@ -68,7 +68,7 @@ public class DictionaryLoader_p {
final Response response = sb.loader.load(sb.loader.request(new DigestURI(LibraryProvider.Dictionary.GEON0.url), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, false);
final byte[] b = response.getContent();
FileUtils.copy(b, LibraryProvider.Dictionary.GEON0.file());
LibraryProvider.geoLoc.activateLocalization(LibraryProvider.Dictionary.GEON0.nickname, new GeonamesLocation(LibraryProvider.Dictionary.GEON0.file()));
LibraryProvider.geoLoc.activateLocation(LibraryProvider.Dictionary.GEON0.nickname, new GeonamesLocation(LibraryProvider.Dictionary.GEON0.file(), null));
LibraryProvider.autotagging.addPlaces(LibraryProvider.geoLoc);
prop.put("geon0Status", LibraryProvider.Dictionary.GEON0.file().exists() ? 1 : 0);
prop.put("geon0ActionLoaded", 1);
@ -98,7 +98,7 @@ public class DictionaryLoader_p {
if (post.containsKey("geon0Activate")) {
LibraryProvider.Dictionary.GEON0.fileDisabled().renameTo(LibraryProvider.Dictionary.GEON0.file());
LibraryProvider.geoLoc.activateLocalization(LibraryProvider.Dictionary.GEON0.nickname, new GeonamesLocation(LibraryProvider.Dictionary.GEON0.file()));
LibraryProvider.geoLoc.activateLocation(LibraryProvider.Dictionary.GEON0.nickname, new GeonamesLocation(LibraryProvider.Dictionary.GEON0.file(), null));
LibraryProvider.autotagging.addPlaces(LibraryProvider.geoLoc);
prop.put("geon0ActionActivated", 1);
}
@ -111,7 +111,7 @@ public class DictionaryLoader_p {
final byte[] b = response.getContent();
FileUtils.copy(b, LibraryProvider.Dictionary.GEODB1.file());
LibraryProvider.geoLoc.deactivateLocalization(LibraryProvider.Dictionary.GEODB0.nickname);
LibraryProvider.geoLoc.activateLocalization(LibraryProvider.Dictionary.GEODB1.nickname, new OpenGeoDBLocation(LibraryProvider.Dictionary.GEODB1.file(), false));
LibraryProvider.geoLoc.activateLocation(LibraryProvider.Dictionary.GEODB1.nickname, new OpenGeoDBLocation(LibraryProvider.Dictionary.GEODB1.file(), null));
LibraryProvider.autotagging.addPlaces(LibraryProvider.geoLoc);
prop.put("geo1Status", LibraryProvider.Dictionary.GEODB1.file().exists() ? 1 : 0);
prop.put("geo1ActionLoaded", 1);
@ -141,7 +141,7 @@ public class DictionaryLoader_p {
if (post.containsKey("geo1Activate")) {
LibraryProvider.Dictionary.GEODB1.fileDisabled().renameTo(LibraryProvider.Dictionary.GEODB1.file());
LibraryProvider.geoLoc.activateLocalization(LibraryProvider.Dictionary.GEODB1.nickname, new OpenGeoDBLocation(LibraryProvider.Dictionary.GEODB1.file(), false));
LibraryProvider.geoLoc.activateLocation(LibraryProvider.Dictionary.GEODB1.nickname, new OpenGeoDBLocation(LibraryProvider.Dictionary.GEODB1.file(), null));
LibraryProvider.autotagging.addPlaces(LibraryProvider.geoLoc);
prop.put("geo1ActionActivated", 1);
}

67
htroot/Vocabulary_p.html
Unescape View File

@ -3,6 +3,59 @@
<head>
<title>YaCy '#[clientname]#': Federated Index</title>
#%env/templates/metas.template%#
<script type="text/javascript">
//<![CDATA[
function xmlhttpPost() {
var searchform = document.getElementById('searchform');
search(searchform.discoverobjectspace.value);
}
function search(query) {
var xmlHttpReq = false;
var self = this;
if (window.XMLHttpRequest) { // Mozilla/Safari
self.xmlHttpReq = new XMLHttpRequest();
}
else if (window.ActiveXObject) { // IE
self.xmlHttpReq = new ActiveXObject("Microsoft.XMLHTTP");
}
self.xmlHttpReq.open('GET', "yacysearch.json?verify=false&resource=local&maximumRecords=100&nav=none&query=" + query + "+inurl:" + query, true);
self.xmlHttpReq.setRequestHeader('Content-Type', 'application/x-www-form-urlencoded');
self.xmlHttpReq.onreadystatechange = function() {
if (self.xmlHttpReq.readyState == 4) {
updatepage(self.xmlHttpReq.responseText);
}
}
self.xmlHttpReq.send(null);
}
function updatepage(str) {
var raw = document.getElementById("raw");
if (raw != null) raw.innerHTML = str;
var rsp = eval("("+str+")");
var firstChannel = rsp.channels[0];
var totalResults = firstChannel.totalResults.replace(/[,.]/,"");
var startIndex = firstChannel.startIndex;
var itemsPerPage = firstChannel.itemsPerPage;
var navigation = firstChannel.navigation;
var html = "";
if (totalResults > 0 && firstChannel.items.length > 0) {
var item;
html += "<table class=\"networkTable\" border=\"0\" cellpadding=\"2\" cellspacing=\"1\" width=\"99%\">";
html += "<tr class=\"TableHeader\" valign=\"bottom\">";
html += "<td>URL from index (total results = " + totalResults + ")<\/td>";
for (var i = 0; i < firstChannel.items.length; i++) {
item = firstChannel.items[i];
html += "<tr class=\"TableCellLight\"><td align=\"left\" onclick=\"document.getElementById('searchform').value='" + item.link + "';\">" + item.link + "<\/td>";
}
html += "<\/table>";
}
document.getElementById("searchresults").innerHTML = html;
}
//]]>
</script>
</head>
<body id="Vocabulary_p" onload="document.getElementById('newterm').focus()">
#%env/templates/header.template%#
@ -31,14 +84,17 @@
</form>
#(create)#::
<form action="Vocabulary_p.html" method="get" accept-charset="UTF-8">
<!--<form action="Vocabulary_p.html" id="searchform" method="get" accept-charset="UTF-8" onkeyup="xmlhttpPost(); return false;">-->
<form action="Vocabulary_p.html" id="searchform" method="get" accept-charset="UTF-8" >
<fieldset><legend>Vocabulary Production</legend>
It is possible to produce a vocabulary out of the existing search index. This is done using a given 'objectspace' which you can enter as a URL Stub.
This stub is used to find all matching URLs. If the remaining path from the matching URLs then denotes a single file, the file name is used as vocabulary term.
This works best with wikis. Try to use a wiki url as objectspace path.
<dl>
<dt>Vocabulary Name</dt><dd><input type="text" name="discovername" value="" size="16" maxlength="128" /></dd>
<dt>Objectspace</dt><dd><input type="text" name="discoverobjectspace" value="http://" size="78" maxlength="128" /></dd>
<dt>Objectspace</dt><dd><input type="text" name="discoverobjectspace" value="http://" size="78" maxlength="128" />
<div id="searchresults"></div></dd>
<dt>Discover Terms from</dt><dd><input type="radio" name="discovermethod" value="path" checked="checked" />object link file name&nbsp;&nbsp;<input type="radio" name="discovermethod" value="title" />object page title&nbsp;&nbsp;<input type="radio" name="discovermethod" value="titlesplitted" />object page title (splitted)&nbsp;&nbsp;<input type="radio" name="discovermethod" value="author" />object page author</dd>
<dt></dt><dd><input type="submit" name="create" value="Create" /></dd>
</dl>
</fieldset>
@ -51,6 +107,7 @@
<dl>
<dt>Vocabulary Name</dt><dd>#[name]#</dd>
<dt>File</dt><dd>#(editable)#[automatically generated, not stored, cannot be edited]::#[file]##(/editable)#</dd>
<dt>Size</dt><dd>#[size]#</dd>
<dt>Namespace</dt><dd>#[namespace]#</dd>
<dt>Predicate</dt><dd>#[predicate]#</dd>
<dt>Prefix</dt><dd>#[prefix]#</dd>
@ -73,10 +130,10 @@
#{terms}#
<tr class="TableCell#(dark)#Light::Dark::Summary#(/dark)#">
<td align="center">#(editable)#&nbsp;::<input type="checkbox" name="modify_#[term]#" id="modify_#[term]#" value="checked" disabled="disabled"/>#(/editable)#</td>
<td align="center">#(editable)#&nbsp;::<input type="checkbox" name="delete_#[term]#" id="delete_#[term]#" value="checked" onchange="this.form.submit()"/>#(/editable)#</td>
<td align="center">#(editable)#&nbsp;::<input type="checkbox" name="delete_#[term]#" id="delete_#[term]#" value="checked" />#(/editable)#</td>
<td align="left">#[term]#</td>
<td align="left">#(editable)##[synonyms]#::<input type="text" name="synonyms_#[term]#" value="#[synonyms]#" size="40" maxlength="1024" onkeyup="document.getElementById('modify_#[term]#').checked='checked'; document.getElementById('modify_#[term]#').disabled=''"/>#(/editable)#</td>
<td align="left">#(editable)##[objectlink]#::<input type="text" name="objectlink_#[term]#" value="#[objectlink]#" size="60" maxlength="1024" onkeyup="document.getElementById('modify_#[term]#').checked='checked'; document.getElementById('modify_#[term]#').disabled=''"/>#(/editable)#</td>
<td align="left">#(editable)##[synonyms]#::<input type="text" name="synonyms_#[term]#" value="#[synonyms]#" size="40" maxlength="1024" onkeyup="document.getElementById('modify_#[term]#').checked='checked'; document.getElementById('modify_#[term]#').disabled=''; document.getElementById('delete_#[term]#').disabled='disabled';"/>#(/editable)#</td>
<td align="left">#(editable)##[objectlink]#::<input type="text" name="objectlink_#[term]#" value="#[objectlink]#" size="60" maxlength="1024" onkeyup="document.getElementById('modify_#[term]#').checked='checked'; document.getElementById('modify_#[term]#').disabled=''; document.getElementById('delete_#[term]#').disabled='disabled';"/>#(/editable)#</td>
</tr>
#{/terms}#
#(editable)#::

63
htroot/Vocabulary_p.java
Unescape View File

@ -35,6 +35,7 @@ import net.yacy.cora.lod.vocabulary.YaCyMetadata;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.document.LibraryProvider;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.data.meta.URIMetadataRow;
import net.yacy.kelondro.logging.Log;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
@ -50,46 +51,73 @@ public class Vocabulary_p {
Collection<Tagging> vocs = LibraryProvider.autotagging.getVocabularies();
String vocabularyName = (post == null) ? null : post.get("vocabulary", null);
String discovername = (post == null) ? null : post.get("discovername", null);
Tagging vocabulary = vocabularyName == null ? null : LibraryProvider.autotagging.getVocabulary(vocabularyName);
if (vocabulary == null) vocabularyName = null;
int count = 0;
for (Tagging v: vocs) {
prop.put("vocabularyset_" + count + "_name", v.getName());
prop.put("vocabularyset_" + count + "_selected", (vocabularyName != null && vocabularyName.equals(v.getName())) ? 1 : 0);
count++;
}
prop.put("vocabularyset", count);
if (post != null) {
try {
if (vocabulary == null) {
// create a vocabulary
String discovername = post.get("discovername", "");
if (discovername.length() > 0) {
if (discovername != null && discovername.length() > 0) {
String discoverobjectspace = post.get("discoverobjectspace", "");
MultiProtocolURI discoveruri = null;
if (discoverobjectspace.length() > 0) try {discoveruri = new MultiProtocolURI(discoverobjectspace);} catch (MalformedURLException e) {}
if (discoveruri == null) discoverobjectspace = "";
Map<String, Tagging.SOTuple> table = new TreeMap<String, Tagging.SOTuple>();
File propFile = LibraryProvider.autotagging.getVocabularyFile(discovername);
boolean discoverFromPath = post.get("discovermethod", "").equals("path");
boolean discoverFromTitle = post.get("discovermethod", "").equals("title");
boolean discoverFromTitleSplitted = post.get("discovermethod", "").equals("titlesplitted");
boolean discoverFromAuthor = post.get("discovermethod", "").equals("author");
if (discoveruri != null) {
String segmentName = sb.getConfig(SwitchboardConstants.SEGMENT_PUBLIC, "default");
Segment segment = sb.indexSegments.segment(segmentName);
Iterator<DigestURI> ui = segment.urlSelector(discoveruri);
String t;
while (ui.hasNext()) {
DigestURI u = ui.next();
String u0 = u.toNormalform(true, false);
String t = u0.substring(discoverobjectspace.length());
t = "";
if (discoverFromPath) {
t = u0.substring(discoverobjectspace.length());
if (t.indexOf('/') >= 0) continue;
int p = t.indexOf('.');
if (p >= 0) t = t.substring(0, p);
while ((p = t.indexOf(':')) >= 0) t = t.substring(p + 1);
while ((p = t.indexOf('=')) >= 0) t = t.substring(p + 1);
if (p >= 0) t = t.substring(p + 1);
}
if (discoverFromTitle || discoverFromTitleSplitted) {
URIMetadataRow m = segment.urlMetadata().load(u.hash());
if (m != null) t = m.dc_title();
if (t.endsWith(".jpg") || t.endsWith(".gif")) continue;
}
if (discoverFromAuthor) {
URIMetadataRow m = segment.urlMetadata().load(u.hash());
if (m != null) t = m.dc_creator();
}
t = t.replaceAll("\"", " ").replaceAll("'", " ").replaceAll(",", " ").replaceAll(" ", " ").trim();
if (t.length() == 0) continue;
if (discoverFromTitleSplitted) {
String[] ts = t.split(" ");
for (String s: ts) {
if (s.length() == 0) continue;
if (s.endsWith(".jpg") || s.endsWith(".gif")) continue;
table.put(s, new Tagging.SOTuple("", u0));
}
} else if (discoverFromAuthor) {
String[] ts = t.split(";"); // author names are often separated by ';'
for (String s: ts) {
if (s.length() == 0) continue;
int p = s.indexOf(','); // check if there is a reversed method to mention the name
if (p >= 0) s = s.substring(p + 1).trim() + " " + s.substring(0, p).trim();
table.put(s, new Tagging.SOTuple("", u0));
}
} else {
table.put(t, new Tagging.SOTuple("", u0));
}
}
}
Tagging newvoc = new Tagging(discovername, propFile, discoverobjectspace, table);
LibraryProvider.autotagging.addVocabulary(newvoc);
vocabularyName = discovername;
@ -143,6 +171,14 @@ public class Vocabulary_p {
}
}
int count = 0;
for (Tagging v: vocs) {
prop.put("vocabularyset_" + count + "_name", v.getName());
prop.put("vocabularyset_" + count + "_selected", ((vocabularyName != null && vocabularyName.equals(v.getName())) || (discovername != null && discovername.equals(v.getName()))) ? 1 : 0);
count++;
}
prop.put("vocabularyset", count);
prop.put("create", vocabularyName == null ? 1 : 0);
if (vocabulary == null) {
@ -164,7 +200,9 @@ public class Vocabulary_p {
prop.putHTML("edit_tripleN", vocabulary.getObjectspace() == null ? "none - missing objectspace" : "<" + yacyurl + "> <" + DCTerms.references.getPredicate() + "> \"[reference-link]#[tag]\" .");
int c = 0;
boolean dark = false;
for (Map.Entry<String, SOTuple> entry: vocabulary.list().entrySet()) {
Map<String, SOTuple> list = vocabulary.list();
prop.put("edit_size", list.size());
for (Map.Entry<String, SOTuple> entry: list.entrySet()) {
prop.put("edit_terms_" + c + "_editable", editable ? 1 : 0);
prop.put("edit_terms_" + c + "_dark", dark ? 1 : 0); dark = !dark;
prop.putHTML("edit_terms_" + c + "_term", entry.getKey());
@ -175,6 +213,7 @@ public class Vocabulary_p {
if (c > 3000) break;
}
prop.put("edit_terms", c);
}
// return rewrite properties

12
htroot/api/yacydoc.java
Unescape View File

@ -25,24 +25,12 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.io.ByteArrayOutputStream;
import java.net.MalformedURLException;
import java.util.Iterator;
import java.util.Map;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.RDFNode;
import com.hp.hpl.jena.rdf.model.Resource;
import net.yacy.cora.date.ISO8601Formatter;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.lod.JenaTripleStore;
import net.yacy.cora.lod.vocabulary.Tagging;
import net.yacy.cora.lod.vocabulary.YaCyMetadata;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.sorting.ConcurrentScoreMap;
import net.yacy.cora.sorting.ScoreMap;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.data.meta.URIMetadataRow;
import net.yacy.kelondro.data.word.Word;

39
htroot/interaction/Triple.java
Unescape View File

@ -32,7 +32,6 @@ package interaction;
import net.yacy.cora.lod.JenaTripleStore;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.interaction.Interaction;
import net.yacy.search.Switchboard;
import de.anomic.data.UserDB;
import de.anomic.server.serverObjects;
@ -99,55 +98,37 @@ public class Triple {
String s = "";
String p = "";
String o = "";
String result = "";
Boolean global = false;
if (post != null) {
if(post.containsKey("s")){
s = post.get("s");
}
if(post.containsKey("sp")){
s = post.get("sp") + "#" + s;
}
if(post.containsKey("p")){
p = post.get("p");
}
if(post.containsKey("pp")){
p = post.get("pp") + "#" + p;
}
s = post.get("s", "");
p = post.get("p", "");
o = post.get("o", "");
if(post.containsKey("o")){
o = post.get("o");
}
if (post.containsKey("sp")) s = post.get("sp") + "#" + s;
if (post.containsKey("pp")) p = post.get("pp") + "#" + p;
global = post.containsKey("global");
}
if (post.containsKey("load")) {
if (global) {
o = JenaTripleStore.getObject(s, p);
result = JenaTripleStore.getObject(s, p);
} else {
o = JenaTripleStore.getPrivateObject(s, p, username);
result = JenaTripleStore.getPrivateObject(s, p, username);
}
} else {
if (global) {
JenaTripleStore.addTriple(s, p, o);
} else {
JenaTripleStore.addTriple(s, p, o, username);
}
}
}
prop.put("result", o);
prop.put("result", result);
return prop;
}

3
htroot/interaction_elements/OverlayInteraction.java
Unescape View File

@ -1,7 +1,6 @@
package interaction_elements;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.search.Switchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
@ -9,8 +8,6 @@ public class OverlayInteraction {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
prop.put("enabled", env.getConfigBool("interaction.overlayinteraction.enabled", false) ? "1" : "0");

5
htroot/interaction_elements/Tag_part.java
Unescape View File

@ -4,11 +4,8 @@ package interaction_elements;
import java.util.Collection;
import net.yacy.cora.lod.vocabulary.Tagging;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.document.LibraryProvider;
import net.yacy.search.Switchboard;
import de.anomic.data.UserDB;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
@ -16,8 +13,6 @@ public class Tag_part {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
prop.put("hash", post.get("hash", ""));

4
htroot/yacysearch.java
Unescape View File

@ -55,7 +55,7 @@ import net.yacy.document.Condenser;
import net.yacy.document.Document;
import net.yacy.document.LibraryProvider;
import net.yacy.document.Parser;
import net.yacy.document.geolocalization.GeoLocation;
import net.yacy.document.geolocation.GeoLocation;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.data.meta.URIMetadataRow;
import net.yacy.kelondro.data.word.Word;
@ -480,7 +480,7 @@ public class yacysearch {
if (p > 0) {
String k = vocabulary.substring(0, p);
String v = vocabulary.substring(p + 1);
metatags.add(LibraryProvider.autotagging.metatag(LibraryProvider.autotagging.prefixChar + k + ":" + v));
metatags.add(LibraryProvider.autotagging.metatag(k, v));
}
}

2
htroot/yacysearch_location.java
Unescape View File

@ -28,7 +28,7 @@ import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.services.federated.opensearch.SRURSSConnector;
import net.yacy.document.LibraryProvider;
import net.yacy.document.geolocalization.GeoLocation;
import net.yacy.document.geolocation.GeoLocation;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import de.anomic.server.serverCore;

2
source/de/anomic/crawler/retrieval/HTTPLoader.java
Unescape View File

@ -67,7 +67,7 @@ public final class HTTPLoader {
this.log = theLog;
// refreshing timeout value
this.socketTimeout = (int) sb.getConfigLong("crawler.clientTimeout", 10000);
this.socketTimeout = (int) sb.getConfigLong("crawler.clientTimeout", 30000);
}
public Response load(final Request entry, final int maxFileSize, final boolean checkBlacklist) throws IOException {

6
source/de/anomic/server/serverSwitch.java
Unescape View File

@ -61,7 +61,7 @@ public class serverSwitch
// configuration management
private final File configFile;
private final String configComment;
private final File dataPath;
public final File dataPath;
public final File appPath;
protected boolean firstInit;
protected Log log;
@ -311,7 +311,7 @@ public class serverSwitch
* the relative path setting.
*/
public File getDataPath(final String key, final String dflt) {
return getFileByPath(key, dflt, dataPath);
return getFileByPath(key, dflt, this.dataPath);
}
/**
@ -321,7 +321,7 @@ public class serverSwitch
* @return
*/
public File getAppPath(final String key, final String dflt) {
return getFileByPath(key, dflt, appPath);
return getFileByPath(key, dflt, this.appPath);
}
private File getFileByPath(String key, String dflt, File prefix) {

82
source/net/yacy/cora/lod/JenaTripleStore.java
Unescape View File

@ -2,12 +2,14 @@
package net.yacy.cora.lod;
import java.io.BufferedOutputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Iterator;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
@ -111,14 +113,27 @@ public class JenaTripleStore {
}
public static void saveFile(String filename, Model model) {
File f = new File(filename);
File ftmp = new File(filename + "." + System.currentTimeMillis());
if (model.size() == 0 && !f.exists()) {
// we don't store zero-size models if they did not exist before
Log.logInfo("TRIPLESTORE", "NOT saving triplestore with " + model.size() + " triples to " + filename);
return;
}
Log.logInfo("TRIPLESTORE", "Saving triplestore with " + model.size() + " triples to " + filename);
FileOutputStream fout;
OutputStream fout;
try {
fout = new FileOutputStream(filename);
fout = new BufferedOutputStream(new FileOutputStream(ftmp));
model.write(fout);
fout.close();
// if something went wrong until here, the original file is not overwritten
// since we are happy here, we can remove the old file and replace it with the new one
f.delete();
if (!f.exists()) {
ftmp.renameTo(f);
}
Log.logInfo("TRIPLESTORE", "Saved triplestore with " + model.size() + " triples to " + filename);
} catch (Exception e) {
// TODO Auto-generated catch block
Log.logWarning("TRIPLESTORE", "Saving to " + filename+" failed");
}
}
@ -164,11 +179,11 @@ public class JenaTripleStore {
}
public static String getObject(final String subject, final String predicate) {
Log.logInfo("TRIPLESTORE", "GET " + subject + " - " + predicate + " ... ");
Iterator<RDFNode> ni = JenaTripleStore.getObjects(subject, predicate);
if (!ni.hasNext()) return "";
return ni.next().toString();
String object = "";
if (ni.hasNext()) object = ni.next().toString();
Log.logInfo("TRIPLESTORE", "GET " + subject + " - " + predicate + " - " + object);
return object;
}
public static Iterator<RDFNode> getObjects(final String subject, final String predicate) {
@ -177,11 +192,11 @@ public class JenaTripleStore {
}
public static String getPrivateObject(final String subject, final String predicate, final String username) {
Log.logInfo("TRIPLESTORE", "GET " + subject + " - " + predicate + " ... ("+username+")");
Iterator<RDFNode> ni = JenaTripleStore.getPrivateObjects(subject, predicate, username);
if (!ni.hasNext()) return "";
return ni.next().toString();
String object = "";
if (ni.hasNext()) object = ni.next().toString();
Log.logInfo("TRIPLESTORE", "GET (" + username + ") " + subject + " - " + predicate + " - " + object);
return object;
}
private static Iterator<RDFNode> getPrivateObjects(final String subject, final String predicate, final String username) {
@ -256,81 +271,64 @@ public class JenaTripleStore {
}
public static void initPrivateStores() {
Switchboard switchboard = Switchboard.getSwitchboard();
Log.logInfo("TRIPLESTORE", "Init private stores");
if (privatestorage == null) privatestorage = new ConcurrentHashMap<String, Model>();
if (privatestorage != null) privatestorage.clear();
try {
Iterator<de.anomic.data.UserDB.Entry> it = switchboard.userDB.iterator(true);
while (it.hasNext()) {
de.anomic.data.UserDB.Entry e = it.next();
String username = e.getUserName();
File triplestore = new File(switchboard.getConfig("triplestore", new File(switchboard.getDataPath(), "DATA/TRIPLESTORE").getAbsolutePath()));
File currentuserfile = new File(triplestore, "private_store_"+username+".rdf");
Log.logInfo("TRIPLESTORE", "Init " + username + " from "+currentuserfile.getAbsolutePath());
Model tmp = ModelFactory.createDefaultModel();
init (tmp);
if (currentuserfile.exists()) {
Log.logInfo("TRIPLESTORE", "Loading from " + currentuserfile.getAbsolutePath());
InputStream is = FileManager.get().open(currentuserfile.getAbsolutePath());
if (is != null) {
// read the RDF/XML file
tmp.read(is, null);
Log.logInfo("TRIPLESTORE", "loaded " + tmp.size() + " triples from " + currentuserfile.getAbsolutePath());
} else {
throw new IOException("cannot read " + currentuserfile.getAbsolutePath());
}
}
if (tmp != null) {
privatestorage.put(username, tmp);
}
}
}
catch (Exception anyex) {
} catch (Exception anyex) {
Log.logException(anyex);
}
}
public static void savePrivateStores(Switchboard switchboard) {
public static void savePrivateStores() {
Switchboard switchboard = Switchboard.getSwitchboard();
Log.logInfo("TRIPLESTORE", "Saving user triplestores");
if (privatestorage == null) return;
for (Entry<String, Model> s : privatestorage.entrySet()) {
File triplestore = new File(switchboard.getConfig("triplestore", new File(switchboard.getDataPath(), "DATA/TRIPLESTORE").getAbsolutePath()));
File currentuserfile = new File(triplestore, "private_store_"+s.getKey()+".rdf");
saveFile (currentuserfile.getAbsolutePath(), s.getValue());
}
}
private static long lastModelSizeStored = -1;
public static void saveAll() {
Switchboard sb = Switchboard.getSwitchboard();
File triplestore = new File(sb.getConfig("triplestore", new File(sb.dataPath, "DATA/TRIPLESTORE").getAbsolutePath()));
if (model.size() != lastModelSizeStored){
JenaTripleStore.saveFile(new File(triplestore, "local.rdf").getAbsolutePath());
lastModelSizeStored = model.size();
}
JenaTripleStore.savePrivateStores();
}
}

61
source/net/yacy/cora/lod/vocabulary/Tagging.java
Unescape View File

@ -36,7 +36,7 @@ import java.util.regex.Pattern;
import net.yacy.cora.storage.Files;
import net.yacy.document.WordCache.Dictionary;
import net.yacy.document.geolocalization.Locations;
import net.yacy.document.geolocation.Locations;
public class Tagging {
@ -95,7 +95,7 @@ public class Tagging {
vocloop: for (Map.Entry<String, SOTuple> e: table.entrySet()) {
if (e.getValue().getSynonymsCSV() == null || e.getValue().getSynonymsCSV().length() == 0) {
term = normalizeKey(e.getKey());
v = normalizeWord(e.getKey());
v = normalizeTerm(e.getKey());
this.synonym2term.put(v, term);
this.term2synonym.put(term, v);
if (e.getValue().getObjectlink() != null && e.getValue().getObjectlink().length() > 0) this.term2objectlink.put(term, e.getValue().getObjectlink());
@ -108,13 +108,13 @@ public class Tagging {
tagloop: for (String synonym: tags) {
if (synonym.length() == 0) continue tagloop;
synonyms.add(synonym);
synonym = normalizeWord(synonym);
synonym = normalizeTerm(synonym);
if (synonym.length() == 0) continue tagloop;
synonyms.add(synonym);
this.synonym2term.put(synonym, term);
this.term2synonym.put(term, synonym);
}
String synonym = normalizeWord(term);
String synonym = normalizeTerm(term);
this.synonym2term.put(synonym, term);
this.term2synonym.put(term, synonym);
if (e.getValue().getObjectlink() != null && e.getValue().getObjectlink().length() > 0) this.term2objectlink.put(term, e.getValue().getObjectlink());
@ -169,9 +169,8 @@ public class Tagging {
}
public void updateTerm(String term, String[] synonyms) {
public int size() {
return this.term2objectlink.size();
}
private File tmpFile() {
@ -402,7 +401,7 @@ public class Tagging {
}
if (pl[1] == null) {
term = normalizeKey(pl[0]);
v = normalizeWord(pl[0]);
v = normalizeTerm(pl[0]);
this.synonym2term.put(v, term);
this.term2synonym.put(term, v);
if (pl[2] != null && pl[2].length() > 0) this.term2objectlink.put(term, pl[2]);
@ -416,13 +415,13 @@ public class Tagging {
tagloop: for (String synonym: tags) {
if (synonym.length() == 0) continue tagloop;
synonyms.add(synonym);
synonym = normalizeWord(synonym);
synonym = normalizeTerm(synonym);
if (synonym.length() == 0) continue tagloop;
synonyms.add(synonym);
this.synonym2term.put(synonym, term);
this.term2synonym.put(term, synonym);
}
String synonym = normalizeWord(term);
String synonym = normalizeTerm(term);
this.synonym2term.put(synonym, term);
this.term2synonym.put(term, synonym);
if (pl[2] != null && pl[2].length() > 0) this.term2objectlink.put(term, pl[2]);
@ -439,8 +438,9 @@ public class Tagging {
this(name);
Set<String> locNames = localization.locationNames();
for (String loc: locNames) {
this.synonym2term.put(loc.toLowerCase(), loc);
this.term2synonym.put(loc, loc.toLowerCase());
String syn = normalizeTerm(loc);
this.synonym2term.put(syn, loc);
this.term2synonym.put(loc, syn);
}
}
@ -493,14 +493,14 @@ public class Tagging {
return this.propFile;
}
public Metatag getMetatagFromSynonym(char prefix, final String word) {
public Metatag getMetatagFromSynonym(final String word) {
String printname = this.synonym2term.get(word);
if (printname == null) return null;
return new Metatag(prefix, printname);
return new Metatag(printname);
}
public Metatag getMetatagFromTerm(char prefix, final String word) {
return new Metatag(prefix, word);
public Metatag getMetatagFromTerm(final String word) {
return new Metatag(word);
}
public Set<String> getSynonyms(String term) {
@ -532,20 +532,23 @@ public class Tagging {
private final static Pattern PATTERN_UE = Pattern.compile("\u00FC");
private final static Pattern PATTERN_SZ = Pattern.compile("\u00DF");
public static final String normalizeWord(String word) {
word = word.trim().toLowerCase();
word = PATTERN_AE.matcher(word).replaceAll("ae");
word = PATTERN_OE.matcher(word).replaceAll("oe");
word = PATTERN_UE.matcher(word).replaceAll("ue");
word = PATTERN_SZ.matcher(word).replaceAll("ss");
return word;
public static final String normalizeTerm(String term) {
term = term.trim().toLowerCase();
term = PATTERN_AE.matcher(term).replaceAll("ae");
term = PATTERN_OE.matcher(term).replaceAll("oe");
term = PATTERN_UE.matcher(term).replaceAll("ue");
term = PATTERN_SZ.matcher(term).replaceAll("ss");
// remove comma
int p;
while ((p = term.indexOf(',')) >= 0) {
term = term.substring(p + 1).trim() + " " + term.substring(0, p);
}
return term;
}
public class Metatag {
private final String object;
private final char prefix;
public Metatag(char prefix, String object) {
this.prefix = prefix;
public Metatag(String object) {
this.object = object;
}
@ -563,7 +566,7 @@ public class Tagging {
@Override
public String toString() {
return this.prefix + Tagging.this.navigatorName + ":" + encodePrintname(this.object);
return Tagging.this.navigatorName + ":" + encodePrintname(this.object);
}
@Override
@ -589,12 +592,12 @@ public class Tagging {
return PATTERN_UL.matcher(maskname).replaceAll(" ");
}
public static String cleanTagFromAutotagging(char prefix, final String tagString) {
public static String cleanTagFromAutotagging(final String tagString) {
if (tagString == null || tagString.length() == 0) return "";
String[] tags = PATTERN_SP.split(tagString);
StringBuilder sb = new StringBuilder(tagString.length());
for (String tag : tags) {
if (tag.length() > 0 && tag.charAt(0) != prefix) {
if (tag.length() > 0) {
sb.append(tag).append(' ');
}
}

70
source/net/yacy/document/Autotagging.java
Unescape View File

@ -20,19 +20,15 @@
package net.yacy.document;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.lod.vocabulary.Tagging;
import net.yacy.document.WordCache.Dictionary;
import net.yacy.document.geolocalization.Locations;
import net.yacy.document.geolocation.Locations;
import net.yacy.kelondro.logging.Log;
/**
@ -44,7 +40,6 @@ public class Autotagging {
private final static Object PRESENT = new Object();
public final char prefixChar;
private final File autotaggingPath;
private final Map<String, Tagging> vocabularies; // mapping from vocabulary name to the tagging vocabulary
private final Map<String, Object> allTags;
@ -58,10 +53,9 @@ public class Autotagging {
* properties without values are allowed (the value is then set to the key)
* also the value can be used as a tag
*/
public Autotagging(final File autotaggingPath, char prefixChar) {
public Autotagging(final File autotaggingPath) {
this.vocabularies = new ConcurrentHashMap<String, Tagging>();
this.autotaggingPath = autotaggingPath;
this.prefixChar = prefixChar;
this.allTags = new ConcurrentHashMap<String, Object>();
if (this.autotaggingPath == null || !this.autotaggingPath.exists()) {
return;
@ -114,16 +108,6 @@ public class Autotagging {
}
}
public void addDictionaries(Map<String, Dictionary> dictionaries) {
for (Map.Entry<String, Dictionary> entry: dictionaries.entrySet()) {
Tagging voc = new Tagging(entry.getKey(), entry.getValue());
this.vocabularies.put(entry.getKey(), voc);
for (String t: voc.tags()) {
this.allTags.put(t, PRESENT);
}
}
}
public void addPlaces(Locations locations) {
if (locations.size() == 0) return; // otherwise we get a navigation that does nothing
Tagging voc = new Tagging("Locations", locations);
@ -137,24 +121,6 @@ public class Autotagging {
}
}
/**
* produce a set of tags for a given text.
* The set contains the names of the tags with a prefix character at the front
* @param text
* @return
*/
public Set<String> getPrintTagsFromText(String text) {
Set<String> as = new HashSet<String>();
if (this.vocabularies.isEmpty()) return as;
final WordTokenizer tokens = new WordTokenizer(new ByteArrayInputStream(UTF8.getBytes(text)), LibraryProvider.dymLib);
String tag;
while (tokens.hasMoreElements()) {
tag = getTagFromTerm(tokens.nextElement().toString()).toString();
if (tag != null) as.add(tag);
}
return as;
}
public int size() {
return this.vocabularies.size();
}
@ -171,41 +137,17 @@ public class Autotagging {
public Tagging.Metatag getTagFromTerm(String term) {
if (this.vocabularies.isEmpty()) return null;
Tagging.Metatag tag;
term = Tagging.normalizeWord(term);
term = Tagging.normalizeTerm(term);
for (Map.Entry<String, Tagging> v: this.vocabularies.entrySet()) {
tag = v.getValue().getMetatagFromSynonym(this.prefixChar, term);
tag = v.getValue().getMetatagFromSynonym(term);
if (tag != null) return tag;
}
return null;
}
public static boolean metatagAppearIn(final Tagging.Metatag metatag, final String[] tags) {
String tag = metatag.toString();
for (String s: tags) {
if (tag.equals(s)) return true;
}
return false;
}
public Tagging.Metatag metatag(String metatag) {
int p = metatag.indexOf(':');
if (p < 0) throw new RuntimeException("bad metatag: metatag = " + metatag);
String vocName = metatag.substring(1, p);
public Tagging.Metatag metatag(String vocName, String term) {
Tagging tagging = this.vocabularies.get(vocName);
return tagging.getMetatagFromTerm(this.prefixChar, Tagging.decodeMaskname(metatag.substring(p + 1)));
}
public String cleanTagFromAutotagging(String tagString) {
return Tagging.cleanTagFromAutotagging(this.prefixChar, tagString);
}
public static void main(String[] args) {
Autotagging a = new Autotagging(new File("DATA/DICTIONARIES/" + LibraryProvider.path_to_autotagging_dictionaries), '$');
for (Map.Entry<String, Tagging> entry: a.vocabularies.entrySet()) {
System.out.println(entry);
}
Set<String> tags = a.getPrintTagsFromText("In die Tueren und Fluchttueren muessen noch Schloesser eingebaut werden");
System.out.println(tags);
return tagging.getMetatagFromTerm(Tagging.decodeMaskname(term));
}
}

4
source/net/yacy/document/Document.java
Unescape View File

@ -227,10 +227,6 @@ dc_rights
String objectspace = vocabulary.getObjectspace();
StringBuilder sb = new StringBuilder(e.getValue().size() * 20);
for (Tagging.Metatag s: e.getValue()) {
String t = s.toString();
if (!this.keywords.contains(t)) {
this.keywords.add(t);
}
sb.append(',').append(s.getObject());
String objectlink = vocabulary.getObjectlink(s.getObject());
if ((objectspace != null && objectspace.length() > 0) || (objectlink != null && objectlink.length() > 0)) {

24
source/net/yacy/document/LibraryProvider.java
Unescape View File

@ -47,9 +47,9 @@ import net.yacy.cora.lod.JenaTripleStore;
import net.yacy.cora.lod.vocabulary.Tagging;
import net.yacy.cora.lod.vocabulary.Tagging.SOTuple;
import net.yacy.cora.storage.Files;
import net.yacy.document.geolocalization.GeonamesLocation;
import net.yacy.document.geolocalization.OpenGeoDBLocation;
import net.yacy.document.geolocalization.OverarchingLocation;
import net.yacy.document.geolocation.GeonamesLocation;
import net.yacy.document.geolocation.OpenGeoDBLocation;
import net.yacy.document.geolocation.OverarchingLocation;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.FileUtils;
@ -57,7 +57,6 @@ import com.hp.hpl.jena.rdf.model.Resource;
public class LibraryProvider {
public static final char tagPrefix = '$';
public static final String path_to_source_dictionaries = "source";
public static final String path_to_did_you_mean_dictionaries = "didyoumean";
public static final String path_to_autotagging_dictionaries = "autotagging";
@ -116,7 +115,7 @@ public class LibraryProvider {
dictRoot = rootPath;
// initialize libraries
initAutotagging(tagPrefix);
initAutotagging();
activateDeReWo();
initDidYouMean();
integrateOpenGeoDB();
@ -136,11 +135,11 @@ public class LibraryProvider {
if ( geo0.exists() ) {
geo0.renameTo(Dictionary.GEODB0.fileDisabled());
}
geoLoc.activateLocalization(Dictionary.GEODB1.nickname, new OpenGeoDBLocation(geo1, false));
geoLoc.activateLocation(Dictionary.GEODB1.nickname, new OpenGeoDBLocation(geo1, dymLib));
return;
}
if ( geo0.exists() ) {
geoLoc.activateLocalization(Dictionary.GEODB0.nickname, new OpenGeoDBLocation(geo0, false));
geoLoc.activateLocation(Dictionary.GEODB0.nickname, new OpenGeoDBLocation(geo0, dymLib));
return;
}
}
@ -148,7 +147,7 @@ public class LibraryProvider {
public static void integrateGeonames() {
final File geon = Dictionary.GEON0.file();
if ( geon.exists() ) {
geoLoc.activateLocalization(Dictionary.GEON0.nickname, new GeonamesLocation(geon));
geoLoc.activateLocation(Dictionary.GEON0.nickname, new GeonamesLocation(geon, dymLib));
return;
}
}
@ -161,12 +160,12 @@ public class LibraryProvider {
dymLib = new WordCache(dymDict);
}
public static void initAutotagging(char prefix) {
public static void initAutotagging() {
final File autotaggingPath = new File(dictRoot, path_to_autotagging_dictionaries);
if ( !autotaggingPath.exists() ) {
autotaggingPath.mkdirs();
}
autotagging = new Autotagging(autotaggingPath, prefix);
autotagging = new Autotagging(autotaggingPath);
}
public static void activateDeReWo() {
@ -220,7 +219,7 @@ public class LibraryProvider {
Resource resource = i.next();
String subject = resource.toString();
// prepare a propert term from the subject uri
// prepare a proper term from the subject uri
int p = subject.lastIndexOf('/');
if (p < 0) continue;
String term = subject.substring(p + 1);
@ -229,9 +228,10 @@ public class LibraryProvider {
if (p >= 0) term = term.substring(0, p);
term = term.replaceAll("_", " ").trim();
if (term.length() == 0) continue;
if (term.indexOf(' ') < 0) continue; // accept only names that have at least two parts
// store the term into the vocabulary map
map.put(term, new SOTuple("", subject));
map.put(term, new SOTuple(Tagging.normalizeTerm(term), subject));
}
try {
Log.logInfo("LibraryProvider", "adding vocabulary to autotagging");

2
source/net/yacy/document/geolocalization/GeoLocation.java → source/net/yacy/document/geolocation/GeoLocation.java
Unescape View File

@ -20,7 +20,7 @@
* If not, see <http://www.gnu.org/licenses/>.
*/
package net.yacy.document.geolocalization;
package net.yacy.document.geolocation;
import java.util.Comparator;

2
source/net/yacy/document/geolocalization/GeoPoint.java → source/net/yacy/document/geolocation/GeoPoint.java
Unescape View File

@ -20,7 +20,7 @@
* If not, see <http://www.gnu.org/licenses/>.
*/
package net.yacy.document.geolocalization;
package net.yacy.document.geolocation;
/**
* Geolocation storage may vary using different data structures for the points.

7
source/net/yacy/document/geolocalization/GeonamesLocation.java → source/net/yacy/document/geolocation/GeonamesLocation.java
Unescape View File

@ -20,7 +20,7 @@
* If not, see <http://www.gnu.org/licenses/>.
*/
package net.yacy.document.geolocalization;
package net.yacy.document.geolocation;
import java.io.BufferedReader;
import java.io.File;
@ -40,6 +40,7 @@ import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import net.yacy.document.StringBuilderComparator;
import net.yacy.document.WordCache;
import net.yacy.kelondro.logging.Log;
public class GeonamesLocation implements Locations
@ -73,7 +74,7 @@ public class GeonamesLocation implements Locations
private final TreeMap<StringBuilder, List<Integer>> name2ids;
private final File file;
public GeonamesLocation(final File file) {
public GeonamesLocation(final File file, WordCache dymLib) {
// this is a processing of the cities1000.zip file from http://download.geonames.org/export/dump/
this.file = file;
@ -117,6 +118,8 @@ public class GeonamesLocation implements Locations
c.setPopulation((int) Long.parseLong(fields[14]));
this.id2loc.put(id, c);
for ( final StringBuilder name : locnames ) {
if (dymLib != null && dymLib.contains(name)) continue;
if (name.length() < OverarchingLocation.MINIMUM_NAME_LENGTH) continue;
List<Integer> locs = this.name2ids.get(name);
if ( locs == null ) {
locs = new ArrayList<Integer>(1);

2
source/net/yacy/document/geolocalization/IntegerGeoPoint.java → source/net/yacy/document/geolocation/IntegerGeoPoint.java
Unescape View File

@ -1,4 +1,4 @@
package net.yacy.document.geolocalization;
package net.yacy.document.geolocation;
/**
* GeoPoint implementation with Integer accuracy

2
source/net/yacy/document/geolocalization/Locations.java → source/net/yacy/document/geolocation/Locations.java
Unescape View File

@ -21,7 +21,7 @@
*/
package net.yacy.document.geolocalization;
package net.yacy.document.geolocation;
import java.util.Set;
import java.util.TreeSet;

27
source/net/yacy/document/geolocalization/OpenGeoDBLocation.java → source/net/yacy/document/geolocation/OpenGeoDBLocation.java
Unescape View File

@ -20,7 +20,7 @@
* If not, see <http://www.gnu.org/licenses/>.
*/
package net.yacy.document.geolocalization;
package net.yacy.document.geolocation;
import java.io.BufferedReader;
import java.io.File;
@ -40,6 +40,7 @@ import java.util.TreeSet;
import java.util.zip.GZIPInputStream;
import net.yacy.document.StringBuilderComparator;
import net.yacy.document.WordCache;
import net.yacy.kelondro.logging.Log;
/**
@ -59,7 +60,7 @@ public class OpenGeoDBLocation implements Locations
private final Map<String, Integer> zip2id;
private final File file;
public OpenGeoDBLocation(final File file, final boolean lonlat) {
public OpenGeoDBLocation(final File file, WordCache dymLib) {
this.file = file;
this.id2loc = new HashMap<Integer, GeoLocation>();
@ -99,21 +100,22 @@ public class OpenGeoDBLocation implements Locations
line = line.substring(18 + 7);
v = line.split(",");
v = line.split(",");
if ( lonlat ) {
lon = Float.parseFloat(v[2]);
lat = Float.parseFloat(v[3]);
} else {
lat = Float.parseFloat(v[2]);
lon = Float.parseFloat(v[3]);
}
this.id2loc.put(Integer.parseInt(v[0]), new GeoLocation(lat, lon));
}
if ( line.startsWith("geodb_textdata ") ) {
line = line.substring(15 + 7);
v = line.split(",");
if ( v[1].equals("500100000") ) { // Ortsname
if (v.length > 10) {
// a ',' is probably inside the location name
v[2] = v[2] + "," + v[3];
}
id = Integer.parseInt(v[0]);
h = removeQuotes(v[2]);
if (h.length() < OverarchingLocation.MINIMUM_NAME_LENGTH) continue;
if (dymLib != null && dymLib.contains(new StringBuilder(h))) continue;
List<Integer> l = this.name2ids.get(new StringBuilder(h));
if ( l == null ) {
l = new ArrayList<Integer>(1);
@ -136,8 +138,8 @@ public class OpenGeoDBLocation implements Locations
} else if ( v[1].equals("400300000") ) { // Ortstyp
id = Integer.parseInt(v[0]);
h = removeQuotes(v[2]);
final Integer hc = h.hashCode();
/*
final Integer hc = h.hashCode();
final byte[] tb = this.locTypeHash2locType.get(hc);
if ( tb == null ) {
this.locTypeHash2locType.put(hc, UTF8.getBytes(h));
@ -173,13 +175,12 @@ public class OpenGeoDBLocation implements Locations
}
private static final String removeQuotes(String s) {
if ( s.length() > 0 && s.charAt(0) != '\'' ) {
return s;
if ( s.length() > 0 && s.charAt(0) == '\'' ) {
s = s.substring(1);
}
if ( s.charAt(s.length() - 1) != '\'' ) {
return s;
if ( s.charAt(s.length() - 1) == '\'' ) {
s = s.substring(0, s.length() - 1);
}
s = s.substring(1, s.length() - 1);
return s;
}

5
source/net/yacy/document/geolocalization/OverarchingLocation.java → source/net/yacy/document/geolocation/OverarchingLocation.java
Unescape View File

@ -20,7 +20,7 @@
* If not, see <http://www.gnu.org/licenses/>.
*/
package net.yacy.document.geolocalization;
package net.yacy.document.geolocation;
import java.util.HashMap;
import java.util.HashSet;
@ -30,6 +30,7 @@ import java.util.TreeSet;
public class OverarchingLocation implements Locations {
public static int MINIMUM_NAME_LENGTH = 4;
private final Map<String, Locations> services;
/**
@ -44,7 +45,7 @@ public class OverarchingLocation implements Locations {
* @param nickname the nickname of the service
* @param service the service
*/
public void activateLocalization(final String nickname, final Locations service) {
public void activateLocation(final String nickname, final Locations service) {
this.services.put(nickname, service);
}

28
source/net/yacy/document/importer/OAIPMHLoader.java
Unescape View File

@ -49,7 +49,20 @@ public class OAIPMHLoader {
// load the file from the net
Log.logInfo("OAIPMHLoader", "loading record from " + source.toNormalform(true, false));
final Response response = loader.load(loader.request(source, false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, true);
Response response = null;
IOException ee = null;
for (int i = 0; i < 5; i++) {
// make some retries if first attempt fails
try {
response = loader.load(loader.request(source, false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, true);
break;
} catch (IOException e) {
Log.logWarning("OAIPMHLoader", "loading failed at attempt " + (i + 1) + ": " + source.toNormalform(true, false));
ee = e;
continue;
}
}
if (response == null) throw ee;
final byte[] b = response.getContent();
this.resumptionToken = new ResumptionToken(source, b);
//System.out.println("*** ResumptionToken = " + this.resumptionToken.toString());
@ -59,19 +72,6 @@ public class OAIPMHLoader {
// transaction-safe writing
FileUtils.copy(b, f0);
f0.renameTo(f1);
/*
SurrogateReader sr = new SurrogateReader(new ByteArrayInputStream(b), 100);
Thread srt = new Thread(sr);
srt.start();
DCEntry dce;
while ((dce = sr.take()) != DCEntry.poison) {
System.out.println(dce.toString());
}
try {
srt.join();
} catch (InterruptedException e) {}
*/
}
public ResumptionToken getResumptionToken() {

6
source/net/yacy/document/parser/html/ContentScraper.java
Unescape View File

@ -99,7 +99,8 @@ public class ContentScraper extends AbstractScraper implements Scraper {
strong(TagType.pair),
i(TagType.pair),
li(TagType.pair),
script(TagType.pair);
script(TagType.pair),
style(TagType.pair);
public TagType type;
private Tag(final TagType type) {
@ -201,6 +202,7 @@ public class ContentScraper extends AbstractScraper implements Scraper {
@Override
public void scrapeText(final char[] newtext, final String insideTag) {
// System.out.println("SCRAPE: " + UTF8.String(newtext));
if (insideTag != null && ("script".equals(insideTag) || "style".equals(insideTag))) return;
int p, pl, q, s = 0;
// match evaluation pattern
@ -434,7 +436,7 @@ public class ContentScraper extends AbstractScraper implements Scraper {
}
@Override
public void scrapeTag1(final String tagname, final Properties tagopts, final char[] text) {
public void scrapeTag1(final String tagname, final Properties tagopts, char[] text) {
// System.out.println("ScrapeTag1: tagname=" + tagname + ", opts=" + tagopts.toString() + ", text=" + UTF8.String(text));
if (tagname.equalsIgnoreCase("a") && text.length < 2048) {
final String href = tagopts.getProperty("href", EMPTY_STRING);

10
source/net/yacy/document/parser/html/TransformerWriter.java
Unescape View File

@ -202,7 +202,7 @@ public final class TransformerWriter extends Writer {
if (tag == null) {
// case (1): this is not a tag opener/closer
if (this.scraper != null) this.scraper.scrapeText(content, null);
if (this.scraper != null && content.length > 0) this.scraper.scrapeText(content, null);
if (this.transformer != null) return this.transformer.transformText(content);
return content;
}
@ -222,7 +222,9 @@ public final class TransformerWriter extends Writer {
// we are collection tag text for the tag 'filterTag' -> case (4) - (7)
if (tag == null || tag.equals("!")) {
// case (4): getting no tag, go on collecting content
if (this.scraper != null) this.scraper.scrapeText(content, this.filterTag);
if (this.scraper != null) {
this.scraper.scrapeText(content, this.filterTag);
}
if (this.transformer != null) {
this.filterCont.append(this.transformer.transformText(content));
} else {
@ -330,7 +332,7 @@ public final class TransformerWriter extends Writer {
if (in[1] == '/') {
// a closing tag
tagend = tagEnd(in, 2);
tag = new String(in, 2, tagend - 2);
tag = new String(in, 2, tagend - 2).toLowerCase();
final char[] text = new char[in.length - tagend - 1];
System.arraycopy(in, tagend, text, 0, in.length - tagend - 1);
return filterTag(tag, false, text, quotechar);
@ -338,7 +340,7 @@ public final class TransformerWriter extends Writer {
// an opening tag
tagend = tagEnd(in, 1);
tag = new String(in, 1, tagend - 1);
tag = new String(in, 1, tagend - 1).toLowerCase();
final char[] text = new char[in.length - tagend - 1];
System.arraycopy(in, tagend, text, 0, in.length - tagend - 1);
return filterTag(tag, true, text, quotechar);

5
source/net/yacy/interaction/AugmentHtmlStream.java
Unescape View File

@ -23,7 +23,6 @@ import net.yacy.search.Switchboard;
import org.htmlparser.Tag;
import org.htmlparser.Text;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.util.NodeList;
import org.htmlparser.visitors.NodeVisitor;
@ -58,7 +57,7 @@ public class AugmentHtmlStream {
// Link
Log.logInfo("AUGMENTATION", tag.getAttribute("href"));
LinkTag lt = (LinkTag)tag;
//LinkTag lt = (LinkTag)tag;
}
@ -90,7 +89,7 @@ public class AugmentHtmlStream {
private int counter;
public VisitorText() {
this.setCounter(0);
this.counter = 0;
}
@Override

7
source/net/yacy/interaction/Interaction.java
Unescape View File

@ -9,7 +9,6 @@ import java.util.Map;
import java.util.Set;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.lod.JenaTripleStore;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.http.HTTPClient;
@ -22,10 +21,6 @@ import net.yacy.search.Switchboard;
import org.apache.http.entity.mime.content.ContentBody;
import com.hp.hpl.jena.rdf.model.Property;
import com.hp.hpl.jena.rdf.model.Resource;
import com.hp.hpl.jena.rdf.model.StmtIterator;
import de.anomic.data.UserDB;
@ -36,7 +31,7 @@ public class Interaction {
UserDB.Entry entry = null;
String result = "anonymous";
//String result = "anonymous";
entry = Switchboard.getSwitchboard().userDB.proxyAuth((requestHeader.get(RequestHeader.AUTHORIZATION, "xxxxxx")));
if(entry != null){

6
source/net/yacy/kelondro/data/meta/URIMetadataRow.java
Unescape View File

@ -37,7 +37,7 @@ import java.util.regex.Pattern;
import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.UTF8;
import net.yacy.document.LibraryProvider;
import net.yacy.cora.lod.vocabulary.Tagging;
import net.yacy.kelondro.data.word.WordReferenceRow;
import net.yacy.kelondro.data.word.WordReferenceVars;
import net.yacy.kelondro.index.Row;
@ -230,7 +230,7 @@ public class URIMetadataRow implements URIMetadata {
String descr = crypt.simpleDecode(prop.getProperty("descr", ""), null); if (descr == null) descr = "";
String dc_creator = crypt.simpleDecode(prop.getProperty("author", ""), null); if (dc_creator == null) dc_creator = "";
String tags = crypt.simpleDecode(prop.getProperty("tags", ""), null); if (tags == null) tags = "";
tags = LibraryProvider.autotagging.cleanTagFromAutotagging(tags);
tags = Tagging.cleanTagFromAutotagging(tags);
String dc_publisher = crypt.simpleDecode(prop.getProperty("publisher", ""), null); if (dc_publisher == null) dc_publisher = "";
String lons = crypt.simpleDecode(prop.getProperty("lon", "0.0"), null); if (lons == null) lons = "0.0";
String lats = crypt.simpleDecode(prop.getProperty("lat", "0.0"), null); if (lats == null) lats = "0.0";
@ -313,7 +313,7 @@ public class URIMetadataRow implements URIMetadata {
assert (s.toString().indexOf(0) < 0);
s.append(",author=").append(crypt.simpleEncode(metadata.dc_creator()));
assert (s.toString().indexOf(0) < 0);
s.append(",tags=").append(crypt.simpleEncode(LibraryProvider.autotagging.cleanTagFromAutotagging(metadata.dc_subject())));
s.append(",tags=").append(crypt.simpleEncode(Tagging.cleanTagFromAutotagging(metadata.dc_subject())));
assert (s.toString().indexOf(0) < 0);
s.append(",publisher=").append(crypt.simpleEncode(metadata.dc_publisher()));
assert (s.toString().indexOf(0) < 0);

5
source/net/yacy/search/Switchboard.java
Unescape View File

@ -2222,6 +2222,11 @@ public final class Switchboard extends serverSwitch
this.tables.cleanFailURLS(getConfigLong("cleanup.failedSearchURLtimeout", -1));
}
// periodically store the triple store
if (getConfigBool("triplestore.persistent", false)) {
JenaTripleStore.saveAll();
}
return true;
} catch ( final InterruptedException e ) {
this.log.logInfo("cleanupJob: Shutdown detected");

34
source/net/yacy/search/query/RWIProcess.java
Unescape View File

@ -716,22 +716,6 @@ public final class RWIProcess extends Thread
}
}
// check vocabulary constraint
/*
final String tags = page.dc_subject();
final String[] taglist = tags == null || tags.length() == 0 ? new String[0] : SPACE_PATTERN.split(page.dc_subject());
if (this.query.metatags != null && this.query.metatags.size() > 0) {
// all metatags must appear in the tags list
for (Tagging.Metatag metatag: this.query.metatags) {
if (!Autotagging.metatagAppearIn(metatag, taglist)) {
this.sortout++;
//Log.logInfo("RWIProcess", "sorted out " + page.url());
continue takeloop;
}
}
}
*/
// evaluate information of metadata for navigation
// author navigation:
if ( pageauthor != null && pageauthor.length() > 0 ) {
@ -787,24 +771,6 @@ public final class RWIProcess extends Thread
this.filetypeNavigator.inc(fileext);
}
// vocabulary navigation
/*
tagharvest: for (String tag: taglist) {
if (tag.length() < 1 || tag.charAt(0) != LibraryProvider.tagPrefix) continue tagharvest;
try {
Tagging.Metatag metatag = LibraryProvider.autotagging.metatag(tag);
ScoreMap<String> voc = this.vocabularyNavigator.get(metatag.getVocabularyName());
if (voc == null) {
voc = new ConcurrentScoreMap<String>();
this.vocabularyNavigator.put(metatag.getVocabularyName(), voc);
}
voc.inc(metatag.getObject());
} catch (RuntimeException e) {
// tag may not be well-formed
}
}
*/
// accept url
return page;
}

38
source/net/yacy/yacy.java
Unescape View File

@ -306,20 +306,27 @@ public final class yacy {
HTTPClient.setDefaultUserAgent(ClientIdentification.getUserAgent());
// initial fill of the triplestore
try {
File triplestore = new File(sb.getConfig("triplestore", new File(dataHome, "DATA/TRIPLESTORE").getAbsolutePath()));
mkdirIfNeseccary(triplestore);
for (String s: triplestore.list()) {
if ((s.endsWith(".rdf") || s.endsWith(".nt")) && !s.equals("local.rdf") && !s.endsWith("_triplestore.rdf") && !s.startsWith("private_store_")) JenaTripleStore.load(new File(triplestore, s).getAbsolutePath());
if ((s.endsWith(".rdf") || s.endsWith(".nt")) && !s.equals("local.rdf") && !s.endsWith("_triplestore.rdf") && !s.startsWith("private_store_")) {
try {
JenaTripleStore.load(new File(triplestore, s).getAbsolutePath());
} catch (IOException e) {
Log.logException(e);
}
}
}
if (sb.getConfigBool("triplestore.persistent", false)) {
File local = new File(triplestore, "local.rdf");
if (local.exists()) JenaTripleStore.load(local.getAbsolutePath());
}
if (local.exists()) {
try {
JenaTripleStore.load(local.getAbsolutePath());
} catch (IOException e) {
Log.logException(e);
}
}
}
// start main threads
final String port = sb.getConfig("port", "8090");
@ -407,26 +414,11 @@ public final class yacy {
server.terminate(false);
server.interrupt();
server.close();
/*
if (server.isAlive()) try {
// TODO only send request, don't read response (cause server is already down resulting in error)
final DigestURI u = new DigestURI((server.withSSL()?"https":"http")+"://localhost:" + serverCore.getPortNr(port), null);
Client.wget(u.toString(), null, 10000); // kick server
Log.logConfig("SHUTDOWN", "sent termination signal to server socket");
} catch (final IOException ee) {
Log.logConfig("SHUTDOWN", "termination signal to server socket missed (server shutdown, ok)");
}
*/
// Client.closeAllConnections();
// MultiThreadedHttpConnectionManager.shutdownAll();
// idle until the processes are down
if (server.isAlive()) {
//Thread.sleep(2000); // wait a while
server.interrupt();
// MultiThreadedHttpConnectionManager.shutdownAll();
}
// MultiThreadedHttpConnectionManager.shutdownAll();
Log.logConfig("SHUTDOWN", "server has terminated");
sb.close();
} catch (final Exception e) {
@ -440,11 +432,9 @@ public final class yacy {
} finally {
}
// save the triple store
if (sb.getConfigBool("triplestore.persistent", false)) {
File triplestore = new File(sb.getConfig("triplestore", new File(dataHome, "DATA/TRIPLESTORE").getAbsolutePath()));
JenaTripleStore.saveFile(new File(triplestore, "local.rdf").getAbsolutePath());
JenaTripleStore.savePrivateStores(sb);
JenaTripleStore.saveAll();
}
Log.logConfig("SHUTDOWN", "goodbye. (this is the last line)");

Write Preview
Loading…
Cancel
Save