diff --git a/defaults/yacy.init b/defaults/yacy.init index e8f2b78c5..d94671b43 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -691,7 +691,7 @@ crawlPause.localsearch=50 crawlPause.remotesearch=10 # Some configuration values for the crawler -crawler.clientTimeout=9000 +crawler.clientTimeout=30000 # http crawler specific settings; size in bytes crawler.http.accept=text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 diff --git a/htroot/DemoServlet.java b/htroot/DemoServlet.java index 98d6ce3e3..42cc2b491 100644 --- a/htroot/DemoServlet.java +++ b/htroot/DemoServlet.java @@ -1,11 +1,6 @@ -import java.util.Iterator; - import net.yacy.yacy; import net.yacy.cora.protocol.RequestHeader; import net.yacy.interaction.Interaction; -import net.yacy.search.Switchboard; -import de.anomic.data.BookmarkHelper; -import de.anomic.data.UserDB; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; @@ -13,17 +8,15 @@ public final class DemoServlet { public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { - + // return variable that accumulates replacements final serverObjects prop = new serverObjects(); - - final Switchboard sb = Switchboard.getSwitchboard(); - + prop.put("temperature", "-10°C"); - + // Display currently logged on user prop.put("username", Interaction.GetLoggedOnUser(header)); - + //Generate Userlist int numUsers = 0; for (String user : Interaction.GetUsers()) { @@ -31,36 +24,36 @@ public final class DemoServlet { numUsers++; } prop.put("users", numUsers); - - - + + + if (post != null) { - + if (post.containsKey("submit")) { - + prop.put("temperature", post.get("textthing")); - + String filename= post.get("textthing"); - + int counter = 0; - + while (counter < 10) { - + prop.put("localimg_"+counter+"_path","/"+filename); - + prop.put("localimg_"+counter+"_checked", "2"); counter++; } - + prop.put("localimg", counter); - - - + + + prop.put("temperature", yacy.homedir+"/DATA/HTDOCS/"+filename); } - + } - + // return rewrite properties return prop; } diff --git a/htroot/DemoServletInteraction.java b/htroot/DemoServletInteraction.java index 2e974e421..f8d34a4b0 100644 --- a/htroot/DemoServletInteraction.java +++ b/htroot/DemoServletInteraction.java @@ -1,5 +1,4 @@ import net.yacy.cora.protocol.RequestHeader; -import de.anomic.data.BookmarkHelper; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; @@ -9,9 +8,9 @@ public final class DemoServletInteraction { final serverObjects post, final serverSwitch env) { // return variable that accumulates replacements final serverObjects prop = new serverObjects(); - + prop.put("temperature", "-10°C"); - + // return rewrite properties return prop; } diff --git a/htroot/DemoServletRDF.java b/htroot/DemoServletRDF.java index 593d6bcbe..25d2143d1 100644 --- a/htroot/DemoServletRDF.java +++ b/htroot/DemoServletRDF.java @@ -1,6 +1,4 @@ -import net.yacy.yacy; import net.yacy.cora.protocol.RequestHeader; -import de.anomic.data.BookmarkHelper; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; @@ -10,37 +8,37 @@ public final class DemoServletRDF { final serverObjects post, final serverSwitch env) { // return variable that accumulates replacements final serverObjects prop = new serverObjects(); - + // prop.put("temperature", "-10°C"); - + if (post != null) { - + if (post.containsKey("submit")) { prop.put("temperature", post.get("textthing")); - + String filename= post.get("textthing"); - + // prop.put("imglink", filename+".jpg"); - + int counter = 0; - + while (counter < 10) { - + prop.put("localimg_"+counter+"_path","/"+filename); - + prop.put("localimg_"+counter+"_checked", "2"); counter++; } - + prop.put("localimg", counter); - - - + + + // prop.put("temperature",yacy.homedir+"/DATA/HTDOCS/"+filename); } - + } - + // return rewrite properties return prop; } diff --git a/htroot/DictionaryLoader_p.html b/htroot/DictionaryLoader_p.html index fd48f6c6a..8d3d0fdfe 100644 --- a/htroot/DictionaryLoader_p.html +++ b/htroot/DictionaryLoader_p.html @@ -65,9 +65,11 @@ Geolocalization will enable YaCy to present locations from OpenStreetMap according to given search words.

GeoNames

-

With this file it is possible to find cities with a population > 1000 all over the world.

+

With this file it is possible to find cities all over the world.

+
Content
+
cities with a population > 1000 all over the world
#[geon0URL]#
@@ -99,6 +101,74 @@
Result
cannot activate dictionary file: #[error]#
#(/geon0ActionActivated)#
+
+
Content
+
cities with a population > 5000 all over the world
+
+
#[geon1URL]#
+
+
#[geon1Storage]#
+
+
#(geon1Status)#
not loaded
::
loaded
::deactivated#(/geon1Status)#
+
Action
+
#(geon1Status)# + :: + + :: + + + #(/geon1Status)#
+ #(geon1ActionLoaded)#:: +
Result
loaded and activated dictionary file
:: +
Result
loading of dictionary file failed: #[error]#
+ #(/geon1ActionLoaded)# + #(geon1ActionRemoved)#:: +
Result
deactivated and removed dictionary file
:: +
Result
cannot remove dictionary file: #[error]#
+ #(/geon1ActionRemoved)# + #(geon1ActionDeactivated)#:: +
Result
deactivated dictionary file
:: +
Result
cannot deactivate dictionary file: #[error]#
+ #(/geon1ActionDeactivated)# + #(geon1ActionActivated)#:: +
Result
activated dictionary file
:: +
Result
cannot activate dictionary file: #[error]#
+ #(/geon1ActionActivated)# +
+
+
Content
+
cities with a population > 100000 all over the world (the set is is reduced to cities > 100000)
+
+
#[geon2URL]#
+
+
#[geon2Storage]#
+
+
#(geon2Status)#
not loaded
::
loaded
::deactivated#(/geon2Status)#
+
Action
+
#(geon2Status)# + :: + + :: + + + #(/geon2Status)#
+ #(geon2ActionLoaded)#:: +
Result
loaded and activated dictionary file
:: +
Result
loading of dictionary file failed: #[error]#
+ #(/geon2ActionLoaded)# + #(geon2ActionRemoved)#:: +
Result
deactivated and removed dictionary file
:: +
Result
cannot remove dictionary file: #[error]#
+ #(/geon2ActionRemoved)# + #(geon2ActionDeactivated)#:: +
Result
deactivated dictionary file
:: +
Result
cannot deactivate dictionary file: #[error]#
+ #(/geon2ActionDeactivated)# + #(geon2ActionActivated)#:: +
Result
activated dictionary file
:: +
Result
cannot activate dictionary file: #[error]#
+ #(/geon2ActionActivated)# +

OpenGeoDB

With this file it is possible to find locations in Germany using the location (city) name, a zip code, a car sign or a telephone pre-dial number.

diff --git a/htroot/DictionaryLoader_p.java b/htroot/DictionaryLoader_p.java index 32e60d18c..70ba014ee 100644 --- a/htroot/DictionaryLoader_p.java +++ b/htroot/DictionaryLoader_p.java @@ -24,8 +24,8 @@ import java.net.MalformedURLException; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.services.federated.yacy.CacheStrategy; import net.yacy.document.LibraryProvider; -import net.yacy.document.geolocalization.GeonamesLocation; -import net.yacy.document.geolocalization.OpenGeoDBLocation; +import net.yacy.document.geolocation.GeonamesLocation; +import net.yacy.document.geolocation.OpenGeoDBLocation; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.util.FileUtils; @@ -68,7 +68,7 @@ public class DictionaryLoader_p { final Response response = sb.loader.load(sb.loader.request(new DigestURI(LibraryProvider.Dictionary.GEON0.url), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, false); final byte[] b = response.getContent(); FileUtils.copy(b, LibraryProvider.Dictionary.GEON0.file()); - LibraryProvider.geoLoc.activateLocalization(LibraryProvider.Dictionary.GEON0.nickname, new GeonamesLocation(LibraryProvider.Dictionary.GEON0.file())); + LibraryProvider.geoLoc.activateLocation(LibraryProvider.Dictionary.GEON0.nickname, new GeonamesLocation(LibraryProvider.Dictionary.GEON0.file(), null, -1)); LibraryProvider.autotagging.addPlaces(LibraryProvider.geoLoc); prop.put("geon0Status", LibraryProvider.Dictionary.GEON0.file().exists() ? 1 : 0); prop.put("geon0ActionLoaded", 1); @@ -98,11 +98,95 @@ public class DictionaryLoader_p { if (post.containsKey("geon0Activate")) { LibraryProvider.Dictionary.GEON0.fileDisabled().renameTo(LibraryProvider.Dictionary.GEON0.file()); - LibraryProvider.geoLoc.activateLocalization(LibraryProvider.Dictionary.GEON0.nickname, new GeonamesLocation(LibraryProvider.Dictionary.GEON0.file())); + LibraryProvider.geoLoc.activateLocation(LibraryProvider.Dictionary.GEON0.nickname, new GeonamesLocation(LibraryProvider.Dictionary.GEON0.file(), null, -1)); LibraryProvider.autotagging.addPlaces(LibraryProvider.geoLoc); prop.put("geon0ActionActivated", 1); } + // GEON1 + if (post.containsKey("geon1Load")) { + // load from the net + try { + final Response response = sb.loader.load(sb.loader.request(new DigestURI(LibraryProvider.Dictionary.GEON1.url), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, false); + final byte[] b = response.getContent(); + FileUtils.copy(b, LibraryProvider.Dictionary.GEON1.file()); + LibraryProvider.geoLoc.activateLocation(LibraryProvider.Dictionary.GEON1.nickname, new GeonamesLocation(LibraryProvider.Dictionary.GEON1.file(), null, -1)); + LibraryProvider.autotagging.addPlaces(LibraryProvider.geoLoc); + prop.put("geon1Status", LibraryProvider.Dictionary.GEON1.file().exists() ? 1 : 0); + prop.put("geon1ActionLoaded", 1); + } catch (final MalformedURLException e) { + Log.logException(e); + prop.put("geon1ActionLoaded", 2); + prop.put("geon1ActionLoaded_error", e.getMessage()); + } catch (final IOException e) { + Log.logException(e); + prop.put("geon1ActionLoaded", 2); + prop.put("geon1ActionLoaded_error", e.getMessage()); + } + } + + if (post.containsKey("geon1Remove")) { + FileUtils.deletedelete(LibraryProvider.Dictionary.GEON1.file()); + FileUtils.deletedelete(LibraryProvider.Dictionary.GEON1.fileDisabled()); + LibraryProvider.geoLoc.deactivateLocalization(LibraryProvider.Dictionary.GEON1.nickname); + prop.put("geon1ActionRemoved", 1); + } + + if (post.containsKey("geon1Deactivate")) { + LibraryProvider.Dictionary.GEON1.file().renameTo(LibraryProvider.Dictionary.GEON1.fileDisabled()); + LibraryProvider.geoLoc.deactivateLocalization(LibraryProvider.Dictionary.GEON1.nickname); + prop.put("geon1ActionDeactivated", 1); + } + + if (post.containsKey("geon1Activate")) { + LibraryProvider.Dictionary.GEON1.fileDisabled().renameTo(LibraryProvider.Dictionary.GEON1.file()); + LibraryProvider.geoLoc.activateLocation(LibraryProvider.Dictionary.GEON1.nickname, new GeonamesLocation(LibraryProvider.Dictionary.GEON1.file(), null, -1)); + LibraryProvider.autotagging.addPlaces(LibraryProvider.geoLoc); + prop.put("geon1ActionActivated", 1); + } + + // GEON2 + if (post.containsKey("geon2Load")) { + // load from the net + try { + final Response response = sb.loader.load(sb.loader.request(new DigestURI(LibraryProvider.Dictionary.GEON2.url), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, false); + final byte[] b = response.getContent(); + FileUtils.copy(b, LibraryProvider.Dictionary.GEON2.file()); + LibraryProvider.geoLoc.activateLocation(LibraryProvider.Dictionary.GEON2.nickname, new GeonamesLocation(LibraryProvider.Dictionary.GEON2.file(), null, 100000)); + LibraryProvider.autotagging.addPlaces(LibraryProvider.geoLoc); + prop.put("geon2Status", LibraryProvider.Dictionary.GEON2.file().exists() ? 1 : 0); + prop.put("geon2ActionLoaded", 1); + } catch (final MalformedURLException e) { + Log.logException(e); + prop.put("geon2ActionLoaded", 2); + prop.put("geon2ActionLoaded_error", e.getMessage()); + } catch (final IOException e) { + Log.logException(e); + prop.put("geon2ActionLoaded", 2); + prop.put("geon2ActionLoaded_error", e.getMessage()); + } + } + + if (post.containsKey("geon2Remove")) { + FileUtils.deletedelete(LibraryProvider.Dictionary.GEON2.file()); + FileUtils.deletedelete(LibraryProvider.Dictionary.GEON2.fileDisabled()); + LibraryProvider.geoLoc.deactivateLocalization(LibraryProvider.Dictionary.GEON2.nickname); + prop.put("geon2ActionRemoved", 1); + } + + if (post.containsKey("geon2Deactivate")) { + LibraryProvider.Dictionary.GEON2.file().renameTo(LibraryProvider.Dictionary.GEON2.fileDisabled()); + LibraryProvider.geoLoc.deactivateLocalization(LibraryProvider.Dictionary.GEON2.nickname); + prop.put("geon2ActionDeactivated", 1); + } + + if (post.containsKey("geon2Activate")) { + LibraryProvider.Dictionary.GEON2.fileDisabled().renameTo(LibraryProvider.Dictionary.GEON2.file()); + LibraryProvider.geoLoc.activateLocation(LibraryProvider.Dictionary.GEON2.nickname, new GeonamesLocation(LibraryProvider.Dictionary.GEON2.file(), null, 100000)); + LibraryProvider.autotagging.addPlaces(LibraryProvider.geoLoc); + prop.put("geon2ActionActivated", 1); + } + // GEO1 if (post.containsKey("geo1Load")) { // load from the net @@ -110,8 +194,8 @@ public class DictionaryLoader_p { final Response response = sb.loader.load(sb.loader.request(new DigestURI(LibraryProvider.Dictionary.GEODB1.url), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, false); final byte[] b = response.getContent(); FileUtils.copy(b, LibraryProvider.Dictionary.GEODB1.file()); - LibraryProvider.geoLoc.deactivateLocalization(LibraryProvider.Dictionary.GEODB0.nickname); - LibraryProvider.geoLoc.activateLocalization(LibraryProvider.Dictionary.GEODB1.nickname, new OpenGeoDBLocation(LibraryProvider.Dictionary.GEODB1.file(), false)); + LibraryProvider.geoLoc.deactivateLocalization(LibraryProvider.Dictionary.GEODB1.nickname); + LibraryProvider.geoLoc.activateLocation(LibraryProvider.Dictionary.GEODB1.nickname, new OpenGeoDBLocation(LibraryProvider.Dictionary.GEODB1.file(), null)); LibraryProvider.autotagging.addPlaces(LibraryProvider.geoLoc); prop.put("geo1Status", LibraryProvider.Dictionary.GEODB1.file().exists() ? 1 : 0); prop.put("geo1ActionLoaded", 1); @@ -141,7 +225,7 @@ public class DictionaryLoader_p { if (post.containsKey("geo1Activate")) { LibraryProvider.Dictionary.GEODB1.fileDisabled().renameTo(LibraryProvider.Dictionary.GEODB1.file()); - LibraryProvider.geoLoc.activateLocalization(LibraryProvider.Dictionary.GEODB1.nickname, new OpenGeoDBLocation(LibraryProvider.Dictionary.GEODB1.file(), false)); + LibraryProvider.geoLoc.activateLocation(LibraryProvider.Dictionary.GEODB1.nickname, new OpenGeoDBLocation(LibraryProvider.Dictionary.GEODB1.file(), null)); LibraryProvider.autotagging.addPlaces(LibraryProvider.geoLoc); prop.put("geo1ActionActivated", 1); } @@ -229,7 +313,7 @@ public class DictionaryLoader_p { LibraryProvider.activatePND(); prop.put("pnd0ActionActivated", 1); } - + // check status again for (final LibraryProvider.Dictionary dictionary: LibraryProvider.Dictionary.values()) { prop.put(dictionary.nickname + "Status", dictionary.file().exists() ? 1 : dictionary.fileDisabled().exists() ? 2 : 0); diff --git a/htroot/ViewFile.java b/htroot/ViewFile.java index a23746628..d9e26ea91 100644 --- a/htroot/ViewFile.java +++ b/htroot/ViewFile.java @@ -40,6 +40,7 @@ import net.yacy.cora.document.ASCII; import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.document.UTF8; import net.yacy.cora.lod.JenaTripleStore; +import net.yacy.cora.lod.vocabulary.YaCyMetadata; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.services.federated.yacy.CacheStrategy; import net.yacy.document.Condenser; @@ -54,6 +55,9 @@ import net.yacy.kelondro.data.meta.URIMetadataRow; import net.yacy.search.Switchboard; import net.yacy.search.index.Segment; import net.yacy.search.index.Segments; + +import com.hp.hpl.jena.rdf.model.Model; + import de.anomic.crawler.Cache; import de.anomic.crawler.retrieval.Response; import de.anomic.server.serverObjects; @@ -344,7 +348,8 @@ public class ViewFile { prop.putNum("error_size", size); prop.put("error_mimeTypeAvailable", (response.getMimeType() == null) ? "0" : "1"); prop.put("error_mimeTypeAvailable_mimeType", response.getMimeType()); - prop.putXML("error_triples", JenaTripleStore.getMetadataByURLHash(url.hash())); + Model model = JenaTripleStore.getSubmodelBySubject(YaCyMetadata.hashURI(url.hash())); + prop.putXML("error_triples", JenaTripleStore.getRDFByModel(model)); return prop; } diff --git a/htroot/Vocabulary_p.html b/htroot/Vocabulary_p.html index 7a2fe2bf4..30226a0ca 100644 --- a/htroot/Vocabulary_p.html +++ b/htroot/Vocabulary_p.html @@ -3,8 +3,76 @@ YaCy '#[clientname]#': Federated Index #%env/templates/metas.template%# + - + + +#(edit)#:: +
+API + +The information that is presented on this page can also be retrieved as XML +Click the API icon to see the RDF Ontology definition for this vocabulary. +To see a list of all APIs, please visit the API wiki page. +
+#(/edit)# + #%env/templates/header.template%# #%env/templates/submenuSemantic.template%#

Vocabulary Administration

@@ -31,14 +99,17 @@ #(create)#:: -
+ +
Vocabulary Production It is possible to produce a vocabulary out of the existing search index. This is done using a given 'objectspace' which you can enter as a URL Stub. This stub is used to find all matching URLs. If the remaining path from the matching URLs then denotes a single file, the file name is used as vocabulary term. This works best with wikis. Try to use a wiki url as objectspace path.
Vocabulary Name
-
Objectspace
+
Objectspace
+
+
Discover Terms from
object link file name  object page title  object page title (splitted)  object page author
@@ -51,6 +122,7 @@
Vocabulary Name
#[name]#
File
#(editable)#[automatically generated, not stored, cannot be edited]::#[file]##(/editable)#
+
Size
#[size]#
Namespace
#[namespace]#
Predicate
#[predicate]#
Prefix
#[prefix]#
@@ -73,10 +145,10 @@ #{terms}# #(editable)# ::#(/editable)# - #(editable)# ::#(/editable)# + #(editable)# ::#(/editable)# #[term]# - #(editable)##[synonyms]#::#(/editable)# - #(editable)##[objectlink]#::#(/editable)# + #(editable)##[synonyms]#::#(/editable)# + #(editable)##[objectlink]#::#(/editable)# #{/terms}# #(editable)#:: diff --git a/htroot/Vocabulary_p.java b/htroot/Vocabulary_p.java index 5a7eeca93..68064fb1b 100644 --- a/htroot/Vocabulary_p.java +++ b/htroot/Vocabulary_p.java @@ -35,6 +35,7 @@ import net.yacy.cora.lod.vocabulary.YaCyMetadata; import net.yacy.cora.protocol.RequestHeader; import net.yacy.document.LibraryProvider; import net.yacy.kelondro.data.meta.DigestURI; +import net.yacy.kelondro.data.meta.URIMetadataRow; import net.yacy.kelondro.logging.Log; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; @@ -50,44 +51,71 @@ public class Vocabulary_p { Collection vocs = LibraryProvider.autotagging.getVocabularies(); String vocabularyName = (post == null) ? null : post.get("vocabulary", null); + String discovername = (post == null) ? null : post.get("discovername", null); Tagging vocabulary = vocabularyName == null ? null : LibraryProvider.autotagging.getVocabulary(vocabularyName); if (vocabulary == null) vocabularyName = null; - int count = 0; - for (Tagging v: vocs) { - prop.put("vocabularyset_" + count + "_name", v.getName()); - prop.put("vocabularyset_" + count + "_selected", (vocabularyName != null && vocabularyName.equals(v.getName())) ? 1 : 0); - count++; - } - prop.put("vocabularyset", count); - if (post != null) { try { if (vocabulary == null) { // create a vocabulary - String discovername = post.get("discovername", ""); - if (discovername.length() > 0) { + if (discovername != null && discovername.length() > 0) { String discoverobjectspace = post.get("discoverobjectspace", ""); MultiProtocolURI discoveruri = null; if (discoverobjectspace.length() > 0) try {discoveruri = new MultiProtocolURI(discoverobjectspace);} catch (MalformedURLException e) {} if (discoveruri == null) discoverobjectspace = ""; Map table = new TreeMap(); File propFile = LibraryProvider.autotagging.getVocabularyFile(discovername); + boolean discoverFromPath = post.get("discovermethod", "").equals("path"); + boolean discoverFromTitle = post.get("discovermethod", "").equals("title"); + boolean discoverFromTitleSplitted = post.get("discovermethod", "").equals("titlesplitted"); + boolean discoverFromAuthor = post.get("discovermethod", "").equals("author"); if (discoveruri != null) { String segmentName = sb.getConfig(SwitchboardConstants.SEGMENT_PUBLIC, "default"); Segment segment = sb.indexSegments.segment(segmentName); Iterator ui = segment.urlSelector(discoveruri); + String t; while (ui.hasNext()) { DigestURI u = ui.next(); String u0 = u.toNormalform(true, false); - String t = u0.substring(discoverobjectspace.length()); - if (t.indexOf('/') >= 0) continue; - int p = t.indexOf('.'); - if (p >= 0) t = t.substring(0, p); - while ((p = t.indexOf(':')) >= 0) t = t.substring(p + 1); - while ((p = t.indexOf('=')) >= 0) t = t.substring(p + 1); - if (p >= 0) t = t.substring(p + 1); + t = ""; + if (discoverFromPath) { + t = u0.substring(discoverobjectspace.length()); + if (t.indexOf('/') >= 0) continue; + int p = t.indexOf('.'); + if (p >= 0) t = t.substring(0, p); + while ((p = t.indexOf(':')) >= 0) t = t.substring(p + 1); + while ((p = t.indexOf('=')) >= 0) t = t.substring(p + 1); + if (p >= 0) t = t.substring(p + 1); + } + if (discoverFromTitle || discoverFromTitleSplitted) { + URIMetadataRow m = segment.urlMetadata().load(u.hash()); + if (m != null) t = m.dc_title(); + if (t.endsWith(".jpg") || t.endsWith(".gif")) continue; + } + if (discoverFromAuthor) { + URIMetadataRow m = segment.urlMetadata().load(u.hash()); + if (m != null) t = m.dc_creator(); + } + t = t.replaceAll("_", " ").replaceAll("\"", " ").replaceAll("'", " ").replaceAll(",", " ").replaceAll(" ", " ").trim(); if (t.length() == 0) continue; - table.put(t, new Tagging.SOTuple("", u0)); + if (discoverFromTitleSplitted) { + String[] ts = t.split(" "); + for (String s: ts) { + if (s.length() == 0) continue; + if (s.endsWith(".jpg") || s.endsWith(".gif")) continue; + table.put(s, new Tagging.SOTuple(Tagging.normalizeTerm(s), u0)); + } + } else if (discoverFromAuthor) { + String[] ts = t.split(";"); // author names are often separated by ';' + for (String s: ts) { + if (s.length() == 0) continue; + int p = s.indexOf(','); // check if there is a reversed method to mention the name + if (p >= 0) s = s.substring(p + 1).trim() + " " + s.substring(0, p).trim(); + table.put(s, new Tagging.SOTuple(Tagging.normalizeTerm(s), u0)); + } + } else { + table.put(t, new Tagging.SOTuple(Tagging.normalizeTerm(t), u0)); + } } } Tagging newvoc = new Tagging(discovername, propFile, discoverobjectspace, table); @@ -143,6 +171,14 @@ public class Vocabulary_p { } } + int count = 0; + for (Tagging v: vocs) { + prop.put("vocabularyset_" + count + "_name", v.getName()); + prop.put("vocabularyset_" + count + "_selected", ((vocabularyName != null && vocabularyName.equals(v.getName())) || (discovername != null && discovername.equals(v.getName()))) ? 1 : 0); + count++; + } + prop.put("vocabularyset", count); + prop.put("create", vocabularyName == null ? 1 : 0); if (vocabulary == null) { @@ -154,27 +190,45 @@ public class Vocabulary_p { prop.put("edit_editable", editable ? 1 : 0); prop.putHTML("edit_editable_file", editable ? vocabulary.getFile().getAbsolutePath() : ""); prop.putHTML("edit_name", vocabulary.getName()); + prop.putXML("edit_namexml", vocabulary.getName()); prop.putHTML("edit_namespace", vocabulary.getNamespace()); + prop.put("edit_size", vocabulary.size()); prop.putHTML("edit_predicate", vocabulary.getPredicate()); prop.putHTML("edit_prefix", Tagging.DEFAULT_PREFIX); prop.putHTML("edit_editable_objectspace", vocabulary.getObjectspace() == null ? "" : vocabulary.getObjectspace()); prop.putHTML("edit_editable_objectspacepredicate", DCTerms.references.getPredicate()); - prop.putHTML("edit_triple1", "<" + yacyurl + "> <" + vocabulary.getPredicate() + "> \"[discovered-tags-commaseparated]\""); - prop.putHTML("edit_triple2", "<" + yacyurl + "> <" + Owl.SameAs.getPredicate() + "> <[document-url]>"); - prop.putHTML("edit_tripleN", vocabulary.getObjectspace() == null ? "none - missing objectspace" : "<" + yacyurl + "> <" + DCTerms.references.getPredicate() + "> \"[reference-link]#[tag]\" ."); + prop.putXML("edit_triple1", "<" + yacyurl + "> <" + vocabulary.getPredicate() + "> \"[discovered-tags-commaseparated]\""); + prop.putXML("edit_triple2", "<" + yacyurl + "> <" + Owl.SameAs.getPredicate() + "> <[document-url]>"); + prop.putXML("edit_tripleN", vocabulary.getObjectspace() == null ? "none - missing objectspace" : "<" + yacyurl + "> <" + DCTerms.references.getPredicate() + "> \"[object-link]#[tag]\" ."); int c = 0; boolean dark = false; - for (Map.Entry entry: vocabulary.list().entrySet()) { + int osl = vocabulary.getObjectspace() == null ? 0 : vocabulary.getObjectspace().length(); + Map list = vocabulary.list(); + prop.put("edit_size", list.size()); + for (Map.Entry entry: list.entrySet()) { prop.put("edit_terms_" + c + "_editable", editable ? 1 : 0); prop.put("edit_terms_" + c + "_dark", dark ? 1 : 0); dark = !dark; + prop.putXML("edit_terms_" + c + "_label", osl > entry.getValue().getObjectlink().length() ? entry.getKey() : entry.getValue().getObjectlink().substring(osl)); prop.putHTML("edit_terms_" + c + "_term", entry.getKey()); + prop.putXML("edit_terms_" + c + "_termxml", entry.getKey()); prop.putHTML("edit_terms_" + c + "_editable_term", entry.getKey()); - prop.putHTML("edit_terms_" + c + "_editable_synonyms", entry.getValue().getSynonymsCSV()); - prop.putHTML("edit_terms_" + c + "_editable_objectlink", entry.getValue().getObjectlink()); + String synonymss = entry.getValue().getSynonymsCSV(); + prop.putHTML("edit_terms_" + c + "_editable_synonyms", synonymss); + if (synonymss.length() > 0) { + String[] synonymsa = entry.getValue().getSynonymsList(); + for (int i = 0; i < synonymsa.length; i++) { + prop.put("edit_terms_" + c + "_synonyms_" + i + "_altLabel", synonymsa[i]); + } + prop.put("edit_terms_" + c + "_synonyms", synonymsa.length); + } else { + prop.put("edit_terms_" + c + "_synonyms", 0); + } + prop.putXML("edit_terms_" + c + "_editable_objectlink", entry.getValue().getObjectlink()); c++; if (c > 3000) break; } prop.put("edit_terms", c); + } // return rewrite properties diff --git a/htroot/Vocabulary_p.xml b/htroot/Vocabulary_p.xml new file mode 100644 index 000000000..afc65ba58 --- /dev/null +++ b/htroot/Vocabulary_p.xml @@ -0,0 +1,25 @@ + + +#(edit)#:: + + #[namexml]# + + + #{terms}# + + #[label]# + + #[termxml]# + #{synonyms}# + #[altLabel]# + #{/synonyms}# + + + #{/terms}# +#(/edit)# + diff --git a/htroot/api/getpageinfo.java b/htroot/api/getpageinfo.java index 126103724..c031cca1a 100644 --- a/htroot/api/getpageinfo.java +++ b/htroot/api/getpageinfo.java @@ -102,7 +102,7 @@ public class getpageinfo { } if (scraper != null) { // put the document title - prop.putXML("title", scraper.dc_title()); + prop.putXML("title", removelinebreaks(scraper.dc_title())); // put the favicon that belongs to the document prop.put("favicon", (scraper.getFavicon()==null) ? "" : scraper.getFavicon().toString()); @@ -119,7 +119,7 @@ public class getpageinfo { } prop.put("tags", count); // put description - prop.putXML("desc", scraper.dc_description()); + prop.putXML("desc", removelinebreaks(scraper.dc_description())); // put language final Set languages = scraper.getContentLanguages(); prop.putXML("lang", (languages == null) ? "unknown" : languages.iterator().next()); @@ -185,7 +185,14 @@ public class getpageinfo { return prop; } - private static String checkOAI(final String url) { + private static String removelinebreaks(String dc_title) { + String newtitle = dc_title.replace ("\r", ""); + newtitle = newtitle.replace ("\n", ""); + newtitle = newtitle.replace ("\r\n", ""); + return newtitle; + } + + private static String checkOAI(final String url) { final DocumentBuilderFactory factory = DocumentBuilderFactory .newInstance(); try { diff --git a/htroot/api/yacydoc.htm b/htroot/api/yacydoc.htm new file mode 100644 index 000000000..8089692e9 --- /dev/null +++ b/htroot/api/yacydoc.htm @@ -0,0 +1,3 @@ +{ + "result": "#[taglinks]#" +} \ No newline at end of file diff --git a/htroot/api/yacydoc.java b/htroot/api/yacydoc.java index a49d5d2bd..2ddfcdfd5 100644 --- a/htroot/api/yacydoc.java +++ b/htroot/api/yacydoc.java @@ -25,24 +25,15 @@ // along with this program; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -import java.io.ByteArrayOutputStream; import java.net.MalformedURLException; import java.util.Iterator; -import java.util.Map; - -import com.hp.hpl.jena.rdf.model.Model; -import com.hp.hpl.jena.rdf.model.RDFNode; -import com.hp.hpl.jena.rdf.model.Resource; import net.yacy.cora.date.ISO8601Formatter; import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; import net.yacy.cora.lod.JenaTripleStore; -import net.yacy.cora.lod.vocabulary.Tagging; import net.yacy.cora.lod.vocabulary.YaCyMetadata; import net.yacy.cora.protocol.RequestHeader; -import net.yacy.cora.sorting.ConcurrentScoreMap; -import net.yacy.cora.sorting.ScoreMap; +import net.yacy.cora.protocol.RequestHeader.FileType; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataRow; import net.yacy.kelondro.data.word.Word; @@ -50,6 +41,10 @@ import net.yacy.kelondro.logging.Log; import net.yacy.search.Switchboard; import net.yacy.search.index.Segment; import net.yacy.search.index.Segments; + +import com.hp.hpl.jena.rdf.model.Model; +import com.hp.hpl.jena.rdf.model.RDFNode; + import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; @@ -139,9 +134,25 @@ public class yacydoc { prop.put("yacy_citations", sb.indexSegments.segment(Segments.Process.PUBLIC).urlCitation().count(entry.hash())); prop.put("yacy_inbound", entry.llocal()); prop.put("yacy_outbound", entry.lother()); - + // extract the submodel from the triplestore - prop.putXML("triples", JenaTripleStore.getMetadataByURLHash(entry.hash())); + Model model = JenaTripleStore.getSubmodelBySubject(YaCyMetadata.hashURI(entry.hash())); + String rdf = JenaTripleStore.getRDFByModel(model); + prop.putXML("triples", rdf); + prop.put("rdf", header.fileType() == FileType.XML ? rdf : ""); + + + String references = ""; + Iterator t = JenaTripleStore.getObjects("http://yacy.net/url#"+urlhash, "http://purl.org/dc/terms/references"); + + while (t.hasNext()) { + RDFNode r = t.next(); + references += r.toString()+","; + } + + Log.logInfo ("TRIPLESTORE", references); + + prop.put("taglinks", references); // return rewrite properties return prop; diff --git a/htroot/env/templates/jqueryheader.template b/htroot/env/templates/jqueryheader.template index c064819e2..e93f8bad3 100644 --- a/htroot/env/templates/jqueryheader.template +++ b/htroot/env/templates/jqueryheader.template @@ -9,4 +9,5 @@ + diff --git a/htroot/interaction/Triple.java b/htroot/interaction/Triple.java index 92f92c841..58913d028 100644 --- a/htroot/interaction/Triple.java +++ b/htroot/interaction/Triple.java @@ -32,7 +32,6 @@ package interaction; import net.yacy.cora.lod.JenaTripleStore; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; -import net.yacy.interaction.Interaction; import net.yacy.search.Switchboard; import de.anomic.data.UserDB; import de.anomic.server.serverObjects; @@ -99,55 +98,37 @@ public class Triple { String s = ""; String p = ""; String o = ""; + String result = ""; Boolean global = false; - if(post != null){ + if (post != null) { - if(post.containsKey("s")){ - s = post.get("s"); - } - - if(post.containsKey("sp")){ - s = post.get("sp") + "#" + s; - } - - if(post.containsKey("p")){ - p = post.get("p"); - } - - if(post.containsKey("pp")){ - p = post.get("pp") + "#" + p; - } + s = post.get("s", ""); + p = post.get("p", ""); + o = post.get("o", ""); - if(post.containsKey("o")){ - o = post.get("o"); - } + if (post.containsKey("sp")) s = post.get("sp") + "#" + s; + if (post.containsKey("pp")) p = post.get("pp") + "#" + p; global = post.containsKey("global"); + if (post.containsKey("load")) { + if (global) { + result = JenaTripleStore.getObject(s, p); + } else { + result = JenaTripleStore.getPrivateObject(s, p, username); + } + } else { + if (global) { + JenaTripleStore.addTriple(s, p, o); + } else { + JenaTripleStore.addTriple(s, p, o, username); + } + } } - if (post.containsKey("load")) { - - if (global) { - o = JenaTripleStore.getObject(s, p); - } else { - o = JenaTripleStore.getPrivateObject(s, p, username); - } - - - } else { - - if (global) { - JenaTripleStore.addTriple(s, p, o); - } else { - JenaTripleStore.addTriple(s, p, o, username); - } - - } - - prop.put("result", o); + prop.put("result", result); return prop; } diff --git a/htroot/interaction_elements/Document_part.html b/htroot/interaction_elements/Document_part.html new file mode 100644 index 000000000..2964b6a17 --- /dev/null +++ b/htroot/interaction_elements/Document_part.html @@ -0,0 +1,41 @@ + + + + + diff --git a/htroot/interaction_elements/Document_part.java b/htroot/interaction_elements/Document_part.java new file mode 100644 index 000000000..48425f2a5 --- /dev/null +++ b/htroot/interaction_elements/Document_part.java @@ -0,0 +1,29 @@ +package interaction_elements; + + +import java.util.Collection; + +import net.yacy.cora.lod.vocabulary.Tagging; +import net.yacy.cora.protocol.HeaderFramework; +import net.yacy.cora.protocol.RequestHeader; +import net.yacy.document.LibraryProvider; +import net.yacy.search.Switchboard; +import de.anomic.data.UserDB; +import de.anomic.server.serverObjects; +import de.anomic.server.serverSwitch; + +public class Document_part { + + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { + + final Switchboard sb = (Switchboard) env; + + final serverObjects prop = new serverObjects(); + + prop.put("hash", post.get("hash", "")); + prop.put("url", post.get("url", "")); + prop.put("action", post.get("action", "")); + + return prop; + } +} diff --git a/htroot/interaction_elements/OverlayInteraction.html b/htroot/interaction_elements/OverlayInteraction.html index 06de6f673..602c9b32b 100644 --- a/htroot/interaction_elements/OverlayInteraction.html +++ b/htroot/interaction_elements/OverlayInteraction.html @@ -25,12 +25,12 @@ color: #5a346e; font: arial,helvetica,sans-serif; font-size: 10px; position: fixed; -right: 10px; +right: 200px; top: 0; background: #ffffff; border:0px; -width: 65px; +width: 200px; height: 20px; padding: 5px 5px 5px 5px; @@ -51,7 +51,7 @@ top: 0; background: #ffffff; border:0px; -width: 30px; +width: 100px; height: 20px; padding: 5px 5px 5px 5px; @@ -178,12 +178,16 @@ z-index:99999; -