From 90c6fc4b638806b5916afdf9e01e97d3c5dd7203 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Sun, 10 Jun 2012 21:49:02 +0200 Subject: [PATCH] load all - but not the persistent local.rdf - triples from DATA/TRIPLESTORE at startup time. The local.rdf is loaded only if the persistent switch is on (as before). --- htroot/Triple_p.java | 9 ++- source/net/yacy/interaction/TripleStore.java | 58 +++++++++++--------- source/net/yacy/yacy.java | 15 +++-- 3 files changed, 48 insertions(+), 34 deletions(-) diff --git a/htroot/Triple_p.java b/htroot/Triple_p.java index c50df2067..c80126b2e 100644 --- a/htroot/Triple_p.java +++ b/htroot/Triple_p.java @@ -1,11 +1,13 @@ import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.net.MalformedURLException; import java.util.List; import net.yacy.cora.protocol.RequestHeader; import net.yacy.interaction.TripleStore; import net.yacy.kelondro.data.meta.DigestURI; +import net.yacy.kelondro.logging.Log; import net.yacy.search.Switchboard; @@ -69,11 +71,12 @@ public class Triple_p { newurl = d.getProtocol()+"://"+HTTPDemon.getAlternativeResolver().resolve(d.getHost())+d.getPath(); System.out.println (newurl); } + TripleStore.Load(newurl); } catch (MalformedURLException e) { - // TODO Auto-generated catch block - e.printStackTrace(); + Log.logException(e); + } catch (IOException e) { + Log.logException(e); } - TripleStore.Load (newurl); } } diff --git a/source/net/yacy/interaction/TripleStore.java b/source/net/yacy/interaction/TripleStore.java index 521a665e9..b2791e774 100644 --- a/source/net/yacy/interaction/TripleStore.java +++ b/source/net/yacy/interaction/TripleStore.java @@ -4,6 +4,7 @@ package net.yacy.interaction; import java.io.File; import java.io.FileOutputStream; +import java.io.IOException; import java.io.InputStream; import java.io.StringBufferInputStream; import java.util.Iterator; @@ -27,26 +28,38 @@ public class TripleStore { public static String file; - public static void Load (String filename) { - + public static void Load(String filename) throws IOException { + if (filename.endsWith(".nt")) LoadNTriples(filename); + else LoadRDF(filename); + } + + public static void LoadRDF(String fileNameOrUri) throws IOException { Model tmp = ModelFactory.createDefaultModel(); - - Log.logInfo("TRIPLESTORE", "Loading from " + filename); - - try { - InputStream in = FileManager.get().open(filename); - - // read the RDF/XML file - tmp.read(in, null); - } - finally - { - model = model.union(tmp); - } - - + Log.logInfo("TRIPLESTORE", "Loading from " + fileNameOrUri); + InputStream is = FileManager.get().open(fileNameOrUri); + if (is != null) { + // read the RDF/XML file + tmp.read(is, null); + Log.logInfo("TRIPLESTORE", "loaded " + tmp.size() + " triples from " + fileNameOrUri); + model = model.union(tmp); + } else { + throw new IOException("cannot read " + fileNameOrUri); + } } + public static void LoadNTriples(String fileNameOrUri) throws IOException { + Model tmp = ModelFactory.createDefaultModel(); + Log.logInfo("TRIPLESTORE", "Loading N-Triples from " + fileNameOrUri); + InputStream is = FileManager.get().open(fileNameOrUri); + if (is != null) { + tmp.read(is, null, "N-TRIPLE"); + Log.logInfo("TRIPLESTORE", "loaded " + tmp.size() + " triples from " + fileNameOrUri); + model = model.union(tmp); + //model.write(System.out, "TURTLE"); + } else { + throw new IOException("cannot read " + fileNameOrUri); + } + } public static void Add (String rdffile) { @@ -68,23 +81,16 @@ public class TripleStore { } public static void Save (String filename) { - - + Log.logInfo("TRIPLESTORE", "Saving triplestore with " + model.size() + " triples to " + filename); FileOutputStream fout; try { - - fout = new FileOutputStream(filename); - model.write(fout); - + Log.logInfo("TRIPLESTORE", "Saved triplestore with " + model.size() + " triples to " + filename); } catch (Exception e) { // TODO Auto-generated catch block Log.logWarning("TRIPLESTORE", "Saving to " + filename+" failed"); } - - - } diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java index ac3f377b0..d168290be 100644 --- a/source/net/yacy/yacy.java +++ b/source/net/yacy/yacy.java @@ -306,9 +306,13 @@ public final class yacy { HTTPClient.setDefaultUserAgent(ClientIdentification.getUserAgent()); // initial fill of the triplestore - mkdirIfNeseccary (new File(sb.getConfig("triplestore", new File (dataHome, "DATA/TRIPLESTORE").toString()))); - if (sb.getConfigBool("triplestore.persistent", false) == true) { - TripleStore.Load(sb.getConfig("triplestore", new File(dataHome, "DATA/TRIPLESTORE").toString())+"/local.rdf"); + File triplestore = new File(sb.getConfig("triplestore", new File(dataHome, "DATA/TRIPLESTORE").getAbsolutePath())); + mkdirIfNeseccary(triplestore); + for (String s: triplestore.list()) { + if ((s.endsWith(".rdf") || s.endsWith(".nt")) && !s.equals("local.rdf")) TripleStore.Load(new File(triplestore, s).getAbsolutePath()); + } + if (sb.getConfigBool("triplestore.persistent", false)) { + TripleStore.Load(new File(triplestore, "local.rdf").getAbsolutePath()); } // start main threads @@ -430,8 +434,9 @@ public final class yacy { } finally { } - if (sb.getConfigBool("triplestore.persistent", false) == true) { - TripleStore.Save(sb.getConfig("triplestore", new File(dataHome, "DATA/TRIPLESTORE").toString())+"/local.rdf"); + if (sb.getConfigBool("triplestore.persistent", false)) { + File triplestore = new File(sb.getConfig("triplestore", new File(dataHome, "DATA/TRIPLESTORE").getAbsolutePath())); + TripleStore.Save(new File(triplestore, "local.rdf").getAbsolutePath()); } Log.logConfig("SHUTDOWN", "goodbye. (this is the last line)");