From 0752983fbd76b4656ad2059f405e9105d3ee1198 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Sun, 17 Jun 2012 10:50:12 +0200 Subject: [PATCH] - automatic periodic saving of triplestore - transaction-safe storage of triplestore --- source/de/anomic/server/serverSwitch.java | 34 +++++----- source/net/yacy/cora/lod/JenaTripleStore.java | 62 +++++++++---------- source/net/yacy/search/Switchboard.java | 9 ++- source/net/yacy/yacy.java | 52 +++++++--------- 4 files changed, 74 insertions(+), 83 deletions(-) diff --git a/source/de/anomic/server/serverSwitch.java b/source/de/anomic/server/serverSwitch.java index 122d8730a..ac175ca20 100644 --- a/source/de/anomic/server/serverSwitch.java +++ b/source/de/anomic/server/serverSwitch.java @@ -61,7 +61,7 @@ public class serverSwitch // configuration management private final File configFile; private final String configComment; - private final File dataPath; + public final File dataPath; public final File appPath; protected boolean firstInit; protected Log log; @@ -167,7 +167,7 @@ public class serverSwitch /** * get my public IP, either set statically or figure out dynamic - * @return + * @return */ public String myPublicIP() { // if a static IP was configured, we have to return it here ... @@ -191,7 +191,7 @@ public class serverSwitch /** * add whole map of key-value pairs to config - * @param otherConfigs + * @param otherConfigs */ public void setConfig(final Map otherConfigs) { final Iterator> i = otherConfigs.entrySet().iterator(); @@ -228,7 +228,7 @@ public class serverSwitch /** * Gets a configuration parameter from the properties. - * + * * @param key name of the configuration parameter * @param dflt default value which will be used in case parameter can not be found or if it is invalid * @return value if the parameter or default value @@ -246,7 +246,7 @@ public class serverSwitch /** * Gets a configuration parameter from the properties. - * + * * @param key name of the configuration parameter * @param dflt default value which will be used in case parameter can not be found or if it is invalid * @return value if the parameter or default value @@ -261,7 +261,7 @@ public class serverSwitch /** * Gets a configuration parameter from the properties. - * + * * @param key name of the configuration parameter * @param dflt default value which will be used in case parameter can not be found or if it is invalid * @return value if the parameter or default value @@ -276,7 +276,7 @@ public class serverSwitch /** * Gets a configuration parameter from the properties. - * + * * @param key name of the configuration parameter * @param dflt default value which will be used in case parameter can not be found or if it is invalid * @return value if the parameter or default value @@ -291,7 +291,7 @@ public class serverSwitch /** * Gets a configuration parameter from the properties. - * + * * @param key name of the configuration parameter * @param dflt default value which will be used in case parameter can not be found or if it is invalid * @return value if the parameter or default value @@ -302,7 +302,7 @@ public class serverSwitch /** * Create a File instance for a configuration setting specifying a path. - * + * * @param key config key * @param dflt default path value, that is used when there is no value key in the * configuration. @@ -311,19 +311,19 @@ public class serverSwitch * the relative path setting. */ public File getDataPath(final String key, final String dflt) { - return getFileByPath(key, dflt, dataPath); + return getFileByPath(key, dflt, this.dataPath); } /** * return file at path from config entry "key", or fallback to default dflt * @param key * @param dflt - * @return + * @return */ public File getAppPath(final String key, final String dflt) { - return getFileByPath(key, dflt, appPath); + return getFileByPath(key, dflt, this.appPath); } - + private File getFileByPath(String key, String dflt, File prefix) { final String path = getConfig(key, dflt).replace('\\', '/'); final File f = new File(path); @@ -345,7 +345,7 @@ public class serverSwitch /** * Gets configuration parameters which have been removed during initialization. - * + * * @return contains parameter name as key and parameter value as value */ public ConcurrentMap getRemoved() { @@ -613,7 +613,7 @@ public class serverSwitch /** * Retrieve text data (e. g. config file) from file file may be an url or a filename with path relative to * rootPath parameter - * + * * @param file url or filename * @param rootPath searchpath for file * @param file file to use when remote fetching fails (null if unused) @@ -665,7 +665,7 @@ public class serverSwitch /** * Generates a random password. - * + * * @return random password which is 20 characters long. */ public String genRandomPassword() { @@ -674,7 +674,7 @@ public class serverSwitch /** * Generates a random password of a given length. - * + * * @param length length o password * @return password of given length */ diff --git a/source/net/yacy/cora/lod/JenaTripleStore.java b/source/net/yacy/cora/lod/JenaTripleStore.java index d0cea5944..7220a8637 100644 --- a/source/net/yacy/cora/lod/JenaTripleStore.java +++ b/source/net/yacy/cora/lod/JenaTripleStore.java @@ -113,12 +113,25 @@ public class JenaTripleStore { } public static void saveFile(String filename, Model model) { + File f = new File(filename); + File ftmp = new File(filename + "." + System.currentTimeMillis()); + if (model.size() == 0 && !f.exists()) { + // we don't store zero-size models if they did not exist before + Log.logInfo("TRIPLESTORE", "NOT saving triplestore with " + model.size() + " triples to " + filename); + return; + } Log.logInfo("TRIPLESTORE", "Saving triplestore with " + model.size() + " triples to " + filename); OutputStream fout; try { - fout = new BufferedOutputStream(new FileOutputStream(filename)); + fout = new BufferedOutputStream(new FileOutputStream(ftmp)); model.write(fout); fout.close(); + // if something went wrong until here, the original file is not overwritten + // since we are happy here, we can remove the old file and replace it with the new one + f.delete(); + if (!f.exists()) { + ftmp.renameTo(f); + } Log.logInfo("TRIPLESTORE", "Saved triplestore with " + model.size() + " triples to " + filename); } catch (Exception e) { Log.logWarning("TRIPLESTORE", "Saving to " + filename+" failed"); @@ -258,81 +271,64 @@ public class JenaTripleStore { } public static void initPrivateStores() { - Switchboard switchboard = Switchboard.getSwitchboard(); - Log.logInfo("TRIPLESTORE", "Init private stores"); - if (privatestorage == null) privatestorage = new ConcurrentHashMap(); - if (privatestorage != null) privatestorage.clear(); try { - Iterator it = switchboard.userDB.iterator(true); - while (it.hasNext()) { de.anomic.data.UserDB.Entry e = it.next(); String username = e.getUserName(); - File triplestore = new File(switchboard.getConfig("triplestore", new File(switchboard.getDataPath(), "DATA/TRIPLESTORE").getAbsolutePath())); - File currentuserfile = new File(triplestore, "private_store_"+username+".rdf"); - Log.logInfo("TRIPLESTORE", "Init " + username + " from "+currentuserfile.getAbsolutePath()); - Model tmp = ModelFactory.createDefaultModel(); - init (tmp); if (currentuserfile.exists()) { - - Log.logInfo("TRIPLESTORE", "Loading from " + currentuserfile.getAbsolutePath()); InputStream is = FileManager.get().open(currentuserfile.getAbsolutePath()); if (is != null) { // read the RDF/XML file tmp.read(is, null); Log.logInfo("TRIPLESTORE", "loaded " + tmp.size() + " triples from " + currentuserfile.getAbsolutePath()); - - } else { throw new IOException("cannot read " + currentuserfile.getAbsolutePath()); } } if (tmp != null) { - privatestorage.put(username, tmp); - } - } - - } - catch (Exception anyex) { - + } catch (Exception anyex) { Log.logException(anyex); - } - } - public static void savePrivateStores(Switchboard switchboard) { - + public static void savePrivateStores() { + Switchboard switchboard = Switchboard.getSwitchboard(); Log.logInfo("TRIPLESTORE", "Saving user triplestores"); - if (privatestorage == null) return; - for (Entry s : privatestorage.entrySet()) { - File triplestore = new File(switchboard.getConfig("triplestore", new File(switchboard.getDataPath(), "DATA/TRIPLESTORE").getAbsolutePath())); - File currentuserfile = new File(triplestore, "private_store_"+s.getKey()+".rdf"); - saveFile (currentuserfile.getAbsolutePath(), s.getValue()); - } } + private static long lastModelSizeStored = -1; + + public static void saveAll() { + Switchboard sb = Switchboard.getSwitchboard(); + File triplestore = new File(sb.getConfig("triplestore", new File(sb.dataPath, "DATA/TRIPLESTORE").getAbsolutePath())); + if (model.size() != lastModelSizeStored){ + JenaTripleStore.saveFile(new File(triplestore, "local.rdf").getAbsolutePath()); + lastModelSizeStored = model.size(); + } + JenaTripleStore.savePrivateStores(); + } + } diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 4c77aeddf..4fd8657cb 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -632,7 +632,7 @@ public final class Switchboard extends serverSwitch + " entries" + ", " + ppRamString(userDbFile.length() / 1024)); - + // init user triplestores JenaTripleStore.initPrivateStores(); @@ -664,7 +664,7 @@ public final class Switchboard extends serverSwitch } } }.start(); - + // define a realtime parsable mimetype list this.log.logConfig("Parser: Initializing Mime Type deny list"); TextParser.setDenyMime(getConfig(SwitchboardConstants.PARSER_MIME_DENY, "")); @@ -2222,6 +2222,11 @@ public final class Switchboard extends serverSwitch this.tables.cleanFailURLS(getConfigLong("cleanup.failedSearchURLtimeout", -1)); } + // periodically store the triple store + if (getConfigBool("triplestore.persistent", false)) { + JenaTripleStore.saveAll(); + } + return true; } catch ( final InterruptedException e ) { this.log.logInfo("cleanupJob: Shutdown detected"); diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java index 21fb8e5d9..2d0f92b9b 100644 --- a/source/net/yacy/yacy.java +++ b/source/net/yacy/yacy.java @@ -306,19 +306,26 @@ public final class yacy { HTTPClient.setDefaultUserAgent(ClientIdentification.getUserAgent()); // initial fill of the triplestore - try { - File triplestore = new File(sb.getConfig("triplestore", new File(dataHome, "DATA/TRIPLESTORE").getAbsolutePath())); - mkdirIfNeseccary(triplestore); - for (String s: triplestore.list()) { - if ((s.endsWith(".rdf") || s.endsWith(".nt")) && !s.equals("local.rdf") && !s.endsWith("_triplestore.rdf") && !s.startsWith("private_store_")) JenaTripleStore.load(new File(triplestore, s).getAbsolutePath()); + File triplestore = new File(sb.getConfig("triplestore", new File(dataHome, "DATA/TRIPLESTORE").getAbsolutePath())); + mkdirIfNeseccary(triplestore); + for (String s: triplestore.list()) { + if ((s.endsWith(".rdf") || s.endsWith(".nt")) && !s.equals("local.rdf") && !s.endsWith("_triplestore.rdf") && !s.startsWith("private_store_")) { + try { + JenaTripleStore.load(new File(triplestore, s).getAbsolutePath()); + } catch (IOException e) { + Log.logException(e); + } + } + } + if (sb.getConfigBool("triplestore.persistent", false)) { + File local = new File(triplestore, "local.rdf"); + if (local.exists()) { + try { + JenaTripleStore.load(local.getAbsolutePath()); + } catch (IOException e) { + Log.logException(e); + } } - if (sb.getConfigBool("triplestore.persistent", false)) { - File local = new File(triplestore, "local.rdf"); - if (local.exists()) JenaTripleStore.load(local.getAbsolutePath()); - } - - } catch (IOException e) { - Log.logException(e); } // start main threads @@ -407,26 +414,11 @@ public final class yacy { server.terminate(false); server.interrupt(); server.close(); - /* - if (server.isAlive()) try { - // TODO only send request, don't read response (cause server is already down resulting in error) - final DigestURI u = new DigestURI((server.withSSL()?"https":"http")+"://localhost:" + serverCore.getPortNr(port), null); - Client.wget(u.toString(), null, 10000); // kick server - Log.logConfig("SHUTDOWN", "sent termination signal to server socket"); - } catch (final IOException ee) { - Log.logConfig("SHUTDOWN", "termination signal to server socket missed (server shutdown, ok)"); - } - */ -// Client.closeAllConnections(); -// MultiThreadedHttpConnectionManager.shutdownAll(); // idle until the processes are down if (server.isAlive()) { - //Thread.sleep(2000); // wait a while server.interrupt(); -// MultiThreadedHttpConnectionManager.shutdownAll(); } -// MultiThreadedHttpConnectionManager.shutdownAll(); Log.logConfig("SHUTDOWN", "server has terminated"); sb.close(); } catch (final Exception e) { @@ -440,11 +432,9 @@ public final class yacy { } finally { } + // save the triple store if (sb.getConfigBool("triplestore.persistent", false)) { - File triplestore = new File(sb.getConfig("triplestore", new File(dataHome, "DATA/TRIPLESTORE").getAbsolutePath())); - JenaTripleStore.saveFile(new File(triplestore, "local.rdf").getAbsolutePath()); - - JenaTripleStore.savePrivateStores(sb); + JenaTripleStore.saveAll(); } Log.logConfig("SHUTDOWN", "goodbye. (this is the last line)");