boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- automatic periodic saving of triplestore

Browse Source 
- transaction-safe storage of triplestore
pull/1/head
Michael Peter Christen 13 years ago
parent f1aa4c4390
commit 0752983fbd
4 changed files with 74 additions and 83 deletions
Show Stats Download Patch File Download Diff File

34
source/de/anomic/server/serverSwitch.java
Unescape View File

@ -61,7 +61,7 @@ public class serverSwitch
// configuration management
private final File configFile;
private final String configComment;
private final File dataPath;
public final File dataPath;
public final File appPath;
protected boolean firstInit;
protected Log log;
@ -167,7 +167,7 @@ public class serverSwitch
/**
* get my public IP, either set statically or figure out dynamic
* @return
* @return
*/
public String myPublicIP() {
// if a static IP was configured, we have to return it here ...
@ -191,7 +191,7 @@ public class serverSwitch
/**
* add whole map of key-value pairs to config
* @param otherConfigs
* @param otherConfigs
*/
public void setConfig(final Map<String, String> otherConfigs) {
final Iterator<Map.Entry<String, String>> i = otherConfigs.entrySet().iterator();
@ -228,7 +228,7 @@ public class serverSwitch
/**
* Gets a configuration parameter from the properties.
*
*
* @param key name of the configuration parameter
* @param dflt default value which will be used in case parameter can not be found or if it is invalid
* @return value if the parameter or default value
@ -246,7 +246,7 @@ public class serverSwitch
/**
* Gets a configuration parameter from the properties.
*
*
* @param key name of the configuration parameter
* @param dflt default value which will be used in case parameter can not be found or if it is invalid
* @return value if the parameter or default value
@ -261,7 +261,7 @@ public class serverSwitch
/**
* Gets a configuration parameter from the properties.
*
*
* @param key name of the configuration parameter
* @param dflt default value which will be used in case parameter can not be found or if it is invalid
* @return value if the parameter or default value
@ -276,7 +276,7 @@ public class serverSwitch
/**
* Gets a configuration parameter from the properties.
*
*
* @param key name of the configuration parameter
* @param dflt default value which will be used in case parameter can not be found or if it is invalid
* @return value if the parameter or default value
@ -291,7 +291,7 @@ public class serverSwitch
/**
* Gets a configuration parameter from the properties.
*
*
* @param key name of the configuration parameter
* @param dflt default value which will be used in case parameter can not be found or if it is invalid
* @return value if the parameter or default value
@ -302,7 +302,7 @@ public class serverSwitch
/**
* Create a File instance for a configuration setting specifying a path.
*
*
* @param key config key
* @param dflt default path value, that is used when there is no value <code>key</code> in the
* configuration.
@ -311,19 +311,19 @@ public class serverSwitch
* the relative path setting.
*/
public File getDataPath(final String key, final String dflt) {
return getFileByPath(key, dflt, dataPath);
return getFileByPath(key, dflt, this.dataPath);
}
/**
* return file at path from config entry "key", or fallback to default dflt
* @param key
* @param dflt
* @return
* @return
*/
public File getAppPath(final String key, final String dflt) {
return getFileByPath(key, dflt, appPath);
return getFileByPath(key, dflt, this.appPath);
}
private File getFileByPath(String key, String dflt, File prefix) {
final String path = getConfig(key, dflt).replace('\\', '/');
final File f = new File(path);
@ -345,7 +345,7 @@ public class serverSwitch
/**
* Gets configuration parameters which have been removed during initialization.
*
*
* @return contains parameter name as key and parameter value as value
*/
public ConcurrentMap<String, String> getRemoved() {
@ -613,7 +613,7 @@ public class serverSwitch
/**
* Retrieve text data (e. g. config file) from file file may be an url or a filename with path relative to
* rootPath parameter
*
*
* @param file url or filename
* @param rootPath searchpath for file
* @param file file to use when remote fetching fails (null if unused)
@ -665,7 +665,7 @@ public class serverSwitch
/**
* Generates a random password.
*
*
* @return random password which is 20 characters long.
*/
public String genRandomPassword() {
@ -674,7 +674,7 @@ public class serverSwitch
/**
* Generates a random password of a given length.
*
*
* @param length length o password
* @return password of given length
*/

62
source/net/yacy/cora/lod/JenaTripleStore.java
Unescape View File

@ -113,12 +113,25 @@ public class JenaTripleStore {
}
public static void saveFile(String filename, Model model) {
File f = new File(filename);
File ftmp = new File(filename + "." + System.currentTimeMillis());
if (model.size() == 0 && !f.exists()) {
// we don't store zero-size models if they did not exist before
Log.logInfo("TRIPLESTORE", "NOT saving triplestore with " + model.size() + " triples to " + filename);
return;
}
Log.logInfo("TRIPLESTORE", "Saving triplestore with " + model.size() + " triples to " + filename);
OutputStream fout;
try {
fout = new BufferedOutputStream(new FileOutputStream(filename));
fout = new BufferedOutputStream(new FileOutputStream(ftmp));
model.write(fout);
fout.close();
// if something went wrong until here, the original file is not overwritten
// since we are happy here, we can remove the old file and replace it with the new one
f.delete();
if (!f.exists()) {
ftmp.renameTo(f);
}
Log.logInfo("TRIPLESTORE", "Saved triplestore with " + model.size() + " triples to " + filename);
} catch (Exception e) {
Log.logWarning("TRIPLESTORE", "Saving to " + filename+" failed");
@ -258,81 +271,64 @@ public class JenaTripleStore {
}
public static void initPrivateStores() {
Switchboard switchboard = Switchboard.getSwitchboard();
Log.logInfo("TRIPLESTORE", "Init private stores");
if (privatestorage == null) privatestorage = new ConcurrentHashMap<String, Model>();
if (privatestorage != null) privatestorage.clear();
try {
Iterator<de.anomic.data.UserDB.Entry> it = switchboard.userDB.iterator(true);
while (it.hasNext()) {
de.anomic.data.UserDB.Entry e = it.next();
String username = e.getUserName();
File triplestore = new File(switchboard.getConfig("triplestore", new File(switchboard.getDataPath(), "DATA/TRIPLESTORE").getAbsolutePath()));
File currentuserfile = new File(triplestore, "private_store_"+username+".rdf");
Log.logInfo("TRIPLESTORE", "Init " + username + " from "+currentuserfile.getAbsolutePath());
Model tmp = ModelFactory.createDefaultModel();
init (tmp);
if (currentuserfile.exists()) {
Log.logInfo("TRIPLESTORE", "Loading from " + currentuserfile.getAbsolutePath());
InputStream is = FileManager.get().open(currentuserfile.getAbsolutePath());
if (is != null) {
// read the RDF/XML file
tmp.read(is, null);
Log.logInfo("TRIPLESTORE", "loaded " + tmp.size() + " triples from " + currentuserfile.getAbsolutePath());
} else {
throw new IOException("cannot read " + currentuserfile.getAbsolutePath());
}
}
if (tmp != null) {
privatestorage.put(username, tmp);
}
}
}
catch (Exception anyex) {
} catch (Exception anyex) {
Log.logException(anyex);
}
}
public static void savePrivateStores(Switchboard switchboard) {
public static void savePrivateStores() {
Switchboard switchboard = Switchboard.getSwitchboard();
Log.logInfo("TRIPLESTORE", "Saving user triplestores");
if (privatestorage == null) return;
for (Entry<String, Model> s : privatestorage.entrySet()) {
File triplestore = new File(switchboard.getConfig("triplestore", new File(switchboard.getDataPath(), "DATA/TRIPLESTORE").getAbsolutePath()));
File currentuserfile = new File(triplestore, "private_store_"+s.getKey()+".rdf");
saveFile (currentuserfile.getAbsolutePath(), s.getValue());
}
}
private static long lastModelSizeStored = -1;
public static void saveAll() {
Switchboard sb = Switchboard.getSwitchboard();
File triplestore = new File(sb.getConfig("triplestore", new File(sb.dataPath, "DATA/TRIPLESTORE").getAbsolutePath()));
if (model.size() != lastModelSizeStored){
JenaTripleStore.saveFile(new File(triplestore, "local.rdf").getAbsolutePath());
lastModelSizeStored = model.size();
}
JenaTripleStore.savePrivateStores();
}
}

9
source/net/yacy/search/Switchboard.java
Unescape View File

@ -632,7 +632,7 @@ public final class Switchboard extends serverSwitch
+ " entries"
+ ", "
+ ppRamString(userDbFile.length() / 1024));
// init user triplestores
JenaTripleStore.initPrivateStores();
@ -664,7 +664,7 @@ public final class Switchboard extends serverSwitch
}
}
}.start();
// define a realtime parsable mimetype list
this.log.logConfig("Parser: Initializing Mime Type deny list");
TextParser.setDenyMime(getConfig(SwitchboardConstants.PARSER_MIME_DENY, ""));
@ -2222,6 +2222,11 @@ public final class Switchboard extends serverSwitch
this.tables.cleanFailURLS(getConfigLong("cleanup.failedSearchURLtimeout", -1));
}
// periodically store the triple store
if (getConfigBool("triplestore.persistent", false)) {
JenaTripleStore.saveAll();
}
return true;
} catch ( final InterruptedException e ) {
this.log.logInfo("cleanupJob: Shutdown detected");

52
source/net/yacy/yacy.java
Unescape View File

@ -306,19 +306,26 @@ public final class yacy {
HTTPClient.setDefaultUserAgent(ClientIdentification.getUserAgent());
// initial fill of the triplestore
try {
File triplestore = new File(sb.getConfig("triplestore", new File(dataHome, "DATA/TRIPLESTORE").getAbsolutePath()));
mkdirIfNeseccary(triplestore);
for (String s: triplestore.list()) {
if ((s.endsWith(".rdf") || s.endsWith(".nt")) && !s.equals("local.rdf") && !s.endsWith("_triplestore.rdf") && !s.startsWith("private_store_")) JenaTripleStore.load(new File(triplestore, s).getAbsolutePath());
File triplestore = new File(sb.getConfig("triplestore", new File(dataHome, "DATA/TRIPLESTORE").getAbsolutePath()));
mkdirIfNeseccary(triplestore);
for (String s: triplestore.list()) {
if ((s.endsWith(".rdf") || s.endsWith(".nt")) && !s.equals("local.rdf") && !s.endsWith("_triplestore.rdf") && !s.startsWith("private_store_")) {
try {
JenaTripleStore.load(new File(triplestore, s).getAbsolutePath());
} catch (IOException e) {
Log.logException(e);
}
}
}
if (sb.getConfigBool("triplestore.persistent", false)) {
File local = new File(triplestore, "local.rdf");
if (local.exists()) {
try {
JenaTripleStore.load(local.getAbsolutePath());
} catch (IOException e) {
Log.logException(e);
}
}
if (sb.getConfigBool("triplestore.persistent", false)) {
File local = new File(triplestore, "local.rdf");
if (local.exists()) JenaTripleStore.load(local.getAbsolutePath());
}
} catch (IOException e) {
Log.logException(e);
}
// start main threads
@ -407,26 +414,11 @@ public final class yacy {
server.terminate(false);
server.interrupt();
server.close();
/*
if (server.isAlive()) try {
// TODO only send request, don't read response (cause server is already down resulting in error)
final DigestURI u = new DigestURI((server.withSSL()?"https":"http")+"://localhost:" + serverCore.getPortNr(port), null);
Client.wget(u.toString(), null, 10000); // kick server
Log.logConfig("SHUTDOWN", "sent termination signal to server socket");
} catch (final IOException ee) {
Log.logConfig("SHUTDOWN", "termination signal to server socket missed (server shutdown, ok)");
}
*/
// Client.closeAllConnections();
// MultiThreadedHttpConnectionManager.shutdownAll();
// idle until the processes are down
if (server.isAlive()) {
//Thread.sleep(2000); // wait a while
server.interrupt();
// MultiThreadedHttpConnectionManager.shutdownAll();
}
// MultiThreadedHttpConnectionManager.shutdownAll();
Log.logConfig("SHUTDOWN", "server has terminated");
sb.close();
} catch (final Exception e) {
@ -440,11 +432,9 @@ public final class yacy {
} finally {
}
// save the triple store
if (sb.getConfigBool("triplestore.persistent", false)) {
File triplestore = new File(sb.getConfig("triplestore", new File(dataHome, "DATA/TRIPLESTORE").getAbsolutePath()));
JenaTripleStore.saveFile(new File(triplestore, "local.rdf").getAbsolutePath());
JenaTripleStore.savePrivateStores(sb);
JenaTripleStore.saveAll();
}
Log.logConfig("SHUTDOWN", "goodbye. (this is the last line)");

Write Preview
Loading…
Cancel
Save