boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Upgraded the OpenGeoDB dump URL

Browse Source 
The status of the library in the DictionaryLoader_p.html page now also
advertises the user that an upgrade can be applied when an older dump is
already loaded.

Upgrade applied as suggested by Niklas Andrus @fapth_gitlab on Gitter
chat.
pull/218/head
luccioman 6 years ago
parent 373edf9eac
commit 8a29551c54
3 changed files with 135 additions and 38 deletions
Show Stats Download Patch File Download Diff File

75
htroot/DictionaryLoader_p.html
Unescape View File

@ -130,24 +130,41 @@
<p>With this file it is possible to find locations in Germany using the location (city) name, a zip code, a car sign or a telephone pre-dial number.</p> <p>With this file it is possible to find locations in Germany using the location (city) name, a zip code, a car sign or a telephone pre-dial number.</p>
<dl> <dl>
<dt><label>Download from</label></dt> #(geo0Status)#::
<dd>#[geo1URL]#</dd> <dt><label>Downloaded from</label></dt>
<dd>#[URL]#</dd>
<dt><label>Storage location</label></dt> <dt><label>Storage location</label></dt>
<dd>#[geo1Storage]#</dd> <dd>#[Storage]#</dd>
<dt><label>Status</label></dt> <dt><label>Status</label></dt>
<dd>#(geo1Status)#<div class="info">not loaded</div>::<div class="commit">loaded</div>::deactivated#(/geo1Status)#</dd> <dd><div class="commit">loaded - can be upgraded using the Load button for the new URL</div></dd>
<dt>Action</dt> <dt>Action</dt>
<dd>#(geo1Status)# <dd>
<input type="submit" name="geo1Load" class="btn btn-sm btn-primary" value="Load" />:: <input type="submit" name="geo0Deactivate" class="btn btn-sm btn-primary" value="Deactivate" />
<input type="submit" name="geo0Remove" class="btn btn-sm btn-danger" value="Remove" />
</dd>::
#(/geo0Status)#
#(geo0ActionRemoved)#::
<dt>Result</dt><dd><div class="commit">deactivated and removed dictionary file</div></dd>::
<dt>Result</dt><dd><div class="error">cannot remove dictionary file: #[error]#</div></dd>
#(/geo0ActionRemoved)#
#(geo0ActionDeactivated)#::
<dt>Result</dt><dd><div class="commit">deactivated dictionary file</div></dd>::
<dt>Result</dt><dd><div class="error">cannot de-activate dictionary file: #[error]#</div></dd>
#(/geo0ActionDeactivated)#
#(geo1Status)#::
<dt><label>Downloaded from</label></dt>
<dd>#[URL]#</dd>
<dt><label>Storage location</label></dt>
<dd>#[Storage]#</dd>
<dt><label>Status</label></dt>
<dd><div class="commit">loaded - can be upgraded using the Load button for the new URL</div></dd>
<dt>Action</dt>
<dd>
<input type="submit" name="geo1Deactivate" class="btn btn-sm btn-primary" value="Deactivate" /> <input type="submit" name="geo1Deactivate" class="btn btn-sm btn-primary" value="Deactivate" />
<input type="submit" name="geo1Remove" class="btn btn-sm btn-danger" value="Remove" />::
<input type="submit" name="geo1Activate" class="btn btn-sm btn-primary" value="Activate" />
<input type="submit" name="geo1Remove" class="btn btn-sm btn-danger" value="Remove" /> <input type="submit" name="geo1Remove" class="btn btn-sm btn-danger" value="Remove" />
#(/geo1Status)#</dd> </dd>::
#(geo1ActionLoaded)#:: #(/geo1Status)#
<dt>Result</dt><dd><div class="commit">loaded and activated dictionary file</div></dd>::
<dt>Result</dt><dd><div class="error">loading of dictionary file failed: #[error]#</div></dd>
#(/geo1ActionLoaded)#
#(geo1ActionRemoved)#:: #(geo1ActionRemoved)#::
<dt>Result</dt><dd><div class="commit">deactivated and removed dictionary file</div></dd>:: <dt>Result</dt><dd><div class="commit">deactivated and removed dictionary file</div></dd>::
<dt>Result</dt><dd><div class="error">cannot remove dictionary file: #[error]#</div></dd> <dt>Result</dt><dd><div class="error">cannot remove dictionary file: #[error]#</div></dd>
@ -156,10 +173,38 @@
<dt>Result</dt><dd><div class="commit">deactivated dictionary file</div></dd>:: <dt>Result</dt><dd><div class="commit">deactivated dictionary file</div></dd>::
<dt>Result</dt><dd><div class="error">cannot de-activate dictionary file: #[error]#</div></dd> <dt>Result</dt><dd><div class="error">cannot de-activate dictionary file: #[error]#</div></dd>
#(/geo1ActionDeactivated)# #(/geo1ActionDeactivated)#
#(geo1ActionActivated)#::
<dt><label>Download from</label></dt>
<dd>#[geo2URL]#</dd>
<dt><label>Storage location</label></dt>
<dd>#[geo2Storage]#</dd>
<dt><label>Status</label></dt>
<dd>#(geo2Status)#<div class="info">not loaded</div>::<div class="commit">loaded</div>::deactivated#(/geo2Status)#</dd>
<dt>Action</dt>
<dd>#(geo2Status)#
<input type="submit" name="geo2Load" class="btn btn-sm btn-primary" value="Load" />::
<input type="submit" name="geo2Deactivate" class="btn btn-sm btn-primary" value="Deactivate" />
<input type="submit" name="geo2Remove" class="btn btn-sm btn-danger" value="Remove" />::
<input type="submit" name="geo2Activate" class="btn btn-sm btn-primary" value="Activate" />
<input type="submit" name="geo2Remove" class="btn btn-sm btn-danger" value="Remove" />
#(/geo2Status)#</dd>
#(geo2ActionLoaded)#::
<dt>Result</dt><dd><div class="commit">loaded and activated dictionary file</div></dd>::
<dt>Result</dt><dd><div class="error">loading of dictionary file failed: #[error]#</div></dd>::
<dt>Result</dt><dd><div class="commit">loaded and upgraded dictionary file</div></dd>
#(/geo2ActionLoaded)#
#(geo2ActionRemoved)#::
<dt>Result</dt><dd><div class="commit">deactivated and removed dictionary file</div></dd>::
<dt>Result</dt><dd><div class="error">cannot remove dictionary file: #[error]#</div></dd>
#(/geo2ActionRemoved)#
#(geo2ActionDeactivated)#::
<dt>Result</dt><dd><div class="commit">deactivated dictionary file</div></dd>::
<dt>Result</dt><dd><div class="error">cannot de-activate dictionary file: #[error]#</div></dd>
#(/geo2ActionDeactivated)#
#(geo2ActionActivated)#::
<dt>Result</dt><dd><div class="commit">activated dictionary file</div></dd>:: <dt>Result</dt><dd><div class="commit">activated dictionary file</div></dd>::
<dt>Result</dt><dd><div class="error">cannot activate dictionary file: #[error]#</div></dd> <dt>Result</dt><dd><div class="error">cannot activate dictionary file: #[error]#</div></dd>
#(/geo1ActionActivated)# #(/geo2ActionActivated)#
</dl> </dl>
</fieldset> </fieldset>
</form> </form>

70
htroot/DictionaryLoader_p.java
Unescape View File

@ -60,7 +60,9 @@ public class DictionaryLoader_p {
for (final LibraryProvider.Dictionary dictionary: LibraryProvider.Dictionary.values()) { for (final LibraryProvider.Dictionary dictionary: LibraryProvider.Dictionary.values()) {
prop.put(dictionary.nickname + "URL", dictionary.url); prop.put(dictionary.nickname + "URL", dictionary.url);
prop.put(dictionary.nickname + "Status_URL", dictionary.url);
prop.put(dictionary.nickname + "Storage", dictionary.file().toString()); prop.put(dictionary.nickname + "Storage", dictionary.file().toString());
prop.put(dictionary.nickname + "Status_Storage", dictionary.file().toString());
prop.put(dictionary.nickname + "Status", dictionary.file().exists() ? 1 : dictionary.fileDisabled().exists() ? 2 : 0); prop.put(dictionary.nickname + "Status", dictionary.file().exists() ? 1 : dictionary.fileDisabled().exists() ? 2 : 0);
prop.put(dictionary.nickname + "ActionLoaded", 0); prop.put(dictionary.nickname + "ActionLoaded", 0);
prop.put(dictionary.nickname + "ActionRemoved", 0); prop.put(dictionary.nickname + "ActionRemoved", 0);
@ -208,31 +210,56 @@ public class DictionaryLoader_p {
prop.put("geon2ActionActivated", 1); prop.put("geon2ActionActivated", 1);
} }
// GEO1 // GEO2
if (post.containsKey("geo1Load")) { if (post.containsKey("geo2Load")) {
// load from the net // load from the net
try { try {
final Response response = sb.loader.load(sb.loader.request(new DigestURL(LibraryProvider.Dictionary.GEODB1.url), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, ClientIdentification.yacyInternetCrawlerAgent); final Response response = sb.loader.load(sb.loader.request(new DigestURL(LibraryProvider.Dictionary.GEODB2.url), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, ClientIdentification.yacyInternetCrawlerAgent);
final byte[] b = response.getContent(); final byte[] b = response.getContent();
FileUtils.copy(b, LibraryProvider.Dictionary.GEODB1.file()); FileUtils.copy(b, LibraryProvider.Dictionary.GEODB2.file());
LibraryProvider.geoLoc.deactivateLocalization(LibraryProvider.Dictionary.GEODB2.nickname);
boolean isUpgrade = false;
if(LibraryProvider.Dictionary.GEODB0.file().exists()) {
isUpgrade = true;
FileUtils.deletedelete(LibraryProvider.Dictionary.GEODB0.file());
FileUtils.deletedelete(LibraryProvider.Dictionary.GEODB0.fileDisabled());
LibraryProvider.geoLoc.deactivateLocalization(LibraryProvider.Dictionary.GEODB0.nickname);
} else if(LibraryProvider.Dictionary.GEODB1.file().exists()) {
isUpgrade = true;
FileUtils.deletedelete(LibraryProvider.Dictionary.GEODB1.file());
FileUtils.deletedelete(LibraryProvider.Dictionary.GEODB1.fileDisabled());
LibraryProvider.geoLoc.deactivateLocalization(LibraryProvider.Dictionary.GEODB1.nickname); LibraryProvider.geoLoc.deactivateLocalization(LibraryProvider.Dictionary.GEODB1.nickname);
LibraryProvider.geoLoc.activateLocation(LibraryProvider.Dictionary.GEODB1.nickname, new OpenGeoDBLocation(LibraryProvider.Dictionary.GEODB1.file(), null)); }
LibraryProvider.geoLoc.activateLocation(LibraryProvider.Dictionary.GEODB2.nickname, new OpenGeoDBLocation(LibraryProvider.Dictionary.GEODB2.file(), null));
LibraryProvider.autotagging.addPlaces(LibraryProvider.geoLoc); LibraryProvider.autotagging.addPlaces(LibraryProvider.geoLoc);
prop.put("geo1Status", LibraryProvider.Dictionary.GEODB1.file().exists() ? 1 : 0); prop.put("geo2Status", LibraryProvider.Dictionary.GEODB2.file().exists() ? 1 : 0);
prop.put("geo1ActionLoaded", 1); prop.put("geo2ActionLoaded", isUpgrade ? 3 : 1);
final String nav = sb.getConfig("search.navigation", ""); final String nav = sb.getConfig("search.navigation", "");
if (nav.indexOf("location") < 0) sb.setConfig("search.navigation", "location,"+nav); if (nav.indexOf("location") < 0) sb.setConfig("search.navigation", "location,"+nav);
} catch (final MalformedURLException e) { } catch (final MalformedURLException e) {
ConcurrentLog.logException(e); ConcurrentLog.logException(e);
prop.put("geo1ActionLoaded", 2); prop.put("geo2ActionLoaded", 2);
prop.put("geo1ActionLoaded_error", e.getMessage()); prop.put("geo2ActionLoaded_error", e.getMessage());
} catch (final IOException e) { } catch (final IOException e) {
ConcurrentLog.logException(e); ConcurrentLog.logException(e);
prop.put("geo1ActionLoaded", 2); prop.put("geo2ActionLoaded", 2);
prop.put("geo1ActionLoaded_error", e.getMessage()); prop.put("geo2ActionLoaded_error", e.getMessage());
} }
} }
if (post.containsKey("geo0Remove")) {
FileUtils.deletedelete(LibraryProvider.Dictionary.GEODB0.file());
FileUtils.deletedelete(LibraryProvider.Dictionary.GEODB0.fileDisabled());
LibraryProvider.geoLoc.deactivateLocalization(LibraryProvider.Dictionary.GEODB0.nickname);
prop.put("geo0ActionRemoved", 1);
}
if (post.containsKey("geo0Deactivate")) {
LibraryProvider.Dictionary.GEODB0.file().renameTo(LibraryProvider.Dictionary.GEODB0.fileDisabled());
LibraryProvider.geoLoc.deactivateLocalization(LibraryProvider.Dictionary.GEODB0.nickname);
prop.put("geo0ActionDeactivated", 1);
}
if (post.containsKey("geo1Remove")) { if (post.containsKey("geo1Remove")) {
FileUtils.deletedelete(LibraryProvider.Dictionary.GEODB1.file()); FileUtils.deletedelete(LibraryProvider.Dictionary.GEODB1.file());
FileUtils.deletedelete(LibraryProvider.Dictionary.GEODB1.fileDisabled()); FileUtils.deletedelete(LibraryProvider.Dictionary.GEODB1.fileDisabled());
@ -246,11 +273,24 @@ public class DictionaryLoader_p {
prop.put("geo1ActionDeactivated", 1); prop.put("geo1ActionDeactivated", 1);
} }
if (post.containsKey("geo1Activate")) { if (post.containsKey("geo2Remove")) {
LibraryProvider.Dictionary.GEODB1.fileDisabled().renameTo(LibraryProvider.Dictionary.GEODB1.file()); FileUtils.deletedelete(LibraryProvider.Dictionary.GEODB2.file());
LibraryProvider.geoLoc.activateLocation(LibraryProvider.Dictionary.GEODB1.nickname, new OpenGeoDBLocation(LibraryProvider.Dictionary.GEODB1.file(), null)); FileUtils.deletedelete(LibraryProvider.Dictionary.GEODB2.fileDisabled());
LibraryProvider.geoLoc.deactivateLocalization(LibraryProvider.Dictionary.GEODB2.nickname);
prop.put("geo2ActionRemoved", 1);
}
if (post.containsKey("geo2Deactivate")) {
LibraryProvider.Dictionary.GEODB2.file().renameTo(LibraryProvider.Dictionary.GEODB2.fileDisabled());
LibraryProvider.geoLoc.deactivateLocalization(LibraryProvider.Dictionary.GEODB2.nickname);
prop.put("geo2ActionDeactivated", 1);
}
if (post.containsKey("geo2Activate")) {
LibraryProvider.Dictionary.GEODB2.fileDisabled().renameTo(LibraryProvider.Dictionary.GEODB2.file());
LibraryProvider.geoLoc.activateLocation(LibraryProvider.Dictionary.GEODB2.nickname, new OpenGeoDBLocation(LibraryProvider.Dictionary.GEODB2.file(), null));
LibraryProvider.autotagging.addPlaces(LibraryProvider.geoLoc); LibraryProvider.autotagging.addPlaces(LibraryProvider.geoLoc);
prop.put("geo1ActionActivated", 1); prop.put("geo2ActionActivated", 1);
} }
// DRW0 // DRW0

26
source/net/yacy/document/LibraryProvider.java
Unescape View File

@ -53,6 +53,8 @@ import net.yacy.kelondro.util.FileUtils;
public class LibraryProvider { public class LibraryProvider {
private final static ConcurrentLog LOG = new ConcurrentLog(LibraryProvider.class.getSimpleName());
public static final String path_to_source_dictionaries = "source"; public static final String path_to_source_dictionaries = "source";
public static final String path_to_did_you_mean_dictionaries = "didyoumean"; public static final String path_to_did_you_mean_dictionaries = "didyoumean";
public static final String path_to_autotagging_dictionaries = "autotagging"; public static final String path_to_autotagging_dictionaries = "autotagging";
@ -69,8 +71,12 @@ public class LibraryProvider {
private static File dictRoot = null; private static File dictRoot = null;
public static enum Dictionary { public static enum Dictionary {
/** Old OpenGeoDB dump */
GEODB0( "geo0", "http://downloads.sourceforge.net/project/opengeodb/Data/0.2.5a/opengeodb-0.2.5a-UTF8-sql.gz" ), GEODB0( "geo0", "http://downloads.sourceforge.net/project/opengeodb/Data/0.2.5a/opengeodb-0.2.5a-UTF8-sql.gz" ),
/** Old OpenGeoDB dump */
GEODB1( "geo1", "http://fa-technik.adfc.de/code/opengeodb/dump/opengeodb-02624_2011-10-17.sql.gz" ), GEODB1( "geo1", "http://fa-technik.adfc.de/code/opengeodb/dump/opengeodb-02624_2011-10-17.sql.gz" ),
/** Latest (2017) OpenGeoDB dump */
GEODB2( "geo2", "http://fa-technik.adfc.de/code/opengeodb/dump/opengeodb-02628_2017-02-07.sql.gz" ),
GEON0( "geon0", "http://download.geonames.org/export/dump/cities1000.zip" ), GEON0( "geon0", "http://download.geonames.org/export/dump/cities1000.zip" ),
GEON1( "geon1", "http://download.geonames.org/export/dump/cities5000.zip" ), GEON1( "geon1", "http://download.geonames.org/export/dump/cities5000.zip" ),
GEON2( "geon2", "http://download.geonames.org/export/dump/cities15000.zip" ), GEON2( "geon2", "http://download.geonames.org/export/dump/cities15000.zip" ),
@ -130,18 +136,24 @@ public class LibraryProvider {
} }
public static void integrateOpenGeoDB() { public static void integrateOpenGeoDB() {
final File geo2 = Dictionary.GEODB2.file();
final File geo1 = Dictionary.GEODB1.file(); final File geo1 = Dictionary.GEODB1.file();
final File geo0 = Dictionary.GEODB0.file(); final File geo0 = Dictionary.GEODB0.file();
if ( geo1.exists() ) { if (geo2.exists()) {
if ( geo0.exists() ) { if (geo1.exists() && !geo1.renameTo(Dictionary.GEODB1.fileDisabled())) {
geo0.renameTo(Dictionary.GEODB0.fileDisabled()); LOG.warn("Could not rename file " + geo1 + " to " + Dictionary.GEODB1.fileDisabled());
} }
geoLoc.activateLocation(Dictionary.GEODB1.nickname, new OpenGeoDBLocation(geo1, dymLib)); if (geo0.exists() && !geo0.renameTo(Dictionary.GEODB0.fileDisabled())) {
return; LOG.warn("Could not rename file " + geo0 + " to " + Dictionary.GEODB0.fileDisabled());
} }
if ( geo0.exists() ) { geoLoc.activateLocation(Dictionary.GEODB2.nickname, new OpenGeoDBLocation(geo2, dymLib));
} else if ( geo1.exists() ) {
if ( geo0.exists() && !geo0.renameTo(Dictionary.GEODB0.fileDisabled())) {
LOG.warn("Could not rename file " + geo0 + " to " + Dictionary.GEODB0.fileDisabled());
}
geoLoc.activateLocation(Dictionary.GEODB1.nickname, new OpenGeoDBLocation(geo1, dymLib));
} else if ( geo0.exists() ) {
geoLoc.activateLocation(Dictionary.GEODB0.nickname, new OpenGeoDBLocation(geo0, dymLib)); geoLoc.activateLocation(Dictionary.GEODB0.nickname, new OpenGeoDBLocation(geo0, dymLib));
return;
} }
} }

Write Preview
Loading…
Cancel
Save