From bec4dbc753b423dff749843ff2e2a3c50335b09d Mon Sep 17 00:00:00 2001 From: orbiter Date: Thu, 12 Jul 2007 16:23:33 +0000 Subject: [PATCH] added options and execution methods for automated updates git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3959 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/ConfigUpdate_p.html | 61 +++++++- htroot/ConfigUpdate_p.java | 147 +++++++++++------- htroot/Network.java | 2 +- htroot/Status.java | 2 +- htroot/Steering.java | 2 +- source/de/anomic/data/htmlTools.java | 1 + .../de/anomic/plasma/plasmaSwitchboard.java | 25 ++- source/de/anomic/yacy/yacyVersion.java | 71 ++++++++- source/yacy.java | 42 +---- yacy.init | 2 +- 10 files changed, 233 insertions(+), 122 deletions(-) diff --git a/htroot/ConfigUpdate_p.html b/htroot/ConfigUpdate_p.html index ab8d47cba..9c1a4cd0c 100644 --- a/htroot/ConfigUpdate_p.html +++ b/htroot/ConfigUpdate_p.html @@ -3,16 +3,18 @@ YaCy '#[clientname]#': System Update #%env/templates/metas.template%# - #%env/templates/header.template%# #%env/templates/submenuConfig.template%#

System Update

- #(candeploy)#:: + #(candeploy)# + This servlet can only be used on operation systems that are currently supported for deploy functions. + If you see this message this means that your operation sytem ist not supported. + ::
- System Update + Manual System Update
Available Releases
@@ -21,10 +23,7 @@ #{/availreleases}# -   

+   
Downloaded Releases
@@ -40,8 +39,54 @@
+ +
+
+ Automated System Update +
+ +
+ manual update   +
+
+ no automatic look-up, updates can be made manually using this interface (see options above)

+
+ +
+ automatic update   +
+
+ updates are made within fixed cycles: +
+
+
Time beween lookup
+
hours
+
Release blacklist
+
(regex on release number strings)
+
Release type
+
+ only main releases   + any release including developer releases +
+
+
+
+
+
#(configCommit)#::
Accepted Changes.
#(/configCommit)#
+
+
+
+ +
+ System Update Statistics +
+
Last System Lookup

#(lastlookup)#never::#[time]##(/lastlookup)#

+
Last Release Download

#(lastdownload)#never::#[time]##(/lastdownload)#

+
Last Deploy

#(lastdeploy)#never::#[time]##(/lastdeploy)#

+
+
#(/candeploy)# #%env/templates/footer.template%# - + \ No newline at end of file diff --git a/htroot/ConfigUpdate_p.java b/htroot/ConfigUpdate_p.java index 1c8f9bafa..a4d0d426f 100644 --- a/htroot/ConfigUpdate_p.java +++ b/htroot/ConfigUpdate_p.java @@ -26,6 +26,7 @@ import java.io.File; import java.io.IOException; +import java.util.Date; import java.util.Iterator; import java.util.TreeSet; @@ -44,6 +45,8 @@ public class ConfigUpdate_p { final serverObjects prop = new serverObjects(); final plasmaSwitchboard sb = (plasmaSwitchboard) env; + prop.put("candeploy_configCommit", 0); + if (post != null) { if (post.containsKey("downloadRelease")) { // download a release @@ -57,79 +60,103 @@ public class ConfigUpdate_p { } } } - + + if (post.containsKey("configSubmit")) { + prop.put("candeploy_configCommit", 1); + sb.setConfig("update.process", (post.get("updateMode", "manual").equals("manual")) ? "manual" : "auto"); + sb.setConfig("update.cycle", Math.max(12, post.getLong("cycle", 168))); + sb.setConfig("update.blacklist", post.get("blacklist", "")); + sb.setConfig("update.concept", (post.get("releaseType", "any").equals("any")) ? "any" : "main"); + } } - + + // set if this should be visible + if (serverSystem.canExecUnix) { + // we can deploy a new system with (i.e.) + // cd DATA/RELEASE;tar xfz $1;cp -Rf yacy/* ../../;rm -Rf yacy + prop.put("candeploy", 1); + } else { + prop.put("candeploy", 0); + } + // version information - String versionstring = yacy.combined2prettyVersion(sb.getConfig("version","0.1")); + String versionstring = yacyVersion.combined2prettyVersion(sb.getConfig("version","0.1")); prop.put("versionpp", versionstring); boolean devenvironment = versionstring.startsWith("dev"); double thisVersion = Double.parseDouble(sb.getConfig("version","0.1")); // cut off the SVN Rev in the Version try {thisVersion = Math.round(thisVersion*1000.0)/1000.0;} catch (NumberFormatException e) {} - if (serverSystem.canExecUnix) { - // we can deploy a new system with (i.e.) - // cd DATA/RELEASE;tar xfz $1;cp -Rf yacy/* ../../;rm -Rf yacy - // list downloaded releases - yacyVersion release, dflt; - String[] downloaded = sb.releasePath.list(); - prop.put("candeploy", 1); - prop.put("candeploy_deployenabled", (downloaded.length == 0) ? 0 : ((devenvironment) ? 1 : 2)); // prevent that a developer-version is over-deployed + // list downloaded releases + yacyVersion release, dflt; + String[] downloaded = sb.releasePath.list(); - TreeSet downloadedreleases = new TreeSet(); - for (int j = 0; j < downloaded.length; j++) { - try { - release = (yacyVersion) new yacyVersion(downloaded[j]); - downloadedreleases.add(release); - } catch (RuntimeException e) { - // not a valid release - new File(sb.releasePath, downloaded[j]).deleteOnExit(); // can be also a restart- or deploy-file - } + prop.put("candeploy_deployenabled", (downloaded.length == 0) ? 0 : ((devenvironment) ? 1 : 2)); // prevent that a developer-version is over-deployed + + TreeSet downloadedreleases = new TreeSet(); + for (int j = 0; j < downloaded.length; j++) { + try { + release = (yacyVersion) new yacyVersion(downloaded[j]); + downloadedreleases.add(release); + } catch (RuntimeException e) { + // not a valid release + new File(sb.releasePath, downloaded[j]).deleteOnExit(); // can be also a restart- or deploy-file } - dflt = (downloadedreleases.size() == 0) ? null : (yacyVersion) downloadedreleases.last(); - Iterator i = downloadedreleases.iterator(); - int relcount = 0; - while (i.hasNext()) { - release = (yacyVersion) i.next(); - prop.put("candeploy_downloadedreleases_" + relcount + "_name", (release.proRelease ? "pro" : "standard") + "/" + ((release.mainRelease) ? "main" : "dev") + " " + release.releaseNr + "/" + release.svn); - prop.put("candeploy_downloadedreleases_" + relcount + "_file", release.name); - prop.put("candeploy_downloadedreleases_" + relcount + "_selected", (release == dflt) ? 1 : 0); - relcount++; - } - prop.put("candeploy_downloadedreleases", relcount); + } + dflt = (downloadedreleases.size() == 0) ? null : (yacyVersion) downloadedreleases.last(); + Iterator i = downloadedreleases.iterator(); + int relcount = 0; + while (i.hasNext()) { + release = (yacyVersion) i.next(); + prop.put("candeploy_downloadedreleases_" + relcount + "_name", (release.proRelease ? "pro" : "standard") + "/" + ((release.mainRelease) ? "main" : "dev") + " " + release.releaseNr + "/" + release.svn); + prop.put("candeploy_downloadedreleases_" + relcount + "_file", release.name); + prop.put("candeploy_downloadedreleases_" + relcount + "_selected", (release == dflt) ? 1 : 0); + relcount++; + } + prop.put("candeploy_downloadedreleases", relcount); - // list remotely available releases - TreeSet[] releasess = yacyVersion.allReleases(false); // {0=promain, 1=prodev, 2=stdmain, 3=stddev} - relcount = 0; - // main - TreeSet releases = releasess[(yacy.pro) ? 0 : 2]; - releases.removeAll(downloadedreleases); - i = releases.iterator(); - while (i.hasNext()) { - release = (yacyVersion) i.next(); - prop.put("candeploy_availreleases_" + relcount + "_name", (release.proRelease ? "pro" : "standard") + "/" + ((release.mainRelease) ? "main" : "dev") + " " + release.releaseNr + "/" + release.svn); - prop.put("candeploy_availreleases_" + relcount + "_url", release.url.toString()); - prop.put("candeploy_availreleases_" + relcount + "_selected", 0); - relcount++; - } - // dev - dflt = (releasess[(yacy.pro) ? 1 : 3].size() == 0) ? null : (yacyVersion) releasess[(yacy.pro) ? 1 : 3].last(); - releases = releasess[(yacy.pro) ? 1 : 3]; - releases.removeAll(downloadedreleases); - i = releases.iterator(); - while (i.hasNext()) { - release = (yacyVersion) i.next(); - prop.put("candeploy_availreleases_" + relcount + "_name", (release.proRelease ? "pro" : "standard") + "/" + ((release.mainRelease) ? "main" : "dev") + " " + release.releaseNr + "/" + release.svn); - prop.put("candeploy_availreleases_" + relcount + "_url", release.url.toString()); - prop.put("candeploy_availreleases_" + relcount + "_selected", (release == dflt) ? 1 : 0); - relcount++; - } - prop.put("candeploy_availreleases", relcount); - } else { - prop.put("candeploy", 0); + // list remotely available releases + TreeSet[] releasess = yacyVersion.allReleases(false); // {0=promain, 1=prodev, 2=stdmain, 3=stddev} + relcount = 0; + // main + TreeSet releases = releasess[(yacy.pro) ? 0 : 2]; + releases.removeAll(downloadedreleases); + i = releases.iterator(); + while (i.hasNext()) { + release = (yacyVersion) i.next(); + prop.put("candeploy_availreleases_" + relcount + "_name", (release.proRelease ? "pro" : "standard") + "/" + ((release.mainRelease) ? "main" : "dev") + " " + release.releaseNr + "/" + release.svn); + prop.put("candeploy_availreleases_" + relcount + "_url", release.url.toString()); + prop.put("candeploy_availreleases_" + relcount + "_selected", 0); + relcount++; + } + // dev + dflt = (releasess[(yacy.pro) ? 1 : 3].size() == 0) ? null : (yacyVersion) releasess[(yacy.pro) ? 1 : 3].last(); + releases = releasess[(yacy.pro) ? 1 : 3]; + releases.removeAll(downloadedreleases); + i = releases.iterator(); + while (i.hasNext()) { + release = (yacyVersion) i.next(); + prop.put("candeploy_availreleases_" + relcount + "_name", (release.proRelease ? "pro" : "standard") + "/" + ((release.mainRelease) ? "main" : "dev") + " " + release.releaseNr + "/" + release.svn); + prop.put("candeploy_availreleases_" + relcount + "_url", release.url.toString()); + prop.put("candeploy_availreleases_" + relcount + "_selected", (release == dflt) ? 1 : 0); + relcount++; } + prop.put("candeploy_availreleases", relcount); + + // properties for automated system update + prop.put("candeploy_manualUpdateChecked", (sb.getConfig("update.process", "manual").equals("manual")) ? 1 : 0); + prop.put("candeploy_autoUpdateChecked", (sb.getConfig("update.process", "manual").equals("auto")) ? 1 : 0); + prop.put("candeploy_cycle", sb.getConfigLong("update.cycle", 168)); + prop.put("candeploy_blacklist", sb.getConfig("update.blacklist", "")); + prop.put("candeploy_releaseTypeMainChecked", (sb.getConfig("update.concept", "any").equals("any")) ? 0 : 1); + prop.put("candeploy_releaseTypeAnyChecked", (sb.getConfig("update.concept", "any").equals("any")) ? 1 : 0); + prop.put("candeploy_lastlookup", (sb.getConfigLong("update.time.lookup", 0) == 0) ? 0 : 1); + prop.put("candeploy_lastlookup_time", new Date(sb.getConfigLong("update.time.lookup", 0)).toString()); + prop.put("candeploy_lastdownload", (sb.getConfigLong("update.time.download", 0) == 0) ? 0 : 1); + prop.put("candeploy_lastdownload_time", new Date(sb.getConfigLong("update.time.download", 0)).toString()); + prop.put("candeploy_lastdeploy", (sb.getConfigLong("update.time.deploy", 0) == 0) ? 0 : 1); + prop.put("candeploy_lastdeploy_time", new Date(sb.getConfigLong("update.time.deploy", 0)).toString()); /* if ((adminaccess) && (yacyVersion.latestRelease >= (thisVersion+0.01))) { // only new Versions(not new SVN) diff --git a/htroot/Network.java b/htroot/Network.java index 094866d13..47be416d9 100644 --- a/htroot/Network.java +++ b/htroot/Network.java @@ -466,7 +466,7 @@ public class Network { String peertags = serverCodings.set2string(seed.getPeerTags(), ",", false); prop.put(STR_TABLE_LIST + conCount + "_dhtreceive_peertags", ((peertags == null) || (peertags.length() == 0)) ? "no tags given" : ("tags = " + peertags)); } - prop.put(STR_TABLE_LIST + conCount + "_version", yacy.combined2prettyVersion(seed.get(yacySeed.VERSION, "0.1"), shortname)); + prop.put(STR_TABLE_LIST + conCount + "_version", yacyVersion.combined2prettyVersion(seed.get(yacySeed.VERSION, "0.1"), shortname)); prop.put(STR_TABLE_LIST + conCount + "_lastSeen", /*seed.getLastSeenString() + " " +*/ lastseen); prop.put(STR_TABLE_LIST + conCount + "_utc", seed.get(yacySeed.UTC, "-")); prop.put(STR_TABLE_LIST + conCount + "_uptime", serverDate.intervalToString(60000 * Long.parseLong(seed.get(yacySeed.UPTIME, "0")))); diff --git a/htroot/Status.java b/htroot/Status.java index a2c061023..90b8fc7c7 100644 --- a/htroot/Status.java +++ b/htroot/Status.java @@ -154,7 +154,7 @@ public class Status { } // version information - String versionstring = yacy.combined2prettyVersion(sb.getConfig("version","0.1")); + String versionstring = yacyVersion.combined2prettyVersion(sb.getConfig("version","0.1")); prop.put("versionpp", versionstring); double thisVersion = Double.parseDouble(sb.getConfig("version","0.1")); diff --git a/htroot/Steering.java b/htroot/Steering.java index 8c18766f5..c46c7021f 100644 --- a/htroot/Steering.java +++ b/htroot/Steering.java @@ -84,7 +84,7 @@ public class Steering { } if (post.containsKey("update")) { - boolean devenvironment = yacy.combined2prettyVersion(sb.getConfig("version","0.1")).startsWith("dev"); + boolean devenvironment = yacyVersion.combined2prettyVersion(sb.getConfig("version","0.1")).startsWith("dev"); String releaseFileName = post.get("releaseinstall", ""); File releaseFile = new File(sb.getRootPath(), "DATA/RELEASE/" + releaseFileName); if ((!devenvironment) && (releaseFile.length() > 0) && (releaseFile.exists())) { diff --git a/source/de/anomic/data/htmlTools.java b/source/de/anomic/data/htmlTools.java index 4249fa82a..ed3b11edb 100644 --- a/source/de/anomic/data/htmlTools.java +++ b/source/de/anomic/data/htmlTools.java @@ -95,6 +95,7 @@ public class htmlTools { //patterns that will be replaced. To add new codes or patterns, just put them at the end //of the list. Codes or patterns in this list can not be escaped with [= or 
     public static final String[] htmlentities={
+        "\\",    "\",  // Backslash
         "\u005E","^",  // Caret
 
         "\u0060","`",  // Accent Grave `
diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java
index f7c345fb0..fe7b7853e 100644
--- a/source/de/anomic/plasma/plasmaSwitchboard.java
+++ b/source/de/anomic/plasma/plasmaSwitchboard.java
@@ -2015,7 +2015,28 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
                 for(int i=0;i System.currentTimeMillis()) return null; // no we have recently made a lookup
         
-        // check if we know that there is a release that is more recent that that which we are using
+        // check if we know that there is a release that is more recent than that which we are using
         TreeSet[] releasess = yacyVersion.allReleases(true); // {0=promain, 1=prodev, 2=stdmain, 3=stddev}
         boolean pro = new File(sb.getRootPath(), "libx").exists();
         yacyVersion latestmain = (yacyVersion) releasess[(pro) ? 0 : 2].first();
@@ -249,12 +254,13 @@ public final class yacyVersion implements Comparator, Comparable {
         // {promainreleases, prodevreleases, stdmainreleases, stddevreleases} 
         TreeSet[] latestRelease = (TreeSet[]) latestReleases.get(location);
         if (force ||
-            (latestRelease == null) ||
+            (latestRelease == null) /*||
             ((latestRelease[0].size() == 0) &&
              (latestRelease[1].size() == 0) &&
              (latestRelease[2].size() == 0) &&
-             (latestRelease[3].size() == 0) )) {
+             (latestRelease[3].size() == 0) )*/) {
             latestRelease = allReleaseFrom(location);
+            latestReleases.put(location, latestRelease);
         }
         return latestRelease;
     }
@@ -388,4 +394,53 @@ public final class yacyVersion implements Comparator, Comparable {
             serverLog.logSevere("UPDATE", "update failed", e);
         }
     }
+    
+    /**
+     * Converts combined version-string to a pretty string, e.g. "0.435/01818" or "dev/01818" (development version) or "dev/00000" (in case of wrong input)
+     *
+     * @param ver Combined version string matching regular expression:  "\A(\d+\.\d{3})(\d{4}|\d{5})\z" 

+     *  (i.e.: start of input, 1 or more digits in front of decimal point, decimal point followed by 3 digits as major version, 4 or 5 digits for SVN-Version, end of input) 
+     * @return If the major version is < 0.11  - major version is separated from SVN-version by '/', e.g. "0.435/01818" 

+     *         If the major version is >= 0.11 - major version is replaced by "dev" and separated SVN-version by '/', e.g."dev/01818" 
 
+     *         "dev/00000" - If the input does not matcht the regular expression above 
+     */
+     public static String combined2prettyVersion(String ver) {
+         return combined2prettyVersion(ver, "");
+     }
+     public static String combined2prettyVersion(String ver, String computerName) {
+         final Matcher matcher = Pattern.compile("\\A(\\d+\\.\\d{1,3})(\\d{0,5})\\z").matcher(ver); 
+
+         if (!matcher.find()) { 
+             serverLog.logWarning("STARTUP", "Peer '"+computerName+"': wrong format of version-string: '" + ver + "'. Using default string 'dev/00000' instead");   
+             return "dev/00000";
+         }
+         
+         String mainversion = (Double.parseDouble(matcher.group(1)) < 0.11 ? "dev" : matcher.group(1));
+        String revision = matcher.group(2);
+        for(int i=revision.length();i<5;++i) revision += "0";
+        return mainversion+"/"+revision;
+     }
+        
+     /**
+     * Combines the version of YaCy with the versionnumber from SVN to a
+     * combined version
+     *
+     * @param version Current given version.
+     * @param svn Current version given from SVN.
+     * @return String with the combined version.
+     */
+     public static double versvn2combinedVersion(double v, int svn) {
+        return (Math.rint((v*100000000.0) + ((double)svn))/100000000);
+     }
+     
+     public static void main(String[] args) {
+         float base = (float) 0.53;
+         String blacklist = "....[123]";
+         String test;
+         for (int i = 0; i < 20; i++) {
+             test = Float.toString(base + (((float) i) / 1000));
+             System.out.println(test + " is " + ((test.matches(blacklist)) ? "blacklisted" : " not blacklisted"));
+         }
+     }
+     
 }
diff --git a/source/yacy.java b/source/yacy.java
index c857ed424..90e530a65 100644
--- a/source/yacy.java
+++ b/source/yacy.java
@@ -160,44 +160,6 @@ public final class yacy {
      * until startup has finished
      */
     private static serverSemaphore startupFinishedSync = new serverSemaphore(0);
-    
-    /**
-    * Converts combined version-string to a pretty string, e.g. "0.435/01818" or "dev/01818" (development version) or "dev/00000" (in case of wrong input)
-    *
-    * @param ver Combined version string matching regular expression:  "\A(\d+\.\d{3})(\d{4}|\d{5})\z" 

-    *  (i.e.: start of input, 1 or more digits in front of decimal point, decimal point followed by 3 digits as major version, 4 or 5 digits for SVN-Version, end of input) 
-    * @return If the major version is < 0.11  - major version is separated from SVN-version by '/', e.g. "0.435/01818" 

-    *         If the major version is >= 0.11 - major version is replaced by "dev" and separated SVN-version by '/', e.g."dev/01818" 
 
-    *         "dev/00000" - If the input does not matcht the regular expression above 
-    */
-    public static String combined2prettyVersion(String ver) {
-        return combined2prettyVersion(ver, "");
-    }
-    public static String combined2prettyVersion(String ver, String computerName) {
-        final Matcher matcher = Pattern.compile("\\A(\\d+\\.\\d{1,3})(\\d{0,5})\\z").matcher(ver); 
-
-        if (!matcher.find()) { 
-            serverLog.logWarning("STARTUP", "Peer '"+computerName+"': wrong format of version-string: '" + ver + "'. Using default string 'dev/00000' instead");   
-            return "dev/00000";
-        }
-        
-        String mainversion = (Double.parseDouble(matcher.group(1)) < 0.11 ? "dev" : matcher.group(1));
-		String revision = matcher.group(2);
-		for(int i=revision.length();i<5;++i) revision += "0";
-		return mainversion+"/"+revision;
-    }
-       
-    /**
-    * Combines the version of YaCy with the versionnumber from SVN to a
-    * combined version
-    *
-    * @param version Current given version.
-    * @param svn Current version given from SVN.
-    * @return String with the combined version.
-    */
-    public static double versvn2combinedVersion(double v, int svn) {
-    	return (Math.rint((v*100000000.0) + ((double)svn))/100000000);
-    }
 
     /**
     * Starts up the whole application. Sets up all datastructures and starts
@@ -294,7 +256,7 @@ public final class yacy {
                         final String svrReleaseNr = matcher.group(1);
                         try {
                             try {version = Double.parseDouble(vString);} catch (NumberFormatException e) {version = (float) 0.1;}
-                            version = versvn2combinedVersion(version, Integer.parseInt(svrReleaseNr));
+                            version = yacyVersion.versvn2combinedVersion(version, Integer.parseInt(svrReleaseNr));
                         } catch (NumberFormatException e) {}
                         sb.setConfig("svnRevision", svrReleaseNr);
                     }
@@ -305,7 +267,7 @@ public final class yacy {
             }
 
             sb.setConfig("version", Double.toString(version));
-            sb.setConfig("vString", combined2prettyVersion(Double.toString(version)));
+            sb.setConfig("vString", yacyVersion.combined2prettyVersion(Double.toString(version)));
             sb.setConfig("vdate", vDATE);
             sb.setConfig("applicationRoot", homePath);
             sb.startupTime = startup;
diff --git a/yacy.init b/yacy.init
index ad96d050b..828271248 100644
--- a/yacy.init
+++ b/yacy.init
@@ -86,7 +86,7 @@ update.cycle = 168
 # a version number blacklist can restrict automatic or guided updates to a specific
 # range of version numbers. The restriction is done with a blacklist (standard regexpr)
 # It is recommended to set this list to low developer version numbers
-update.blacklist = .\...[123]
+update.blacklist = ....[123]
 # a update can also restricted with a concept property, which can decide if an
 # update is only valid if it either is a main release or any svn release including new development releases
 # Valid keywords are 'main' and 'any'