diff --git a/build.xml b/build.xml index 3f0ffc0b1..17a8679d4 100644 --- a/build.xml +++ b/build.xml @@ -1103,7 +1103,7 @@ replace="yacy (*auto-svn-version*) unstable; urgency=low" /> - + @@ -1111,4 +1111,20 @@ + + + + + + + + + + + + + + + + diff --git a/htroot/ConfigUpdate_p.java b/htroot/ConfigUpdate_p.java index 0afafd5f9..1625417b6 100644 --- a/htroot/ConfigUpdate_p.java +++ b/htroot/ConfigUpdate_p.java @@ -27,7 +27,6 @@ import java.io.File; import java.io.IOException; import java.util.Date; -import java.util.Iterator; import java.util.TreeSet; import de.anomic.http.httpRequestHeader; @@ -63,7 +62,19 @@ public class ConfigUpdate_p { final String release = post.get("releasedownload", ""); if (release.length() > 0) { try { - yacyVersion.downloadRelease(new yacyVersion(new yacyURL(release, null))); + yacyVersion versionToDownload = new yacyVersion(new yacyURL(release, null)); + // replace this version with version which contains public key + yacyVersion.DevAndMainVersions releases = yacyVersion.allReleases(false); + if(versionToDownload.mainRelease) { + yacyVersion repVersionToDownload = releases.main.ceiling(versionToDownload); + if(repVersionToDownload.equals(versionToDownload)) + versionToDownload = repVersionToDownload; + } else { + yacyVersion repVersionToDownload = releases.dev.ceiling(versionToDownload); + if(repVersionToDownload.equals(versionToDownload)) + versionToDownload = repVersionToDownload; + } + versionToDownload.downloadRelease(); } catch (final IOException e) { // TODO Auto-generated catch block e.printStackTrace(); @@ -92,14 +103,14 @@ public class ConfigUpdate_p { } else { // there is a version that is more recent. Load it and re-start with it sb.getLog().logInfo("AUTO-UPDATE: downloading more recent release " + updateVersion.url); - final File downloaded = yacyVersion.downloadRelease(updateVersion); + final File downloaded = updateVersion.downloadRelease(); prop.putHTML("candeploy_autoUpdate_downloadedRelease", updateVersion.name); final boolean devenvironment = yacyVersion.combined2prettyVersion(sb.getConfig("version","0.1")).startsWith("dev"); if (devenvironment) { sb.getLog().logInfo("AUTO-UPDATE: omiting update because this is a development environment"); prop.put("candeploy_autoUpdate", "3"); } else if ((downloaded == null) || (!downloaded.exists()) || (downloaded.length() == 0)) { - sb.getLog().logInfo("AUTO-UPDATE: omiting update because download failed (file cannot be found or is too small)"); + sb.getLog().logInfo("AUTO-UPDATE: omiting update because download failed (file cannot be found, is too small or signature was bad)"); prop.put("candeploy_autoUpdate", "4"); } else { yacyVersion.deployRelease(downloaded); @@ -138,29 +149,27 @@ public class ConfigUpdate_p { // list downloaded releases - yacyVersion release, dflt; - final String[] downloaded = sb.releasePath.list(); + final File[] downloadedFiles = sb.releasePath.listFiles(); - prop.put("candeploy_deployenabled", (downloaded.length == 0) ? "0" : ((devenvironment) ? "1" : "2")); // prevent that a developer-version is over-deployed + prop.put("candeploy_deployenabled", (downloadedFiles.length == 0) ? "0" : ((devenvironment) ? "1" : "2")); // prevent that a developer-version is over-deployed - final TreeSet downloadedreleases = new TreeSet(); - for (int j = 0; j < downloaded.length; j++) { + final TreeSet downloadedReleases = new TreeSet(); + for(File downloaded : downloadedFiles) { try { - release = new yacyVersion(downloaded[j]); - downloadedreleases.add(release); + yacyVersion release = new yacyVersion(downloaded); + downloadedReleases.add(release); } catch (final RuntimeException e) { // not a valid release // can be also a restart- or deploy-file - final File invalid = new File(sb.releasePath, downloaded[j]); + final File invalid = downloaded; if (!(invalid.getName().endsWith(".bat") || invalid.getName().endsWith(".sh"))) // Windows & Linux don't like deleted scripts while execution! invalid.deleteOnExit(); } } - dflt = (downloadedreleases.size() == 0) ? null : downloadedreleases.last(); - Iterator i = downloadedreleases.iterator(); + // latest downloaded release + yacyVersion dflt = (downloadedReleases.size() == 0) ? null : downloadedReleases.last(); int relcount = 0; - while (i.hasNext()) { - release = i.next(); + for(yacyVersion release : downloadedReleases) { prop.put("candeploy_downloadedreleases_" + relcount + "_name", ((release.mainRelease) ? "main" : "dev") + " " + release.releaseNr + "/" + release.svn); prop.putHTML("candeploy_downloadedreleases_" + relcount + "_file", release.name); prop.put("candeploy_downloadedreleases_" + relcount + "_selected", (release == dflt) ? "1" : "0"); @@ -169,15 +178,13 @@ public class ConfigUpdate_p { prop.put("candeploy_downloadedreleases", relcount); // list remotely available releases - final yacyVersion.DevMain releasess = yacyVersion.allReleases(false); + final yacyVersion.DevAndMainVersions releasess = yacyVersion.allReleases(false); relcount = 0; // main - TreeSet releases = releasess.main; - releases.removeAll(downloadedreleases); - i = releases.iterator(); - while (i.hasNext()) { - release = i.next(); + final TreeSet remoteMainReleases = releasess.main; + remoteMainReleases.removeAll(downloadedReleases); + for (yacyVersion release : remoteMainReleases) { prop.put("candeploy_availreleases_" + relcount + "_name", ((release.mainRelease) ? "main" : "dev") + " " + release.releaseNr + "/" + release.svn); prop.put("candeploy_availreleases_" + relcount + "_url", release.url.toString()); prop.put("candeploy_availreleases_" + relcount + "_selected", "0"); @@ -186,11 +193,9 @@ public class ConfigUpdate_p { // dev dflt = (releasess.dev.size() == 0) ? null : releasess.dev.last(); - releases = releasess.dev; - releases.removeAll(downloadedreleases); - i = releases.iterator(); - while (i.hasNext()) { - release = i.next(); + final TreeSet remoteDevReleases = releasess.dev; + remoteDevReleases.removeAll(downloadedReleases); + for(yacyVersion release : remoteDevReleases) { prop.put("candeploy_availreleases_" + relcount + "_name", ((release.mainRelease) ? "main" : "dev") + " " + release.releaseNr + "/" + release.svn); prop.put("candeploy_availreleases_" + relcount + "_url", release.url.toString()); prop.put("candeploy_availreleases_" + relcount + "_selected", (release == dflt) ? "1" : "0"); diff --git a/source/de/anomic/kelondro/util/FileUtils.java b/source/de/anomic/kelondro/util/FileUtils.java index 6ef9ada5b..e905c9ce2 100644 --- a/source/de/anomic/kelondro/util/FileUtils.java +++ b/source/de/anomic/kelondro/util/FileUtils.java @@ -573,6 +573,21 @@ public final class FileUtils { } return v; } + public static ArrayList strings(final Reader reader) { + if (reader == null) return new ArrayList(); + BufferedReader bufreader = new BufferedReader(reader); + final ArrayList list = new ArrayList(); + String line = null; + try { + while ((line = bufreader.readLine()) != null) { + list.add(line); + } + } catch (IOException e) { + e.printStackTrace(); + return null; + } + return list; + } /** diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index 3e8812290..74fab373f 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -93,6 +93,9 @@ import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.lang.reflect.Constructor; import java.net.MalformedURLException; +import java.security.NoSuchAlgorithmException; +import java.security.PublicKey; +import java.security.spec.InvalidKeySpecException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; @@ -145,6 +148,7 @@ import de.anomic.http.httpdRobotsTxtConfig; import de.anomic.kelondro.order.Digest; import de.anomic.kelondro.order.NaturalOrder; import de.anomic.kelondro.text.metadataPrototype.URLMetadataRow; +import de.anomic.kelondro.order.Base64Order; import de.anomic.kelondro.util.DateFormatter; import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.Log; @@ -168,6 +172,7 @@ import de.anomic.server.serverSemaphore; import de.anomic.server.serverSwitch; import de.anomic.server.serverThread; import de.anomic.tools.crypt; +import de.anomic.tools.CryptoLib; import de.anomic.xml.SurrogateReader; import de.anomic.yacy.yacyClient; import de.anomic.yacy.yacyCore; @@ -176,6 +181,7 @@ import de.anomic.yacy.yacyNewsRecord; import de.anomic.yacy.yacySeed; import de.anomic.yacy.yacyTray; import de.anomic.yacy.yacyURL; +import de.anomic.yacy.yacyUpdateLocation; import de.anomic.yacy.yacyVersion; import de.anomic.yacy.dht.Dispatcher; import de.anomic.yacy.dht.PeerSelection; @@ -706,17 +712,38 @@ public final class plasmaSwitchboard extends serverAbstractSwitch +// first published 16.04.2009 on http://yacy.net +// +// This is a part of YaCy, a peer-to-peer based web search engine +// +// $LastChangedDate: 2009-03-30 17:31:25 +0200 (Mo, 30. Mär 2009) $ +// $LastChangedRevision: 5756 $ +// $LastChangedBy: orbiter $ +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package de.anomic.tools; + +import java.io.IOException; +import java.io.FilterOutputStream; +import java.io.OutputStream; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.security.PublicKey; +import java.security.Signature; +import java.security.SignatureException; + +/** + * A SignatureOuputStream is composed of a Signature and a OutputStream so that + * write()-methods first update the signature and then pass the data to the + * underlying OutputStream. + * + * @author flori + * + */ +public class SignatureOutputStream extends FilterOutputStream { + + private Signature signature; + + /** + * create new SignatureOutputStream and setup the Signature + * @param stream OutputStream to pass data on + * @param algorithm Algorithm to use for Signature + * @param publicKey Public key to verify Signature against + * @throws NoSuchAlgorithmException + */ + public SignatureOutputStream(OutputStream stream, String algorithm, PublicKey publicKey) throws NoSuchAlgorithmException { + super(stream); + try { + signature = Signature.getInstance(algorithm); + signature.initVerify(publicKey); + } catch (InvalidKeyException e) { + System.out.println("Internal Error at signature:" + e.getMessage()); + } + } + + /** + * write byte + * @see FilterOutputStream.write(int b) + */ + public void write(int b) throws IOException { + try { + signature.update((byte)b); + } catch (SignatureException e) { + throw new IOException("Signature update failed: "+ e.getMessage()); + } + out.write(b); + } + + public void write(byte[] b, int off, int len) throws IOException { + try { + signature.update(b, off, len); + } catch (SignatureException e) { + throw new IOException("Signature update failed: "+ e.getMessage()); + } + out.write(b, off, len); + } + + /** + * verify signature, don't use this stream for another signature afterwards + * @param sign signature as bytes + * @return true, when signature was right + * @throws SignatureException + */ + public boolean verify(byte[] sign) throws SignatureException { + return signature.verify(sign); + } + +} diff --git a/source/de/anomic/yacy/yacyUpdateLocation.java b/source/de/anomic/yacy/yacyUpdateLocation.java new file mode 100644 index 000000000..dac18991b --- /dev/null +++ b/source/de/anomic/yacy/yacyUpdateLocation.java @@ -0,0 +1,52 @@ +// yacyUpdateLocation.java +// ---------------- +// (C) 2009 by Florian Richter +// first published 5.03.2009 on http://yacy.net +// +// This is a part of YaCy, a peer-to-peer based web search engine +// +// $LastChangedDate: 2009-02-19 17:24:46 +0100 (Do, 19. Feb 2009) $ +// $LastChangedRevision: 5621 $ +// $LastChangedBy: orbiter $ +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package de.anomic.yacy; + +import java.security.PublicKey; + + +/** + * Holds a update location with url and public key + * + */ +public class yacyUpdateLocation { + private yacyURL locationURL; + private PublicKey publicKey; + + public yacyUpdateLocation(yacyURL locationURL, PublicKey publicKey) { + this.locationURL = locationURL; + this.publicKey = publicKey; + } + + public yacyURL getLocationURL() { + return this.locationURL; + } + public PublicKey getPublicKey() { + return this.publicKey; + } +} diff --git a/source/de/anomic/yacy/yacyVersion.java b/source/de/anomic/yacy/yacyVersion.java index 524c2fa53..5ba2a1ca9 100644 --- a/source/de/anomic/yacy/yacyVersion.java +++ b/source/de/anomic/yacy/yacyVersion.java @@ -32,10 +32,13 @@ import java.io.BufferedOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.security.NoSuchAlgorithmException; +import java.security.PublicKey; +import java.security.SignatureException; import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; -import java.util.Iterator; import java.util.Map; import java.util.SortedSet; import java.util.TreeSet; @@ -48,11 +51,14 @@ import de.anomic.http.httpClient; import de.anomic.http.httpResponse; import de.anomic.http.httpResponseHeader; import de.anomic.http.httpRequestHeader; +import de.anomic.kelondro.order.Base64Order; import de.anomic.kelondro.util.Log; import de.anomic.kelondro.util.FileUtils; import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverCore; import de.anomic.server.serverSystem; +import de.anomic.tools.CryptoLib; +import de.anomic.tools.SignatureOutputStream; import de.anomic.tools.tarTools; public final class yacyVersion implements Comparator, Comparable { @@ -71,8 +77,8 @@ public final class yacyVersion implements Comparator, Comparable latestReleases = new HashMap(); - public final static ArrayList latestReleaseLocations = new ArrayList(); // will be initialized with value in defaults/yacy.network.freeworld.unit + private static HashMap latestReleases = new HashMap(); + public final static ArrayList latestReleaseLocations = new ArrayList(); // will be initialized with value in defaults/yacy.network.freeworld.unit // private static release info about this release; is generated only once and can be retrieved by thisVersion() private static yacyVersion thisVersion = null; @@ -84,12 +90,25 @@ public final class yacyVersion implements Comparator, Comparable, Comparable dev, main; - public DevMain(final TreeSet dev, final TreeSet main) { + public DevAndMainVersions(final TreeSet dev, final TreeSet main) { this.dev = dev; this.main = main; } @@ -213,9 +232,9 @@ public final class yacyVersion implements Comparator, Comparable, Comparable, Comparable, Comparable anchors = scraper.getAnchors(); // a url (String) / name (String) relation - final Iterator i = anchors.keySet().iterator(); - final TreeSet devreleases = new TreeSet(); - final TreeSet mainreleases = new TreeSet(); - yacyVersion release; - while (i.hasNext()) { - url = i.next(); + final TreeSet mainReleases = new TreeSet(); + final TreeSet devReleases = new TreeSet(); + for(yacyURL url : anchors.keySet()) { try { - release = new yacyVersion(url); + yacyVersion release = new yacyVersion(url, location.getPublicKey()); //System.out.println("r " + release.toAnchor()); - if ( release.mainRelease) mainreleases.add(release); - if (!release.mainRelease) devreleases.add(release); + if(release.mainRelease) { + mainReleases.add(release); + } else { + devReleases.add(release); + } } catch (final RuntimeException e) { // the release string was not well-formed. // that might have been another link - // just dont care + // just don't care continue; } } plasmaSwitchboard.getSwitchboard().setConfig("update.time.lookup", System.currentTimeMillis()); - return new DevMain(devreleases, mainreleases); + return new DevAndMainVersions(devReleases, mainReleases); } - public static File downloadRelease(final yacyVersion release) { + /** + *

download this release and if public key is know, download signature and check it. + *

The signature is named $releaseurl.sig and contains the base64 encoded signature + * (@see de.anomic.tools.CryptoLib) + * @return file object of release file, null in case of failure + */ + public File downloadRelease() { final File storagePath = plasmaSwitchboard.getSwitchboard().releasePath; - // load file File download = null; - final httpRequestHeader header = new httpRequestHeader(); - header.put(httpResponseHeader.USER_AGENT, HTTPLoader.yacyUserAgent); - final httpClient client = new httpClient(120000, header); + // setup httpClient + final httpRequestHeader reqHeader = new httpRequestHeader(); + reqHeader.put(httpResponseHeader.USER_AGENT, HTTPLoader.yacyUserAgent); + httpResponse res = null; - final String name = release.url.getFileName(); - try { - res = client.GET(release.url.toString()); - final boolean unzipped = res.getResponseHeader().gzip() && (res.getResponseHeader().mime().toLowerCase().equals("application/x-tar")); // if true, then the httpc has unzipped the file - if ((unzipped) && (name.endsWith(".tar.gz"))) { - download = new File(storagePath, name.substring(0, name.length() - 3)); - } else { - download = new File(storagePath, name); - } - try { - FileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), new BufferedOutputStream(new FileOutputStream(download))); - } catch(IOException ie) { - // Saving file failed, abort download - res.abort(); - throw ie; - } finally { - res.closeStream(); - } - if ((!download.exists()) || (download.length() == 0)) throw new IOException("wget of url " + release.url + " failed"); - } catch (final IOException e) { - Log.logSevere("yacyVersion", "download of " + release.name + " failed: " + e.getMessage()); - if (download != null && download.exists()) { - FileUtils.deletedelete(download); - if (download.exists()) - Log.logWarning("yacyVersion", "could not delete file "+ download); - } - download = null; - } finally { - if (res != null) { - // release connection - res.closeStream(); - } - } + final String name = this.url.getFileName(); + byte[] signatureBytes = null; + // download signature first, if public key is available + if(this.publicKey != null) { + final byte[] signatureData = httpClient.wget(this.url.toString() + ".sig", reqHeader, 6000); + if(signatureData == null) { + Log.logSevere("yacyVersion", "download of signature " + this.url.toString() + " failed"); + return null; + } + try { + signatureBytes = Base64Order.standardCoder.decode(new String(signatureData, "UTF8"), "decode signature"); + } catch (UnsupportedEncodingException e) { + Log.logSevere("yacyVersion", "download of signature " + this.url.toString() + " failed: unsupported encoding"); + return null; + } + } + try { + final httpClient client = new httpClient(120000, reqHeader); + res = client.GET(this.url.toString()); + + final boolean unzipped = res.getResponseHeader().gzip() && (res.getResponseHeader().mime().toLowerCase().equals("application/x-tar")); // if true, then the httpc has unzipped the file + if ((unzipped) && (name.endsWith(".tar.gz"))) { + download = new File(storagePath, name.substring(0, name.length() - 3)); + } else { + download = new File(storagePath, name); + } + if(this.publicKey != null) { + // copy to file and check signature + SignatureOutputStream verifyOutput = null; + try { + verifyOutput = new SignatureOutputStream(new FileOutputStream(download), CryptoLib.signAlgorithm, publicKey); + FileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), new BufferedOutputStream(verifyOutput)); + + if(!verifyOutput.verify(signatureBytes)) { + throw new IOException("Bad Signature!"); + } + } catch (NoSuchAlgorithmException e) { + throw new IOException("No such algorithm"); + } catch (SignatureException e) { + throw new IOException("Signature exception"); + } finally { + if(verifyOutput != null) + verifyOutput.close(); + } + } else { + // just copy into file + FileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), new BufferedOutputStream(new FileOutputStream(download))); + } + if ((!download.exists()) || (download.length() == 0)) throw new IOException("wget of url " + this.url + " failed"); + } catch (final IOException e) { + // Saving file failed, abort download + res.abort(); + Log.logSevere("yacyVersion", "download of " + this.name + " failed: " + e.getMessage()); + if (download != null && download.exists()) { + FileUtils.deletedelete(download); + if (download.exists()) + Log.logWarning("yacyVersion", "could not delete file "+ download); + } + download = null; + } finally { + if (res != null) { + // release connection + res.closeStream(); + } + } + this.releaseFile = ((download != null) && (download.exists())) ? download : null; + // check signature plasmaSwitchboard.getSwitchboard().setConfig("update.time.download", System.currentTimeMillis()); - return ((download != null) && (download.exists())) ? download : null; + return this.releaseFile; } - + /** + * restart yacy by stopping yacy and previously running a batch + * script, which waits until yacy is terminated and starts it again + */ public static void restart() { final plasmaSwitchboard sb = plasmaSwitchboard.getSwitchboard(); final String apphome = sb.getRootPath().toString(); @@ -458,6 +531,10 @@ public final class yacyVersion implements Comparator, Comparable