boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

* introduce signatures to autoupdate

Browse Source 
as long as there aren't publickeys for the updatelocations set,
  no signatures are checked
* wiki-article follows...


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5822 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
f1ori 16 years ago
parent 538e375901
commit 44daec7936
8 changed files with 412 additions and 117 deletions
Show Stats Download Patch File Download Diff File

18
build.xml
Unescape View File

@ -1103,7 +1103,7 @@
replace="yacy (*auto-svn-version*) unstable; urgency=low" /> replace="yacy (*auto-svn-version*) unstable; urgency=low" />
</target> </target>
<target name="sign" depends="readBuildProperties" description="sign all files in RELEASE/ with $privateKey"> <target name="sign" depends="readBuildProperties" description="sign current release file in RELEASE/ with $privateKey">
<java classname="de.anomic.tools.CryptoLib" failonerror="true"> <java classname="de.anomic.tools.CryptoLib" failonerror="true">
<classpath> <classpath>
<pathelement location="${build}"/> <pathelement location="${build}"/>
@ -1111,4 +1111,20 @@
<arg line="--sign ${privateKeyFile} ${release}/${stdReleaseFile}"/> <arg line="--sign ${privateKeyFile} ${release}/${stdReleaseFile}"/>
</java> </java>
</target> </target>
<target name="genkey" depends="readBuildProperties" description="generate a pair of keys and write it to $privateKey and $privateKey.pub">
<!-- TODO: check doesn't work! why?? -->
<fail message="There is already a private Key!!">
<condition>
<available file="$privateKey" type="file" />
</condition>
</fail>
<java classname="de.anomic.tools.CryptoLib" failonerror="true">
<classpath>
<pathelement location="${build}"/>
</classpath>
<arg line="--gen-key ${privateKeyFile} ${privateKeyFile}.pub"/>
</java>
<chmod file="${privateKeyFile}" perm="600"/>
</target>
</project> </project>

59
htroot/ConfigUpdate_p.java
Unescape View File

@ -27,7 +27,6 @@
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.Date; import java.util.Date;
import java.util.Iterator;
import java.util.TreeSet; import java.util.TreeSet;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
@ -63,7 +62,19 @@ public class ConfigUpdate_p {
final String release = post.get("releasedownload", ""); final String release = post.get("releasedownload", "");
if (release.length() > 0) { if (release.length() > 0) {
try { try {
yacyVersion.downloadRelease(new yacyVersion(new yacyURL(release, null))); yacyVersion versionToDownload = new yacyVersion(new yacyURL(release, null));
// replace this version with version which contains public key
yacyVersion.DevAndMainVersions releases = yacyVersion.allReleases(false);
if(versionToDownload.mainRelease) {
yacyVersion repVersionToDownload = releases.main.ceiling(versionToDownload);
if(repVersionToDownload.equals(versionToDownload))
versionToDownload = repVersionToDownload;
} else {
yacyVersion repVersionToDownload = releases.dev.ceiling(versionToDownload);
if(repVersionToDownload.equals(versionToDownload))
versionToDownload = repVersionToDownload;
}
versionToDownload.downloadRelease();
} catch (final IOException e) { } catch (final IOException e) {
// TODO Auto-generated catch block // TODO Auto-generated catch block
e.printStackTrace(); e.printStackTrace();
@ -92,14 +103,14 @@ public class ConfigUpdate_p {
} else { } else {
// there is a version that is more recent. Load it and re-start with it // there is a version that is more recent. Load it and re-start with it
sb.getLog().logInfo("AUTO-UPDATE: downloading more recent release " + updateVersion.url); sb.getLog().logInfo("AUTO-UPDATE: downloading more recent release " + updateVersion.url);
final File downloaded = yacyVersion.downloadRelease(updateVersion); final File downloaded = updateVersion.downloadRelease();
prop.putHTML("candeploy_autoUpdate_downloadedRelease", updateVersion.name); prop.putHTML("candeploy_autoUpdate_downloadedRelease", updateVersion.name);
final boolean devenvironment = yacyVersion.combined2prettyVersion(sb.getConfig("version","0.1")).startsWith("dev"); final boolean devenvironment = yacyVersion.combined2prettyVersion(sb.getConfig("version","0.1")).startsWith("dev");
if (devenvironment) { if (devenvironment) {
sb.getLog().logInfo("AUTO-UPDATE: omiting update because this is a development environment"); sb.getLog().logInfo("AUTO-UPDATE: omiting update because this is a development environment");
prop.put("candeploy_autoUpdate", "3"); prop.put("candeploy_autoUpdate", "3");
} else if ((downloaded == null) || (!downloaded.exists()) || (downloaded.length() == 0)) { } else if ((downloaded == null) || (!downloaded.exists()) || (downloaded.length() == 0)) {
sb.getLog().logInfo("AUTO-UPDATE: omiting update because download failed (file cannot be found or is too small)"); sb.getLog().logInfo("AUTO-UPDATE: omiting update because download failed (file cannot be found, is too small or signature was bad)");
prop.put("candeploy_autoUpdate", "4"); prop.put("candeploy_autoUpdate", "4");
} else { } else {
yacyVersion.deployRelease(downloaded); yacyVersion.deployRelease(downloaded);
@ -138,29 +149,27 @@ public class ConfigUpdate_p {
// list downloaded releases // list downloaded releases
yacyVersion release, dflt; final File[] downloadedFiles = sb.releasePath.listFiles();
final String[] downloaded = sb.releasePath.list();
prop.put("candeploy_deployenabled", (downloaded.length == 0) ? "0" : ((devenvironment) ? "1" : "2")); // prevent that a developer-version is over-deployed prop.put("candeploy_deployenabled", (downloadedFiles.length == 0) ? "0" : ((devenvironment) ? "1" : "2")); // prevent that a developer-version is over-deployed
final TreeSet<yacyVersion> downloadedreleases = new TreeSet<yacyVersion>(); final TreeSet<yacyVersion> downloadedReleases = new TreeSet<yacyVersion>();
for (int j = 0; j < downloaded.length; j++) { for(File downloaded : downloadedFiles) {
try { try {
release = new yacyVersion(downloaded[j]); yacyVersion release = new yacyVersion(downloaded);
downloadedreleases.add(release); downloadedReleases.add(release);
} catch (final RuntimeException e) { } catch (final RuntimeException e) {
// not a valid release // not a valid release
// can be also a restart- or deploy-file // can be also a restart- or deploy-file
final File invalid = new File(sb.releasePath, downloaded[j]); final File invalid = downloaded;
if (!(invalid.getName().endsWith(".bat") || invalid.getName().endsWith(".sh"))) // Windows & Linux don't like deleted scripts while execution! if (!(invalid.getName().endsWith(".bat") || invalid.getName().endsWith(".sh"))) // Windows & Linux don't like deleted scripts while execution!
invalid.deleteOnExit(); invalid.deleteOnExit();
} }
} }
dflt = (downloadedreleases.size() == 0) ? null : downloadedreleases.last(); // latest downloaded release
Iterator<yacyVersion> i = downloadedreleases.iterator(); yacyVersion dflt = (downloadedReleases.size() == 0) ? null : downloadedReleases.last();
int relcount = 0; int relcount = 0;
while (i.hasNext()) { for(yacyVersion release : downloadedReleases) {
release = i.next();
prop.put("candeploy_downloadedreleases_" + relcount + "_name", ((release.mainRelease) ? "main" : "dev") + " " + release.releaseNr + "/" + release.svn); prop.put("candeploy_downloadedreleases_" + relcount + "_name", ((release.mainRelease) ? "main" : "dev") + " " + release.releaseNr + "/" + release.svn);
prop.putHTML("candeploy_downloadedreleases_" + relcount + "_file", release.name); prop.putHTML("candeploy_downloadedreleases_" + relcount + "_file", release.name);
prop.put("candeploy_downloadedreleases_" + relcount + "_selected", (release == dflt) ? "1" : "0"); prop.put("candeploy_downloadedreleases_" + relcount + "_selected", (release == dflt) ? "1" : "0");
@ -169,15 +178,13 @@ public class ConfigUpdate_p {
prop.put("candeploy_downloadedreleases", relcount); prop.put("candeploy_downloadedreleases", relcount);
// list remotely available releases // list remotely available releases
final yacyVersion.DevMain releasess = yacyVersion.allReleases(false); final yacyVersion.DevAndMainVersions releasess = yacyVersion.allReleases(false);
relcount = 0; relcount = 0;
// main // main
TreeSet<yacyVersion> releases = releasess.main; final TreeSet<yacyVersion> remoteMainReleases = releasess.main;
releases.removeAll(downloadedreleases); remoteMainReleases.removeAll(downloadedReleases);
i = releases.iterator(); for (yacyVersion release : remoteMainReleases) {
while (i.hasNext()) {
release = i.next();
prop.put("candeploy_availreleases_" + relcount + "_name", ((release.mainRelease) ? "main" : "dev") + " " + release.releaseNr + "/" + release.svn); prop.put("candeploy_availreleases_" + relcount + "_name", ((release.mainRelease) ? "main" : "dev") + " " + release.releaseNr + "/" + release.svn);
prop.put("candeploy_availreleases_" + relcount + "_url", release.url.toString()); prop.put("candeploy_availreleases_" + relcount + "_url", release.url.toString());
prop.put("candeploy_availreleases_" + relcount + "_selected", "0"); prop.put("candeploy_availreleases_" + relcount + "_selected", "0");
@ -186,11 +193,9 @@ public class ConfigUpdate_p {
// dev // dev
dflt = (releasess.dev.size() == 0) ? null : releasess.dev.last(); dflt = (releasess.dev.size() == 0) ? null : releasess.dev.last();
releases = releasess.dev; final TreeSet<yacyVersion> remoteDevReleases = releasess.dev;
releases.removeAll(downloadedreleases); remoteDevReleases.removeAll(downloadedReleases);
i = releases.iterator(); for(yacyVersion release : remoteDevReleases) {
while (i.hasNext()) {
release = i.next();
prop.put("candeploy_availreleases_" + relcount + "_name", ((release.mainRelease) ? "main" : "dev") + " " + release.releaseNr + "/" + release.svn); prop.put("candeploy_availreleases_" + relcount + "_name", ((release.mainRelease) ? "main" : "dev") + " " + release.releaseNr + "/" + release.svn);
prop.put("candeploy_availreleases_" + relcount + "_url", release.url.toString()); prop.put("candeploy_availreleases_" + relcount + "_url", release.url.toString());
prop.put("candeploy_availreleases_" + relcount + "_selected", (release == dflt) ? "1" : "0"); prop.put("candeploy_availreleases_" + relcount + "_selected", (release == dflt) ? "1" : "0");

15
source/de/anomic/kelondro/util/FileUtils.java
Unescape View File

@ -573,6 +573,21 @@ public final class FileUtils {
} }
return v; return v;
} }
public static ArrayList<String> strings(final Reader reader) {
if (reader == null) return new ArrayList<String>();
BufferedReader bufreader = new BufferedReader(reader);
final ArrayList<String> list = new ArrayList<String>();
String line = null;
try {
while ((line = bufreader.readLine()) != null) {
list.add(line);
}
} catch (IOException e) {
e.printStackTrace();
return null;
}
return list;
}
/** /**

53
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -93,6 +93,9 @@ import java.io.InputStream;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.lang.reflect.Constructor; import java.lang.reflect.Constructor;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.security.NoSuchAlgorithmException;
import java.security.PublicKey;
import java.security.spec.InvalidKeySpecException;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date; import java.util.Date;
@ -145,6 +148,7 @@ import de.anomic.http.httpdRobotsTxtConfig;
import de.anomic.kelondro.order.Digest; import de.anomic.kelondro.order.Digest;
import de.anomic.kelondro.order.NaturalOrder; import de.anomic.kelondro.order.NaturalOrder;
import de.anomic.kelondro.text.metadataPrototype.URLMetadataRow; import de.anomic.kelondro.text.metadataPrototype.URLMetadataRow;
import de.anomic.kelondro.order.Base64Order;
import de.anomic.kelondro.util.DateFormatter; import de.anomic.kelondro.util.DateFormatter;
import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.FileUtils;
import de.anomic.kelondro.util.Log; import de.anomic.kelondro.util.Log;
@ -168,6 +172,7 @@ import de.anomic.server.serverSemaphore;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
import de.anomic.server.serverThread; import de.anomic.server.serverThread;
import de.anomic.tools.crypt; import de.anomic.tools.crypt;
import de.anomic.tools.CryptoLib;
import de.anomic.xml.SurrogateReader; import de.anomic.xml.SurrogateReader;
import de.anomic.yacy.yacyClient; import de.anomic.yacy.yacyClient;
import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacyCore;
@ -176,6 +181,7 @@ import de.anomic.yacy.yacyNewsRecord;
import de.anomic.yacy.yacySeed; import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyTray; import de.anomic.yacy.yacyTray;
import de.anomic.yacy.yacyURL; import de.anomic.yacy.yacyURL;
import de.anomic.yacy.yacyUpdateLocation;
import de.anomic.yacy.yacyVersion; import de.anomic.yacy.yacyVersion;
import de.anomic.yacy.dht.Dispatcher; import de.anomic.yacy.dht.Dispatcher;
import de.anomic.yacy.dht.PeerSelection; import de.anomic.yacy.dht.PeerSelection;
@ -706,17 +712,38 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
// set release locations // set release locations
int i = 0; int i = 0;
String location; CryptoLib cryptoLib;
while (true) { try {
location = getConfig("network.unit.update.location" + i, ""); cryptoLib = new CryptoLib();
if (location.length() == 0) break; while (true) {
try { String location = getConfig("network.unit.update.location" + i, "");
yacyVersion.latestReleaseLocations.add(new yacyURL(location, null)); if (location.length() == 0) break;
} catch (final MalformedURLException e) { yacyURL locationURL;
break; try {
} // try to parse url
i++; locationURL = new yacyURL(location, null);
} } catch (final MalformedURLException e) {
break;
}
PublicKey publicKey = null;
// get public key if it's in config
try {
String publicKeyString = getConfig("network.unit.update.location" + i + ".key", null);
if(publicKeyString != null) {
byte[] publicKeyBytes = Base64Order.standardCoder.decode(publicKeyString, "decode public Key");
publicKey = cryptoLib.getPublicKeyFromBytes(publicKeyBytes);
}
} catch (InvalidKeySpecException e) {
e.printStackTrace();
}
yacyUpdateLocation updateLocation = new yacyUpdateLocation(locationURL, publicKey);
yacyVersion.latestReleaseLocations.add(updateLocation);
i++;
}
} catch (NoSuchAlgorithmException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
// initiate url license object // initiate url license object
licensedURLs = new URLLicense(8); licensedURLs = new URLLicense(8);
@ -1469,12 +1496,12 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
if (updateVersion != null) { if (updateVersion != null) {
// there is a version that is more recent. Load it and re-start with it // there is a version that is more recent. Load it and re-start with it
log.logInfo("AUTO-UPDATE: downloading more recent release " + updateVersion.url); log.logInfo("AUTO-UPDATE: downloading more recent release " + updateVersion.url);
final File downloaded = yacyVersion.downloadRelease(updateVersion); final File downloaded = updateVersion.downloadRelease();
final boolean devenvironment = yacyVersion.combined2prettyVersion(sb.getConfig("version","0.1")).startsWith("dev"); final boolean devenvironment = yacyVersion.combined2prettyVersion(sb.getConfig("version","0.1")).startsWith("dev");
if (devenvironment) { if (devenvironment) {
log.logInfo("AUTO-UPDATE: omiting update because this is a development environment"); log.logInfo("AUTO-UPDATE: omiting update because this is a development environment");
} else if ((downloaded == null) || (!downloaded.exists()) || (downloaded.length() == 0)) { } else if ((downloaded == null) || (!downloaded.exists()) || (downloaded.length() == 0)) {
log.logInfo("AUTO-UPDATE: omiting update because download failed (file cannot be found or is too small)"); log.logInfo("AUTO-UPDATE: omiting update because download failed (file cannot be found, is too small or signature is bad)");
} else { } else {
yacyVersion.deployRelease(downloaded); yacyVersion.deployRelease(downloaded);
terminate(5000); terminate(5000);

21
source/de/anomic/tools/CryptoLib.java
Unescape View File

@ -70,22 +70,23 @@ public class CryptoLib {
" Verify signatur\n" + " Verify signatur\n" +
" --gen-key privatekey publickey\n"; " --gen-key privatekey publickey\n";
private static final String algorithm = "DSA"; public static final String algorithm = "DSA";
private static final int bitkey = 1024; public static final int bitkey = 1024;
public static final String signAlgorithm = "SHA1with"+algorithm;
private KeyFactory keyFact; private KeyFactory keyFact;
private Signature sign; private Signature sign;
public CryptoLib() throws NoSuchAlgorithmException { public CryptoLib() throws NoSuchAlgorithmException {
keyFact = KeyFactory.getInstance(algorithm); keyFact = KeyFactory.getInstance(algorithm);
sign = Signature.getInstance("SHA1with"+algorithm); sign = Signature.getInstance(signAlgorithm);
} }
public PrivateKey getPrivateKeyFromBytes(byte[] keyBuffer) throws IOException, InvalidKeySpecException { public PrivateKey getPrivateKeyFromBytes(byte[] keyBuffer) throws InvalidKeySpecException {
return keyFact.generatePrivate(new PKCS8EncodedKeySpec(keyBuffer)); return keyFact.generatePrivate(new PKCS8EncodedKeySpec(keyBuffer));
} }
public PublicKey getPublicKeyFromBytes(byte[] keyBuffer) throws IOException, InvalidKeySpecException { public PublicKey getPublicKeyFromBytes(byte[] keyBuffer) throws InvalidKeySpecException {
return keyFact.generatePublic(new X509EncodedKeySpec(keyBuffer)); return keyFact.generatePublic(new X509EncodedKeySpec(keyBuffer));
} }
@ -104,8 +105,9 @@ public class CryptoLib {
public byte[] getSignature(PrivateKey privKey, InputStream dataStream) throws InvalidKeyException, SignatureException, IOException { public byte[] getSignature(PrivateKey privKey, InputStream dataStream) throws InvalidKeyException, SignatureException, IOException {
sign.initSign(privKey); sign.initSign(privKey);
byte[] buffer = new byte[1024]; byte[] buffer = new byte[1024];
while(dataStream.read(buffer) != -1) { int count = 0;
sign.update(buffer); while((count = dataStream.read(buffer)) != -1) {
sign.update(buffer, 0, count);
} }
dataStream.close(); dataStream.close();
return sign.sign(); return sign.sign();
@ -115,8 +117,9 @@ public class CryptoLib {
sign.initVerify(pubKey); sign.initVerify(pubKey);
byte[] buffer = new byte[1024]; byte[] buffer = new byte[1024];
while(dataStream.read(buffer) != -1) { int count = 0;
sign.update(buffer); while((count = dataStream.read(buffer)) != -1) {
sign.update(buffer, 0, count);
} }
dataStream.close(); dataStream.close();

100
source/de/anomic/tools/SignatureOutputStream.java
Unescape View File

@ -0,0 +1,100 @@
// SignatureOutputStream.java
// ----------------
// (C) 2009 by Florian Richter <mail@f1ori.de>
// first published 16.04.2009 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2009-03-30 17:31:25 +0200 (Mo, 30. Mär 2009) $
// $LastChangedRevision: 5756 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.tools;
import java.io.IOException;
import java.io.FilterOutputStream;
import java.io.OutputStream;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.PublicKey;
import java.security.Signature;
import java.security.SignatureException;
/**
* A SignatureOuputStream is composed of a Signature and a OutputStream so that
* write()-methods first update the signature and then pass the data to the
* underlying OutputStream.
*
* @author flori
*
*/
public class SignatureOutputStream extends FilterOutputStream {
private Signature signature;
/**
* create new SignatureOutputStream and setup the Signature
* @param stream OutputStream to pass data on
* @param algorithm Algorithm to use for Signature
* @param publicKey Public key to verify Signature against
* @throws NoSuchAlgorithmException
*/
public SignatureOutputStream(OutputStream stream, String algorithm, PublicKey publicKey) throws NoSuchAlgorithmException {
super(stream);
try {
signature = Signature.getInstance(algorithm);
signature.initVerify(publicKey);
} catch (InvalidKeyException e) {
System.out.println("Internal Error at signature:" + e.getMessage());
}
}
/**
* write byte
* @see FilterOutputStream.write(int b)
*/
public void write(int b) throws IOException {
try {
signature.update((byte)b);
} catch (SignatureException e) {
throw new IOException("Signature update failed: "+ e.getMessage());
}
out.write(b);
}
public void write(byte[] b, int off, int len) throws IOException {
try {
signature.update(b, off, len);
} catch (SignatureException e) {
throw new IOException("Signature update failed: "+ e.getMessage());
}
out.write(b, off, len);
}
/**
* verify signature, don't use this stream for another signature afterwards
* @param sign signature as bytes
* @return true, when signature was right
* @throws SignatureException
*/
public boolean verify(byte[] sign) throws SignatureException {
return signature.verify(sign);
}
}

52
source/de/anomic/yacy/yacyUpdateLocation.java
Unescape View File

@ -0,0 +1,52 @@
// yacyUpdateLocation.java
// ----------------
// (C) 2009 by Florian Richter
// first published 5.03.2009 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2009-02-19 17:24:46 +0100 (Do, 19. Feb 2009) $
// $LastChangedRevision: 5621 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.yacy;
import java.security.PublicKey;
/**
* Holds a update location with url and public key
*
*/
public class yacyUpdateLocation {
private yacyURL locationURL;
private PublicKey publicKey;
public yacyUpdateLocation(yacyURL locationURL, PublicKey publicKey) {
this.locationURL = locationURL;
this.publicKey = publicKey;
}
public yacyURL getLocationURL() {
return this.locationURL;
}
public PublicKey getPublicKey() {
return this.publicKey;
}
}

211
source/de/anomic/yacy/yacyVersion.java
Unescape View File

@ -32,10 +32,13 @@ import java.io.BufferedOutputStream;
import java.io.File; import java.io.File;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.security.NoSuchAlgorithmException;
import java.security.PublicKey;
import java.security.SignatureException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Comparator; import java.util.Comparator;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator;
import java.util.Map; import java.util.Map;
import java.util.SortedSet; import java.util.SortedSet;
import java.util.TreeSet; import java.util.TreeSet;
@ -48,11 +51,14 @@ import de.anomic.http.httpClient;
import de.anomic.http.httpResponse; import de.anomic.http.httpResponse;
import de.anomic.http.httpResponseHeader; import de.anomic.http.httpResponseHeader;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.order.Base64Order;
import de.anomic.kelondro.util.Log; import de.anomic.kelondro.util.Log;
import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.FileUtils;
import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore; import de.anomic.server.serverCore;
import de.anomic.server.serverSystem; import de.anomic.server.serverSystem;
import de.anomic.tools.CryptoLib;
import de.anomic.tools.SignatureOutputStream;
import de.anomic.tools.tarTools; import de.anomic.tools.tarTools;
public final class yacyVersion implements Comparator<yacyVersion>, Comparable<yacyVersion> { public final class yacyVersion implements Comparator<yacyVersion>, Comparable<yacyVersion> {
@ -71,8 +77,8 @@ public final class yacyVersion implements Comparator<yacyVersion>, Comparable<ya
// information about latest release, retrieved from download pages // information about latest release, retrieved from download pages
// this static information should be overwritten by network-specific locations // this static information should be overwritten by network-specific locations
// for details see defaults/yacy.network.freeworld.unit // for details see defaults/yacy.network.freeworld.unit
private static HashMap<yacyURL, DevMain> latestReleases = new HashMap<yacyURL, DevMain>(); private static HashMap<yacyUpdateLocation, DevAndMainVersions> latestReleases = new HashMap<yacyUpdateLocation, DevAndMainVersions>();
public final static ArrayList<yacyURL> latestReleaseLocations = new ArrayList<yacyURL>(); // will be initialized with value in defaults/yacy.network.freeworld.unit public final static ArrayList<yacyUpdateLocation> latestReleaseLocations = new ArrayList<yacyUpdateLocation>(); // will be initialized with value in defaults/yacy.network.freeworld.unit
// private static release info about this release; is generated only once and can be retrieved by thisVersion() // private static release info about this release; is generated only once and can be retrieved by thisVersion()
private static yacyVersion thisVersion = null; private static yacyVersion thisVersion = null;
@ -84,12 +90,25 @@ public final class yacyVersion implements Comparator<yacyVersion>, Comparable<ya
public boolean mainRelease; public boolean mainRelease;
public yacyURL url; public yacyURL url;
public String name; public String name;
private File releaseFile;
private PublicKey publicKey;
public yacyVersion(final yacyURL url) { public yacyVersion(final yacyURL url) {
this(url.getFileName()); this(url.getFileName());
this.url = url; this.url = url;
} }
public yacyVersion(final yacyURL url, PublicKey publicKey) {
this(url.getFileName());
this.url = url;
this.publicKey = publicKey;
}
public yacyVersion(final File releaseFile) {
this(releaseFile.getName());
this.releaseFile = releaseFile;
}
public yacyVersion(String release) { public yacyVersion(String release) {
// parse a release file name // parse a release file name
// the have the following form: // the have the following form:
@ -137,9 +156,9 @@ public final class yacyVersion implements Comparator<yacyVersion>, Comparable<ya
// finished! we parsed a relase string // finished! we parsed a relase string
} }
public static final class DevMain { public static final class DevAndMainVersions {
public TreeSet<yacyVersion> dev, main; public TreeSet<yacyVersion> dev, main;
public DevMain(final TreeSet<yacyVersion> dev, final TreeSet<yacyVersion> main) { public DevAndMainVersions(final TreeSet<yacyVersion> dev, final TreeSet<yacyVersion> main) {
this.dev = dev; this.dev = dev;
this.main = main; this.main = main;
} }
@ -213,9 +232,9 @@ public final class yacyVersion implements Comparator<yacyVersion>, Comparable<ya
} }
// check if we know that there is a release that is more recent than that which we are using // check if we know that there is a release that is more recent than that which we are using
final DevMain releasess = yacyVersion.allReleases(true); final DevAndMainVersions releases = yacyVersion.allReleases(true);
final yacyVersion latestmain = (releasess.main.size() == 0) ? null : releasess.main.last(); final yacyVersion latestmain = (releases.main.size() == 0) ? null : releases.main.last();
final yacyVersion latestdev = (releasess.dev.size() == 0) ? null : releasess.dev.last(); final yacyVersion latestdev = (releases.dev.size() == 0) ? null : releases.dev.last();
final String concept = sb.getConfig("update.concept", "any"); final String concept = sb.getConfig("update.concept", "any");
String blacklist = sb.getConfig("update.blacklist", "...[123]"); String blacklist = sb.getConfig("update.blacklist", "...[123]");
if (blacklist.equals("....[123]")) { if (blacklist.equals("....[123]")) {
@ -275,9 +294,9 @@ public final class yacyVersion implements Comparator<yacyVersion>, Comparable<ya
return null; return null;
} }
public static DevMain allReleases(final boolean force) { public static DevAndMainVersions allReleases(final boolean force) {
// join the release infos // join the release infos
final DevMain[] a = new DevMain[latestReleaseLocations.size()]; final DevAndMainVersions[] a = new DevAndMainVersions[latestReleaseLocations.size()];
for (int j = 0; j < latestReleaseLocations.size(); j++) { for (int j = 0; j < latestReleaseLocations.size(); j++) {
a[j] = getReleases(latestReleaseLocations.get(j), force); a[j] = getReleases(latestReleaseLocations.get(j), force);
} }
@ -286,12 +305,18 @@ public final class yacyVersion implements Comparator<yacyVersion>, Comparable<ya
for (int j = 0; j < a.length; j++) if ((a[j] != null) && (a[j].dev != null)) alldev.addAll(a[j].dev); for (int j = 0; j < a.length; j++) if ((a[j] != null) && (a[j].dev != null)) alldev.addAll(a[j].dev);
for (int j = 0; j < a.length; j++) if ((a[j] != null) && (a[j].main != null)) allmain.addAll(a[j].main); for (int j = 0; j < a.length; j++) if ((a[j] != null) && (a[j].main != null)) allmain.addAll(a[j].main);
return new DevMain(alldev, allmain); return new DevAndMainVersions(alldev, allmain);
} }
private static DevMain getReleases(final yacyURL location, final boolean force) { /**
// get release info from a internet resource * get all Releases from update location using cache
DevMain locLatestRelease = latestReleases.get(location); * @param location Update location
* @param force when true, don't fetch from cache
* @return
*/
private static DevAndMainVersions getReleases(final yacyUpdateLocation location, final boolean force) {
// get release info from a Internet resource
DevAndMainVersions locLatestRelease = latestReleases.get(location);
if (force || if (force ||
(locLatestRelease == null) /*|| (locLatestRelease == null) /*||
((latestRelease[0].size() == 0) && ((latestRelease[0].size() == 0) &&
@ -304,88 +329,136 @@ public final class yacyVersion implements Comparator<yacyVersion>, Comparable<ya
return locLatestRelease; return locLatestRelease;
} }
private static DevMain allReleaseFrom(yacyURL url) { /**
* get all releases from update location
* @param location
* @return
*/
private static DevAndMainVersions allReleaseFrom(yacyUpdateLocation location) {
// retrieves the latest info about releases // retrieves the latest info about releases
// this is done by contacting a release location, // this is done by contacting a release location,
// parsing the content and filtering+parsing links // parsing the content and filtering+parsing links
// returns the version info if successful, null otherwise // returns the version info if successful, null otherwise
htmlFilterContentScraper scraper; htmlFilterContentScraper scraper;
try { try {
scraper = htmlFilterContentScraper.parseResource(url); scraper = htmlFilterContentScraper.parseResource(location.getLocationURL());
} catch (final IOException e) { } catch (final IOException e) {
return null; return null;
} }
// analyse links in scraper resource, and find link to latest release in it // analyse links in scraper resource, and find link to latest release in it
final Map<yacyURL, String> anchors = scraper.getAnchors(); // a url (String) / name (String) relation final Map<yacyURL, String> anchors = scraper.getAnchors(); // a url (String) / name (String) relation
final Iterator<yacyURL> i = anchors.keySet().iterator(); final TreeSet<yacyVersion> mainReleases = new TreeSet<yacyVersion>();
final TreeSet<yacyVersion> devreleases = new TreeSet<yacyVersion>(); final TreeSet<yacyVersion> devReleases = new TreeSet<yacyVersion>();
final TreeSet<yacyVersion> mainreleases = new TreeSet<yacyVersion>(); for(yacyURL url : anchors.keySet()) {
yacyVersion release;
while (i.hasNext()) {
url = i.next();
try { try {
release = new yacyVersion(url); yacyVersion release = new yacyVersion(url, location.getPublicKey());
//System.out.println("r " + release.toAnchor()); //System.out.println("r " + release.toAnchor());
if ( release.mainRelease) mainreleases.add(release); if(release.mainRelease) {
if (!release.mainRelease) devreleases.add(release); mainReleases.add(release);
} else {
devReleases.add(release);
}
} catch (final RuntimeException e) { } catch (final RuntimeException e) {
// the release string was not well-formed. // the release string was not well-formed.
// that might have been another link // that might have been another link
// just dont care // just don't care
continue; continue;
} }
} }
plasmaSwitchboard.getSwitchboard().setConfig("update.time.lookup", System.currentTimeMillis()); plasmaSwitchboard.getSwitchboard().setConfig("update.time.lookup", System.currentTimeMillis());
return new DevMain(devreleases, mainreleases); return new DevAndMainVersions(devReleases, mainReleases);
} }
public static File downloadRelease(final yacyVersion release) { /**
* <p>download this release and if public key is know, download signature and check it.
* <p>The signature is named $releaseurl.sig and contains the base64 encoded signature
* (@see de.anomic.tools.CryptoLib)
* @return file object of release file, null in case of failure
*/
public File downloadRelease() {
final File storagePath = plasmaSwitchboard.getSwitchboard().releasePath; final File storagePath = plasmaSwitchboard.getSwitchboard().releasePath;
// load file
File download = null; File download = null;
final httpRequestHeader header = new httpRequestHeader(); // setup httpClient
header.put(httpResponseHeader.USER_AGENT, HTTPLoader.yacyUserAgent); final httpRequestHeader reqHeader = new httpRequestHeader();
final httpClient client = new httpClient(120000, header); reqHeader.put(httpResponseHeader.USER_AGENT, HTTPLoader.yacyUserAgent);
httpResponse res = null; httpResponse res = null;
final String name = release.url.getFileName(); final String name = this.url.getFileName();
try { byte[] signatureBytes = null;
res = client.GET(release.url.toString()); // download signature first, if public key is available
final boolean unzipped = res.getResponseHeader().gzip() && (res.getResponseHeader().mime().toLowerCase().equals("application/x-tar")); // if true, then the httpc has unzipped the file if(this.publicKey != null) {
if ((unzipped) && (name.endsWith(".tar.gz"))) { final byte[] signatureData = httpClient.wget(this.url.toString() + ".sig", reqHeader, 6000);
download = new File(storagePath, name.substring(0, name.length() - 3)); if(signatureData == null) {
} else { Log.logSevere("yacyVersion", "download of signature " + this.url.toString() + " failed");
download = new File(storagePath, name); return null;
} }
try { try {
FileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), new BufferedOutputStream(new FileOutputStream(download))); signatureBytes = Base64Order.standardCoder.decode(new String(signatureData, "UTF8"), "decode signature");
} catch(IOException ie) { } catch (UnsupportedEncodingException e) {
// Saving file failed, abort download Log.logSevere("yacyVersion", "download of signature " + this.url.toString() + " failed: unsupported encoding");
res.abort(); return null;
throw ie; }
} finally { }
res.closeStream(); try {
} final httpClient client = new httpClient(120000, reqHeader);
if ((!download.exists()) || (download.length() == 0)) throw new IOException("wget of url " + release.url + " failed"); res = client.GET(this.url.toString());
} catch (final IOException e) {
Log.logSevere("yacyVersion", "download of " + release.name + " failed: " + e.getMessage()); final boolean unzipped = res.getResponseHeader().gzip() && (res.getResponseHeader().mime().toLowerCase().equals("application/x-tar")); // if true, then the httpc has unzipped the file
if (download != null && download.exists()) { if ((unzipped) && (name.endsWith(".tar.gz"))) {
FileUtils.deletedelete(download); download = new File(storagePath, name.substring(0, name.length() - 3));
if (download.exists()) } else {
Log.logWarning("yacyVersion", "could not delete file "+ download); download = new File(storagePath, name);
} }
download = null; if(this.publicKey != null) {
} finally { // copy to file and check signature
if (res != null) { SignatureOutputStream verifyOutput = null;
// release connection try {
res.closeStream(); verifyOutput = new SignatureOutputStream(new FileOutputStream(download), CryptoLib.signAlgorithm, publicKey);
} FileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), new BufferedOutputStream(verifyOutput));
}
if(!verifyOutput.verify(signatureBytes)) {
throw new IOException("Bad Signature!");
}
} catch (NoSuchAlgorithmException e) {
throw new IOException("No such algorithm");
} catch (SignatureException e) {
throw new IOException("Signature exception");
} finally {
if(verifyOutput != null)
verifyOutput.close();
}
} else {
// just copy into file
FileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), new BufferedOutputStream(new FileOutputStream(download)));
}
if ((!download.exists()) || (download.length() == 0)) throw new IOException("wget of url " + this.url + " failed");
} catch (final IOException e) {
// Saving file failed, abort download
res.abort();
Log.logSevere("yacyVersion", "download of " + this.name + " failed: " + e.getMessage());
if (download != null && download.exists()) {
FileUtils.deletedelete(download);
if (download.exists())
Log.logWarning("yacyVersion", "could not delete file "+ download);
}
download = null;
} finally {
if (res != null) {
// release connection
res.closeStream();
}
}
this.releaseFile = ((download != null) && (download.exists())) ? download : null;
// check signature
plasmaSwitchboard.getSwitchboard().setConfig("update.time.download", System.currentTimeMillis()); plasmaSwitchboard.getSwitchboard().setConfig("update.time.download", System.currentTimeMillis());
return ((download != null) && (download.exists())) ? download : null; return this.releaseFile;
} }
/**
* restart yacy by stopping yacy and previously running a batch
* script, which waits until yacy is terminated and starts it again
*/
public static void restart() { public static void restart() {
final plasmaSwitchboard sb = plasmaSwitchboard.getSwitchboard(); final plasmaSwitchboard sb = plasmaSwitchboard.getSwitchboard();
final String apphome = sb.getRootPath().toString(); final String apphome = sb.getRootPath().toString();
@ -458,6 +531,10 @@ public final class yacyVersion implements Comparator<yacyVersion>, Comparable<ya
} }
} }
/**
* stop yacy and run a batch script, applies a new release and restarts yacy
* @param releaseFile
*/
public static void deployRelease(final File releaseFile) { public static void deployRelease(final File releaseFile) {
//byte[] script = ("cd " + plasmaSwitchboard.getSwitchboard().getRootPath() + ";while [ -e ../yacy.running ]; do sleep 1;done;tar xfz " + release + ";cp -Rf yacy/* ../../;rm -Rf yacy;cd ../../;startYACY.sh").getBytes(); //byte[] script = ("cd " + plasmaSwitchboard.getSwitchboard().getRootPath() + ";while [ -e ../yacy.running ]; do sleep 1;done;tar xfz " + release + ";cp -Rf yacy/* ../../;rm -Rf yacy;cd ../../;startYACY.sh").getBytes();
try { try {

Write Preview
Loading…
Cancel
Save