0N - basic dump upload servlet infrastructure, to share index dumps

within an experimental new sharing model
pull/46/head
Michael Peter Christen 9 years ago
parent f12a900f3e
commit b89465d952

@ -0,0 +1,47 @@
<html>
<head>
</head>
<body>
<h1>File Share</h1>
#(mode)#
<!-- mode = 0 - display an upload form -->
<p>This form can be used to share a (index) file</p>
<form id="push" action="share.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8">
<dl>
<dt>Files to process:<dt>
<dt>&nbsp;<dt><dd>&nbsp;</dd>
<dd>
<dl>
<dt>data=</dt>
<dd><input name="data" type="file"></dd>
</dl>
</dd>
</dl>
<input type="submit" value="Submit">
</form>
::
<!-- mode 1 - display the push results -->
Result for the recently submitted file(s). You can also submit the same form using the servlet share.json to get push confirmations in json format.
<dl>
<dt>successall</dt><dd>#(successall)#false::true#(/successall)#</dd>
<dt>countsuccess</dt><dd>#[countsuccess]#</dd>
<dt>countfail</dt><dd>#[countfail]#</dd>
</dl>
<table border="1">
<tr><th>Item</th><th>URL</th><th>Success</th><th>Message</th></tr>
<tr>
<td>#[item]#</td>
<td><a href="#[url]#">#[url]#</a></td>
<td>#(success)#fail::ok#(/success)#</td>
<td>#(success)##[message]#::<a href="#[message]#" target="_blank">#[message]#</a>#(/success)#</td>
</tr>
</table>
<p>
If you want to push again files, use this form to pre-define a number of upload forms:
<form id="push" action="share.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8">
<input name="c" type="text" value="1" size="4" maxlength="8"><input type="submit" value="Submit">
</form>
</p>
#(/mode)#
</body>
</html>

@ -0,0 +1,106 @@
/**
* share
* Copyright 2016 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany
* First released 24.02.2016 at http://yacy.net
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
import java.io.File;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import net.yacy.yacy;
import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.order.Base64Order;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.search.Switchboard;
import net.yacy.search.index.Fulltext;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
public class share {
/**
* Servlet to share any kind of binary to this peer.
* That mean you can upload 'things'. While this is the generic view,
* it will operate in the beginning only for full solr export files.
* The servlet will decide if it wants that kind of data and if the sender is valid,
* i.e. if the sender is within the own network and known.
* Index dumps which are uploaded are placed to a specific folder
* where they can be downloaded again by peers.
* An optional operation is the immediate indexing of the shared index.
* @param header
* @param post
* @param env
* @return
*/
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
final serverObjects prop = new serverObjects();
// display mode: this only helps to display a nice input form for test cases
int c = post == null ? 1 : post.getInt("c", 0);
if (c > 0) {
prop.put("mode", 0);
return prop;
}
// push mode: this does a document upload
prop.put("mode", 1);
prop.put("success", 0);
if (post == null) return prop;
// check file name
String filename = post.get("data", "");
if (!filename.startsWith(Fulltext.yacy_dump_prefix) || !filename.endsWith(".xml.gz")) return prop;
// check data
String dataString = post.get("data$file", "");
if (dataString.length() == 0) return prop;
byte[] data;
if (filename.endsWith(".base64")) {
data = Base64Order.standardCoder.decode(dataString);
filename = filename.substring(0, filename.length() - 7);
} else {
data = UTF8.getBytes(dataString);
}
if (data == null || data.length == 0) return prop;
// modify the file name; ignore and replace the used transaction token
int ttp = filename.indexOf("_t");
if (ttp < 0) return prop;
if (filename.charAt(ttp + 3) != '.') return prop;
filename = filename.substring(0, ttp) + "_ts" + filename.substring(ttp + 3); // transaction token: 's' as 'shared'.
// process the data
File tmpFile = new File(yacy.shareDumpDefaultPath, filename + ".tmp");
File finalFile = new File(yacy.shareDumpDefaultPath, filename);
try {
Files.copy(new ByteArrayInputStream(data), tmpFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
tmpFile.renameTo(finalFile);
} catch (IOException e) {
ConcurrentLog.logException(e);
return prop;
}
prop.put("success", 1);
return prop;
}
}

@ -627,6 +627,7 @@ public final class Fulltext {
public String getExt() {return this.ext;}
}
public final static String yacy_dump_prefix = "yacy_dump_";
public Export export(Fulltext.ExportFormat format, String filter, String query, final int maxseconds, File path, boolean dom, boolean text) throws IOException {
// modify query according to maxseconds
@ -662,11 +663,11 @@ public final class Fulltext {
Object lastdateobject = lastdoc.getFieldValue(CollectionSchema.load_date_dt.getSolrFieldName());
Date firstdate = (Date) firstdateobject;
Date lastdate = (Date) lastdateobject;
String s = new File(path, "yacy_dump_" +
String s = new File(path, yacy_dump_prefix +
"f" + GenericFormatter.FORMAT_SHORT_MINUTE.format(firstdate) + "_" +
"l" + GenericFormatter.FORMAT_SHORT_MINUTE.format(lastdate) + "_" +
"n" + GenericFormatter.FORMAT_SHORT_MINUTE.format(new Date(now)) + "_" +
"c" + String.format("%1$012d", doccount)).getAbsolutePath();
"c" + String.format("%1$012d", doccount)).getAbsolutePath() + "_tc"; // the name ends with the transaction token ('c' = 'created')
// create export file name
if (s.indexOf('.',0) < 0) s += "." + format.getExt();

@ -114,6 +114,10 @@ public final class yacy {
public static final String hline = "-------------------------------------------------------------------------------";
public static final Semaphore shutdownSemaphore = new Semaphore(0);
public static File htDocsPath = null;
public static File shareDefaultPath = null;
public static File shareDumpDefaultPath = null;
/**
* a reference to the {@link Switchboard} created by the
* {@link yacy#startup(String, long, long)} method.
@ -244,7 +248,7 @@ public final class yacy {
// create some directories
final File htRootPath = new File(appHome, sb.getConfig(SwitchboardConstants.HTROOT_PATH, SwitchboardConstants.HTROOT_PATH_DEFAULT));
mkdirIfNeseccary(htRootPath);
final File htDocsPath = sb.getDataPath(SwitchboardConstants.HTDOCS_PATH, SwitchboardConstants.HTDOCS_PATH_DEFAULT);
htDocsPath = sb.getDataPath(SwitchboardConstants.HTDOCS_PATH, SwitchboardConstants.HTDOCS_PATH_DEFAULT);
mkdirIfNeseccary(htDocsPath);
//final File htTemplatePath = new File(homePath, sb.getConfig("htTemplatePath","htdocs"));
@ -287,8 +291,10 @@ public final class yacy {
System.out.println("Error creating htdocs readme: " + e.getMessage());
}
final File shareDefaultPath = new File(htDocsPath, "share");
shareDefaultPath = new File(htDocsPath, "share");
mkdirIfNeseccary(shareDefaultPath);
shareDumpDefaultPath = new File(shareDefaultPath, "dump");
mkdirIfNeseccary(shareDumpDefaultPath);
migration.migrate(sb, oldRev, newRev);

Loading…
Cancel
Save