boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- it is now possible to start several OAI imports concurrently

Browse Source 
(still not possible to start them with one single request, that will be next)
- added a monitor for all running and finished OAI imports (with a little bit of animation..)



git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6447 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent 176e334aa4
commit 0f63de8236
4 changed files with 119 additions and 70 deletions
Show Stats Download Patch File Download Diff File

29
htroot/IndexImportOAIPMHList_p.html
Unescape View File

@ -0,0 +1,29 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>YaCy '#[clientname]#': OAI-PMH source import list</title>
#%env/templates/metas.template%#
<meta http-equiv="REFRESH" content="6" />
</head>
<body>
<h3>Import List</h3>
<table cellpadding="2" cellspacing="1" >
<tr class="TableHeader">
<td>Thread</td>
<td>Source</td>
<td>Processed<br />Chunks</td>
<td>Imported<br />Records</td>
<td>Speed<br />(records/second)</td>
</tr>
#{table}#
<tr class="TableCell#(dark)#Light::Dark#(/dark)#">
<td>#[thread]#</td>
<td>#[source]#</td>
<td>#[chunkCount]#</td>
<td>#[recordsCount]#</td>
<td>#[speed]#</td>
</tr>
#{/table}#
</table>
</body>
</html>

59
htroot/IndexImportOAIPMHList_p.java
Unescape View File

@ -0,0 +1,59 @@
// IndexImportOAIPMHList.java
// -------------------------
// (C) 2009 by Michael Peter Christen; mc@yacy.net
// first published 03.11.2009 on http://yacy.net
// Frankfurt, Germany
//
// $LastChangedDate: 2009-10-11 23:29:18 +0200 (So, 11 Okt 2009) $
// $LastChangedRevision: 6400 $
// $LastChangedBy: orbiter $
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.util.ArrayList;
import net.yacy.document.importer.OAIPMHImporter;
import de.anomic.http.server.RequestHeader;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
public class IndexImportOAIPMHList_p {
public static serverObjects respond(final RequestHeader header, serverObjects post, final serverSwitch env) {
final serverObjects prop = new serverObjects();
ArrayList<OAIPMHImporter> jobs = new ArrayList<OAIPMHImporter>();
for (OAIPMHImporter job: OAIPMHImporter.runningJobs) jobs.add(job);
for (OAIPMHImporter job: OAIPMHImporter.startedJobs) jobs.add(job);
for (OAIPMHImporter job: OAIPMHImporter.finishedJobs) jobs.add(job);
boolean dark = false;
int cnt = 0;
for (OAIPMHImporter job: jobs) {
prop.put("table_" + cnt + "_dark", (dark) ? "1" : "0");
prop.put("table_" + cnt + "_thread", (job.isAlive()) ? "<img src=\"/env/grafics/crawl.gif\" alt=\"running\" />" : "finished");
prop.put("table_" + cnt + "_source", job.source());
prop.put("table_" + cnt + "_chunkCount", job.chunkCount());
prop.put("table_" + cnt + "_recordsCount", job.count());
prop.put("table_" + cnt + "_speed", job.speed());
dark = !dark;
cnt++;
}
prop.put("table", cnt);
return prop;
}
}

58
htroot/IndexImportOAIPMH_p.html
Unescape View File

@ -11,49 +11,41 @@
<h2>OAI-PMH Import</h2>
<p>Results from the import can be monitored in the <a href="/CrawlResults.html?process=7">indexing results for surrogates</a></p>
<form action="IndexImportOAIPMH_p.html" method="get">
<fieldset>
<legend>Single request import</legend>
This will submit only a single request as given here to a OAI-PMH server and imports records into the index<br />
<input name="urlstartone" type="text" value="#[defaulturl]#" size="100" />
<input name="submit" type="submit" value="Import OAI-PMH source" />
<form action="IndexImportOAIPMH_p.html" method="post" enctype="multipart/form-data">
<fieldset>
<legend>Single request import</legend>
This will submit only a single request as given here to a OAI-PMH server and imports records into the index<br />
<input name="urlstartone" type="text" value="#[defaulturl]#" size="100" />
<input name="submit" type="submit" value="Import OAI-PMH source" />
#(import-one)#::
<p><dl>
<dt>Source:</dt><dd>#[source]#</dd>
<dt>Processed:</dt><dd>#[count]# records</dd>
<dt>ResumptionToken:</dt><dd>#[rt]#</dd>
</dl></p>
::<p>Import failed: #[error]#</p>
#(/import-one)#
</fieldset>
</form>
#(import-one)#
::
<form><fieldset><legend>Import Process</legend>
<dl>
<dt>Source:</dt><dd>#[source]#</dd>
<dt>Processed:</dt><dd>#[count]# records</dd>
<dt>ResumptionToken:</dt><dd>#[rt]#</dd>
</dl>
</fieldset></form>
::
Import failed: #[error]#
#(/import-one)#
#(import-all)#
<p>#(status)#::Bad input data: #[message]# #(/status)#</p>
<form action="IndexImportOAIPMH_p.html" method="get">
<form action="IndexImportOAIPMH_p.html" method="post" enctype="multipart/form-data">
<fieldset>
<legend>Import all Records from a server</legend>
Import all records that follow acording to resumption elements into index<br />
<input name="urlstartall" type="text" value="" size="100" />
<input name="submit" type="submit" value="Import OAI-PMH source" />
#(status)#::<p>Import started!</p>::<p>Bad input data: #[message]# </p>#(/status)#
</fieldset>
</form>
::
<form><fieldset><legend>Import Process</legend>
<dl>
<dt>Thread:</dt><dd>#[thread]#</dd>
<dt>Source:</dt><dd>#[source]#</dd>
<dt>Processed Chunks:</dt><dd>#[chunkCount]#</dd>
<dt>Imported Records:</dt><dd>#[recordsCount]# records</dd>
<dt>Speed:</dt><dd>#[speed]# records per second</dd>
</dl>
</fieldset></form>
#(/import-all)#
<iframe name="OAI-PMH Import List"
src="/IndexImportOAIPMHList_p.html"
width="100%"
height="340"
frameborder="0"
scrolling="auto"
id="list">
</iframe>
#%env/templates/footer.template%#
</body>
</html>

43
htroot/IndexImportOAIPMH_p.java
Unescape View File

@ -1,7 +1,7 @@
// IndexImportOAIPMH.java
// -------------------------
// (C) 2009 by Michael Peter Christen; mc@yacy.net
// first published 04.05.2009 on http://yacy.net
// first published 30.10.2009 on http://yacy.net
// Frankfurt, Germany
//
// $LastChangedDate: 2009-10-11 23:29:18 +0200 (So, 11 Okt 2009) $
@ -24,7 +24,6 @@
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.NoSuchElementException;
import net.yacy.document.importer.OAIPMHImporter;
import net.yacy.document.importer.OAIPMHReader;
@ -43,33 +42,9 @@ public class IndexImportOAIPMH_p {
final Switchboard sb = (Switchboard) env;
prop.put("import-one", 0);
prop.put("import-all", 0);
prop.put("import-all_status", 0);
prop.put("status", 0);
prop.put("defaulturl", "");
OAIPMHImporter job = null;
try {
job = OAIPMHImporter.runningJobs.first();
} catch (NoSuchElementException e0) {
try {
job = OAIPMHImporter.startedJobs.first();
} catch (NoSuchElementException e1) {
try {
job = OAIPMHImporter.finishedJobs.first();
} catch (NoSuchElementException e2) {}
}
}
if (job != null) {
// one import is running, no option to insert anything
prop.put("import-all", 1);
prop.put("import-all_thread", (job.isAlive()) ? "running" : "finished");
prop.put("import-all_source", job.source());
prop.put("import-all_chunkCount", job.chunkCount());
prop.put("import-all_recordsCount", job.count());
prop.put("import-all_speed", job.speed());
return prop;
}
if (post != null) {
if (post.containsKey("urlstartone")) {
String oaipmhurl = post.get("urlstartone");
@ -110,19 +85,13 @@ public class IndexImportOAIPMH_p {
DigestURI url = null;
try {
url = new DigestURI(oaipmhurl, null);
job = new OAIPMHImporter(sb.loader, url);
OAIPMHImporter job = new OAIPMHImporter(sb.loader, url);
job.start();
prop.put("import-all", 1);
prop.put("import-all_thread", "started");
prop.put("import-all_source", job.source());
prop.put("import-all_chunkCount", 0);
prop.put("import-all_recordsCount", 0);
prop.put("import-all_speed", 0);
prop.put("status", 1);
} catch (MalformedURLException e) {
e.printStackTrace();
prop.put("import-all", 0);
prop.put("import-all_status", 1);
prop.put("import-all_status_message", e.getMessage());
prop.put("status", 2);
prop.put("status_message", e.getMessage());
}
}
}

Write Preview
Loading…
Cancel
Save