- some redesign in UI menu structure to make room for new 'Content Integration' main menu containing import servlets for Wikimedia Dumps, phpbb3 forum imports and OAI-PMH imports
- extended the OAI-PMH test applet and integrated it into the menu. Does still not import OAI-PMH records, but shows that it is able to read and parse this data - some redesign in ZURL storage: refactoring of access methods, better concurrency, less synchronization - added a limitation to the LURL metadata database table cache to 20 million entries: this cache was until now not limited and only limited by the available RAM which may have caused a memory-leak-like behavior. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6440 6c8d7289-2bf4-0310-a012-ef5d649a1542pull/1/head
orbiter
16 years ago
parent
8a1046feaa
commit
a0e891c63d
@ -0,0 +1,8 @@
|
||||
<div class="SubMenu">
|
||||
<h3>External Content Integration</h3>
|
||||
<ul class="SubMenu">
|
||||
<li><a href="/ContentIntegrationPHPBB3_p.html" class="MenuItemLink lock">Import phpBB3 forum</a></li>
|
||||
<li><a href="/IndexImportWikimedia_p.html" class="MenuItemLink lock">Import Wikimedia dumps</a></li>
|
||||
<li><a href="/IndexImportOAIPMH_p.html" class="MenuItemLink lock">Import OAI-PMH Sources</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
@ -0,0 +1,82 @@
|
||||
package net.yacy.document.importer;
|
||||
|
||||
import java.text.Collator;
|
||||
import java.text.ParseException;
|
||||
import java.util.Date;
|
||||
import java.util.Locale;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import net.yacy.kelondro.util.DateFormatter;
|
||||
|
||||
public class ResumptionToken extends TreeMap<String, String> {
|
||||
|
||||
private static final long serialVersionUID = -8389462290545629792L;
|
||||
|
||||
// use a collator to relax when distinguishing between lowercase und uppercase letters
|
||||
private static final Collator insensitiveCollator = Collator.getInstance(Locale.US);
|
||||
static {
|
||||
insensitiveCollator.setStrength(Collator.SECONDARY);
|
||||
insensitiveCollator.setDecomposition(Collator.NO_DECOMPOSITION);
|
||||
}
|
||||
|
||||
public ResumptionToken(
|
||||
Date expirationDate,
|
||||
int completeListSize,
|
||||
int cursor,
|
||||
int token
|
||||
) {
|
||||
super((Collator) insensitiveCollator.clone());
|
||||
this.put("expirationDate", DateFormatter.formatISO8601(expirationDate));
|
||||
this.put("completeListSize", Integer.toString(completeListSize));
|
||||
this.put("cursor", Integer.toString(cursor));
|
||||
this.put("token", Integer.toString(token));
|
||||
}
|
||||
|
||||
public ResumptionToken(
|
||||
String expirationDate,
|
||||
int completeListSize,
|
||||
int cursor,
|
||||
int token
|
||||
) {
|
||||
super((Collator) insensitiveCollator.clone());
|
||||
this.put("expirationDate", expirationDate);
|
||||
this.put("completeListSize", Integer.toString(completeListSize));
|
||||
this.put("cursor", Integer.toString(cursor));
|
||||
this.put("token", Integer.toString(token));
|
||||
}
|
||||
|
||||
public Date getExpirationDate() {
|
||||
String d = this.get("expirationDate");
|
||||
if (d == null) return null;
|
||||
try {
|
||||
return DateFormatter.parseISO8601(d);
|
||||
} catch (ParseException e) {
|
||||
e.printStackTrace();
|
||||
return new Date();
|
||||
}
|
||||
}
|
||||
|
||||
public int getCompleteListSize() {
|
||||
String t = this.get("completeListSize");
|
||||
if (t == null) return 0;
|
||||
return Integer.parseInt(t);
|
||||
}
|
||||
|
||||
public int getCursor() {
|
||||
String t = this.get("cursor");
|
||||
if (t == null) return 0;
|
||||
return Integer.parseInt(t);
|
||||
}
|
||||
|
||||
public int getToken() {
|
||||
String t = this.get("token");
|
||||
if (t == null) return 0;
|
||||
return Integer.parseInt(t);
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
return "expirationDate=" + DateFormatter.formatISO8601(this.getExpirationDate()) + ", completeListSize=" + getCompleteListSize() +
|
||||
", cursor=" + this.getCursor() + ", token=" + this.getToken();
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,90 @@
|
||||
package net.yacy.document.importer;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
import javax.xml.parsers.ParserConfigurationException;
|
||||
import javax.xml.parsers.SAXParser;
|
||||
import javax.xml.parsers.SAXParserFactory;
|
||||
|
||||
import org.xml.sax.Attributes;
|
||||
import org.xml.sax.SAXException;
|
||||
import org.xml.sax.helpers.DefaultHandler;
|
||||
|
||||
public class ResumptionTokenReader extends DefaultHandler {
|
||||
|
||||
// class variables
|
||||
private final StringBuilder buffer;
|
||||
private boolean parsingValue;
|
||||
private ResumptionToken token;
|
||||
private SAXParser saxParser;
|
||||
private InputStream stream;
|
||||
private Attributes atts;
|
||||
|
||||
public ResumptionTokenReader(final InputStream stream) throws IOException {
|
||||
this.buffer = new StringBuilder();
|
||||
this.parsingValue = false;
|
||||
this.token = null;
|
||||
this.stream = stream;
|
||||
this.atts = null;
|
||||
final SAXParserFactory factory = SAXParserFactory.newInstance();
|
||||
try {
|
||||
this.saxParser = factory.newSAXParser();
|
||||
this.saxParser.parse(this.stream, this);
|
||||
} catch (SAXException e) {
|
||||
e.printStackTrace();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
} catch (ParserConfigurationException e) {
|
||||
e.printStackTrace();
|
||||
throw new IOException(e.getMessage());
|
||||
} finally {
|
||||
try {
|
||||
this.stream.close();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public ResumptionToken getToken() {
|
||||
return this.token;
|
||||
}
|
||||
|
||||
/*
|
||||
<resumptionToken expirationDate="2009-10-31T22:52:14Z"
|
||||
completeListSize="226"
|
||||
cursor="0">688</resumptionToken>
|
||||
*/
|
||||
|
||||
public void run() {
|
||||
|
||||
}
|
||||
|
||||
public void startElement(final String uri, final String name, final String tag, final Attributes atts) throws SAXException {
|
||||
if ("resumptionToken".equals(tag)) {
|
||||
this.parsingValue = true;
|
||||
this.atts = atts;
|
||||
}
|
||||
}
|
||||
|
||||
public void endElement(final String uri, final String name, final String tag) {
|
||||
if (tag == null) return;
|
||||
if ("resumptionToken".equals(tag)) {
|
||||
this.token = new ResumptionToken(
|
||||
atts.getValue("expirationDate"),
|
||||
Integer.parseInt(atts.getValue("completeListSize")),
|
||||
Integer.parseInt(atts.getValue("cursor")),
|
||||
Integer.parseInt(buffer.toString().trim()));
|
||||
this.buffer.setLength(0);
|
||||
this.parsingValue = false;
|
||||
}
|
||||
}
|
||||
|
||||
public void characters(final char ch[], final int start, final int length) {
|
||||
if (parsingValue) {
|
||||
buffer.append(ch, start, length);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in new issue