- yacy can import phpbb3 posts without crawling - all data is written as surrogate - indexed surrogate files can be re-used git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5985 6c8d7289-2bf4-0310-a012-ef5d649a1542pull/1/head
orbiter
16 years ago
parent
f1a9253baa
commit
4b4bddca00
@ -0,0 +1,83 @@
|
|||||||
|
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
|
||||||
|
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||||
|
<head>
|
||||||
|
<title>YaCy '#[clientname]#': Content Integration: Retrieval from phpBB3 Databases</title>
|
||||||
|
#%env/templates/metas.template%#
|
||||||
|
</head>
|
||||||
|
<body id="ContentIntegrationPHPBB3">
|
||||||
|
#%env/templates/header.template%#
|
||||||
|
#%env/templates/submenuIndexCreate.template%#
|
||||||
|
<h2>Content Integration: Retrieval from phpBB3 Databases</h2>
|
||||||
|
<p>
|
||||||
|
It is possible to extract texts directly from mySQL and postgreSQL databases.
|
||||||
|
Each extraction is specific to the data that is hosted in the database.
|
||||||
|
This interface gives you access to the phpBB3 forums software content.
|
||||||
|
</p>
|
||||||
|
<p>
|
||||||
|
When a export is started, surrogate files are generated into DATA/SURROGATE/in which are automatically fetched by an indexer thread.
|
||||||
|
All indexed surrogate files are then moved to DATA/SURROGATE/out and can be re-cycled when a index is deleted.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<form action="ContentIntegrationPHPBB3_p.html">
|
||||||
|
<fieldset>
|
||||||
|
<dl>
|
||||||
|
<dt><b>The URL stub</b>,<br />like http://forum.yacy-websuche.de<br />this must be the path right in front of '/viewtopic.php?'</dt>
|
||||||
|
<dd><input type="text" name="content.phpbb3.urlstub" value="#[content.phpbb3.urlstub]#" size="60" /></dd>
|
||||||
|
|
||||||
|
<dt><b>Type</b> of database<br />(use either 'mysql' or 'pgsql')</dt>
|
||||||
|
<dd><input type="text" name="content.phpbb3.dbtype" value="#[content.phpbb3.dbtype]#" size="6" /></dd>
|
||||||
|
|
||||||
|
<dt><b>Host</b> of the database</dt>
|
||||||
|
<dd><input type="text" name="content.phpbb3.dbhost" value="#[content.phpbb3.dbhost]#" size="40" /></dd>
|
||||||
|
|
||||||
|
<dt><b>Port</b> of database service<br />(usually 3306 for mySQL)</dt>
|
||||||
|
<dd><input type="text" name="content.phpbb3.dbport" value="#[content.phpbb3.dbport]#" size="6" /></dd>
|
||||||
|
|
||||||
|
<dt><b>Name of the database</b> on the host</dt>
|
||||||
|
<dd><input type="text" name="content.phpbb3.dbname" value="#[content.phpbb3.dbname]#" size="20" /></dd>
|
||||||
|
|
||||||
|
<dt><b>User</b> that can access the database</dt>
|
||||||
|
<dd><input type="text" name="content.phpbb3.dbuser" value="#[content.phpbb3.dbuser]#" size="20" /></dd>
|
||||||
|
|
||||||
|
<dt><b>Password</b> for the account of that user given above</dt>
|
||||||
|
<dd><input type="text" name="content.phpbb3.dbpw" value="#[content.phpbb3.dbpw]#" size="20" /></dd>
|
||||||
|
|
||||||
|
<dt><b>Posts per file</b><br />in exported surrogates</dt>
|
||||||
|
<dd><input type="text" name="content.phpbb3.ppf" value="#[content.phpbb3.ppf]#" size="20" /></dd>
|
||||||
|
|
||||||
|
<dt></dt>
|
||||||
|
<dd>
|
||||||
|
<input type="submit" name="check" value="Check database connection" />
|
||||||
|
<input type="submit" name="export" value="Export Content to Surrogates" />
|
||||||
|
</dd>
|
||||||
|
</dl>
|
||||||
|
</fieldset>
|
||||||
|
</form>
|
||||||
|
|
||||||
|
#(check)#::
|
||||||
|
<form>
|
||||||
|
<fieldset>
|
||||||
|
<dl>
|
||||||
|
<dt>Posts in database</dt>
|
||||||
|
<dd>#[posts]#</dd>
|
||||||
|
|
||||||
|
<dt>first entry</dt>
|
||||||
|
<dd>#[first]#</dd>
|
||||||
|
|
||||||
|
<dt>last entry</dt>
|
||||||
|
<dd>#[last]#</dd>
|
||||||
|
|
||||||
|
</dl>
|
||||||
|
</fieldset>
|
||||||
|
</form>::
|
||||||
|
<p>Info failed: #[error]#</p>
|
||||||
|
#(/check)#
|
||||||
|
|
||||||
|
#(export)#::
|
||||||
|
<p>Export successful! Wrote #[files]# files in DATA/SURROGATES/in</p>::
|
||||||
|
<p>Export failed: #[error]#</p>
|
||||||
|
#(/export)#
|
||||||
|
|
||||||
|
#%env/templates/footer.template%#
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@ -0,0 +1,123 @@
|
|||||||
|
// ContentIntegrationPHPBB3_p.java
|
||||||
|
// (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
||||||
|
// first published 27.05.2009 on http://yacy.net
|
||||||
|
//
|
||||||
|
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
|
||||||
|
// $LastChangedRevision: 1986 $
|
||||||
|
// $LastChangedBy: orbiter $
|
||||||
|
//
|
||||||
|
// LICENSE
|
||||||
|
//
|
||||||
|
// This program is free software; you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU General Public License as published by
|
||||||
|
// the Free Software Foundation; either version 2 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU General Public License
|
||||||
|
// along with this program; if not, write to the Free Software
|
||||||
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||||
|
|
||||||
|
import de.anomic.content.dao.Dao;
|
||||||
|
import de.anomic.content.dao.PhpBB3Dao;
|
||||||
|
import de.anomic.http.httpRequestHeader;
|
||||||
|
import de.anomic.kelondro.util.DateFormatter;
|
||||||
|
import de.anomic.plasma.plasmaSwitchboard;
|
||||||
|
import de.anomic.server.serverObjects;
|
||||||
|
import de.anomic.server.serverSwitch;
|
||||||
|
|
||||||
|
public class ContentIntegrationPHPBB3_p {
|
||||||
|
|
||||||
|
public static serverObjects respond(final httpRequestHeader header, final serverObjects post, final serverSwitch<?> env) {
|
||||||
|
final serverObjects prop = new serverObjects();
|
||||||
|
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
|
||||||
|
|
||||||
|
prop.put("check", 0);
|
||||||
|
prop.put("export", 0);
|
||||||
|
|
||||||
|
if (post != null) {
|
||||||
|
|
||||||
|
String urlstub = post.get("content.phpbb3.urlstub", "");
|
||||||
|
String dbtype = post.get("content.phpbb3.dbtype", "");
|
||||||
|
String dbhost = post.get("content.phpbb3.dbhost", "");
|
||||||
|
int dbport = post.getInt("content.phpbb3.dbport", 3306);
|
||||||
|
String dbname = post.get("content.phpbb3.dbname", "");
|
||||||
|
String dbuser = post.get("content.phpbb3.dbuser", "");
|
||||||
|
String dbpw = post.get("content.phpbb3.dbpw", "");
|
||||||
|
int ppf = post.getInt("content.phpbb3.ppf", 1000);
|
||||||
|
|
||||||
|
|
||||||
|
sb.setConfig("content.phpbb3.urlstub", urlstub);
|
||||||
|
sb.setConfig("content.phpbb3.dbtype", dbtype);
|
||||||
|
sb.setConfig("content.phpbb3.dbhost", dbhost);
|
||||||
|
sb.setConfig("content.phpbb3.dbport", dbport);
|
||||||
|
sb.setConfig("content.phpbb3.dbname", dbname);
|
||||||
|
sb.setConfig("content.phpbb3.dbuser", dbuser);
|
||||||
|
sb.setConfig("content.phpbb3.dbpw", dbpw);
|
||||||
|
sb.setConfig("content.phpbb3.ppf", ppf);
|
||||||
|
|
||||||
|
if (post.containsKey("check")) {
|
||||||
|
try {
|
||||||
|
Dao db = new PhpBB3Dao(
|
||||||
|
urlstub,
|
||||||
|
dbtype,
|
||||||
|
dbhost,
|
||||||
|
dbport,
|
||||||
|
dbname,
|
||||||
|
dbuser,
|
||||||
|
dbpw
|
||||||
|
);
|
||||||
|
prop.put("check", 1);
|
||||||
|
prop.put("check_posts", db.size());
|
||||||
|
prop.putHTML("check_first", db.first().toString());
|
||||||
|
prop.putHTML("check_last", db.latest().toString());
|
||||||
|
db.close();
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
prop.put("check", 2);
|
||||||
|
prop.put("check_error", e.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
if (post.containsKey("export")) {
|
||||||
|
try {
|
||||||
|
Dao db = new PhpBB3Dao(
|
||||||
|
urlstub,
|
||||||
|
dbtype,
|
||||||
|
dbhost,
|
||||||
|
dbport,
|
||||||
|
dbname,
|
||||||
|
dbuser,
|
||||||
|
dbpw
|
||||||
|
);
|
||||||
|
|
||||||
|
int files = db.writeSurrogates(db.query(0, -1, 100), sb.surrogatesInPath, "fullexport-" + DateFormatter.formatShortSecond(), ppf);
|
||||||
|
prop.put("export", 1);
|
||||||
|
prop.put("export_files", files);
|
||||||
|
db.close();
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
prop.put("export", 2);
|
||||||
|
prop.put("export_error", e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
prop.putHTML("content.phpbb3.urlstub", sb.getConfig("content.phpbb3.urlstub", ""));
|
||||||
|
prop.putHTML("content.phpbb3.dbtype", sb.getConfig("content.phpbb3.dbtype", ""));
|
||||||
|
prop.putHTML("content.phpbb3.dbhost", sb.getConfig("content.phpbb3.dbhost", ""));
|
||||||
|
prop.putHTML("content.phpbb3.dbport", sb.getConfig("content.phpbb3.dbport", ""));
|
||||||
|
prop.putHTML("content.phpbb3.dbname", sb.getConfig("content.phpbb3.dbname", ""));
|
||||||
|
prop.putHTML("content.phpbb3.dbuser", sb.getConfig("content.phpbb3.dbuser", ""));
|
||||||
|
prop.putHTML("content.phpbb3.dbpw", sb.getConfig("content.phpbb3.dbpw", ""));
|
||||||
|
prop.putHTML("content.phpbb3.ppf", sb.getConfig("content.phpbb3.ppf", ""));
|
||||||
|
|
||||||
|
return prop;
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,393 @@
|
|||||||
|
// PhpBB3Dao.java
|
||||||
|
// (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
||||||
|
// first published 26.05.2009 on http://yacy.net
|
||||||
|
//
|
||||||
|
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
|
||||||
|
// $LastChangedRevision: 1986 $
|
||||||
|
// $LastChangedBy: orbiter $
|
||||||
|
//
|
||||||
|
// LICENSE
|
||||||
|
//
|
||||||
|
// This program is free software; you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU General Public License as published by
|
||||||
|
// the Free Software Foundation; either version 2 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU General Public License
|
||||||
|
// along with this program; if not, write to the Free Software
|
||||||
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||||
|
|
||||||
|
package de.anomic.content.dao;
|
||||||
|
|
||||||
|
import java.io.BufferedOutputStream;
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileOutputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.OutputStreamWriter;
|
||||||
|
import java.io.UnsupportedEncodingException;
|
||||||
|
import java.net.MalformedURLException;
|
||||||
|
import java.sql.Connection;
|
||||||
|
import java.sql.DriverManager;
|
||||||
|
import java.sql.ResultSet;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.sql.Statement;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.concurrent.ArrayBlockingQueue;
|
||||||
|
import java.util.concurrent.BlockingQueue;
|
||||||
|
|
||||||
|
import de.anomic.content.DCEntry;
|
||||||
|
import de.anomic.yacy.yacyURL;
|
||||||
|
|
||||||
|
public class PhpBB3Dao implements Dao {
|
||||||
|
|
||||||
|
private Connection conn = null;
|
||||||
|
private String urlstub;
|
||||||
|
private HashMap<Integer, String> users;
|
||||||
|
|
||||||
|
public PhpBB3Dao(
|
||||||
|
String urlstub,
|
||||||
|
String dbType,
|
||||||
|
String host,
|
||||||
|
int port,
|
||||||
|
String dbname,
|
||||||
|
String user,
|
||||||
|
String pw) throws Exception {
|
||||||
|
this.conn = getConnection(dbType, host, port, dbname, user, pw);
|
||||||
|
this.urlstub = urlstub;
|
||||||
|
this.users = new HashMap<Integer, String>();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void finalize() throws Throwable {
|
||||||
|
closeConnection();
|
||||||
|
}
|
||||||
|
|
||||||
|
private Connection getConnection(final String dbType, String host, int port, String dbname, String user, String pw) throws Exception {
|
||||||
|
String dbDriverStr = null, dbConnStr = null;
|
||||||
|
if (dbType.equalsIgnoreCase("mysql")) {
|
||||||
|
dbDriverStr = "com.mysql.jdbc.Driver";
|
||||||
|
dbConnStr = "jdbc:mysql://" + host + ":" + port + "/" + dbname;
|
||||||
|
} else if (dbType.equalsIgnoreCase("pgsql")) {
|
||||||
|
dbDriverStr = "org.postgresql.Driver";
|
||||||
|
dbConnStr = "jdbc:postgresql://" + host + ":" + port + "/" + dbname;
|
||||||
|
} else throw new IllegalArgumentException();
|
||||||
|
|
||||||
|
try {
|
||||||
|
Class.forName(dbDriverStr).newInstance();
|
||||||
|
} catch (final Exception e) {
|
||||||
|
throw new Exception("Unable to load the jdbc driver: " + e.getMessage(),e);
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
return DriverManager.getConnection(dbConnStr, user, pw);
|
||||||
|
} catch (final Exception e) {
|
||||||
|
throw new Exception("Unable to establish a database connection: " + e.getMessage(),e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void closeConnection() {
|
||||||
|
if (conn != null) {
|
||||||
|
try {
|
||||||
|
conn.close();
|
||||||
|
conn = null;
|
||||||
|
} catch (SQLException e) {
|
||||||
|
System.out.println("PhpBB3Dao: " + e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public Date first() {
|
||||||
|
StringBuilder sql = new StringBuilder(256);
|
||||||
|
sql.append("select min(post_time) from phpbb_posts");
|
||||||
|
Statement stmt = null;
|
||||||
|
ResultSet rs = null;
|
||||||
|
try {
|
||||||
|
stmt = conn.createStatement();
|
||||||
|
rs = stmt.executeQuery(sql.toString());
|
||||||
|
if (rs.next()) {
|
||||||
|
return new Date(rs.getLong(1) * 1000L);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
} catch (SQLException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
return null;
|
||||||
|
} finally {
|
||||||
|
if (rs != null) try {rs.close();} catch (SQLException e) {}
|
||||||
|
if (stmt != null) try {stmt.close();} catch (SQLException e) {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public Date latest() {
|
||||||
|
StringBuilder sql = new StringBuilder(256);
|
||||||
|
sql.append("select max(post_time) from phpbb_posts");
|
||||||
|
Statement stmt = null;
|
||||||
|
ResultSet rs = null;
|
||||||
|
try {
|
||||||
|
stmt = conn.createStatement();
|
||||||
|
rs = stmt.executeQuery(sql.toString());
|
||||||
|
if (rs.next()) {
|
||||||
|
return new Date(rs.getLong(1) * 1000L);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
} catch (SQLException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
return null;
|
||||||
|
} finally {
|
||||||
|
if (rs != null) try {rs.close();} catch (SQLException e) {}
|
||||||
|
if (stmt != null) try {stmt.close();} catch (SQLException e) {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public int size() {
|
||||||
|
StringBuilder sql = new StringBuilder(256);
|
||||||
|
sql.append("select count(*) from phpbb_posts");
|
||||||
|
Statement stmt = null;
|
||||||
|
ResultSet rs = null;
|
||||||
|
try {
|
||||||
|
stmt = conn.createStatement();
|
||||||
|
rs = stmt.executeQuery(sql.toString());
|
||||||
|
if (rs.next()) {
|
||||||
|
return rs.getInt(1);
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
} catch (SQLException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
return 0;
|
||||||
|
} finally {
|
||||||
|
if (rs != null) try {rs.close();} catch (SQLException e) {}
|
||||||
|
if (stmt != null) try {stmt.close();} catch (SQLException e) {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public DCEntry get(int item) {
|
||||||
|
StringBuilder sql = new StringBuilder(256);
|
||||||
|
sql.append("select * from phpbb_posts where post_id = ");
|
||||||
|
sql.append(item);
|
||||||
|
return getOne(sql);
|
||||||
|
}
|
||||||
|
|
||||||
|
public BlockingQueue<DCEntry> query(int from, int until, int queueSize) {
|
||||||
|
// define the sql query
|
||||||
|
final StringBuilder sql = new StringBuilder(256);
|
||||||
|
sql.append("select * from phpbb_posts where post_id >= ");
|
||||||
|
sql.append(from);
|
||||||
|
if (until > from) {
|
||||||
|
sql.append(" and post_id < ");
|
||||||
|
sql.append(until);
|
||||||
|
}
|
||||||
|
sql.append(" order by post_id");
|
||||||
|
|
||||||
|
// execute the query and push entries to a queue concurrently
|
||||||
|
return toQueue(sql, queueSize);
|
||||||
|
}
|
||||||
|
|
||||||
|
public BlockingQueue<DCEntry> query(Date from, int queueSize) {
|
||||||
|
// define the sql query
|
||||||
|
final StringBuilder sql = new StringBuilder(256);
|
||||||
|
sql.append("select * from phpbb_posts where post_time >= ");
|
||||||
|
sql.append(from.getTime() / 1000);
|
||||||
|
sql.append(" order by post_id");
|
||||||
|
|
||||||
|
// execute the query and push entries to a queue concurrently
|
||||||
|
return toQueue(sql, queueSize);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private DCEntry getOne(StringBuilder sql) {
|
||||||
|
Statement stmt = null;
|
||||||
|
ResultSet rs = null;
|
||||||
|
try {
|
||||||
|
stmt = conn.createStatement();
|
||||||
|
rs = stmt.executeQuery(sql.toString());
|
||||||
|
if (rs.next()) {
|
||||||
|
try {
|
||||||
|
return parseResultSet(rs);
|
||||||
|
} catch (MalformedURLException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
} catch (SQLException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
return null;
|
||||||
|
} finally {
|
||||||
|
if (rs != null) try {rs.close();} catch (SQLException e) {}
|
||||||
|
if (stmt != null) try {stmt.close();} catch (SQLException e) {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private BlockingQueue<DCEntry> toQueue(final StringBuilder sql, int queueSize) {
|
||||||
|
// execute the query and push entries to a queue concurrently
|
||||||
|
final BlockingQueue<DCEntry> queue = new ArrayBlockingQueue<DCEntry>(queueSize);
|
||||||
|
Thread dbreader = new Thread() {
|
||||||
|
public void run() {
|
||||||
|
Statement stmt = null;
|
||||||
|
ResultSet rs = null;
|
||||||
|
try {
|
||||||
|
stmt = conn.createStatement();
|
||||||
|
rs = stmt.executeQuery(sql.toString());
|
||||||
|
while (rs.next()) {
|
||||||
|
try {
|
||||||
|
queue.put(parseResultSet(rs));
|
||||||
|
} catch (MalformedURLException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
queue.put(DCEntry.poison);
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
} catch (SQLException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
} finally {
|
||||||
|
if (rs != null) try {rs.close();} catch (SQLException e) {}
|
||||||
|
if (stmt != null) try {stmt.close();} catch (SQLException e) {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
dbreader.start();
|
||||||
|
return queue;
|
||||||
|
}
|
||||||
|
|
||||||
|
private DCEntry parseResultSet(ResultSet rs) throws SQLException, MalformedURLException {
|
||||||
|
yacyURL url;
|
||||||
|
int item = rs.getInt("post_id");
|
||||||
|
url = new yacyURL(this.urlstub + "/viewtopic.php?t=" + item);
|
||||||
|
String subject = rs.getString("post_subject");
|
||||||
|
String text = xmlCleaner(rs.getString("post_text"));
|
||||||
|
String user = getUser(rs.getInt("poster_id"));
|
||||||
|
Date date = new Date(rs.getLong("post_time") * 1000L);
|
||||||
|
return new DCEntry(url, date, subject, user, text);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String xmlCleaner(String s) {
|
||||||
|
if (s == null) return null;
|
||||||
|
|
||||||
|
StringBuilder sbOutput = new StringBuilder(s.length());
|
||||||
|
char c;
|
||||||
|
|
||||||
|
for (int i = 0; i < s.length(); i++ ) {
|
||||||
|
c = s.charAt(i);
|
||||||
|
if ((c >= 0x0020 && c <= 0xD7FF) ||
|
||||||
|
(c >= 0xE000 && c <= 0xFFFD) ||
|
||||||
|
c == 0x0009 ||
|
||||||
|
c == 0x000A ||
|
||||||
|
c == 0x000D ) {
|
||||||
|
sbOutput.append(c);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sbOutput.toString().trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
private String getUser(int poster_id) {
|
||||||
|
String nick = this.users.get(poster_id);
|
||||||
|
if (nick != null) return nick;
|
||||||
|
|
||||||
|
StringBuilder sql = new StringBuilder(256);
|
||||||
|
sql.append("select * from phpbb_users where user_id = ");
|
||||||
|
sql.append(poster_id);
|
||||||
|
Statement stmt = null;
|
||||||
|
ResultSet rs = null;
|
||||||
|
try {
|
||||||
|
stmt = conn.createStatement();
|
||||||
|
rs = stmt.executeQuery(sql.toString());
|
||||||
|
if (rs.next()) nick = rs.getString("username");
|
||||||
|
if (nick == null) nick = "";
|
||||||
|
this.users.put(poster_id, nick);
|
||||||
|
return nick;
|
||||||
|
} catch (SQLException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
return "";
|
||||||
|
} finally {
|
||||||
|
if (rs != null) try {rs.close();} catch (SQLException e) {}
|
||||||
|
if (stmt != null) try {stmt.close();} catch (SQLException e) {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public int writeSurrogates(
|
||||||
|
BlockingQueue<DCEntry> queue,
|
||||||
|
File targetdir,
|
||||||
|
String versioninfo,
|
||||||
|
int maxEntriesInFile
|
||||||
|
) {
|
||||||
|
try {
|
||||||
|
// generate output file name and attributes
|
||||||
|
String targethost = new yacyURL(this.urlstub, null).getHost();
|
||||||
|
int fc = 0;
|
||||||
|
File outputfiletmp = null, outputfile = null;
|
||||||
|
|
||||||
|
// write the result from the query concurrently in a file
|
||||||
|
OutputStreamWriter osw = null;
|
||||||
|
DCEntry e;
|
||||||
|
int c = 0;
|
||||||
|
while ((e = queue.take()) != DCEntry.poison) {
|
||||||
|
if (osw == null) {
|
||||||
|
outputfiletmp = new File(targetdir, targethost + "." + versioninfo + "." + fc + ".xml.tmp");
|
||||||
|
outputfile = new File(targetdir, targethost + "." + versioninfo + "." + fc + ".xml");
|
||||||
|
if (outputfiletmp.exists()) outputfiletmp.delete();
|
||||||
|
if (outputfile.exists()) outputfile.delete();
|
||||||
|
osw = new OutputStreamWriter(new BufferedOutputStream(new FileOutputStream(outputfiletmp)), "UTF-8");
|
||||||
|
osw.write("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<surrogates xmlns:dc=\"http://purl.org/dc/elements/1.1/\">\n");
|
||||||
|
}
|
||||||
|
e.writeXML(osw);
|
||||||
|
c++;
|
||||||
|
if (c >= maxEntriesInFile) {
|
||||||
|
osw.write("</surrogates>\n");
|
||||||
|
osw.close();
|
||||||
|
outputfiletmp.renameTo(outputfile);
|
||||||
|
osw = null;
|
||||||
|
c = 0;
|
||||||
|
fc++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
osw.write("</surrogates>\n");
|
||||||
|
osw.close();
|
||||||
|
outputfiletmp.renameTo(outputfile);
|
||||||
|
return fc + 1;
|
||||||
|
} catch (MalformedURLException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
} catch (UnsupportedEncodingException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
} catch (IOException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void close() {
|
||||||
|
try {
|
||||||
|
this.conn.close();
|
||||||
|
} catch (SQLException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void main(String[] args) {
|
||||||
|
PhpBB3Dao db;
|
||||||
|
try {
|
||||||
|
db = new PhpBB3Dao(
|
||||||
|
"http://forum.yacy-websuche.de",
|
||||||
|
"mysql",
|
||||||
|
"localhost",
|
||||||
|
3306,
|
||||||
|
"forum",
|
||||||
|
"root",
|
||||||
|
""
|
||||||
|
);
|
||||||
|
System.out.println("Posts in database : " + db.size());
|
||||||
|
System.out.println("First entry : " + db.first());
|
||||||
|
System.out.println("Last entry : " + db.latest());
|
||||||
|
File targetdir = new File("x").getParentFile();
|
||||||
|
db.writeSurrogates(db.query(0, -1, 100), targetdir, "id0-current", 3000);
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
Loading…
Reference in new issue