From f348190566ab69a5efbdc33a2c2b4b567c33452e Mon Sep 17 00:00:00 2001 From: orbiter Date: Sat, 13 Jun 2009 23:03:40 +0000 Subject: [PATCH] tried to insert a database dump import method to the phpBB3 import function. Reason: imports or large database dumps are cannot be handled with phpMyAdmin and this should be an easy way to the database dumps into a mySQL database where it can be exported again with the phpBB3 content integration adapter. Completion or removal of this function stub will follow before next main release. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6065 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- defaults/yacy.init | 1 + htroot/ContentIntegrationPHPBB3_p.html | 15 +++ htroot/ContentIntegrationPHPBB3_p.java | 33 +++++- source/de/anomic/content/dao/Dao.java | 4 +- .../content/dao/DatabaseConnection.java | 99 +++++++++++++++++ source/de/anomic/content/dao/ImportDump.java | 105 ++++++++++++++++++ source/de/anomic/content/dao/PhpBB3Dao.java | 77 ++----------- 7 files changed, 266 insertions(+), 68 deletions(-) create mode 100644 source/de/anomic/content/dao/DatabaseConnection.java create mode 100644 source/de/anomic/content/dao/ImportDump.java diff --git a/defaults/yacy.init b/defaults/yacy.init index 8ed45d7c9..93f3fd8b7 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -910,3 +910,4 @@ content.phpbb3.tableprefix = phpbb_ content.phpbb3.dbuser = notroot content.phpbb3.dbpw = joshua content.phpbb3.ppf = 1000 +content.phpbb3.dumpfile = diff --git a/htroot/ContentIntegrationPHPBB3_p.html b/htroot/ContentIntegrationPHPBB3_p.html index d49992b70..6c165e26e 100644 --- a/htroot/ContentIntegrationPHPBB3_p.html +++ b/htroot/ContentIntegrationPHPBB3_p.html @@ -55,6 +55,16 @@ +
+
+
Import a database dump,
+
+
+
+
+ +
+
#(check)#:: @@ -81,6 +91,11 @@

Export failed: #[error]#

#(/export)# + #(import)#:: +

Import successful!:: +

Import failed: #[error]#

+ #(/import)# + #%env/templates/footer.template%# diff --git a/htroot/ContentIntegrationPHPBB3_p.java b/htroot/ContentIntegrationPHPBB3_p.java index c17f941d6..9321285fa 100644 --- a/htroot/ContentIntegrationPHPBB3_p.java +++ b/htroot/ContentIntegrationPHPBB3_p.java @@ -22,7 +22,10 @@ // along with this program; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +import java.io.File; + import de.anomic.content.dao.Dao; +import de.anomic.content.dao.ImportDump; import de.anomic.content.dao.PhpBB3Dao; import de.anomic.http.httpRequestHeader; import de.anomic.kelondro.util.DateFormatter; @@ -38,6 +41,7 @@ public class ContentIntegrationPHPBB3_p { prop.put("check", 0); prop.put("export", 0); + prop.put("import", 0); if (post != null) { @@ -50,6 +54,7 @@ public class ContentIntegrationPHPBB3_p { String dbuser = post.get("content.phpbb3.dbuser", ""); String dbpw = post.get("content.phpbb3.dbpw", ""); int ppf = post.getInt("content.phpbb3.ppf", 1000); + String dumpfile = post.get("content.phpbb3.dumpfile", ""); sb.setConfig("content.phpbb3.urlstub", urlstub); @@ -61,6 +66,7 @@ public class ContentIntegrationPHPBB3_p { sb.setConfig("content.phpbb3.dbuser", dbuser); sb.setConfig("content.phpbb3.dbpw", dbpw); sb.setConfig("content.phpbb3.ppf", ppf); + sb.setConfig("content.phpbb3.dumpfile", dumpfile); if (post.containsKey("check")) { try { @@ -84,7 +90,6 @@ public class ContentIntegrationPHPBB3_p { prop.put("check", 2); prop.put("check_error", e.getMessage()); } - } if (post.containsKey("export")) { @@ -111,6 +116,31 @@ public class ContentIntegrationPHPBB3_p { } } + if (post.containsKey("import")) { + File f = new File(dumpfile); + if (!f.exists()) { + prop.put("import", 2); + prop.put("import_error", "file " + dumpfile + " does not exist"); + } else try { + ImportDump importer = new ImportDump( + dbtype, + dbhost, + dbport, + dbname, + dbuser, + dbpw + ); + + importer.imp(f); + prop.put("import", 1); + importer.close(); + } catch (Exception e) { + e.printStackTrace(); + prop.put("import", 2); + prop.put("import_error", e.getMessage()); + } + } + } prop.putHTML("content.phpbb3.urlstub", sb.getConfig("content.phpbb3.urlstub", "")); @@ -122,6 +152,7 @@ public class ContentIntegrationPHPBB3_p { prop.putHTML("content.phpbb3.dbuser", sb.getConfig("content.phpbb3.dbuser", "")); prop.putHTML("content.phpbb3.dbpw", sb.getConfig("content.phpbb3.dbpw", "")); prop.putHTML("content.phpbb3.ppf", sb.getConfig("content.phpbb3.ppf", "")); + prop.putHTML("content.phpbb3.dumpfile", sb.getConfig("content.phpbb3.dumpfile", "")); return prop; } diff --git a/source/de/anomic/content/dao/Dao.java b/source/de/anomic/content/dao/Dao.java index 2c3e719f4..7c5305f2f 100644 --- a/source/de/anomic/content/dao/Dao.java +++ b/source/de/anomic/content/dao/Dao.java @@ -25,6 +25,7 @@ package de.anomic.content.dao; import java.io.File; +import java.sql.SQLException; import java.util.Date; import java.util.concurrent.BlockingQueue; @@ -39,8 +40,9 @@ public interface Dao { /** * get the maximum number of possible DCEntry items in the database + * @throws SQLException */ - public int size(); + public int size() throws SQLException; /** * retrieve a single item from the database diff --git a/source/de/anomic/content/dao/DatabaseConnection.java b/source/de/anomic/content/dao/DatabaseConnection.java new file mode 100644 index 000000000..8b6f8bf91 --- /dev/null +++ b/source/de/anomic/content/dao/DatabaseConnection.java @@ -0,0 +1,99 @@ +// DatabaseConnection.java +// (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany +// first published 11.06.2009 on http://yacy.net +// +// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $ +// $LastChangedRevision: 1986 $ +// $LastChangedBy: orbiter $ +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package de.anomic.content.dao; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +public class DatabaseConnection { + + private Connection connection; + + public DatabaseConnection(final String dbType, String host, int port, String dbname, String user, String pw) throws SQLException { + String dbDriverStr = null, dbConnStr = null; + if (dbType.equalsIgnoreCase("mysql")) { + dbDriverStr = "com.mysql.jdbc.Driver"; + dbConnStr = "jdbc:mysql://" + host + ":" + port + "/" + dbname; + } else if (dbType.equalsIgnoreCase("pgsql")) { + dbDriverStr = "org.postgresql.Driver"; + dbConnStr = "jdbc:postgresql://" + host + ":" + port + "/" + dbname; + } else throw new IllegalArgumentException(); + + try { + Class.forName(dbDriverStr).newInstance(); + } catch (final Exception e) { + throw new SQLException("Unable to load the jdbc driver: " + e.getMessage()); + } + + try { + this.connection = DriverManager.getConnection(dbConnStr, user, pw); + } catch (final Exception e) { + throw new SQLException("Unable to establish a database connection: " + e.getMessage()); + } + } + + public void setAutoCommit(boolean b) { + try { + this.connection.setAutoCommit(b); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + public int count(String tablename) throws SQLException { + Statement stmt = null; + ResultSet rs = null; + try { + stmt = this.connection.createStatement(); + rs = stmt.executeQuery("select count(*) from " + tablename); + if (rs.next()) { + return rs.getInt(1); + } + return 0; + } catch (SQLException e) { + throw e; + } finally { + if (rs != null) try {rs.close();} catch (SQLException e) {} + if (stmt != null) try {stmt.close();} catch (SQLException e) {} + } + } + + public void close() { + if (connection != null) { + try { + connection.close(); + connection = null; + } catch (SQLException e) { + } + } + } + + public Statement statement() throws SQLException { + return this.connection.createStatement(); + } +} diff --git a/source/de/anomic/content/dao/ImportDump.java b/source/de/anomic/content/dao/ImportDump.java new file mode 100644 index 000000000..9687ea223 --- /dev/null +++ b/source/de/anomic/content/dao/ImportDump.java @@ -0,0 +1,105 @@ +// PhpBB3Dao.java +// (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany +// first published 26.05.2009 on http://yacy.net +// +// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $ +// $LastChangedRevision: 1986 $ +// $LastChangedBy: orbiter $ +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package de.anomic.content.dao; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.sql.SQLException; +import java.sql.Statement; + +import de.anomic.kelondro.util.FileUtils; + +public class ImportDump { + + private DatabaseConnection conn = null; + + public ImportDump( + String dbType, + String host, + int port, + String dbname, + String user, + String pw) throws Exception { + this.conn = new DatabaseConnection(dbType, host, port, dbname, user, pw); + this.conn.setAutoCommit(true); + } + + public void imp(File dump) throws SQLException { + Statement statement = null; + //int maxBatch = 1048576; + try { + statement = conn.statement(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + FileUtils.copy(dump, baos); + + String s = new String(baos.toByteArray(), "UTF-8"); + int batchSize = 0; + int p, q; + String t; + loop: while (s.length() > 0) { + p = s.indexOf("INSERT INTO", 1); + q = s.indexOf("CREATE TABLE", 1); + if (q >= 0 && q < p) p = q; + if (p < 0) { + // finalize process + statement.executeBatch(); + System.out.println(s); + statement.addBatch(s); + statement.executeBatch(); + break loop; + } + t = s.substring(0, p); + s = s.substring(p); + //if (batchSize + t.length() >= maxBatch) { + statement.executeBatch(); + batchSize = 0; + //} + System.out.println(t); + statement.addBatch(t); + batchSize += t.length(); + } + statement.executeBatch(); + } catch (SQLException e) { + e.printStackTrace(); + throw e; + } catch (IOException e) { + e.printStackTrace(); + throw new SQLException(e.getMessage()); + } finally { + if (statement != null) try {statement.close();} catch (SQLException e) {} + } + } + + protected void finalize() throws Throwable { + close(); + } + + public void close() { + this.conn.close(); + } + + +} diff --git a/source/de/anomic/content/dao/PhpBB3Dao.java b/source/de/anomic/content/dao/PhpBB3Dao.java index ae02f6a65..6a407eb22 100644 --- a/source/de/anomic/content/dao/PhpBB3Dao.java +++ b/source/de/anomic/content/dao/PhpBB3Dao.java @@ -31,8 +31,6 @@ import java.io.IOException; import java.io.OutputStreamWriter; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; -import java.sql.Connection; -import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -46,7 +44,7 @@ import de.anomic.yacy.yacyURL; public class PhpBB3Dao implements Dao { - private Connection conn = null; + private DatabaseConnection conn = null; private String urlstub, prefix; private HashMap users; @@ -59,55 +57,21 @@ public class PhpBB3Dao implements Dao { String prefix, String user, String pw) throws Exception { - this.conn = getConnection(dbType, host, port, dbname, user, pw); + this.conn = new DatabaseConnection(dbType, host, port, dbname, user, pw); this.urlstub = urlstub; this.prefix = prefix; this.users = new HashMap(); } protected void finalize() throws Throwable { - closeConnection(); - } - - private Connection getConnection(final String dbType, String host, int port, String dbname, String user, String pw) throws Exception { - String dbDriverStr = null, dbConnStr = null; - if (dbType.equalsIgnoreCase("mysql")) { - dbDriverStr = "com.mysql.jdbc.Driver"; - dbConnStr = "jdbc:mysql://" + host + ":" + port + "/" + dbname; - } else if (dbType.equalsIgnoreCase("pgsql")) { - dbDriverStr = "org.postgresql.Driver"; - dbConnStr = "jdbc:postgresql://" + host + ":" + port + "/" + dbname; - } else throw new IllegalArgumentException(); - - try { - Class.forName(dbDriverStr).newInstance(); - } catch (final Exception e) { - throw new Exception("Unable to load the jdbc driver: " + e.getMessage(),e); - } - - try { - return DriverManager.getConnection(dbConnStr, user, pw); - } catch (final Exception e) { - throw new Exception("Unable to establish a database connection: " + e.getMessage(),e); - } - } - - public void closeConnection() { - if (conn != null) { - try { - conn.close(); - conn = null; - } catch (SQLException e) { - System.out.println("PhpBB3Dao: " + e); - } - } + close(); } public Date first() { Statement stmt = null; ResultSet rs = null; try { - stmt = conn.createStatement(); + stmt = conn.statement(); rs = stmt.executeQuery("select min(post_time) from " + prefix + "posts"); if (rs.next()) { return new Date(rs.getLong(1) * 1000L); @@ -126,7 +90,7 @@ public class PhpBB3Dao implements Dao { Statement stmt = null; ResultSet rs = null; try { - stmt = conn.createStatement(); + stmt = conn.statement(); rs = stmt.executeQuery("select max(post_time) from " + prefix + "posts"); if (rs.next()) { return new Date(rs.getLong(1) * 1000L); @@ -141,23 +105,8 @@ public class PhpBB3Dao implements Dao { } } - public int size() { - Statement stmt = null; - ResultSet rs = null; - try { - stmt = conn.createStatement(); - rs = stmt.executeQuery("select count(*) from " + prefix + "posts"); - if (rs.next()) { - return rs.getInt(1); - } - return 0; - } catch (SQLException e) { - e.printStackTrace(); - return 0; - } finally { - if (rs != null) try {rs.close();} catch (SQLException e) {} - if (stmt != null) try {stmt.close();} catch (SQLException e) {} - } + public int size() throws SQLException { + return this.conn.count(prefix + "posts"); } public DCEntry get(int item) { @@ -195,7 +144,7 @@ public class PhpBB3Dao implements Dao { Statement stmt = null; ResultSet rs = null; try { - stmt = conn.createStatement(); + stmt = conn.statement(); rs = stmt.executeQuery(sql); if (rs.next()) { try { @@ -222,7 +171,7 @@ public class PhpBB3Dao implements Dao { Statement stmt = null; ResultSet rs = null; try { - stmt = conn.createStatement(); + stmt = conn.statement(); rs = stmt.executeQuery(sql.toString()); while (rs.next()) { try { @@ -286,7 +235,7 @@ public class PhpBB3Dao implements Dao { Statement stmt = null; ResultSet rs = null; try { - stmt = conn.createStatement(); + stmt = conn.statement(); rs = stmt.executeQuery(sql.toString()); if (rs.next()) nick = rs.getString("username"); if (nick == null) nick = ""; @@ -354,11 +303,7 @@ public class PhpBB3Dao implements Dao { } public void close() { - try { - this.conn.close(); - } catch (SQLException e) { - e.printStackTrace(); - } + this.conn.close(); } public static void main(String[] args) {