fix for Malformed URL Exception in url migration

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2825 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent 9eecc9a888
commit 11843bba7f

@ -1122,9 +1122,13 @@ public final class yacy {
} }
private static void migratelurls(File root, File urlHash) { private static void migratelurls(File root, File urlHash) {
try {
plasmaURLPool pool = new plasmaURLPool(new File(root, "DATA/PLASMADB"), new File(root, "DATA/INDEX"), 16000, true, 1000, true, 1000, true, 10000); plasmaURLPool pool = new plasmaURLPool(new File(root, "DATA/PLASMADB"), new File(root, "DATA/INDEX"), 16000, true, 1000, true, 1000, true, 10000);
kelondroTree oldindex = new kelondroTree(urlHash, 1000, -1, kelondroTree.defaultObjectCachePercent, plasmaCrawlLURLOldEntry.rowdef); kelondroTree oldindex = null;
try {
oldindex = new kelondroTree(urlHash, 1000, -1, kelondroTree.defaultObjectCachePercent, plasmaCrawlLURLOldEntry.rowdef);
} catch (IOException e) {
System.out.println("ERROR: CANNOT OPEN OLD INDEX: " + e.getMessage());
}
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
long last = start; long last = start;
@ -1137,7 +1141,7 @@ public final class yacy {
byte[] dummymd5 = new byte[0]; byte[] dummymd5 = new byte[0];
while (eiter.hasNext()) { while (eiter.hasNext()) {
oldrow = (kelondroRow.Entry) eiter.next(); oldrow = (kelondroRow.Entry) eiter.next();
if (oldrow != null) { if (oldrow != null) try {
oldentry = new plasmaCrawlLURLOldEntry(oldrow, null); oldentry = new plasmaCrawlLURLOldEntry(oldrow, null);
comp = oldentry.comp(); comp = oldentry.comp();
newentry = pool.loadedURL.newEntry( newentry = pool.loadedURL.newEntry(
@ -1158,19 +1162,18 @@ public final class yacy {
oldentry.language(), oldentry.language(),
0, 0, 0, 0, 0, 0); 0, 0, 0, 0, 0, 0);
pool.loadedURL.store(newentry); pool.loadedURL.store(newentry);
}
c++; c++;
} catch (IOException e) {
// ignore
}
if (System.currentTimeMillis() - last > 60000) { if (System.currentTimeMillis() - last > 60000) {
System.out.println("Migrated " + c + " from " + tc + " urls. Estimated remaining time: " + ((System.currentTimeMillis() - start) * (tc - c) / c / 60000) + " minutes"); System.out.println("Migrated " + c + " from " + tc + " urls. Estimated remaining time: " + ((System.currentTimeMillis() - start) * (tc - c) / c / 60000) + " minutes");
last = System.currentTimeMillis(); last = System.currentTimeMillis();
} }
} }
pool.close(); pool.close();
oldindex.close(); try { oldindex.close(); } catch (IOException e) { }
System.out.println("MIGRATION OF " + c + " URLs FINISHED"); System.out.println("MIGRATION OF " + c + " URLs FINISHED");
} catch (IOException e) {
e.printStackTrace();
}
} }
private static String[] shift(String[] args, int pos, int count) { private static String[] shift(String[] args, int pos, int count) {

Loading…
Cancel
Save