From 11f7da06ed8e3f430d692ef94f5367677434a648 Mon Sep 17 00:00:00 2001
From: orbiter <orbiter@6c8d7289-2bf4-0310-a012-ef5d649a1542>
Date: Tue, 3 Nov 2009 21:18:19 +0000
Subject: [PATCH] - fixes to csv parser - automatic OAI-PMH import by just
 clicking on one link from the provided resource list

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6449 6c8d7289-2bf4-0310-a012-ef5d649a1542
---
 htroot/IndexImportOAIPMHList_p.java           |  2 +-
 htroot/IndexImportOAIPMH_p.html               |  4 +--
 htroot/IndexImportOAIPMH_p.java               |  7 +++--
 .../document/importer/OAIPMHImporter.java     |  4 ++-
 .../net/yacy/document/parser/csvParser.java   | 26 ++++++++++++++++++-
 5 files changed, 36 insertions(+), 7 deletions(-)

diff --git a/htroot/IndexImportOAIPMHList_p.java b/htroot/IndexImportOAIPMHList_p.java
index 4f39cda82..7ff7592d6 100644
--- a/htroot/IndexImportOAIPMHList_p.java
+++ b/htroot/IndexImportOAIPMHList_p.java
@@ -49,7 +49,7 @@ public class IndexImportOAIPMHList_p {
             int cnt = 0;
             for (String root: oaiRoots) {
                 prop.put("source_table_" + cnt + "_dark", (dark) ? "1" : "0");
-                prop.put("source_table_" + cnt + "_source", root);
+                prop.put("source_table_" + cnt + "_source", "<a href=\"/IndexImportOAIPMH_p.html?importroot=&urlstartall=" + root + "\" target=\"_top\">" + root+ "</a>");
                 dark = !dark;
                 cnt++;
             }
diff --git a/htroot/IndexImportOAIPMH_p.html b/htroot/IndexImportOAIPMH_p.html
index 00049abe4..325bedf01 100644
--- a/htroot/IndexImportOAIPMH_p.html
+++ b/htroot/IndexImportOAIPMH_p.html
@@ -35,14 +35,14 @@
           Import all records that follow according to resumption elements into index<br />
           <input name="urlstartall" type="text" value="" size="80" />
           <input name="importroot" type="submit" value="import this source" />
-          or&nbsp;<input name="getlist" type="submit" value="import from a list" />
+          #(optiongetlist)#::or&nbsp;<input name="getlist" type="submit" value="import from a list" />#(/optiongetlist)#
           #(status)#::<p>Import started!</p>::<p>Bad input data: #[message]# </p>#(/status)#
         </fieldset>
     </form>
     <iframe name="OAI-PMH Import List"
        src="/IndexImportOAIPMHList_p.html#(iframetype)#::?import=::?source=#(/iframetype)#"
        width="100%"
-       height="340"
+       height="420"
        frameborder="0"
        scrolling="auto"
        id="list">
diff --git a/htroot/IndexImportOAIPMH_p.java b/htroot/IndexImportOAIPMH_p.java
index 077e9c6c8..415a11798 100644
--- a/htroot/IndexImportOAIPMH_p.java
+++ b/htroot/IndexImportOAIPMH_p.java
@@ -44,8 +44,9 @@ public class IndexImportOAIPMH_p {
         prop.put("import-one", 0);
         prop.put("status", 0);
         prop.put("defaulturl", "");
-        prop.put("iframetype", (OAIPMHImporter.runningJobs.size() + OAIPMHImporter.startedJobs.size() + OAIPMHImporter.finishedJobs.size() == 0) ? 0 : 1);
-        
+        int jobcount = OAIPMHImporter.runningJobs.size() + OAIPMHImporter.startedJobs.size() + OAIPMHImporter.finishedJobs.size();
+        prop.put("iframetype", (jobcount == 0) ? 2 : 1);
+        prop.put("optiongetlist", (jobcount == 0) ? 0 : 1);
         if (post != null) {
             if (post.containsKey("urlstartone")) {
                 String oaipmhurl = post.get("urlstartone");
@@ -89,6 +90,8 @@ public class IndexImportOAIPMH_p {
                     OAIPMHImporter job = new OAIPMHImporter(sb.loader, url);
                     job.start();
                     prop.put("status", 1);
+                    prop.put("optiongetlist", 1);
+                    prop.put("iframetype", 1);
                 } catch (MalformedURLException e) {
                     e.printStackTrace();
                     prop.put("status", 2);
diff --git a/source/net/yacy/document/importer/OAIPMHImporter.java b/source/net/yacy/document/importer/OAIPMHImporter.java
index 22996fbd7..9ecaef117 100644
--- a/source/net/yacy/document/importer/OAIPMHImporter.java
+++ b/source/net/yacy/document/importer/OAIPMHImporter.java
@@ -196,7 +196,9 @@ public class OAIPMHImporter extends Thread implements Importer, Comparable<OAIPM
             try {
                 List<String[]> table = parser.getTable(roarSource, "", "UTF-8", new FileInputStream(roar));
                 for (String[] row: table) {
-                    list.add(row[2]);
+                    if (row.length > 2 && (row[2].startsWith("http://") || row[2].startsWith("https://"))) {
+                        list.add(row[2]);
+                    }
                 }
             } catch (FileNotFoundException e) {
                 e.printStackTrace();
diff --git a/source/net/yacy/document/parser/csvParser.java b/source/net/yacy/document/parser/csvParser.java
index bca44582e..57208dea9 100644
--- a/source/net/yacy/document/parser/csvParser.java
+++ b/source/net/yacy/document/parser/csvParser.java
@@ -134,6 +134,8 @@ public class csvParser extends AbstractParser implements Idiom {
                     if (cols.length >= colc.length && cols.length >= colt.length) separator = ";";
                     if (colt.length >= cols.length && colt.length >= colc.length) separator = "\t";
                 }
+                row = stripQuotes(row, '\"', separator.charAt(0), ' ');
+                row = stripQuotes(row, '\'', separator.charAt(0), ' ');
                 String[] cols = row.split(separator);
                 if (columns == -1) columns = cols.length;
                 //if (cols.length != columns) continue; // skip lines that have the wrong number of columns
@@ -144,6 +146,28 @@ public class csvParser extends AbstractParser implements Idiom {
         return rows;
     }
 
-    
+    /**
+     * remove quotes AND separator characters within the quotes
+     * to make it possible to split the line using the String.split method
+     * @param line
+     * @param quote
+     * @param separator
+     * @param replacement
+     * @return the line without the quotes
+     */
+    public static String stripQuotes(String line, char quote, char separator, char replacement) {
+        int p, q;
+        // find left quote
+        while ((p = line.indexOf(quote)) >= 0) {
+            q = line.indexOf(quote, p + 1);
+            if (q < 0) {
+                // there is only a single quote but no 'right' quote.
+                // This data is not well-formed. Just remove the quote and give up.
+                return line.substring(0, p) + line.substring(p + 1);
+            }
+            line = line.substring(0, p) + line.substring(p + 1, q).replace(separator, replacement) + line.substring(q + 1);
+        }
+        return line;
+    }
 
 }