From 0769517129f4e0432d634051697acea9c5f8d65a Mon Sep 17 00:00:00 2001 From: orbiter Date: Tue, 9 Mar 2010 11:31:15 +0000 Subject: [PATCH] added a robots.txt monitor in the crawler monitor submenu git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6733 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/Table_RobotsTxt_p.html | 30 +++++ htroot/Table_RobotsTxt_p.java | 9 ++ htroot/Tables_p.java | 4 +- htroot/api/table_p.html | 40 ++++++ htroot/api/table_p.java | 115 ++++++++++++++++++ htroot/api/table_p.xml | 18 +++ .../templates/submenuCrawlMonitor.template | 1 + .../net/yacy/kelondro/blob/BEncodedHeap.java | 5 +- source/net/yacy/kelondro/blob/Tables.java | 4 +- 9 files changed, 219 insertions(+), 7 deletions(-) create mode 100644 htroot/Table_RobotsTxt_p.html create mode 100644 htroot/Table_RobotsTxt_p.java create mode 100644 htroot/api/table_p.html create mode 100644 htroot/api/table_p.java create mode 100644 htroot/api/table_p.xml diff --git a/htroot/Table_RobotsTxt_p.html b/htroot/Table_RobotsTxt_p.html new file mode 100644 index 000000000..b7b707518 --- /dev/null +++ b/htroot/Table_RobotsTxt_p.html @@ -0,0 +1,30 @@ + + + + YaCy '#[clientname]#': Table Viewer + + #%env/templates/metas.template%# + + + +
+ +API +The information that is presented on this page can also be retrieved as XML +Click the API icon to see the XML. +To see a list of all APIs, please visit the API wiki page. +
+ + #%env/templates/header.template%# + #%env/templates/submenuCrawlMonitor.template%# + +

robots.txt table

+
+ + #%env/templates/footer.template%# + + diff --git a/htroot/Table_RobotsTxt_p.java b/htroot/Table_RobotsTxt_p.java new file mode 100644 index 000000000..892b42751 --- /dev/null +++ b/htroot/Table_RobotsTxt_p.java @@ -0,0 +1,9 @@ +import de.anomic.http.server.RequestHeader; +import de.anomic.server.serverObjects; +import de.anomic.server.serverSwitch; + +public class Table_RobotsTxt_p { + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { + return new serverObjects(); + } +} diff --git a/htroot/Tables_p.java b/htroot/Tables_p.java index babaaf073..d4266ca8f 100644 --- a/htroot/Tables_p.java +++ b/htroot/Tables_p.java @@ -146,8 +146,8 @@ public class Tables_p { // check if we can find a key String pk = null; for (Map.Entry entry: post.entrySet()) { - if (entry.getKey().startsWith("mark_") && entry.getValue().equals("on")) { - pk = entry.getKey().substring(5); + if (entry.getValue().startsWith("mark_")) { + pk = entry.getValue().substring(5); break; } } diff --git a/htroot/api/table_p.html b/htroot/api/table_p.html new file mode 100644 index 000000000..6bdde6c81 --- /dev/null +++ b/htroot/api/table_p.html @@ -0,0 +1,40 @@ + + + + YaCy '#[clientname]#': Table Viewer + #(showtable)#:: + + #(/showtable)# + #%env/templates/metas.template%# + + + #(showtable)#:: +
+
+ + + + #(showpk)#::#(/showpk)# + #{columns}# + + #{/columns}# + + #{list}# + + #(showpk)#::#(/showpk)# + #{columns}# + + #{/columns}# + + #{/list}# +
PK#[header]#
#[pk]##[cell]#
+

+ + + +

+
+
+ #(/showtable)# + + diff --git a/htroot/api/table_p.java b/htroot/api/table_p.java new file mode 100644 index 000000000..4cc808869 --- /dev/null +++ b/htroot/api/table_p.java @@ -0,0 +1,115 @@ +// Table_p.java +// ----------------------- +// (C) 2010 by Michael Peter Christen; mc@yacy.net +// first published 21.01.2010 in Frankfurt, Germany on http://yacy.net +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; + +import net.yacy.kelondro.blob.Tables; +import net.yacy.kelondro.logging.Log; + +import de.anomic.http.server.RequestHeader; +import de.anomic.search.Switchboard; +import de.anomic.server.serverObjects; +import de.anomic.server.serverSwitch; + +public class table_p { + + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { + final Switchboard sb = (Switchboard) env; + final serverObjects prop = new serverObjects(); + String table = (post == null) ? null : post.get("table", null); + if (table != null && !sb.tables.hasHeap(table)) table = null; + if (table == null) { + prop.put("showtable", 0); + return prop; + } + + boolean showpk = post.containsKey("pk"); + + ArrayList columns = null; + try { + columns = sb.tables.columns(table); + } catch (IOException e) { + Log.logException(e); + columns = new ArrayList(); + } + + // if a row attribute is given + // then order the columns according to the given order + String[] row = post.get("row", "").split(","); + for (int i = 0; i < row.length; i++) { + if (columns.contains(row[i])) { + columns.remove(row[i]); + columns.add(i, row[i]); + } + } + + // generate table + prop.put("showtable", 1); + prop.put("showtable_table", table); + + // insert the columns + prop.put("showtable_showpk", showpk ? 1 : 0); + for (int i = 0; i < columns.size(); i++) { + prop.putHTML("showtable_columns_" + i + "_header", columns.get(i)); + } + prop.put("showtable_columns", columns.size()); + + // insert all rows + int maxCount; + try { + maxCount = Math.min(1000, sb.tables.size(table)); + } catch (IOException e) { + Log.logException(e); + maxCount = 0; + } + int count = 0; + try { + final Iterator mapIterator = sb.tables.orderByPK(table, maxCount).iterator(); + Tables.Row trow; + boolean dark = true; + byte[] cell; + while ((mapIterator.hasNext()) && (count < maxCount)) { + trow = mapIterator.next(); + if (row == null) continue; + prop.put("showtable_list_" + count + "_dark", ((dark) ? 1 : 0) ); dark=!dark; + prop.put("showtable_list_" + count + "_showpk", showpk ? 1 : 0); + prop.put("showtable_list_" + count + "_showpk_pk", new String(trow.getPK())); + prop.put("showtable_list_" + count + "_count", count); + for (int i = 0; i < columns.size(); i++) { + cell = trow.from(columns.get(i)); + prop.putHTML("showtable_list_" + count + "_columns_" + i + "_column", columns.get(i)); + prop.putHTML("showtable_list_" + count + "_columns_" + i + "_cell", cell == null ? "" : new String(cell)); + } + prop.put("showtable_list_" + count + "_columns", columns.size()); + + count++; + } + } catch (IOException e) { + Log.logException(e); + } + prop.put("showtable_list", count); + prop.put("showtable_num", count); + + // return rewrite properties + return prop; + } + +} diff --git a/htroot/api/table_p.xml b/htroot/api/table_p.xml new file mode 100644 index 000000000..d465d8891 --- /dev/null +++ b/htroot/api/table_p.xml @@ -0,0 +1,18 @@ +#(showtable)#:: + + + #{columns}# + + #{/columns}# + + + #{list}# + + #{columns}# + #[cell]# + #{/columns}# + + #{/list}# + +
+#(/showtable)# \ No newline at end of file diff --git a/htroot/env/templates/submenuCrawlMonitor.template b/htroot/env/templates/submenuCrawlMonitor.template index 7c1f1b3a6..5c880851e 100644 --- a/htroot/env/templates/submenuCrawlMonitor.template +++ b/htroot/env/templates/submenuCrawlMonitor.template @@ -24,6 +24,7 @@

Crawler Steering

\ No newline at end of file diff --git a/source/net/yacy/kelondro/blob/BEncodedHeap.java b/source/net/yacy/kelondro/blob/BEncodedHeap.java index 0436a381d..7161f983a 100644 --- a/source/net/yacy/kelondro/blob/BEncodedHeap.java +++ b/source/net/yacy/kelondro/blob/BEncodedHeap.java @@ -30,7 +30,6 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashSet; -import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -314,13 +313,13 @@ public class BEncodedHeap implements Iterable columns() { + public ArrayList columns() { if (this.columnames.size() == 0) { for (Map.Entry> row: this) { this.columnames.addAll(row.getValue().keySet()); } } - List l = new ArrayList(); + ArrayList l = new ArrayList(); l.addAll(this.columnames); return l; } diff --git a/source/net/yacy/kelondro/blob/Tables.java b/source/net/yacy/kelondro/blob/Tables.java index 70f2db963..f67076a11 100644 --- a/source/net/yacy/kelondro/blob/Tables.java +++ b/source/net/yacy/kelondro/blob/Tables.java @@ -29,10 +29,10 @@ package net.yacy.kelondro.blob; import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.Iterator; -import java.util.List; import java.util.Map; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; @@ -371,7 +371,7 @@ public class Tables { return sortTree.values(); } - public List columns(String table) throws IOException { + public ArrayList columns(String table) throws IOException { BEncodedHeap heap = getHeap(table); return heap.columns(); }