added a robots.txt monitor in the crawler monitor submenu

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6733 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent 48995e71c4
commit 0769517129

@ -0,0 +1,30 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>YaCy '#[clientname]#': Table Viewer</title>
<link rel="alternate" type="application/xml" title="robots.txt Table" href="/api/table_p.xml?table=robots&pk=" />
#%env/templates/metas.template%#
</head>
<body id="Tables">
<div id="api">
<script type="text/javascript">
//<![CDATA[
document.write("\<a href=\"/api/table_p.xml?table=robots&pk=\"\>")
//]]>
</script>
<img src="env/grafics/api.png" width="60" height="40" alt="API"/></a>
<span>The information that is presented on this page can also be retrieved as XML
Click the API icon to see the XML.
To see a list of all APIs, please visit the <a href="http://www.yacy-websuche.de/wiki/index.php/Dev:API">API wiki page</a>.</span>
</div>
#%env/templates/header.template%#
#%env/templates/submenuCrawlMonitor.template%#
<h2>robots.txt table</h2>
<iframe src="/api/table_p.html?table=robots&row=hostname,modDate,date,disallow,etag,sitemap" width="100%" height="600" frameborder="0" scrolling="auto"></iframe><br />
#%env/templates/footer.template%#
</body>
</html>

@ -0,0 +1,9 @@
import de.anomic.http.server.RequestHeader;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
public class Table_RobotsTxt_p {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
return new serverObjects();
}
}

@ -146,8 +146,8 @@ public class Tables_p {
// check if we can find a key
String pk = null;
for (Map.Entry<String, String> entry: post.entrySet()) {
if (entry.getKey().startsWith("mark_") && entry.getValue().equals("on")) {
pk = entry.getKey().substring(5);
if (entry.getValue().startsWith("mark_")) {
pk = entry.getValue().substring(5);
break;
}
}

@ -0,0 +1,40 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>YaCy '#[clientname]#': Table Viewer</title>
#(showtable)#::
<link rel="alternate" type="application/xml" title="Table" href="table_p.xml?table=#[table]#&pk=" />
#(/showtable)#
#%env/templates/metas.template%#
</head>
<body id="Table">
#(showtable)#::
<form action="/Tables_p.html" method="post" target="_blank" enctype="multipart/form-data" accept-charset="UTF-8" name="tablelisting">
<fieldset>
<legend><label for="table">Table: '#[table]#'</label></legend>
<table border="0" cellpadding="2" cellspacing="1">
<tr class="TableHeader" valign="bottom">
#(showpk)#::<td>PK</td>#(/showpk)#
#{columns}#
<td>#[header]#</td>
#{/columns}#
</tr>
#{list}#
<tr class="TableCell#(dark)#Light::Dark::Summary#(/dark)#">
#(showpk)#::<td>#[pk]#</td>#(/showpk)#
#{columns}#
<td align="left">#[cell]#</td>
#{/columns}#
</tr>
#{/list}#
</table>
<p>
<input type="hidden" name="table" value="#[table]#" />
<input type="hidden" name="num" value="#[num]#" />
<input type="submit" name="edittable" value="Edit Table" />
</p>
</fieldset>
</form>
#(/showtable)#
</body>
</html>

@ -0,0 +1,115 @@
// Table_p.java
// -----------------------
// (C) 2010 by Michael Peter Christen; mc@yacy.net
// first published 21.01.2010 in Frankfurt, Germany on http://yacy.net
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import net.yacy.kelondro.blob.Tables;
import net.yacy.kelondro.logging.Log;
import de.anomic.http.server.RequestHeader;
import de.anomic.search.Switchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
public class table_p {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
String table = (post == null) ? null : post.get("table", null);
if (table != null && !sb.tables.hasHeap(table)) table = null;
if (table == null) {
prop.put("showtable", 0);
return prop;
}
boolean showpk = post.containsKey("pk");
ArrayList<String> columns = null;
try {
columns = sb.tables.columns(table);
} catch (IOException e) {
Log.logException(e);
columns = new ArrayList<String>();
}
// if a row attribute is given
// then order the columns according to the given order
String[] row = post.get("row", "").split(",");
for (int i = 0; i < row.length; i++) {
if (columns.contains(row[i])) {
columns.remove(row[i]);
columns.add(i, row[i]);
}
}
// generate table
prop.put("showtable", 1);
prop.put("showtable_table", table);
// insert the columns
prop.put("showtable_showpk", showpk ? 1 : 0);
for (int i = 0; i < columns.size(); i++) {
prop.putHTML("showtable_columns_" + i + "_header", columns.get(i));
}
prop.put("showtable_columns", columns.size());
// insert all rows
int maxCount;
try {
maxCount = Math.min(1000, sb.tables.size(table));
} catch (IOException e) {
Log.logException(e);
maxCount = 0;
}
int count = 0;
try {
final Iterator<Tables.Row> mapIterator = sb.tables.orderByPK(table, maxCount).iterator();
Tables.Row trow;
boolean dark = true;
byte[] cell;
while ((mapIterator.hasNext()) && (count < maxCount)) {
trow = mapIterator.next();
if (row == null) continue;
prop.put("showtable_list_" + count + "_dark", ((dark) ? 1 : 0) ); dark=!dark;
prop.put("showtable_list_" + count + "_showpk", showpk ? 1 : 0);
prop.put("showtable_list_" + count + "_showpk_pk", new String(trow.getPK()));
prop.put("showtable_list_" + count + "_count", count);
for (int i = 0; i < columns.size(); i++) {
cell = trow.from(columns.get(i));
prop.putHTML("showtable_list_" + count + "_columns_" + i + "_column", columns.get(i));
prop.putHTML("showtable_list_" + count + "_columns_" + i + "_cell", cell == null ? "" : new String(cell));
}
prop.put("showtable_list_" + count + "_columns", columns.size());
count++;
}
} catch (IOException e) {
Log.logException(e);
}
prop.put("showtable_list", count);
prop.put("showtable_num", count);
// return rewrite properties
return prop;
}
}

@ -0,0 +1,18 @@
#(showtable)#::
<table name="#[table]#" count="#[num]#">
<columns>
#{columns}#
<column name="#[header]#" />
#{/columns}#
</columns>
<rows>
#{list}#
<row#(showpk)#:: key="#[pk]#"#(/showpk)#>
#{columns}#
<column name="#[column]#">#[cell]#</column>
#{/columns}#
</row>
#{/list}#
</rows>
</table>
#(/showtable)#

@ -24,6 +24,7 @@
<h3>Crawler Steering</h3>
<ul class="SubMenu">
<li><a href="/CrawlProfileEditor_p.html" class="MenuItemLink lock">Crawl Profile Editor</a></li>
<li><a href="/Table_RobotsTxt_p.html" class="MenuItemLink lock">robots.txt Monitor</a></li>
</ul>
</div>
</div>

@ -30,7 +30,6 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@ -314,13 +313,13 @@ public class BEncodedHeap implements Iterable<Map.Entry<byte[], Map<String, byte
}
}
public List<String> columns() {
public ArrayList<String> columns() {
if (this.columnames.size() == 0) {
for (Map.Entry<byte[], Map<String, byte[]>> row: this) {
this.columnames.addAll(row.getValue().keySet());
}
}
List<String> l = new ArrayList<String>();
ArrayList<String> l = new ArrayList<String>();
l.addAll(this.columnames);
return l;
}

@ -29,10 +29,10 @@ package net.yacy.kelondro.blob;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
@ -371,7 +371,7 @@ public class Tables {
return sortTree.values();
}
public List<String> columns(String table) throws IOException {
public ArrayList<String> columns(String table) throws IOException {
BEncodedHeap heap = getHeap(table);
return heap.columns();
}

Loading…
Cancel
Save