orbiter 15 years ago
parent 7fdf59a77f
commit 24060885b6

@ -43,6 +43,7 @@ import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.repository.Blacklist;
import de.anomic.data.Tables;
import de.anomic.data.listManager;
import de.anomic.http.server.RequestHeader;
import de.anomic.search.SearchEventCache;
@ -253,7 +254,7 @@ public class Blacklist_p {
String blentry = post.get("newEntry", "").trim();
// store this call as api call
listManager.switchboard.recordAPICall(post, "Blacklist_p.html", "blacklist", "add to blacklist: " + blentry);
listManager.switchboard.tables.recordAPICall(post, "Blacklist_p.html", Tables.API_TYPE_CONFIGURATION, "add to blacklist: " + blentry);
final String temp = addBlacklistEntry(blacklistToUse, blentry, header, supportedBlacklistTypes);
if (temp != null) {

@ -34,6 +34,7 @@ import java.util.regex.Pattern;
import net.yacy.kelondro.util.Domains;
import net.yacy.kelondro.workflow.InstantBusyThread;
import de.anomic.data.Tables;
import de.anomic.data.translator;
import de.anomic.http.server.HTTPDemon;
import de.anomic.http.server.HTTPDFileHandler;
@ -72,7 +73,7 @@ public class ConfigBasic {
// store this call as api call
if (post != null && post.containsKey("set")) {
sb.recordAPICall(post, "ConfigBasic.html", "configuration", "basic settings");
sb.tables.recordAPICall(post, "ConfigBasic.html", Tables.API_TYPE_CONFIGURATION, "basic settings");
}
//boolean doPeerPing = false;

@ -42,6 +42,7 @@ import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.util.FileUtils;
import de.anomic.crawler.retrieval.HTTPLoader;
import de.anomic.data.Tables;
import de.anomic.data.translator;
import de.anomic.http.client.Client;
import de.anomic.http.server.HeaderFramework;
@ -74,7 +75,7 @@ public class ConfigLanguage_p {
String selectedLanguage = post.get("language");
// store this call as api call
((Switchboard) env).recordAPICall(post, "ConfigLanguage.html", "configuration", "language settings: " + selectedLanguage);
((Switchboard) env).tables.recordAPICall(post, "ConfigLanguage.html", Tables.API_TYPE_CONFIGURATION, "language settings: " + selectedLanguage);
//change language
if(post.containsKey("use_button") && selectedLanguage != null){

@ -32,6 +32,7 @@ import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.MapTools;
import net.yacy.kelondro.workflow.BusyThread;
import de.anomic.data.Tables;
import de.anomic.http.server.HTTPDemon;
import de.anomic.http.server.RequestHeader;
import de.anomic.search.Switchboard;
@ -55,7 +56,7 @@ public class ConfigNetwork_p {
if (post != null) {
// store this call as api call
sb.recordAPICall(post, "ConfigNetwork.html", "configuration", "network settings");
sb.tables.recordAPICall(post, "ConfigNetwork.html", Tables.API_TYPE_CONFIGURATION, "network settings");
if (post.containsKey("changeNetwork")) {
final String networkDefinition = post.get("networkDefinition", "defaults/yacy.network.freeworld.unit");

@ -25,6 +25,7 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import de.anomic.data.Tables;
import de.anomic.http.server.RequestHeader;
import de.anomic.search.Switchboard;
import de.anomic.search.SwitchboardConstants;
@ -59,7 +60,7 @@ public class ConfigPortal {
if (post.containsKey("searchpage_set")) {
String newGreeting = post.get(SwitchboardConstants.GREETING, "");
// store this call as api call
sb.recordAPICall(post, "ConfigPortal.html", "appearance", "new portal design. greeting: " + newGreeting);
sb.tables.recordAPICall(post, "ConfigPortal.html", Tables.API_TYPE_CONFIGURATION, "new portal design. greeting: " + newGreeting);
sb.setConfig(SwitchboardConstants.GREETING, newGreeting);
sb.setConfig(SwitchboardConstants.GREETING_HOMEPAGE, post.get(SwitchboardConstants.GREETING_HOMEPAGE, ""));

@ -45,6 +45,7 @@ import net.yacy.kelondro.util.FileUtils;
import de.anomic.crawler.CrawlProfile;
import de.anomic.crawler.SitemapImporter;
import de.anomic.crawler.retrieval.Request;
import de.anomic.data.Tables;
import de.anomic.data.bookmarksDB;
import de.anomic.data.listManager;
import de.anomic.http.server.RequestHeader;
@ -141,7 +142,7 @@ public class Crawler_p {
crawlingStart = (crawlingStartURL == null) ? null : crawlingStartURL.toNormalform(true, true);
// store this call as api call
sb.recordAPICall(post, "Crawler_p.html", "crawler", "crawl start for " + crawlingStartURL.getHost());
sb.tables.recordAPICall(post, "Crawler_p.html", Tables.API_TYPE_CRAWLER, "crawl start for " + crawlingStart);
// set new properties
final boolean fullDomain = post.get("range", "wide").equals("domain"); // special property in simple crawl start

@ -17,15 +17,11 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log;
import de.anomic.http.server.RequestHeader;
import de.anomic.search.Switchboard;
import de.anomic.server.serverObjects;
@ -37,7 +33,7 @@ public class Tables_p {
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
String table = (post == null) ? null : post.get("table", null);
if (table != null && !sb.tables.hasHeap(table)) table = null;
if (table != null && !sb.tables.has(table)) table = null;
// show table selection
int count = 0;
@ -52,30 +48,20 @@ public class Tables_p {
prop.put("tables", count);
List<String> columns = null;
if (table != null) try {
if (table != null) {
columns = sb.tables.columns(table);
} catch (IOException e) {
Log.logException(e);
}
// apply deletion requests
if (post != null && post.get("deletetable", "").length() > 0) {
try {
sb.tables.clear(table);
} catch (IOException e) {
Log.logException(e);
}
sb.tables.clear(table);
}
if (post != null && post.get("deleterows", "").length() > 0) {
try {
for (Map.Entry<String, String> entry: post.entrySet()) {
if (entry.getKey().startsWith("mark_") && entry.getValue().equals("on")) {
sb.tables.delete(table, entry.getKey().substring(5).getBytes());
}
for (Map.Entry<String, String> entry: post.entrySet()) {
if (entry.getKey().startsWith("mark_") && entry.getValue().equals("on")) {
sb.tables.delete(table, entry.getKey().substring(5).getBytes());
}
} catch (IOException e) {
Log.logException(e);
}
}
@ -87,20 +73,14 @@ public class Tables_p {
map.put(entry.getKey().substring(4), entry.getValue().getBytes());
}
}
try {
sb.tables.insert(table, pk.getBytes(), map);
} catch (IOException e) {
Log.logException(e);
} catch (RowSpaceExceededException e) {
Log.logException(e);
}
sb.tables.insert(table, pk.getBytes(), map);
}
// generate table
prop.put("showtable", 0);
prop.put("showedit", 0);
if (table != null && !post.containsKey("editrow") && !post.containsKey("addrow")) try {
if (table != null && !post.containsKey("editrow") && !post.containsKey("addrow")) {
prop.put("showtable", 1);
prop.put("showtable_table", table);
@ -136,9 +116,9 @@ public class Tables_p {
count++;
}
prop.put("showtable_list", count);
} catch (IOException e) {}
}
if (post != null && table != null && post.containsKey("editrow")) try {
if (post != null && table != null && post.containsKey("editrow")) {
// check if we can find a key
String pk = null;
for (Map.Entry<String, String> entry: post.entrySet()) {
@ -150,16 +130,12 @@ public class Tables_p {
if (pk != null && sb.tables.has(table, pk.getBytes())) {
setEdit(sb, prop, table, pk, columns);
}
} catch (IOException e) {}
}
if (post != null && table != null && post.containsKey("addrow")) try {
if (post != null && table != null && post.containsKey("addrow")) {
// get a new key
String pk = new String(sb.tables.insert(table, new HashMap<String, byte[]>()));
String pk = sb.tables.createRow(table);
setEdit(sb, prop, table, pk, columns);
} catch (IOException e) {
Log.logException(e);
} catch (RowSpaceExceededException e) {
Log.logException(e);
}
// adding the peer address
@ -169,7 +145,7 @@ public class Tables_p {
return prop;
}
private static void setEdit(final Switchboard sb, final serverObjects prop, final String table, final String pk, List<String> columns) throws IOException {
private static void setEdit(final Switchboard sb, final serverObjects prop, final String table, final String pk, List<String> columns) {
prop.put("showedit", 1);
prop.put("showedit_table", table);
prop.put("showedit_pk", pk);

@ -0,0 +1,144 @@
package de.anomic.data;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import net.yacy.kelondro.blob.BEncodedHeapArray;
import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.DateFormatter;
import de.anomic.server.serverObjects;
public class Tables {
public final static String API_TYPE_CONFIGURATION = "configuration";
public final static String API_TYPE_CRAWLER = "crawler";
private BEncodedHeapArray tables;
public Tables(File workPath) {
this.tables = new BEncodedHeapArray(workPath, 12);
}
public boolean has(String table) {
return tables.hasHeap(table);
}
public boolean has(String table, byte[] pk) {
try {
return tables.has(table, pk);
} catch (IOException e) {
Log.logException(e);
return false;
}
}
public Iterator<String> tables() {
return this.tables.tables();
}
public List<String> columns(String table) {
try {
return this.tables.columns(table);
} catch (IOException e) {
Log.logException(e);
return new ArrayList<String>(0);
}
}
public void clear(String table) {
try {
this.tables.clear(table);
} catch (IOException e) {
Log.logException(e);
}
}
public void delete(String table, byte[] pk) {
try {
this.tables.delete(table, pk);
} catch (IOException e) {
Log.logException(e);
}
}
public int size(String table) {
try {
return this.tables.size(table);
} catch (IOException e) {
Log.logException(e);
return 0;
}
}
public Iterator<Map.Entry<byte[], Map<String, byte[]>>> iterator(String table) {
try {
return this.tables.iterator(table);
} catch (IOException e) {
Log.logException(e);
return new TreeMap<byte[], Map<String, byte[]>>().entrySet().iterator();
}
}
public void close() {
this.tables.close();
}
public Map<String, byte[]> select(String table, byte[] pk) {
try {
return tables.select(table, pk);
} catch (IOException e) {
Log.logException(e);
return new TreeMap<String, byte[]>();
}
}
public void insert(String table, byte[] pk, Map<String, byte[]> map) {
try {
this.tables.insert(table, pk, map);
} catch (RowSpaceExceededException e) {
Log.logException(e);
} catch (IOException e) {
Log.logException(e);
}
}
public String createRow(String table) {
try {
return new String(this.tables.insert(table, new HashMap<String, byte[]>()));
} catch (RowSpaceExceededException e) {
Log.logException(e);
return null;
} catch (IOException e) {
Log.logException(e);
return null;
}
}
public void recordAPICall(final serverObjects post, final String servletName, String type, String comment) {
String apiurl = /*"http://localhost:" + getConfig("port", "8080") +*/ "/" + servletName + "?" + post.toString();
try {
this.tables.insert(
"api",
"type", type.getBytes(),
"comment", comment.getBytes(),
"date", DateFormatter.formatShortMilliSecond(new Date()).getBytes(),
"url", apiurl.getBytes()
);
} catch (RowSpaceExceededException e2) {
Log.logException(e2);
} catch (IOException e2) {
Log.logException(e2);
}
Log.logInfo("APICALL", apiurl);
}
}

@ -70,12 +70,10 @@ import net.yacy.document.content.RSSMessage;
import net.yacy.document.content.SurrogateReader;
import net.yacy.document.parser.html.ImageEntry;
import net.yacy.document.parser.xml.RSSFeed;
import net.yacy.kelondro.blob.BEncodedHeapArray;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.data.meta.URIMetadataRow;
import net.yacy.kelondro.data.meta.URIMetadataRow.Components;
import net.yacy.kelondro.data.word.Word;
import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.order.Digest;
@ -112,6 +110,7 @@ import de.anomic.crawler.retrieval.HTTPLoader;
import de.anomic.crawler.retrieval.Request;
import de.anomic.crawler.retrieval.Response;
import de.anomic.data.LibraryProvider;
import de.anomic.data.Tables;
import de.anomic.data.URLLicense;
import de.anomic.data.blogBoard;
import de.anomic.data.blogBoardComments;
@ -132,7 +131,6 @@ import de.anomic.http.server.ResponseHeader;
import de.anomic.http.server.RobotsTxtConfig;
import de.anomic.net.UPnP;
import de.anomic.search.blockrank.CRDistribution;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.server.serverCore;
import de.anomic.tools.crypt;
@ -218,7 +216,7 @@ public final class Switchboard extends serverSwitch {
public Dispatcher dhtDispatcher;
public List<String> trail;
public yacySeedDB peers;
public BEncodedHeapArray tables;
public Tables tables;
public WorkflowProcessor<indexingQueueEntry> indexingDocumentProcessor;
public WorkflowProcessor<indexingQueueEntry> indexingCondensementProcessor;
@ -279,7 +277,7 @@ public final class Switchboard extends serverSwitch {
this.log.logConfig("Dictionaries Path:" + this.dictionariesPath.toString());
// init tables
this.tables = new BEncodedHeapArray(this.workPath, 12);
this.tables = new Tables(this.workPath);
// init libraries
this.log.logConfig("initializing libraries");
@ -2100,26 +2098,7 @@ public final class Switchboard extends serverSwitch {
}
yacyCore.log.logInfo("BOOTSTRAP: " + (peers.sizeConnected() - sc) + " new seeds while bootstraping.");
}
public void recordAPICall(final serverObjects post, final String servletName, String type, String comment) {
String apiurl = /*"http://localhost:" + getConfig("port", "8080") +*/ "/" + servletName + "?" + post.toString();
try {
sb.tables.insert(
"api",
"type", type.getBytes(),
"comment", comment.getBytes(),
"date", DateFormatter.formatShortMilliSecond(new Date()).getBytes(),
"url", apiurl.getBytes()
);
} catch (RowSpaceExceededException e2) {
Log.logException(e2);
} catch (IOException e2) {
Log.logException(e2);
}
Log.logInfo("APICALL", apiurl);
}
public void checkInterruption() throws InterruptedException {
final Thread curThread = Thread.currentThread();
if ((curThread instanceof WorkflowThread) && ((WorkflowThread)curThread).shutdownInProgress()) throw new InterruptedException("Shutdown in progress ...");

@ -283,7 +283,7 @@ public class serverSwitch {
if (f == null) {
ret = null;
} else {
ret = (f.isAbsolute() ? f : new File(this.rootPath, path));
ret = (f.isAbsolute() ? new File(f.getAbsolutePath()) : new File(this.rootPath, path));
}
return ret;

@ -52,6 +52,7 @@ public class BEncodedHeapArray {
public BEncodedHeapArray(final File location, final int keymaxlen) {
this.location = new File(location.getAbsolutePath());
if (!this.location.exists()) this.location.mkdirs();
this.keymaxlen = keymaxlen;
this.tables = new ConcurrentHashMap<String, BEncodedHeap>();
String[] files = this.location.list();

@ -555,17 +555,22 @@ public class HeapReader {
final int keylen1 = this.keylen - 1;
while (true) {
int len = is.readInt();
if (len == 0) continue;
b = is.readByte(); // check for empty record
if (len == 0) continue; // rare, but possible: zero length record (takes 4 bytes)
b = is.readByte(); // read a single by te to check for empty record
if (b == 0) {
// this is empty
// read some more bytes to consume the empty record
is.skip(len - 1);
is.skip(len - 1); // all that is remaining
continue;
}
// we are now ahead of remaining this.keylen - 1 bytes of the key
key = new byte[this.keylen];
key[0] = b;
if (is.read(key, 1, keylen1) < keylen1) return null;
payload = new byte[len - this.keylen];
key[0] = b; // the first entry that we know already
if (is.read(key, 1, keylen1) < keylen1) return null; // read remaining key bytes
// so far we have read this.keylen - 1 + 1 = this.keylen bytes.
// there must be a remaining number of len - this.keylen bytes left for the BLOB
if (len < this.keylen) return null; // a strange case that can only happen in case of corrupted data
payload = new byte[len - this.keylen]; // the remaining record entries
if (is.read(payload) < payload.length) return null;
return new entry(key, payload);
}
@ -574,6 +579,22 @@ public class HeapReader {
}
}
/* the old code:
private Map.Entry<byte[], byte[]> next0() {
try {
while (true) {
int len = is.readInt();
byte[] key = new byte[this.keylen];
if (is.read(key) < key.length) return null;
byte[] payload = new byte[len - this.keylen];
if (is.read(payload) < payload.length) return null;
if (key[0] == 0) continue; // this is an empty gap
return new entry(key, payload);
}
}
*/
public Map.Entry<byte[], byte[]> next() {
final Map.Entry<byte[], byte[]> n = this.nextEntry;
this.nextEntry = next0();

Loading…
Cancel
Save