- modified and enhanced the crawl balancer: better list export, fixing of damaged crawl queue at start-up, re-sorting at start-up to enhance domain order
- added option to set minimum crawl delta for domains in balancer - added default values to crawl deltas in yacy.init - added configuration for these deltas in performance queues - enhanced performance setting computation (more time for indexing queue for a faster flush - remote crawling is now enabled during local crawling if indexer has space and time for more links - added database stub for new distributed file system - refactoring of time computation to get an abstraction level that will be used by a TTL rule in new distributed file system git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4966 6c8d7289-2bf4-0310-a012-ef5d649a1542pull/1/head
orbiter
17 years ago
parent
1afc36d9a8
commit
474659a71f
@ -0,0 +1,64 @@
|
||||
// microDate.java
|
||||
// (C) 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
||||
// first published 3.7.2008 on http://yacy.net
|
||||
//
|
||||
// This is a part of YaCy, a peer-to-peer based web search engine
|
||||
//
|
||||
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
|
||||
// $LastChangedRevision: 1986 $
|
||||
// $LastChangedBy: orbiter $
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package de.anomic.kelondro;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
public class kelondroMicroDate {
|
||||
|
||||
private static final long hour = 3600000L; // milliseconds of a hour
|
||||
private static final long day = 86400000L; // milliseconds of a day
|
||||
|
||||
public static int microDateDays(Date modified) {
|
||||
return microDateDays(modified.getTime());
|
||||
}
|
||||
|
||||
public static int microDateDays(long modified) {
|
||||
// this calculates a virtual age from a given date
|
||||
// the purpose is to have an age in days of a given modified date
|
||||
// from a fixed standpoint in the past
|
||||
// one day has 60*60*24 seconds = 86400 seconds
|
||||
// we take mod 64**3 = 262144, this is the mask of the storage
|
||||
return (int) ((modified / day) % 262144L);
|
||||
}
|
||||
|
||||
public static String microDateHoursStr(long time) {
|
||||
return kelondroBase64Order.enhancedCoder.encodeLong(microDateHoursInt(time), 3);
|
||||
}
|
||||
|
||||
public static int microDateHoursInt(long time) {
|
||||
return (int) ((time / hour) % 262144L);
|
||||
}
|
||||
|
||||
public static int microDateHoursAge(String mdhs) {
|
||||
return microDateHoursInt(System.currentTimeMillis()) - (int) kelondroBase64Order.enhancedCoder.decodeLong(mdhs);
|
||||
}
|
||||
|
||||
public static long reverseMicroDateDays(long microDateDays) {
|
||||
return Math.min(System.currentTimeMillis(), microDateDays * day);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,142 @@
|
||||
package de.anomic.kelondro;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
|
||||
public class kelondroRelations {
|
||||
|
||||
private File baseDir;
|
||||
private HashMap<String, kelondroIndex> relations;
|
||||
|
||||
public kelondroRelations(File location) {
|
||||
this.baseDir = location;
|
||||
}
|
||||
|
||||
private static kelondroRow rowdef(String filename) {
|
||||
int p = filename.lastIndexOf('.');
|
||||
if (p >= 0) filename = filename.substring(0, p);
|
||||
p = filename.lastIndexOf('-');
|
||||
assert p >= 0;
|
||||
int payloadsize = Integer.parseInt(filename.substring(p + 1));
|
||||
filename = filename.substring(0, p);
|
||||
p = filename.lastIndexOf('-');
|
||||
assert p >= 0;
|
||||
int keysize = Integer.parseInt(filename.substring(p + 1));
|
||||
return rowdef(keysize, payloadsize);
|
||||
}
|
||||
|
||||
private static kelondroRow rowdef(int keysize, int payloadsize) {
|
||||
return new kelondroRow(
|
||||
"byte[] key-" + keysize + ", " +
|
||||
"long time-8" + keysize + ", " +
|
||||
"int ttl-4" + keysize + ", " +
|
||||
"byte[] node-" + payloadsize,
|
||||
kelondroNaturalOrder.naturalOrder, 0);
|
||||
}
|
||||
|
||||
private static String filename(String tablename, int keysize, int payloadsize) {
|
||||
return tablename + "-" + keysize + "-" + payloadsize + ".eco";
|
||||
}
|
||||
|
||||
public void declareRelation(String name, int keysize, int payloadsize) {
|
||||
// try to get the relation from the relation-cache
|
||||
kelondroIndex relation = relations.get(name);
|
||||
if (relation != null) return;
|
||||
// try to find the relation as stored on file
|
||||
String[] list = baseDir.list();
|
||||
String targetfilename = filename(name, keysize, payloadsize);
|
||||
for (int i = 0; i < list.length; i++) {
|
||||
if (list[i].startsWith(name)) {
|
||||
if (!list[i].equals(targetfilename)) continue;
|
||||
kelondroRow row = rowdef(list[i]);
|
||||
if (row.primaryKeyLength != keysize || row.column(1).cellwidth != payloadsize) continue; // a wrong table
|
||||
kelondroIndex table = new kelondroEcoTable(new File(baseDir, list[i]), row, kelondroEcoTable.tailCacheUsageAuto, 1024*1024, 0);
|
||||
relations.put(name, table);
|
||||
return;
|
||||
}
|
||||
}
|
||||
// the relation does not exist, create it
|
||||
kelondroRow row = rowdef(keysize, payloadsize);
|
||||
kelondroIndex table = new kelondroEcoTable(new File(baseDir, targetfilename), row, kelondroEcoTable.tailCacheUsageAuto, 1024*1024, 0);
|
||||
relations.put(name, table);
|
||||
}
|
||||
|
||||
public kelondroIndex getRelation(String name) {
|
||||
// try to get the relation from the relation-cache
|
||||
kelondroIndex relation = relations.get(name);
|
||||
if (relation != null) return relation;
|
||||
// try to find the relation as stored on file
|
||||
String[] list = baseDir.list();
|
||||
for (int i = 0; i < list.length; i++) {
|
||||
if (list[i].startsWith(name)) {
|
||||
kelondroRow row = rowdef(list[i]);
|
||||
kelondroIndex table = new kelondroEcoTable(new File(baseDir, list[i]), row, kelondroEcoTable.tailCacheUsageAuto, 1024*1024, 0);
|
||||
relations.put(name, table);
|
||||
return table;
|
||||
}
|
||||
}
|
||||
// the relation does not exist
|
||||
return null;
|
||||
}
|
||||
|
||||
public String putRelation(String name, String key, String value) throws IOException {
|
||||
byte[] r = putRelation(name, key.getBytes(), value.getBytes());
|
||||
if (r == null) return null;
|
||||
return new String(r);
|
||||
}
|
||||
|
||||
public byte[] putRelation(String name, byte[] key, byte[] value) throws IOException {
|
||||
kelondroIndex table = getRelation(name);
|
||||
if (table == null) return null;
|
||||
kelondroRow.Entry entry = table.row().newEntry();
|
||||
entry.setCol(0, key);
|
||||
entry.setCol(1, System.currentTimeMillis());
|
||||
entry.setCol(2, 1000000);
|
||||
entry.setCol(3, value);
|
||||
kelondroRow.Entry oldentry = table.put(entry);
|
||||
if (oldentry == null) return null;
|
||||
return oldentry.getColBytes(3);
|
||||
}
|
||||
|
||||
public String getRelation(String name, String key) throws IOException {
|
||||
byte[] r = getRelation(name, key.getBytes());
|
||||
if (r == null) return null;
|
||||
return new String(r);
|
||||
}
|
||||
|
||||
public byte[] getRelation(String name, byte[] key) throws IOException {
|
||||
kelondroIndex table = getRelation(name);
|
||||
if (table == null) return null;
|
||||
kelondroRow.Entry entry = table.get(key);
|
||||
if (entry == null) return null;
|
||||
return entry.getColBytes(3);
|
||||
}
|
||||
|
||||
public boolean hasRelation(String name, byte[] key) throws IOException {
|
||||
kelondroIndex table = getRelation(name);
|
||||
if (table == null) return false;
|
||||
return table.has(key);
|
||||
}
|
||||
|
||||
public byte[] removeRelation(String name, byte[] key) throws IOException {
|
||||
kelondroIndex table = getRelation(name);
|
||||
if (table == null) return null;
|
||||
kelondroRow.Entry entry = table.remove(key, false);
|
||||
if (entry == null) return null;
|
||||
return entry.getColBytes(3);
|
||||
}
|
||||
|
||||
public static void main(String args[]) {
|
||||
kelondroRelations r = new kelondroRelations(new File("/Users/admin/"));
|
||||
try {
|
||||
String table1 = "test1";
|
||||
r.declareRelation(table1, 12, 30);
|
||||
r.putRelation(table1, "abcdefg", "eineintrag");
|
||||
r.putRelation(table1, "abcdefg", "eineintrag");
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in new issue