boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- extended news size from 512 to 1024 characters

Browse Source 
- a new news db will be created (news1024.db), the old one (news.db) can be deleted
- peers with too large news payload are not ignored any more (they may have been invisible because they had a too large news payload!)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6917 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent 353a924760
commit 56ff9d5fd4
15 changed files with 77 additions and 57 deletions
Show Stats Download Patch File Download Diff File

1
.classpath
Unescape View File

@ -24,7 +24,6 @@
<classpathentry exported="true" kind="lib" path="lib/webcat-0.1-swf.jar"/>
<classpathentry exported="true" kind="lib" path="lib/activation.jar"/>
<classpathentry exported="true" kind="lib" path="lib/commons-jxpath-1.3.jar"/>
<classpathentry exported="true" kind="lib" path="libt/junit-4.7.jar"/>
<classpathentry kind="lib" path="lib/fontbox-1.0.0.jar"/>
<classpathentry kind="lib" path="lib/pdfbox-1.0.0.jar"/>
<classpathentry kind="lib" path="lib/poi-3.6-20091214.jar"/>

3
htroot/Blog.java
Unescape View File

@ -46,7 +46,6 @@ import de.anomic.http.server.RequestHeader;
import de.anomic.search.Switchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyNewsDB;
import de.anomic.yacy.yacyNewsPool;
import java.util.List;
import java.util.Map;
@ -175,7 +174,7 @@ public class Blog {
map.put("page", pagename);
map.put("subject", StrSubject.replace(',', ' '));
map.put("author", StrAuthor.replace(',', ' '));
sb.peers.newsPool.publishMyNews(new yacyNewsDB.Record(sb.peers.mySeed(), yacyNewsPool.CATEGORY_BLOG_ADD, map));
sb.peers.newsPool.publishMyNews(sb.peers.mySeed(), yacyNewsPool.CATEGORY_BLOG_ADD, map);
}
page = sb.blogDB.readBlogEntry(pagename); //maybe "if(page == null)"

3
htroot/Bookmarks.java
Unescape View File

@ -52,7 +52,6 @@ import de.anomic.search.Segments;
import de.anomic.search.Switchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyNewsDB;
import de.anomic.yacy.yacyNewsPool;
@ -434,7 +433,7 @@ public class Bookmarks {
map.put("title", title.replace(',', ' '));
map.put("description", description.replace(',', ' '));
map.put("tags", tagsString.replace(',', ' '));
sb.peers.newsPool.publishMyNews(new yacyNewsDB.Record(sb.peers.mySeed(), yacyNewsPool.CATEGORY_BOOKMARK_ADD, map));
sb.peers.newsPool.publishMyNews(sb.peers.mySeed(), yacyNewsPool.CATEGORY_BOOKMARK_ADD, map);
}
}

3
htroot/ConfigProfile_p.java
Unescape View File

@ -39,7 +39,6 @@ import de.anomic.http.server.RequestHeader;
import de.anomic.search.Switchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyNewsDB;
import de.anomic.yacy.yacyNewsPool;
public class ConfigProfile_p {
@ -94,7 +93,7 @@ public class ConfigProfile_p {
// generate a news message
final Properties news = profile;
news.remove("comment");
sb.peers.newsPool.publishMyNews(yacyNewsDB.newRecord(sb.peers.mySeed(), yacyNewsPool.CATEGORY_PROFILE_UPDATE, news));
sb.peers.newsPool.publishMyNews(sb.peers.mySeed(), yacyNewsPool.CATEGORY_PROFILE_UPDATE, news);
//yacyCore.newsPool.publishMyNews(new yacyNewsRecord(yacyNewsRecord.CATEGORY_PROFILE_UPDATE, profile));
} catch(final IOException e) {
} finally {

3
htroot/Crawler_p.java
Unescape View File

@ -57,7 +57,6 @@ import de.anomic.search.Switchboard;
import de.anomic.search.SwitchboardConstants;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyNewsDB;
import de.anomic.yacy.yacyNewsPool;
public class Crawler_p {
@ -289,7 +288,7 @@ public class Crawler_p {
m.remove("generalFilter");
m.remove("specificFilter");
m.put("intention", post.get("intention", "").replace(',', '/'));
sb.peers.newsPool.publishMyNews(new yacyNewsDB.Record(sb.peers.mySeed(), yacyNewsPool.CATEGORY_CRAWL_START, m));
sb.peers.newsPool.publishMyNews(sb.peers.mySeed(), yacyNewsPool.CATEGORY_CRAWL_START, m);
}
} else {
prop.put("info", "5"); //Crawling failed

4
htroot/Supporter.java
Unescape View File

@ -79,7 +79,7 @@ public class Supporter {
map.put("urlhash", hash);
map.put("vote", "negative");
map.put("refid", post.get("refid", ""));
sb.peers.newsPool.publishMyNews(new yacyNewsDB.Record(sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map));
sb.peers.newsPool.publishMyNews(sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map);
}
if ((post != null) && ((hash = post.get("votePositive", null)) != null)) {
if (!sb.verifyAuthentication(header, false)) {
@ -95,7 +95,7 @@ public class Supporter {
map.put("vote", "positive");
map.put("refid", post.get("refid", ""));
map.put("comment", post.get("comment", ""));
sb.peers.newsPool.publishMyNews(new yacyNewsDB.Record(sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map));
sb.peers.newsPool.publishMyNews(sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map);
}
// create Supporter

4
htroot/Surftips.java
Unescape View File

@ -87,7 +87,7 @@ public class Surftips {
map.put("urlhash", hash);
map.put("vote", "negative");
map.put("refid", post.get("refid", ""));
sb.peers.newsPool.publishMyNews(new yacyNewsDB.Record(sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map));
sb.peers.newsPool.publishMyNews(sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map);
}
if ((post != null) && ((hash = post.get("votePositive", null)) != null)) {
if (!sb.verifyAuthentication(header, false)) {
@ -103,7 +103,7 @@ public class Surftips {
map.put("vote", "positive");
map.put("refid", post.get("refid", ""));
map.put("comment", post.get("comment", ""));
sb.peers.newsPool.publishMyNews(new yacyNewsDB.Record(sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map));
sb.peers.newsPool.publishMyNews(sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map);
}
// create surftips

3
htroot/Wiki.java
Unescape View File

@ -45,7 +45,6 @@ import de.anomic.http.server.RequestHeader;
import de.anomic.search.Switchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyNewsDB;
import de.anomic.yacy.yacyNewsPool;
public class Wiki {
@ -121,7 +120,7 @@ public class Wiki {
map.put("page", pagename);
map.put("author", author.replace(',', ' '));
if (post.get("content", "").trim().length() > 0 && !page.page().equals(content))
sb.peers.newsPool.publishMyNews(new yacyNewsDB.Record(sb.peers.mySeed(), yacyNewsPool.CATEGORY_WIKI_UPDATE, map));
sb.peers.newsPool.publishMyNews(sb.peers.mySeed(), yacyNewsPool.CATEGORY_WIKI_UPDATE, map);
page = newEntry;
prop.putHTML("LOCATION", "/Wiki.html?page=" + pagename);
}

3
htroot/api/bookmarks/posts/add_p.java
Unescape View File

@ -10,7 +10,6 @@ import de.anomic.http.server.RequestHeader;
import de.anomic.search.Switchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyNewsDB;
import de.anomic.yacy.yacyNewsPool;
@ -83,6 +82,6 @@ public class add_p {
map.put("title", title.replace(',', ' '));
map.put("description", description.replace(',', ' '));
map.put("tags", tagsString.replace(',', ' '));
sb.peers.newsPool.publishMyNews(new yacyNewsDB.Record(sb.peers.mySeed(), yacyNewsPool.CATEGORY_BOOKMARK_ADD, map));
sb.peers.newsPool.publishMyNews(sb.peers.mySeed(), yacyNewsPool.CATEGORY_BOOKMARK_ADD, map);
}
}

4
htroot/yacysearch.java
Unescape View File

@ -412,7 +412,7 @@ public class yacysearch {
map.put("urlhash", delHash);
map.put("vote", "negative");
map.put("refid", "");
sb.peers.newsPool.publishMyNews(new yacyNewsDB.Record(sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map));
sb.peers.newsPool.publishMyNews(sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map);
} catch (IOException e) {
Log.logException(e);
}
@ -437,7 +437,7 @@ public class yacysearch {
map.put("description", document.dc_title().replace(',', ' '));
map.put("author", document.dc_creator());
map.put("tags", document.dc_subject(' '));
sb.peers.newsPool.publishMyNews(new yacyNewsDB.Record(sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_ADD, map));
sb.peers.newsPool.publishMyNews(sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_ADD, map);
document.close();
}
}

3
source/de/anomic/data/bookmarksDB.java
Unescape View File

@ -56,7 +56,6 @@ import de.anomic.crawler.CrawlProfile;
import de.anomic.crawler.retrieval.Request;
import de.anomic.search.Segments;
import de.anomic.search.Switchboard;
import de.anomic.yacy.yacyNewsDB;
import de.anomic.yacy.yacyNewsPool;
public class bookmarksDB {
@ -291,7 +290,7 @@ public class bookmarksDB {
m.remove("generalFilter");
m.remove("specificFilter");
m.put("intention", "Automatic ReCrawl!");
sb.peers.newsPool.publishMyNews(new yacyNewsDB.Record(sb.peers.mySeed(), yacyNewsPool.CATEGORY_CRAWL_START, m));
sb.peers.newsPool.publishMyNews(sb.peers.mySeed(), yacyNewsPool.CATEGORY_CRAWL_START, m);
}
} catch (MalformedURLException e1) {}
} // if

3
source/de/anomic/search/Switchboard.java
Unescape View File

@ -150,7 +150,6 @@ import de.anomic.tools.CryptoLib;
import de.anomic.yacy.yacyBuildProperties;
import de.anomic.yacy.yacyClient;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNewsDB;
import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.Tray;
@ -1608,7 +1607,7 @@ public final class Switchboard extends serverSwitch {
if ((homepage != null) && (homepage.length() > 10)) {
final Properties news = new Properties();
news.put("homepage", profile.get("homepage"));
this.peers.newsPool.publishMyNews(yacyNewsDB.newRecord(peers.mySeed(), yacyNewsPool.CATEGORY_PROFILE_BROADCAST, news));
this.peers.newsPool.publishMyNews(peers.mySeed(), yacyNewsPool.CATEGORY_PROFILE_BROADCAST, news);
}
}

80
source/de/anomic/yacy/yacyNewsDB.java
Unescape View File

@ -71,33 +71,32 @@ import net.yacy.kelondro.util.MapTools;
public class yacyNewsDB {
private final File path;
private final Row rowdef;
protected final int attributesMaxLength;
protected ObjectIndex news;
private static final int maxNewsRecordLength = 512;
private static final int categoryStringLength = 8;
public static final int idLength = DateFormatter.PATTERN_SHORT_SECOND.length() + Word.commonHashLength;
private static final int attributesMaxLength =
maxNewsRecordLength
- idLength
- categoryStringLength
- DateFormatter.PATTERN_SHORT_SECOND.length()
- 2;
private static final Row rowdef = new Row(
"String idx-" + idLength + " \"id = created + originator\"," +
"String cat-" + categoryStringLength + "," +
"String rec-" + DateFormatter.PATTERN_SHORT_SECOND.length() + "," +
"short dis-2 {b64e}," +
"String att-" + attributesMaxLength,
NaturalOrder.naturalOrder
);
public yacyNewsDB(
final File path,
final int maxNewsRecordLength,
final boolean useTailCache,
final boolean exceed134217727) {
this.path = path;
this.attributesMaxLength = maxNewsRecordLength
- idLength
- categoryStringLength
- DateFormatter.PATTERN_SHORT_SECOND.length()
- 2;
this.rowdef = new Row(
"String idx-" + idLength + " \"id = created + originator\"," +
"String cat-" + categoryStringLength + "," +
"String rec-" + DateFormatter.PATTERN_SHORT_SECOND.length() + "," +
"short dis-2 {b64e}," +
"String att-" + attributesMaxLength,
NaturalOrder.naturalOrder
);
try {
this.news = new Table(path, rowdef, 10, 0, useTailCache, exceed134217727);
} catch (RowSpaceExceededException e) {
@ -185,7 +184,7 @@ public class yacyNewsDB {
}
}
protected final static Record b2r(final Row.Entry b) {
protected Record b2r(final Row.Entry b) {
if (b == null) return null;
return new yacyNewsDB.Record(
b.getColString(0, null),
@ -199,8 +198,11 @@ public class yacyNewsDB {
protected final Row.Entry r2b(final Record r) {
if (r == null) return null;
try {
final String attributes = r.attributes().toString();
if (attributes.length() > attributesMaxLength) throw new IllegalArgumentException("attribute length=" + attributes.length() + " exceeds maximum size=" + attributesMaxLength);
String attributes = r.attributes().toString();
if (attributes.length() > attributesMaxLength) {
Log.logWarning("yacyNewsDB", "attribute length=" + attributes.length() + " exceeds maximum size=" + attributesMaxLength);
attributes = new HashMap<String, String>().toString();
}
final Row.Entry entry = this.news.row().newEntry();
entry.setCol(0, r.id().getBytes());
entry.setCol(1, r.category().getBytes("UTF-8"));
@ -214,7 +216,7 @@ public class yacyNewsDB {
}
}
public static Record newRecord(final yacySeed mySeed, final String category, final Properties attributes) {
public Record newRecord(final yacySeed mySeed, final String category, final Properties attributes) {
try {
final Map<String, String> m = new HashMap<String, String>();
final Iterator<Entry<Object, Object>> e = attributes.entrySet().iterator();
@ -225,24 +227,42 @@ public class yacyNewsDB {
}
return new Record(mySeed, category, m);
} catch (final IllegalArgumentException e) {
yacyCore.log.logWarning("rejected bad yacy news record: " + e.getMessage());
yacyCore.log.logWarning("rejected bad yacy news record (1): " + e.getMessage());
return null;
}
}
public static class Record {
public Record newRecord(final yacySeed mySeed, final String category, final Map<String, String> attributes) {
try {
return new Record(mySeed, category, attributes);
} catch (final IllegalArgumentException e) {
yacyCore.log.logWarning("rejected bad yacy news record (2): " + e.getMessage());
return null;
}
}
public Record newRecord(String external) {
try {
return new Record(external);
} catch (final IllegalArgumentException e) {
yacyCore.log.logWarning("rejected bad yacy news record (3): " + e.getMessage());
return null;
}
}
public class Record {
private final String originator; // hash of originating peer
private final Date created; // Date when news was created by originator
private final Date received; // Date when news was received here at this peer
private final String category; // keyword that adresses possible actions
private final String category; // keyword that addresses possible actions
private int distributed; // counter that counts number of distributions of this news record
private final Map<String, String> attributes; // elemets of the news for a special category
private final Map<String, String> attributes; // elements of the news for a special category
public Record(final String newsString) {
this.attributes = MapTools.string2map(newsString, ",");
if (attributes.toString().length() > yacyNewsDB.attributesMaxLength) throw new IllegalArgumentException("attributes length (" + attributes.toString().length() + ") exceeds maximum (" + yacyNewsDB.attributesMaxLength + ")");
if (attributes.toString().length() > attributesMaxLength) throw new IllegalArgumentException("attributes length (" + attributes.toString().length() + ") exceeds maximum (" + attributesMaxLength + ")");
this.category = (attributes.containsKey("cat")) ? attributes.get("cat") : "";
if (category.length() > yacyNewsDB.categoryStringLength) throw new IllegalArgumentException("category length (" + category.length() + ") exceeds maximum (" + yacyNewsDB.categoryStringLength + ")");
this.received = (attributes.containsKey("rec")) ? DateFormatter.parseShortSecond(attributes.get("rec"), DateFormatter.UTCDiffString()) : new Date();
@ -254,7 +274,7 @@ public class yacyNewsDB {
public Record(final yacySeed mySeed, final String category, final Map<String, String> attributes) {
if (category.length() > yacyNewsDB.categoryStringLength) throw new IllegalArgumentException("category length (" + category.length() + ") exceeds maximum (" + yacyNewsDB.categoryStringLength + ")");
if (attributes.toString().length() > yacyNewsDB.attributesMaxLength) throw new IllegalArgumentException("attributes length (" + attributes.toString().length() + ") exceeds maximum (" + yacyNewsDB.attributesMaxLength + ")");
if (attributes.toString().length() > attributesMaxLength) throw new IllegalArgumentException("attributes length (" + attributes.toString().length() + ") exceeds maximum (" + attributesMaxLength + ")");
this.attributes = attributes;
this.received = null;
this.created = new Date();
@ -266,7 +286,7 @@ public class yacyNewsDB {
protected Record(final String id, final String category, final Date received, final int distributed, final Map<String, String> attributes) {
if (category.length() > yacyNewsDB.categoryStringLength) throw new IllegalArgumentException("category length (" + category.length() + ") exceeds maximum (" + yacyNewsDB.categoryStringLength + ")");
if (attributes.toString().length() > yacyNewsDB.attributesMaxLength) throw new IllegalArgumentException("attributes length (" + attributes.toString().length() + ") exceeds maximum (" + yacyNewsDB.attributesMaxLength + ")");
if (attributes.toString().length() > attributesMaxLength) throw new IllegalArgumentException("attributes length (" + attributes.toString().length() + ") exceeds maximum (" + attributesMaxLength + ")");
this.attributes = attributes;
this.received = received;
this.created = DateFormatter.parseShortSecond(id.substring(0, DateFormatter.PATTERN_SHORT_SECOND.length()), DateFormatter.UTCDiffString());
@ -334,10 +354,6 @@ public class yacyNewsDB {
if ((s == null) || (s.length() == 0)) return dflt;
return s;
}
public static void main(final String[] args) {
System.out.println((new yacyNewsDB.Record(args[0])).toString());
}
}
}

15
source/de/anomic/yacy/yacyNewsPool.java
Unescape View File

@ -49,6 +49,7 @@ import java.io.IOException;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Properties;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.index.RowSpaceExceededException;
@ -271,7 +272,7 @@ public class yacyNewsPool {
final File yacyDBPath,
final boolean useTailCache,
final boolean exceed134217727) {
newsDB = new yacyNewsDB(new File(yacyDBPath, "news.db"), useTailCache, exceed134217727);
newsDB = new yacyNewsDB(new File(yacyDBPath, "news1024.db"), 1024, useTailCache, exceed134217727);
outgoingNews = new yacyNewsQueue(new File(yacyDBPath, "newsOut.table"), newsDB);
publishedNews = new yacyNewsQueue(new File(yacyDBPath, "newsPublished.table"), newsDB);
incomingNews = new yacyNewsQueue(new File(yacyDBPath, "newsIn.table"), newsDB);
@ -297,6 +298,18 @@ public class yacyNewsPool {
return queue.records(up);
}
public yacyNewsDB.Record parseExternal(String external) {
return newsDB.newRecord(external);
}
public void publishMyNews(final yacySeed mySeed, final String category, final Map<String, String> attributes) {
publishMyNews(newsDB.newRecord(mySeed, category, attributes));
}
public void publishMyNews(final yacySeed mySeed, final String category, final Properties attributes) {
publishMyNews(newsDB.newRecord(mySeed, category, attributes));
}
public void publishMyNews(final yacyNewsDB.Record record) {
// this shall be called if our peer generated a new news record and wants to publish it
if (record == null) return;

2
source/de/anomic/yacy/yacyPeerActions.java
Unescape View File

@ -228,7 +228,7 @@ public class yacyPeerActions {
//System.out.println("### triggered news arrival from peer " + peer.getName() + ", news " + ((recordString == null) ? "empty" : "attached"));
if ((recordString == null) || (recordString.length() == 0)) return;
final String decodedString = de.anomic.tools.crypt.simpleDecode(recordString, "");
final yacyNewsDB.Record record = new yacyNewsDB.Record(decodedString);
final yacyNewsDB.Record record = this.newsPool.parseExternal(decodedString);
if (record != null) {
//System.out.println("### news arrival from peer " + peer.getName() + ", decoded=" + decodedString + ", record=" + recordString + ", news=" + record.toString());
final String cre1 = MapTools.string2map(decodedString, ",").get("cre");

Write Preview
Loading…
Cancel
Save