boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

many YaCyNews fixes

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@461 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 20 years ago
parent 13abd8b6e7
commit 1022fbeb65
12 changed files with 66 additions and 29 deletions
Show Stats Download Patch File Download Diff File

2
htroot/IndexControl_p.java
Unescape View File

@ -91,7 +91,7 @@ public class IndexControl_p {
String urlstring = ((String) post.get("urlstring")).trim();
String urlhash = ((String) post.get("urlhash")).trim();
if (!(urlstring.startsWith("http://"))) urlstring = "http://" + urlstring;
if ((!(urlstring.startsWith("http://"))) && (!(urlstring.startsWith("https://")))) urlstring = "http://" + urlstring;
prop.put("keystring", keystring);
prop.put("keyhash", keyhash);

14
htroot/IndexCreate_p.java
Unescape View File

@ -144,6 +144,7 @@ public class IndexCreate_p {
String urlhash = plasmaURL.urlHash(crawlingStart);
switchboard.urlPool.loadedURL.remove(urlhash);
switchboard.urlPool.noticeURL.remove(urlhash);
switchboard.urlPool.errorURL.remove(urlhash);
// stack url
plasmaCrawlProfile.entry pe = switchboard.profiles.newEntry(crawlingStartURL.getHost(), crawlingStart, newcrawlingfilter, newcrawlingfilter, newcrawlingdepth, newcrawlingdepth, crawlingQ, storeHTCache, true, localIndexing, crawlOrder, xsstopw, xdstopw, xpstopw);
@ -156,7 +157,18 @@ public class IndexCreate_p {
// generate a YaCyNews if the global flag was set
if (crawlOrder) {
yacyCore.newsPool.publishMyNews(new yacyNewsRecord("crwlstrt", pe.map()));
Map m = new HashMap(pe.map()); // must be cloned
m.remove("specificDepth");
m.remove("localIndexing");
m.remove("remoteIndexing");
m.remove("xsstopw");
m.remove("xpstopw");
m.remove("xdstopw");
m.remove("storeTXCache");
m.remove("storeHTCache");
m.remove("generalFilter");
m.remove("specificFilter");
yacyCore.newsPool.publishMyNews(new yacyNewsRecord("crwlstrt", m));
}
} else {

4
htroot/News.html
Unescape View File

@ -42,7 +42,9 @@ This is the news system (currently under testing).
<p>
<form action="News.html" method="post" enctype="multipart/form-data">
<input type="hidden" name="page" value="#[page]#">
<input type="submit" name="delete" value="#(page)#::Process Selected News::Delete Selected News::Abort Publication of Selected News::Delete Selected News#(/page)#"><br><br>
<input type="submit" name="deletespecific" value="#(page)#::Process Selected News::Delete Selected News::Abort Publication of Selected News::Delete Selected News#(/page)#">
<input type="submit" name="deleteall" value="#(page)#::Process All News::Delete All News::Abort Publication of All News::Delete All News#(/page)#">
<br><br>
<table border="0" cellpadding="2" cellspacing="1">
<tr class="TableHeader" valign="bottom">
<td class="small"></td>

36
htroot/News.java
Unescape View File

@ -72,19 +72,31 @@ public class News {
return prop;
}
if ((post.containsKey("delete")) && (tableID >= 0)) {
Enumeration e = post.keys();
String check;
String id;
while (e.hasMoreElements()) {
check = (String) e.nextElement();
if ((check.startsWith("del_")) && (post.get(check, "off").equals("on"))) {
id = check.substring(4);
try {
yacyCore.newsPool.moveOff(tableID, id);
} catch (IOException ee) {ee.printStackTrace();}
if ((post.containsKey("deletespecific")) && (tableID >= 0)) {
Enumeration e = post.keys();
String check;
String id;
while (e.hasMoreElements()) {
check = (String) e.nextElement();
if ((check.startsWith("del_")) && (post.get(check, "off").equals("on"))) {
id = check.substring(4);
try {
yacyCore.newsPool.moveOff(tableID, id);
} catch (IOException ee) {ee.printStackTrace();}
}
}
}
if ((post.containsKey("deleteall")) && (tableID >= 0)) {
yacyNewsRecord record;
try {
while (yacyCore.newsPool.size(tableID) > 0) {
record = yacyCore.newsPool.get(tableID, 0);
yacyCore.newsPool.moveOff(tableID, record.id());
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
@ -115,7 +127,7 @@ public class News {
prop.put("table_list_" + i + "_ori", (seed == null) ? record.originator() : seed.getName());
prop.put("table_list_" + i + "_cre", yacyCore.universalDateShortString(record.created()));
prop.put("table_list_" + i + "_cat", record.category());
prop.put("table_list_" + i + "_rec", yacyCore.universalDateShortString(record.received()));
prop.put("table_list_" + i + "_rec", (record.received() == null) ? "-" : yacyCore.universalDateShortString(record.received()));
prop.put("table_list_" + i + "_dis", record.distributed());
prop.put("table_list_" + i + "_att", record.attributes().toString());
} catch (IOException e) {e.printStackTrace();}

9
source/de/anomic/kelondro/kelondroRecords.java
Unescape View File

@ -588,7 +588,7 @@ public class kelondroRecords {
for (int j = values[i].length; j < COLWIDTHS[i]; j++)
entryFile.writeByte(0);
}
seek = seek + COLWIDTHS[i];
seek += COLWIDTHS[i];
}
}
updateNodeCache();
@ -630,7 +630,8 @@ public class kelondroRecords {
entryFile.seek(seek);
values[i] = new byte[COLWIDTHS[i]];
entryFile.read(values[i], 0, values[i].length);
seek = seek + COLWIDTHS[i];
if (values[i][0] == 0) values[i] = null;
seek += COLWIDTHS[i];
}
}
return values;
@ -643,7 +644,8 @@ public class kelondroRecords {
entryFile.seek(seek);
values[i] = new byte[COLWIDTHS[i]];
entryFile.read(values[i], 0, values[i].length);
seek = seek + COLWIDTHS[i];
if (values[i][0] == 0) values[i] = null;
seek += COLWIDTHS[i];
}
}
return values;
@ -865,6 +867,7 @@ public class kelondroRecords {
}
public static long bytes2long(byte[] b) {
if (b == null) return 0;
long x = 0;
for (int i = 0; i < b.length; i++) x = (x << 8) | (0xff & (int) b[i]);
return x;

2
source/de/anomic/plasma/plasmaCrawlLURL.java
Unescape View File

@ -370,7 +370,7 @@ public class plasmaCrawlLURL extends plasmaURL {
byte[][] entry = urlHashCache.get(urlHash.getBytes());
if (entry != null) {
this.url = new URL(new String(entry[1]).trim());
this.descr = new String(entry[2]).trim();
this.descr = (entry[2] == null) ? "" : new String(entry[2]).trim();
this.moddate = new Date(86400000 * serverCodings.enhancedCoder.decodeBase64Long(new String(entry[3])));
this.loaddate = new Date(86400000 * serverCodings.enhancedCoder.decodeBase64Long(new String(entry[4])));
this.referrerHash = new String(entry[5]);

4
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -701,6 +701,10 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
}
String profileHandle = urlEntry.profileHandle();
//System.out.println("DEBUG plasmaSwitchboard.processCrawling: profileHandle = " + profileHandle + ", urlEntry.url = " + urlEntry.url());
if (profileHandle == null) {
log.logError(stats + ": NULL PROFILE HANDLE '" + urlEntry.profileHandle() + "' (must be internal error) for URL " + urlEntry.url());
return true;
}
plasmaCrawlProfile.entry profile = profiles.getEntry(profileHandle);
if (profile == null) {
log.logError(stats + ": LOST PROFILE HANDLE '" + urlEntry.profileHandle() + "' (must be internal error) for URL " + urlEntry.url());

2
source/de/anomic/plasma/plasmaSwitchboardQueue.java
Unescape View File

@ -159,7 +159,7 @@ public class plasmaSwitchboardQueue {
public Entry(byte[][] row) {
long ims = serverCodings.enhancedCoder.decodeBase64Long(new String(row[2]));
byte flags = row[3][0];
byte flags = (row[3] == null) ? 0 : row[3][0];
try {
this.url = new URL(new String(row[0]));
} catch (MalformedURLException e) {

4
source/de/anomic/yacy/yacyNewsDB.java
Unescape View File

@ -159,7 +159,7 @@ public class yacyNewsDB {
return new yacyNewsRecord(
new String(b[0]),
new String(b[1]),
yacyCore.parseUniversalDate(new String(b[2])),
(b[2] == null) ? null : yacyCore.parseUniversalDate(new String(b[2])),
(int) serverCodings.enhancedCoder.decodeBase64Long(new String(b[3])),
serverCodings.string2map(new String(b[4]))
);
@ -172,7 +172,7 @@ public class yacyNewsDB {
byte[][] b = new byte[5][];
b[0] = r.id().getBytes();
b[1] = r.category().getBytes();
b[2] = yacyCore.universalDateShortString(r.created()).getBytes();
b[2] = (r.received() == null) ? null : yacyCore.universalDateShortString(r.received()).getBytes();
b[3] = serverCodings.enhancedCoder.encodeBase64Long(r.distributed(), 2).getBytes();
b[4] = attributes.getBytes();
return b;

6
source/de/anomic/yacy/yacyNewsPool.java
Unescape View File

@ -111,7 +111,9 @@ public class yacyNewsPool {
public void enqueueIncomingNews(yacyNewsRecord record) throws IOException {
// called if a news is attached to a seed
if (newsDB.get(record.id()) == null) incomingNews.push(record);
if (record.created().getTime() == 0) return;
if (newsDB.get(record.id()) != null) return;
incomingNews.push(record);
}
public synchronized boolean commitIncomingNews(String id) throws IOException {
@ -202,5 +204,5 @@ public class yacyNewsPool {
case PUBLISHED_DB: deletePublishedNews(id); break;
}
}
}

9
source/de/anomic/yacy/yacyNewsRecord.java
Unescape View File

@ -61,6 +61,7 @@ public class yacyNewsRecord {
public yacyNewsRecord(String newsString) {
this.attributes = serverCodings.string2map(newsString);
this.received = (attributes.containsKey("rec")) ? yacyCore.parseUniversalDate((String) attributes.get("rec")) : new Date();
//this.received = new Date();
this.created = (attributes.containsKey("cre")) ? yacyCore.parseUniversalDate((String) attributes.get("cre")) : new Date();
this.category = (attributes.containsKey("cat")) ? (String) attributes.get("cat") : null;
this.distributed = (attributes.containsKey("dis")) ? Integer.parseInt((String) attributes.get("dis")) : 0;
@ -101,10 +102,10 @@ public class yacyNewsRecord {
// this creates the string that shall be distributed
// attention: this has no additional encoding
if (this.originator != null) attributes.put("ori", this.originator);
if (this.category != null) attributes.put("cat", this.category);
attributes.put("cre", yacyCore.universalDateShortString(this.created));
attributes.put("rec", yacyCore.universalDateShortString(this.received));
attributes.put("dis", "" +this. distributed);
if (this.category != null) attributes.put("cat", this.category);
if (this.created != null) attributes.put("cre", yacyCore.universalDateShortString(this.created));
if (this.received != null) attributes.put("rec", yacyCore.universalDateShortString(this.received));
attributes.put("dis", "" + this.distributed);
String theString = attributes.toString();
removeStandards();
return theString;

3
source/de/anomic/yacy/yacyPeerActions.java
Unescape View File

@ -94,7 +94,8 @@ public class yacyPeerActions {
if (sb.getConfig("peerName", "nameless").equals("nameless")) {
// generate new peer name
String newPeerName = serverCore.publicIP().getHostName() + yacyCore.speedKey + serverSystem.infoKey() + (System.currentTimeMillis() & 99);
newPeerName = newPeerName.replace('.', '_');
newPeerName = newPeerName.replace('.', '-');
newPeerName = newPeerName.replace('_', '-');
sb.setConfig("peerName", newPeerName);
}
seedDB.mySeed.put("Name", sb.getConfig("peerName", "nameless"));

Write Preview
Loading…
Cancel
Save