boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

some more fixes for new plasmaCrawlLURL.load behavior

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2511 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
hermens 19 years ago
parent 7aeadbe7cc
commit ff4362b02d
5 changed files with 54 additions and 22 deletions
Show Stats Download Patch File Download Diff File

8
htroot/Bookmarks.java
Unescape View File

@ -138,9 +138,11 @@ public class Bookmarks {
try { try {
plasmaCrawlLURL.Entry urlentry = switchboard.urlPool.loadedURL.load(urlHash, null); plasmaCrawlLURL.Entry urlentry = switchboard.urlPool.loadedURL.load(urlHash, null);
prop.put("mode_edit", 0); // create mode prop.put("mode_edit", 0); // create mode
prop.put("mode_title", urlentry.descr()); if (urlentry != null) {
prop.put("mode_description", urlentry.descr()); prop.put("mode_title", urlentry.descr());
prop.put("mode_url", urlentry.url()); prop.put("mode_description", urlentry.descr());
prop.put("mode_url", urlentry.url());
}
prop.put("mode_tags", ""); prop.put("mode_tags", "");
prop.put("mode_public", 0); prop.put("mode_public", 0);
} catch (IOException e) { } catch (IOException e) {

46
htroot/IndexControl_p.java
Unescape View File

@ -213,11 +213,15 @@ public class IndexControl_p {
if (post.containsKey("urlhashdelete")) { if (post.containsKey("urlhashdelete")) {
try { try {
plasmaCrawlLURL.Entry entry = switchboard.urlPool.loadedURL.load(urlhash, null); plasmaCrawlLURL.Entry entry = switchboard.urlPool.loadedURL.load(urlhash, null);
URL url = entry.url(); if (entry != null) {
urlstring = url.toNormalform(); URL url = entry.url();
prop.put("urlstring", ""); urlstring = url.toNormalform();
switchboard.urlPool.loadedURL.remove(urlhash); prop.put("urlstring", "");
prop.put("result", "Removed URL " + urlstring); switchboard.urlPool.loadedURL.remove(urlhash);
prop.put("result", "Removed URL " + urlstring);
} else {
prop.put("result", "No Entry for URL hash " + urlhash + "; nothing deleted.");
}
} catch (IOException e) { } catch (IOException e) {
prop.put("result", "No Entry for URL hash " + urlhash + "; nothing deleted."); prop.put("result", "No Entry for URL hash " + urlhash + "; nothing deleted.");
} }
@ -263,7 +267,7 @@ public class IndexControl_p {
iEntry = (indexEntry) urlIter.next(); iEntry = (indexEntry) urlIter.next();
try { try {
lurl = switchboard.urlPool.loadedURL.load(iEntry.urlHash(), null); lurl = switchboard.urlPool.loadedURL.load(iEntry.urlHash(), null);
if (lurl.toString() == null) { if ((lurl == null)||(lurl.toString() == null)) {
unknownURLEntries.add(iEntry.urlHash()); unknownURLEntries.add(iEntry.urlHash());
urlIter.remove(); urlIter.remove();
} else { } else {
@ -325,10 +329,14 @@ public class IndexControl_p {
if (post.containsKey("urlhashsearch")) { if (post.containsKey("urlhashsearch")) {
try { try {
plasmaCrawlLURL.Entry entry = switchboard.urlPool.loadedURL.load(urlhash, null); plasmaCrawlLURL.Entry entry = switchboard.urlPool.loadedURL.load(urlhash, null);
URL url = entry.url(); if (entry != null) {
urlstring = url.toString(); URL url = entry.url();
prop.put("urlstring", urlstring); urlstring = url.toString();
prop.put("result", genUrlProfile(switchboard, entry, urlhash)); prop.put("urlstring", urlstring);
prop.put("result", genUrlProfile(switchboard, entry, urlhash));
} else {
prop.put("result", "No Entry for URL hash " + urlhash);
}
} catch (IOException e) { } catch (IOException e) {
prop.put("result", "No Entry for URL hash " + urlhash); prop.put("result", "No Entry for URL hash " + urlhash);
} }
@ -387,7 +395,12 @@ public class IndexControl_p {
URL url = entry.url(); URL url = entry.url();
String referrer = null; String referrer = null;
try { try {
referrer = switchboard.urlPool.loadedURL.load(entry.referrerHash(), null).url().toString(); plasmaCrawlLURL.Entry referrerEntry = switchboard.urlPool.loadedURL.load(entry.referrerHash(), null);
if (referrerEntry != null) {
referrer = referrerEntry.url().toString();
} else {
referrer = "<unknown>";
}
} catch (IOException e) { } catch (IOException e) {
referrer = "<unknown>"; referrer = "<unknown>";
} }
@ -444,8 +457,13 @@ public class IndexControl_p {
xi = (indexEntry) en.next(); xi = (indexEntry) en.next();
uh = new String[]{xi.urlHash(), Integer.toString(xi.posintext())}; uh = new String[]{xi.urlHash(), Integer.toString(xi.posintext())};
try { try {
us = switchboard.urlPool.loadedURL.load(uh[0], null).url().toString(); plasmaCrawlLURL.Entry entry = switchboard.urlPool.loadedURL.load(uh[0], null);
tm.put(us, uh); if (entry != null) {
us = entry.url().toString();
tm.put(us, uh);
} else {
tm.put(uh[0], uh);
}
} catch (IOException e) { } catch (IOException e) {
tm.put(uh[0], uh); tm.put(uh[0], uh);
} }
@ -498,4 +516,4 @@ public class IndexControl_p {
} }
} }
} }

11
htroot/yacy/crawlOrder.java
Unescape View File

@ -251,9 +251,14 @@ public final class crawlOrder {
// send lurl-Entry as response // send lurl-Entry as response
try { try {
plasmaCrawlLURL.Entry entry = switchboard.urlPool.loadedURL.load(indexURL.urlHash(url), null); plasmaCrawlLURL.Entry entry = switchboard.urlPool.loadedURL.load(indexURL.urlHash(url), null);
response = "double"; if (entry != null) {
switchboard.urlPool.loadedURL.notifyGCrawl(entry.hash(), iam, youare); response = "double";
lurl = crypt.simpleEncode(entry.toString()); switchboard.urlPool.loadedURL.notifyGCrawl(entry.hash(), iam, youare);
lurl = crypt.simpleEncode(entry.toString());
} else {
response = "rejected";
lurl = "";
}
} catch (IOException e) { } catch (IOException e) {
response = "rejected"; response = "rejected";
lurl = ""; lurl = "";

2
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -2159,6 +2159,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
// determine the url string // determine the url string
try { try {
plasmaCrawlLURL.Entry entry = urlPool.loadedURL.load(urlhash, null); plasmaCrawlLURL.Entry entry = urlPool.loadedURL.load(urlhash, null);
if (entry == null)
return 0;
URL url = entry.url(); URL url = entry.url();
if (url == null) if (url == null)
return 0; return 0;

9
source/de/anomic/plasma/plasmaWordIndex.java
Unescape View File

@ -692,8 +692,13 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI {
// System.out.println("Wordhash: "+wordHash+" UrlHash: // System.out.println("Wordhash: "+wordHash+" UrlHash:
// "+entry.getUrlHash()); // "+entry.getUrlHash());
try { try {
url = lurl.load(entry.urlHash(), null).url(); plasmaCrawlLURL.Entry lurlEntry = lurl.load(entry.urlHash(), null);
if ((url == null) || (plasmaSwitchboard.urlBlacklist.isListed(plasmaURLPattern.BLACKLIST_CRAWLER, url) == true)) { if (lurlEntry != null) {
url = lurlEntry.url();
if ((url == null) || (plasmaSwitchboard.urlBlacklist.isListed(plasmaURLPattern.BLACKLIST_CRAWLER, url) == true)) {
urlHashs.add(entry.urlHash());
}
} else {
urlHashs.add(entry.urlHash()); urlHashs.add(entry.urlHash());
} }
} catch (IOException e) { } catch (IOException e) {

Write Preview
Loading…
Cancel
Save