diff --git a/htroot/IndexCreate_p.html b/htroot/IndexCreate_p.html index d420b741c..95b9831a0 100644 --- a/htroot/IndexCreate_p.html +++ b/htroot/IndexCreate_p.html @@ -263,7 +263,7 @@ Continue crawling. #[startURL]# #[intention]# #[generalDepth]# - #(crawlingQ)#no::yes#(/withQuery)# + #(crawlingQ)#no::yes#(/crawlingQ)# #{/otherCrawlStart}# diff --git a/htroot/IndexCreate_p.java b/htroot/IndexCreate_p.java index d7bd44025..67ab68904 100644 --- a/htroot/IndexCreate_p.java +++ b/htroot/IndexCreate_p.java @@ -170,7 +170,7 @@ public class IndexCreate_p { m.remove("storeHTCache"); m.remove("generalFilter"); m.remove("specificFilter"); - m.put("intention", ((String) post.get("intention", ""))); + m.put("intention", ((String) post.get("intention", "")).replace(',', '/')); yacyCore.newsPool.publishMyNews(new yacyNewsRecord("crwlstrt", m)); } diff --git a/htroot/Network.html b/htroot/Network.html index 4a32d3edd..1680c70a0 100644 --- a/htroot/Network.html +++ b/htroot/Network.html @@ -47,7 +47,7 @@ Type
  Release/
SVN
  Contact
  -Last Seen
  < > +Last
Seen
  < > Uptime
  < > #Links
  < > #RWIs
  < > diff --git a/htroot/ViewProfile.java b/htroot/ViewProfile.java index 59b87656c..d537cc5d3 100644 --- a/htroot/ViewProfile.java +++ b/htroot/ViewProfile.java @@ -45,6 +45,7 @@ // javac -classpath .:../Classes Blacklist_p.java // if the shell's current path is HTROOT +import java.io.IOException; import java.util.HashMap; import java.util.Iterator; import java.util.Map; @@ -57,6 +58,8 @@ import de.anomic.server.serverSwitch; import de.anomic.yacy.yacyClient; import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacySeed; +import de.anomic.yacy.yacyNewsPool; +import de.anomic.yacy.yacyNewsRecord; import de.anomic.tools.bbCode; public class ViewProfile { @@ -79,6 +82,13 @@ public class ViewProfile { } } else { prop.put("success","3"); // everything ok + // process news if existent + try { + yacyNewsRecord record = yacyCore.newsPool.getByOriginator(yacyNewsPool.INCOMING_DB, "prfleupd", seed.hash); + if (record != null) yacyCore.newsPool.moveOff(yacyNewsPool.INCOMING_DB, record.id()); + } catch (IOException e) {} + + // read profile from other peer HashMap profile = yacyClient.getProfile(seed); yacyCore.log.logInfo("fetched profile:" + profile); Iterator i = profile.entrySet().iterator(); diff --git a/source/de/anomic/htmlFilter/htmlFilterContentScraper.java b/source/de/anomic/htmlFilter/htmlFilterContentScraper.java index 263645a84..747bc820b 100644 --- a/source/de/anomic/htmlFilter/htmlFilterContentScraper.java +++ b/source/de/anomic/htmlFilter/htmlFilterContentScraper.java @@ -68,6 +68,7 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen linkTags0 = new TreeSet(insensitiveCollator); linkTags0.add("img"); linkTags0.add("base"); + linkTags0.add("frame"); linkTags1 = new TreeSet(insensitiveCollator); linkTags1.add("a"); @@ -127,6 +128,7 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen public void scrapeTag0(String tagname, Properties tagopts) { if (tagname.equalsIgnoreCase("img")) images.put(absolutePath(tagopts.getProperty("src", "")), tagopts.getProperty("alt","")); if (tagname.equalsIgnoreCase("base")) try {root = new URL(tagopts.getProperty("href", ""));} catch (MalformedURLException e) {} + if (tagname.equalsIgnoreCase("frame")) anchors.put(absolutePath(tagopts.getProperty("src", "")), tagopts.getProperty("name","")); } public void scrapeTag1(String tagname, Properties tagopts, byte[] text) { @@ -136,7 +138,6 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen if ((tagname.equalsIgnoreCase("title")) && (text.length < 1024)) title = super.stripAll(new serverByteBuffer(text)).toString(); } - public String getHeadline() { String hl = ""; diff --git a/source/de/anomic/kelondro/kelondroMScoreCluster.java b/source/de/anomic/kelondro/kelondroMScoreCluster.java index 8390edbd9..06d2c5325 100644 --- a/source/de/anomic/kelondro/kelondroMScoreCluster.java +++ b/source/de/anomic/kelondro/kelondroMScoreCluster.java @@ -97,35 +97,35 @@ public class kelondroMScoreCluster { return (((long) (elementCount & 0xFFFFFFFFL)) << 32) | ((long) (elementNr & 0xFFFFFFFFL)); } - public long totalCount() { + public synchronized long totalCount() { return gcount; } - public int size() { + public synchronized int size() { return refkeyDB.size(); } - public void incScore(Object[] objs) { + public synchronized void incScore(Object[] objs) { addScore(objs, 1); } - public void addScore(Object[] objs, int count) { + public synchronized void addScore(Object[] objs, int count) { if (objs != null) for (int i = 0; i < objs.length; i++) addScore(objs[i], count); } - public void setScore(Object[] objs, int count) { + public synchronized void setScore(Object[] objs, int count) { if (objs != null) for (int i = 0; i < objs.length; i++) setScore(objs[i], count); } - public void incScore(Object obj) { + public synchronized void incScore(Object obj) { addScore(obj, 1); } - public void addScore(Object obj, int count) { + public synchronized void addScore(Object obj, int count) { if (obj == null) return; Long cs = (Long) refkeyDB.get(obj); long c; @@ -154,7 +154,7 @@ public class kelondroMScoreCluster { gcount += count; } - public void setScore(Object obj, int count) { + public synchronized void setScore(Object obj, int count) { if (obj == null) return; //System.out.println("setScore " + obj.getClass().getName()); Long cs = (Long) refkeyDB.get(obj); @@ -183,7 +183,7 @@ public class kelondroMScoreCluster { gcount += count; } - public int deleteScore(Object obj) { + public synchronized int deleteScore(Object obj) { if (obj == null) return -1; Long cs = (Long) refkeyDB.get(obj); if (cs == null) { @@ -199,11 +199,11 @@ public class kelondroMScoreCluster { } } - public boolean existsScore(Object obj) { + public synchronized boolean existsScore(Object obj) { return (refkeyDB.get(obj) != null); } - public int getScore(Object obj) { + public synchronized int getScore(Object obj) { if (obj == null) return 0; Long cs = (Long) refkeyDB.get(obj); if (cs == null) { @@ -213,7 +213,7 @@ public class kelondroMScoreCluster { } } - public int getMaxScore() { + public synchronized int getMaxScore() { if (refkeyDB.size() == 0) return -1; return (int) ((((Long) keyrefDB.lastKey()).longValue() & 0xFFFFFFFF00000000L) >> 32); } @@ -223,23 +223,23 @@ public class kelondroMScoreCluster { return (int) ((((Long) keyrefDB.firstKey()).longValue() & 0xFFFFFFFF00000000L) >> 32); } - public Object getMaxObject() { + public synchronized Object getMaxObject() { if (refkeyDB.size() == 0) return null; //return getScores(1, false)[0]; return keyrefDB.get((Long) keyrefDB.lastKey()); } - public Object getMinObject() { + public synchronized Object getMinObject() { if (refkeyDB.size() == 0) return null; //return getScores(1, true)[0]; return keyrefDB.get((Long) keyrefDB.firstKey()); } - public Object[] getScores(int maxCount, boolean up) { + public synchronized Object[] getScores(int maxCount, boolean up) { return getScores(maxCount, up, Integer.MIN_VALUE, Integer.MAX_VALUE); } - public Object[] getScores(int maxCount, boolean up, int minScore, int maxScore) { + public synchronized Object[] getScores(int maxCount, boolean up, int minScore, int maxScore) { if (maxCount > refkeyDB.size()) maxCount = refkeyDB.size(); Object[] s = new Object[maxCount]; Iterator it = scores(up, minScore, maxScore); @@ -255,12 +255,12 @@ public class kelondroMScoreCluster { return s; } - public Iterator scores(boolean up) { + public synchronized Iterator scores(boolean up) { if (up) return new simpleScoreIterator(); else return scores(false, Integer.MIN_VALUE, Integer.MAX_VALUE); } - public Iterator scores(boolean up, int minScore, int maxScore) { + public synchronized Iterator scores(boolean up, int minScore, int maxScore) { return new komplexScoreIterator(up, minScore, maxScore); } diff --git a/source/de/anomic/kelondro/kelondroMScoreIndex.java b/source/de/anomic/kelondro/kelondroMScoreIndex.java index e40d3f895..ee4dd62e2 100644 --- a/source/de/anomic/kelondro/kelondroMScoreIndex.java +++ b/source/de/anomic/kelondro/kelondroMScoreIndex.java @@ -74,35 +74,35 @@ public class kelondroMScoreIndex { scoreCluster = new kelondroMScoreCluster(); // scores for int-handles } - public long totalCount() { + public synchronized long totalCount() { return scoreCluster.totalCount(); } - public int size() { + public synchronized int size() { return handles.size(); } - public void incScore(Object[] objs) { + public synchronized void incScore(Object[] objs) { addScore(objs, 1); } - public void addScore(Object[] objs, int count) { + public synchronized void addScore(Object[] objs, int count) { if (objs != null) for (int i = 0; i < objs.length; i++) addScore(objs[i], count); } - public void setScore(Object[] objs, int count) { + public synchronized void setScore(Object[] objs, int count) { if (objs != null) for (int i = 0; i < objs.length; i++) setScore(objs[i], count); } - public void incScore(Object obj) { + public synchronized void incScore(Object obj) { addScore(obj, 1); } - public void addScore(Object obj, int count) { + public synchronized void addScore(Object obj, int count) { // get handle Integer handle = (Integer) handles.get(obj); if (handle == null) { @@ -115,7 +115,7 @@ public class kelondroMScoreIndex { scoreCluster.addScore(handle, count); } - public void setScore(Object obj, int count) { + public synchronized void setScore(Object obj, int count) { // get handle Integer handle = (Integer) handles.get(obj); if (handle == null) { @@ -128,7 +128,7 @@ public class kelondroMScoreIndex { scoreCluster.setScore(handle, count); } - public void deleteScore(Object obj) { + public synchronized void deleteScore(Object obj) { // get handle Integer handle = (Integer) handles.get(obj); if (handle != null) { @@ -138,22 +138,22 @@ public class kelondroMScoreIndex { } } - public int getScore(Object obj) { + public synchronized int getScore(Object obj) { // get handle Integer handle = (Integer) handles.get(obj); if (handle == null) return -1; return scoreCluster.getScore(handle); } - public Object[] getScores(int count, boolean up, boolean weight, char weightsep) { + public synchronized Object[] getScores(int count, boolean up, boolean weight, char weightsep) { return new Object[1]; } - public Object[] getScores(int maxCount, boolean up) { + public synchronized Object[] getScores(int maxCount, boolean up) { return getScores(maxCount, up, Integer.MIN_VALUE, Integer.MAX_VALUE); } - public Object[] getScores(int maxCount, boolean up, int minScore, int maxScore) { + public synchronized Object[] getScores(int maxCount, boolean up, int minScore, int maxScore) { if (maxCount > handles.size()) maxCount = handles.size(); Object[] s = new Object[maxCount]; Iterator it = scores(up, minScore, maxScore); @@ -169,11 +169,11 @@ public class kelondroMScoreIndex { return s; } - public Iterator scores(boolean up) { + public synchronized Iterator scores(boolean up) { return scores(up, Integer.MIN_VALUE, Integer.MAX_VALUE); } - public Iterator scores(boolean up, int minScore, int maxScore) { + public synchronized Iterator scores(boolean up, int minScore, int maxScore) { return new scoreIterator(up, minScore, maxScore); } diff --git a/source/de/anomic/plasma/plasmaHTCache.java b/source/de/anomic/plasma/plasmaHTCache.java index 79991255d..2ff996750 100644 --- a/source/de/anomic/plasma/plasmaHTCache.java +++ b/source/de/anomic/plasma/plasmaHTCache.java @@ -217,13 +217,13 @@ public final class plasmaHTCache { } private String ageString(long date, File f) { - String s = Integer.toHexString(f.hashCode()); - StringBuffer sb = new StringBuffer(32); - for (int i = s.length(); i < 8; i++) sb.append('0'); - sb.append(s); - s = Long.toHexString(date); + StringBuffer sb = new StringBuffer(32); + String s = Long.toHexString(date); for (int i = s.length(); i < 16; i++) sb.append('0'); sb.append(s); + s = Integer.toHexString(f.hashCode()); + for (int i = s.length(); i < 8; i++) sb.append('0'); + sb.append(s); return sb.toString(); } diff --git a/source/de/anomic/plasma/plasmaWordIndexAssortment.java b/source/de/anomic/plasma/plasmaWordIndexAssortment.java index d5449bb26..4a490b2ba 100644 --- a/source/de/anomic/plasma/plasmaWordIndexAssortment.java +++ b/source/de/anomic/plasma/plasmaWordIndexAssortment.java @@ -207,6 +207,11 @@ public final class plasmaWordIndexAssortment { e.printStackTrace(); resetDatabase(); return null; + } catch (kelondroException e) { + log.logFailure("iterateAssortment/kelondro-error: " + e.getMessage() + " - reset assortment-DB"); + e.printStackTrace(); + resetDatabase(); + return null; } } diff --git a/source/de/anomic/yacy/yacyNewsPool.java b/source/de/anomic/yacy/yacyNewsPool.java index 5e3aeb3bb..27d31dc7e 100644 --- a/source/de/anomic/yacy/yacyNewsPool.java +++ b/source/de/anomic/yacy/yacyNewsPool.java @@ -134,44 +134,56 @@ public class yacyNewsPool { } public int size(int dbKey) { - switch (dbKey) { - case INCOMING_DB: return incomingNews.size(); - case PROCESSED_DB: return processedNews.size(); - case OUTGOING_DB: return outgoingNews.size(); - case PUBLISHED_DB: return publishedNews.size(); - default: return -1; - } + return switchQueue(dbKey).size(); } public yacyNewsRecord get(int dbKey, int element) throws IOException { + yacyNewsQueue queue = switchQueue(dbKey); yacyNewsRecord record; - switch (dbKey) { - case INCOMING_DB: - synchronized (incomingNews) { - record = incomingNews.top(element); - if (record == null) incomingNews.pop(element); - } - return record; - case PROCESSED_DB: - synchronized (processedNews) { - record = processedNews.top(element); - if (record == null) processedNews.pop(element); - } - return record; - case OUTGOING_DB: - synchronized (outgoingNews) { - record = outgoingNews.top(element); - if (record == null) outgoingNews.pop(element); - } - return record; - case PUBLISHED_DB: - synchronized (publishedNews) { - record = publishedNews.top(element); - if (record == null) publishedNews.pop(element); - } + synchronized (queue) { + record = queue.top(element); + if (record == null) queue.pop(element); + } + return record; + } + + public synchronized yacyNewsRecord getSpecific(int dbKey, String category, String key, String value) throws IOException { + yacyNewsQueue queue = switchQueue(dbKey); + yacyNewsRecord record; + String s; + for (int i = queue.size() - 1; i >= 0; i--) { + record = queue.top(i); + if ((record != null) && (record.category().equals(category))) { + s = (String) record.attributes().get(key); + if ((s != null) && (s.equals(value))) return record; + } + } + return null; + } + + public synchronized yacyNewsRecord getByOriginator(int dbKey, String category, String originatorHash) throws IOException { + yacyNewsQueue queue = switchQueue(dbKey); + yacyNewsRecord record; + String s; + for (int i = queue.size() - 1; i >= 0; i--) { + record = queue.top(i); + if ((record != null) && + (record.category().equals(category)) && + (record.originator().equals(originatorHash))) { return record; - default: return null; + } + } + return null; + } + + private yacyNewsQueue switchQueue(int dbKey) { + switch (dbKey) { + case INCOMING_DB: return incomingNews; + case PROCESSED_DB: return processedNews; + case OUTGOING_DB: return outgoingNews; + case PUBLISHED_DB: return publishedNews; } + return null; } public void moveOff(int dbKey, String id) throws IOException {