re-try to fetch url after a soft commit

12 years ago · f425b2c61c
parent b8f57f7703
commit f425b2c61c
1 changed files with 6 additions and 1 deletions
--- a/htroot/CrawlResults.java
+++ b/htroot/CrawlResults.java
@ -182,7 +182,12 @@ public class CrawlResults {
            while (i.hasNext()) {
                entry = i.next();
                try {
-                    urle = sb.index.fulltext().getMetadata(UTF8.getBytes(entry.getKey()));
+                    byte[] urlhash = UTF8.getBytes(entry.getKey());
+                    urle = sb.index.fulltext().getMetadata(urlhash);
+                    if (urle == null) {
+                        sb.index.fulltext().commit(true);
+                        urle = sb.index.fulltext().getMetadata(urlhash);
+                    }
                    if (urle == null) {
                        ConcurrentLog.warn("PLASMA", "CrawlResults: URL not in index with url hash " + entry.getKey());
                        urlstr = null;