From a4e8ad95abdd6c50456951e56b4a01c597055e85 Mon Sep 17 00:00:00 2001 From: orbiter Date: Fri, 13 Jul 2007 13:00:18 +0000 Subject: [PATCH] enhancements to news and switchboard queue processing removed direct access and replaced by iteration git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3961 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/IndexCreateIndexingQueue_p.java | 86 +++++++-------- htroot/xml/queues_p.java | 77 ++++++------- source/de/anomic/kelondro/kelondroStack.java | 101 ++++++------------ .../de/anomic/plasma/plasmaCrawlBalancer.java | 4 +- .../de/anomic/plasma/plasmaSwitchboard.java | 20 ++-- .../anomic/plasma/plasmaSwitchboardQueue.java | 75 +++++++------ source/de/anomic/yacy/yacyNewsPool.java | 17 +-- source/de/anomic/yacy/yacyNewsQueue.java | 30 +++--- 8 files changed, 187 insertions(+), 223 deletions(-) diff --git a/htroot/IndexCreateIndexingQueue_p.java b/htroot/IndexCreateIndexingQueue_p.java index 2fa92c213..0ea83cc64 100644 --- a/htroot/IndexCreateIndexingQueue_p.java +++ b/htroot/IndexCreateIndexingQueue_p.java @@ -46,6 +46,7 @@ import java.io.IOException; import java.text.DecimalFormat; import java.util.ArrayList; +import java.util.Iterator; import de.anomic.data.htmlTools; import de.anomic.http.httpHeader; @@ -95,19 +96,7 @@ public class IndexCreateIndexingQueue_p { } else if (post.containsKey("deleteEntry")) { String urlHash = (String) post.get("deleteEntry"); try { - synchronized (switchboard.sbQueue) { - ArrayList entries = switchboard.sbQueue.list(0); - for (int i=entries.size()-1; i >= 0; i--) { - plasmaSwitchboardQueue.Entry pcentry = (plasmaSwitchboardQueue.Entry) entries.get(i); - if (pcentry.urlHash().equals(urlHash)) { - plasmaSwitchboardQueue.Entry entry = switchboard.sbQueue.remove(i); - if ((entry != null) && (entry.profile() != null) && (!(entry.profile().storeHTCache()))) { - switchboard.cacheManager.deleteFile(entry.url()); - } - break; - } - } - } + switchboard.sbQueue.remove(urlHash); } catch (Exception e) {} prop.put("LOCATION",""); return prop; @@ -126,44 +115,43 @@ public class IndexCreateIndexingQueue_p { plasmaSwitchboardQueue.Entry pcentry; int inProcessCount = 0, entryCount = 0, totalCount = 0; long totalSize = 0; - try { - ArrayList entryList = new ArrayList(); - - // getting all entries that are currently in process - synchronized (switchboard.indexingTasksInProcess) { - inProcessCount = switchboard.indexingTasksInProcess.size(); - entryList.addAll(switchboard.indexingTasksInProcess.values()); - } - - // getting all enqueued entries - if ((switchboard.sbQueue.size() > 0)) { - entryList.addAll(switchboard.sbQueue.list(0)); - } - - int count=entryList.size(); - totalCount = count; - for (int i = 0; (i < count) && (entryCount < showLimit); i++) { + ArrayList entryList = new ArrayList(); + + // getting all entries that are currently in process + synchronized (switchboard.indexingTasksInProcess) { + inProcessCount = switchboard.indexingTasksInProcess.size(); + entryList.addAll(switchboard.indexingTasksInProcess.values()); + } + + // getting all enqueued entries + if ((switchboard.sbQueue.size() > 0)) { + Iterator i = switchboard.sbQueue.entryIterator(false); + while (i.hasNext()) entryList.add((plasmaSwitchboardQueue.Entry) i.next()); + } + + int count=entryList.size(); + totalCount = count; + for (int i = 0; (i < count) && (entryCount < showLimit); i++) { - boolean inProcess = i < inProcessCount; - pcentry = (plasmaSwitchboardQueue.Entry) entryList.get(i); - if ((pcentry != null)&&(pcentry.url() != null)) { - long entrySize = pcentry.size(); - totalSize += entrySize; - initiator = yacyCore.seedDB.getConnected(pcentry.initiator()); - prop.put("indexing-queue_list_"+entryCount+"_dark", (inProcess)? 2: ((dark) ? 1 : 0)); - prop.put("indexing-queue_list_"+entryCount+"_initiator", ((initiator == null) ? "proxy" : htmlTools.replaceHTML(initiator.getName()))); - prop.put("indexing-queue_list_"+entryCount+"_depth", pcentry.depth()); - prop.put("indexing-queue_list_"+entryCount+"_modified", pcentry.getModificationDate()); - prop.put("indexing-queue_list_"+entryCount+"_anchor", (pcentry.anchorName()==null)?"":htmlTools.replaceHTML(pcentry.anchorName())); - prop.put("indexing-queue_list_"+entryCount+"_url", htmlTools.replaceHTML(pcentry.normalizedURLString())); - prop.put("indexing-queue_list_"+entryCount+"_size", bytesToString(entrySize)); - prop.put("indexing-queue_list_"+entryCount+"_inProcess", (inProcess)?1:0); - prop.put("indexing-queue_list_"+entryCount+"_inProcess_hash", pcentry.urlHash()); - dark = !dark; - entryCount++; - } + boolean inProcess = i < inProcessCount; + pcentry = (plasmaSwitchboardQueue.Entry) entryList.get(i); + if ((pcentry != null)&&(pcentry.url() != null)) { + long entrySize = pcentry.size(); + totalSize += entrySize; + initiator = yacyCore.seedDB.getConnected(pcentry.initiator()); + prop.put("indexing-queue_list_"+entryCount+"_dark", (inProcess)? 2: ((dark) ? 1 : 0)); + prop.put("indexing-queue_list_"+entryCount+"_initiator", ((initiator == null) ? "proxy" : htmlTools.replaceHTML(initiator.getName()))); + prop.put("indexing-queue_list_"+entryCount+"_depth", pcentry.depth()); + prop.put("indexing-queue_list_"+entryCount+"_modified", pcentry.getModificationDate()); + prop.put("indexing-queue_list_"+entryCount+"_anchor", (pcentry.anchorName()==null)?"":htmlTools.replaceHTML(pcentry.anchorName())); + prop.put("indexing-queue_list_"+entryCount+"_url", htmlTools.replaceHTML(pcentry.normalizedURLString())); + prop.put("indexing-queue_list_"+entryCount+"_size", bytesToString(entrySize)); + prop.put("indexing-queue_list_"+entryCount+"_inProcess", (inProcess)?1:0); + prop.put("indexing-queue_list_"+entryCount+"_inProcess_hash", pcentry.urlHash()); + dark = !dark; + entryCount++; } - } catch (IOException e) {} + } prop.put("indexing-queue_show", entryCount);//show shown entries prop.put("indexing-queue_num", totalCount);//num entries in queue diff --git a/htroot/xml/queues_p.java b/htroot/xml/queues_p.java index ac5e5cd57..a6ef8b572 100644 --- a/htroot/xml/queues_p.java +++ b/htroot/xml/queues_p.java @@ -46,10 +46,11 @@ //package xml.queues; package xml; -import java.io.IOException; + import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; +import java.util.Iterator; import java.util.Locale; import de.anomic.data.htmlTools; @@ -98,44 +99,44 @@ public class queues_p { int inProcessCount = 0; long totalSize = 0; int i=0; //counter - try { - ArrayList entryList = new ArrayList(); - - // getting all entries that are currently in process - synchronized (switchboard.indexingTasksInProcess) { - inProcessCount = switchboard.indexingTasksInProcess.size(); - entryList.addAll(switchboard.indexingTasksInProcess.values()); - } - - // getting all enqueued entries - if ((switchboard.sbQueue.size() > 0)) { - entryList.addAll(switchboard.sbQueue.list(0)); - } - int size = (post == null) ? entryList.size() : post.getInt("num", entryList.size()); - if (size > entryList.size()) size = entryList.size(); - - int ok = 0; - for (i = 0; i < size; i++) { - boolean inProcess = i < inProcessCount; - pcentry = (plasmaSwitchboardQueue.Entry) entryList.get(i); - if ((pcentry != null)&&(pcentry.url() != null)) { - long entrySize = pcentry.size(); - totalSize += entrySize; - initiator = yacyCore.seedDB.getConnected(pcentry.initiator()); - prop.put("list-indexing_"+i+"_profile", (pcentry.profile() != null) ? pcentry.profile().name() : "deleted"); - prop.putSafeXML("list-indexing_"+i+"_initiator", ((initiator == null) ? "proxy" : htmlTools.replaceHTML(initiator.getName()))); - prop.put("list-indexing_"+i+"_depth", pcentry.depth()); - prop.put("list-indexing_"+i+"_modified", pcentry.getModificationDate()); - prop.putSafeXML("list-indexing_"+i+"_anchor", (pcentry.anchorName()==null)?"":htmlTools.replaceHTML(pcentry.anchorName())); - prop.putSafeXML("list-indexing_"+i+"_url", pcentry.normalizedURLString()); - prop.put("list-indexing_"+i+"_size", entrySize); - prop.put("list-indexing_"+i+"_inProcess", (inProcess)?1:0); - prop.put("list-indexing_"+i+"_hash", pcentry.urlHash()); - ok++; - } + ArrayList entryList = new ArrayList(); + + // getting all entries that are currently in process + synchronized (switchboard.indexingTasksInProcess) { + inProcessCount = switchboard.indexingTasksInProcess.size(); + entryList.addAll(switchboard.indexingTasksInProcess.values()); + } + + // getting all enqueued entries + if ((switchboard.sbQueue.size() > 0)) { + Iterator i1 = switchboard.sbQueue.entryIterator(false); + while (i1.hasNext()) entryList.add((plasmaSwitchboardQueue.Entry) i1.next()); + } + + int size = (post == null) ? entryList.size() : post.getInt("num", entryList.size()); + if (size > entryList.size()) size = entryList.size(); + + int ok = 0; + for (i = 0; i < size; i++) { + boolean inProcess = i < inProcessCount; + pcentry = (plasmaSwitchboardQueue.Entry) entryList.get(i); + if ((pcentry != null)&&(pcentry.url() != null)) { + long entrySize = pcentry.size(); + totalSize += entrySize; + initiator = yacyCore.seedDB.getConnected(pcentry.initiator()); + prop.put("list-indexing_"+i+"_profile", (pcentry.profile() != null) ? pcentry.profile().name() : "deleted"); + prop.putSafeXML("list-indexing_"+i+"_initiator", ((initiator == null) ? "proxy" : htmlTools.replaceHTML(initiator.getName()))); + prop.put("list-indexing_"+i+"_depth", pcentry.depth()); + prop.put("list-indexing_"+i+"_modified", pcentry.getModificationDate()); + prop.putSafeXML("list-indexing_"+i+"_anchor", (pcentry.anchorName()==null)?"":htmlTools.replaceHTML(pcentry.anchorName())); + prop.putSafeXML("list-indexing_"+i+"_url", pcentry.normalizedURLString()); + prop.put("list-indexing_"+i+"_size", entrySize); + prop.put("list-indexing_"+i+"_inProcess", (inProcess)?1:0); + prop.put("list-indexing_"+i+"_hash", pcentry.urlHash()); + ok++; } - prop.put("list-indexing", ok); - } catch (IOException e) {} + } + prop.put("list-indexing", ok); } //loader queue diff --git a/source/de/anomic/kelondro/kelondroStack.java b/source/de/anomic/kelondro/kelondroStack.java index 8a9f0c41d..cd2f91595 100644 --- a/source/de/anomic/kelondro/kelondroStack.java +++ b/source/de/anomic/kelondro/kelondroStack.java @@ -50,7 +50,6 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; import java.io.RandomAccessFile; -import java.util.ArrayList; import java.util.Iterator; import java.util.StringTokenizer; @@ -107,18 +106,18 @@ public final class kelondroStack extends kelondroRecords { return open(f, row); } - public Iterator iterator(boolean up) { + public Iterator stackIterator(boolean up) { // iterates the elements in an ordered way. - // returns kelondroRecords.Node - type Objects - return new nodeIterator(up); + // returns kelondroRow.Entry - type Objects + return new stackIterator(up); } - public class nodeIterator implements Iterator { + public class stackIterator implements Iterator { Handle nextHandle = null; Handle lastHandle = null; boolean up; - public nodeIterator(boolean up) { + public stackIterator(boolean up) { this.up = up; nextHandle = getHandle((up) ? root : toor); } @@ -131,7 +130,7 @@ public final class kelondroStack extends kelondroRecords { lastHandle = nextHandle; try { nextHandle = getNode(nextHandle, null, 0, false).getOHHandle((up) ? right : left); - return getNode(lastHandle, null, 0, true); + return row().newEntry(getNode(lastHandle, null, 0, true).getValueRow()); } catch (IOException e) { throw new kelondroException(filename, "IO error at Counter:next()"); } @@ -180,12 +179,7 @@ public final class kelondroStack extends kelondroRecords { public synchronized kelondroRow.Entry pop() throws IOException { // return row ontop of the stack and shrink stack by one - return pop(0); - } - - public synchronized kelondroRow.Entry pop(int dist) throws IOException { - // return row relative to top of the stack and remove addressed element - Node n = topNode(dist); + Node n = topNode(); if (n == null) return null; kelondroRow.Entry ret = row().newEntry(n.getValueRow()); @@ -198,25 +192,14 @@ public final class kelondroStack extends kelondroRecords { public synchronized kelondroRow.Entry top() throws IOException { // return row ontop of the stack - return top(0); - } - - public synchronized kelondroRow.Entry top(int dist) throws IOException { - // return row ontop of the stack - // with dist == 0 this is the same function as with top() - Node n = topNode(dist); + Node n = topNode(); if (n == null) return null; return row().newEntry(n.getValueRow()); } public synchronized kelondroRow.Entry pot() throws IOException { // return row on the bottom of the stack and remove record - return pot(0); - } - - public synchronized kelondroRow.Entry pot(int dist) throws IOException { - // return row relative to the bottom of the stack and remove addressed element - Node n = botNode(dist); + Node n = botNode(); if (n == null) return null; kelondroRow.Entry ret = row().newEntry(n.getValueRow()); @@ -226,28 +209,14 @@ public final class kelondroStack extends kelondroRecords { return ret; } - + public synchronized kelondroRow.Entry bot() throws IOException { // return row on the bottom of the stack - return bot(0); - } - - public synchronized kelondroRow.Entry bot(int dist) throws IOException { - // return row on bottom of the stack - // with dist == 0 this is the same function as with bot() - Node n = botNode(dist); + Node n = botNode(); if (n == null) return null; return row().newEntry(n.getValueRow()); } - public synchronized ArrayList botList(int dist) throws IOException { - ArrayList botList = new ArrayList(size()); - for (int i=dist; i < size(); i++) { - botList.add(bot(i)); - } - return botList; - } - private void unlinkNode(Node n) throws IOException { // join chaines over node Handle l = n.getOHHandle(left); @@ -276,26 +245,32 @@ public final class kelondroStack extends kelondroRecords { } } - private Node topNode(int dist) throws IOException { + private Node topNode() throws IOException { // return node ontop of the stack - return queueNode(dist, toor, left); + if (size() == 0) return null; + Handle h = getHandle(toor); + if (h == null) return null; + return getNode(h, true); } - private Node botNode(int dist) throws IOException { + private Node botNode() throws IOException { // return node on bottom of the stack - return queueNode(dist, root, right); + if (size() == 0) return null; + Handle h = getHandle(root); + if (h == null) return null; + return getNode(h, true); } - + /* private Node queueNode(int dist, int side, int dir) throws IOException { // with dist == 0 this is the same function as with // getNode(getHandle(side), null, 0) + if (dist >= size()) return null; // that would exceed the stack Handle h = getHandle(side); if (h == null) return null; - if (dist >= size()) return null; // that would exceed the stack while ((dist-- > 0) && (h != null)) h = getNode(h, false).getOHHandle(dir); // track through elements if (h == null) return null; else return getNode(h, true); } - +*/ public int imp(File file, String separator) throws IOException { // imports a value-separated file, returns number of records that have been read RandomAccessFile f = null; @@ -341,26 +316,16 @@ public final class kelondroStack extends kelondroRecords { public void print() throws IOException { super.print(false); - Node n; - try { - Iterator it = iterator(true); - kelondroRow.Entry r; - while (it.hasNext()) { - n = (Node) it.next(); - r = row().newEntry(n.getValueRow()); - // n = getNode(h, null, 0); - System.out.println("> NODE " + hp(n.handle()) + "; left " - + hp(n.getOHHandle(left)) + ", right " - + hp(n.getOHHandle(right))); - System.out.print(" KEY:'" + r.getColString(0, null) + "'"); - for (int j = 1; j < row().columns(); j++) - System.out.print(", V[" + j + "]:'" + r.getColString(j, null) + "'"); - System.out.println(); - } + Iterator it = stackIterator(true); + kelondroRow.Entry r; + while (it.hasNext()) { + r = (kelondroRow.Entry) it.next(); + System.out.print(" KEY:'" + r.getColString(0, null) + "'"); + for (int j = 1; j < row().columns(); j++) + System.out.print(", V[" + j + "]:'" + r.getColString(j, null) + "'"); System.out.println(); - } catch (IOException e) { - System.out.println("File error: " + e.getMessage()); } + System.out.println(); } private static void cmd(String[] args) { @@ -413,7 +378,7 @@ public final class kelondroStack extends kelondroRecords { } } else if (args[0].equals("-g")) { kelondroStack fm = new kelondroStack(new File(args[2]), lens); - kelondroRow.Entry ret2 = fm.pop(Integer.parseInt(args[1])); + kelondroRow.Entry ret2 = fm.pop(); ret = ((ret2 == null) ? null : ret2.getColBytes(1)); fm.close(); } diff --git a/source/de/anomic/plasma/plasmaCrawlBalancer.java b/source/de/anomic/plasma/plasmaCrawlBalancer.java index 27dade343..579eddecb 100644 --- a/source/de/anomic/plasma/plasmaCrawlBalancer.java +++ b/source/de/anomic/plasma/plasmaCrawlBalancer.java @@ -166,9 +166,9 @@ public class plasmaCrawlBalancer { // iterate through the file stack // in general this is a bad idea. But this can only be avoided by avoidance of this method - i = urlFileStack.iterator(true); + i = urlFileStack.stackIterator(true); while (i.hasNext()) { - h = new String(((kelondroRecords.Node) i.next()).getKey()); + h = new String(((kelondroRow.Entry) i.next()).getColBytes(0)); if (h.equals(urlhash)) { i.remove(); return new plasmaCrawlEntry(entry); diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index 456a8c012..91e72d63b 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -1167,19 +1167,17 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser // going through the sbQueue Entries and registering all content files as in use int count = 0; - try { - ArrayList sbQueueEntries = this.sbQueue.list(); - for (i = 0; i < sbQueueEntries.size(); i++) { - plasmaSwitchboardQueue.Entry entry = (plasmaSwitchboardQueue.Entry) sbQueueEntries.get(i); - if ((entry != null) && (entry.url() != null) && (entry.cacheFile().exists())) { - plasmaHTCache.filesInUse.add(entry.cacheFile()); - count++; - } + plasmaSwitchboardQueue.Entry queueEntry; + Iterator i1 = sbQueue.entryIterator(true); + while (i1.hasNext()) { + queueEntry = (plasmaSwitchboardQueue.Entry) i1.next(); + if ((queueEntry != null) && (queueEntry.url() != null) && (queueEntry.cacheFile().exists())) { + plasmaHTCache.filesInUse.add(queueEntry.cacheFile()); + count++; } - this.log.logConfig(count + " files in htcache reported to the cachemanager as in use."); - } catch (IOException e) { - this.log.logSevere("cannot find any files in htcache reported to the cachemanager: " + e.getMessage()); } + this.log.logConfig(count + " files in htcache reported to the cachemanager as in use."); + // define an extension-blacklist log.logConfig("Parser: Initializing Extension Mappings for Media/Parser"); plasmaParser.initMediaExt(plasmaParser.extString2extList(getConfig(PARSER_MEDIA_EXT,""))); diff --git a/source/de/anomic/plasma/plasmaSwitchboardQueue.java b/source/de/anomic/plasma/plasmaSwitchboardQueue.java index 80a6c5e58..2e4f863da 100644 --- a/source/de/anomic/plasma/plasmaSwitchboardQueue.java +++ b/source/de/anomic/plasma/plasmaSwitchboardQueue.java @@ -47,12 +47,11 @@ package de.anomic.plasma; import java.io.File; import java.io.IOException; import java.net.MalformedURLException; -import java.util.ArrayList; import java.util.Date; +import java.util.Iterator; import de.anomic.index.indexURLEntry; import de.anomic.kelondro.kelondroBase64Order; -import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroNaturalOrder; import de.anomic.kelondro.kelondroRow; import de.anomic.kelondro.kelondroStack; @@ -94,17 +93,18 @@ public class plasmaSwitchboardQueue { sbQueueStack = kelondroStack.open(sbQueueStackPath, rowdef); } + /* private void resetQueueStack() { try {sbQueueStack.close();} catch (Exception e) {} if (sbQueueStackPath.exists()) sbQueueStackPath.delete(); initQueueStack(); } - + */ public int size() { return sbQueueStack.size(); } - public void push(Entry entry) throws IOException { + public synchronized void push(Entry entry) throws IOException { sbQueueStack.push(sbQueueStack.row().newEntry(new byte[][]{ entry.url.toString().getBytes(), (entry.referrerHash == null) ? plasmaURL.dummyHash.getBytes() : entry.referrerHash.getBytes(), @@ -117,42 +117,26 @@ public class plasmaSwitchboardQueue { })); } - public Entry pop() throws IOException { + public synchronized Entry pop() throws IOException { if (sbQueueStack.size() == 0) return null; kelondroRow.Entry b = sbQueueStack.pot(); if (b == null) return null; return new Entry(b); } - public Entry remove(int index) throws IOException { - if (sbQueueStack.size() == 0) return null; - return new Entry(sbQueueStack.pot(index)); - } - - public Entry get(int index) throws IOException { - if ((index < 0) || (index >= sbQueueStack.size())) throw new ArrayIndexOutOfBoundsException(); - return new Entry(sbQueueStack.bot(index)); - } - - public ArrayList list() throws IOException { - return list(0); - } - - public ArrayList list(int index) throws IOException { - if ((index == 0) && (sbQueueStack.size() == 0)) return new ArrayList(0); - if ((index < 0) || (index >= sbQueueStack.size())) throw new ArrayIndexOutOfBoundsException(); - try { - ArrayList list = sbQueueStack.botList(index); - kelondroRow.Entry entry; - for (int i = 0; i < list.size(); i++) { - entry = (kelondroRow.Entry) list.get(i); - list.set(i, (entry == null) ? null : new Entry(entry)); + public synchronized Entry remove(String urlHash) throws IOException { + Iterator i = sbQueueStack.stackIterator(true); + kelondroRow.Entry rowentry; + Entry entry; + while (i.hasNext()) { + rowentry = (kelondroRow.Entry) i.next(); + entry = new Entry(rowentry); + if (entry.urlHash().equals(urlHash)) { + i.remove(); + return entry; } - return list; - } catch (kelondroException e) { - resetQueueStack(); - return new ArrayList(); } + return null; } public void clear() { @@ -175,6 +159,33 @@ public class plasmaSwitchboardQueue { super.finalize(); } + public Iterator entryIterator(boolean up) { + // iterates the elements in an ordered way. + // returns plasmaSwitchboardQueue.Entry - type Objects + return new entryIterator(up); + } + + public class entryIterator implements Iterator { + + Iterator rows; + + public entryIterator(boolean up) { + rows = sbQueueStack.stackIterator(up); + } + + public boolean hasNext() { + return rows.hasNext(); + } + + public Object next() { + return new Entry((kelondroRow.Entry) rows.next()); + } + + public void remove() { + rows.remove(); + } + } + public Entry newEntry(URL url, String referrer, Date ifModifiedSince, boolean requestWithCookie, String initiator, int depth, String profilehandle, String anchorName) { return new Entry(url, referrer, ifModifiedSince, requestWithCookie, initiator, depth, profilehandle, anchorName); diff --git a/source/de/anomic/yacy/yacyNewsPool.java b/source/de/anomic/yacy/yacyNewsPool.java index a2abc4383..65441c634 100644 --- a/source/de/anomic/yacy/yacyNewsPool.java +++ b/source/de/anomic/yacy/yacyNewsPool.java @@ -310,7 +310,7 @@ public class yacyNewsPool { yacyNewsRecord record = outgoingNews.topInc(); if ((record != null) && (record.distributed() >= maxDistribution)) { // move record to its final position. This is only for history - publishedNews.push(outgoingNews.pop(0)); + publishedNews.push(outgoingNews.pop()); } return record; } @@ -353,16 +353,16 @@ public class yacyNewsPool { yacyNewsRecord record; int pc = 0; synchronized (this.incomingNews) { - for (int i = this.incomingNews.size() - 1; i >= 0; i--) { + Iterator i = incomingNews.records(true); + while (i.hasNext()) { // check for interruption if (Thread.currentThread().isInterrupted()) throw new InterruptedException("Shutdown in progress"); // get next news record - record = this.incomingNews.top(i); - if ((i > 500) || (automaticProcessP(record))) { - this.incomingNews.pop(i); + record = (yacyNewsRecord) i.next(); + if (automaticProcessP(record)) { this.processedNews.push(record); - //newsDB.remove(id); + i.remove(); pc++; } } @@ -421,8 +421,9 @@ public class yacyNewsPool { public synchronized yacyNewsRecord getByOriginator(int dbKey, String category, String originatorHash) throws IOException { yacyNewsQueue queue = switchQueue(dbKey); yacyNewsRecord record; - for (int i = queue.size() - 1; i >= 0; i--) { - record = queue.top(i); + Iterator i = queue.records(true); + while (i.hasNext()) { + record = (yacyNewsRecord) i.next(); if ((record != null) && (record.category().equals(category)) && (record.originator().equals(originatorHash))) { diff --git a/source/de/anomic/yacy/yacyNewsQueue.java b/source/de/anomic/yacy/yacyNewsQueue.java index b35c29707..b9fabe4e6 100644 --- a/source/de/anomic/yacy/yacyNewsQueue.java +++ b/source/de/anomic/yacy/yacyNewsQueue.java @@ -51,7 +51,6 @@ import java.util.Iterator; import de.anomic.kelondro.kelondroColumn; import de.anomic.kelondro.kelondroNaturalOrder; -import de.anomic.kelondro.kelondroRecords; import de.anomic.kelondro.kelondroRow; import de.anomic.kelondro.kelondroStack; @@ -101,19 +100,19 @@ public class yacyNewsQueue { queueStack.push(r2b(entry, true)); } - public synchronized yacyNewsRecord pop(int dist) throws IOException { + public synchronized yacyNewsRecord pop() throws IOException { if (queueStack.size() == 0) return null; - return b2r(queueStack.pop(dist)); + return b2r(queueStack.pop()); } - public synchronized yacyNewsRecord top(int dist) throws IOException { + public synchronized yacyNewsRecord top() throws IOException { if (queueStack.size() == 0) return null; - return b2r(queueStack.top(dist)); + return b2r(queueStack.top()); } public synchronized yacyNewsRecord topInc() throws IOException { if (queueStack.size() == 0) return null; - yacyNewsRecord entry = pop(0); + yacyNewsRecord entry = pop(); if (entry != null) { entry.incDistribution(); push(entry); @@ -123,8 +122,9 @@ public class yacyNewsQueue { public synchronized yacyNewsRecord get(String id) throws IOException { yacyNewsRecord record; - for (int i = 0; i < size(); i++) { - record = top(i); + Iterator i = records(true); + while (i.hasNext()) { + record = (yacyNewsRecord) i.next(); if ((record != null) && (record.id().equals(id))) return record; } return null; @@ -132,10 +132,11 @@ public class yacyNewsQueue { public synchronized yacyNewsRecord remove(String id) throws IOException { yacyNewsRecord record; - for (int i = 0; i < size(); i++) { - record = top(i); + Iterator i = records(true); + while (i.hasNext()) { + record = (yacyNewsRecord) i.next(); if ((record != null) && (record.id().equals(id))) { - pop(i); + i.remove(); return record; } } @@ -174,7 +175,7 @@ public class yacyNewsQueue { Iterator stackNodeIterator; public newsIterator(boolean up) { - stackNodeIterator = queueStack.iterator(up); + stackNodeIterator = queueStack.stackIterator(up); } public boolean hasNext() { @@ -182,10 +183,9 @@ public class yacyNewsQueue { } public Object next() { - kelondroRecords.Node n = (kelondroRecords.Node) stackNodeIterator.next(); + kelondroRow.Entry row = (kelondroRow.Entry) stackNodeIterator.next(); try { - kelondroRow.Entry entry = queueStack.row().newEntry(n.getValueRow()); - return b2r(entry); + return b2r(row); } catch (IOException e) { return null; }