fixed news receipt and added processing buttons on News page

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@458 6c8d7289-2bf4-0310-a012-ef5d649a1542

build.properties

@@ -3,7 +3,7 @@ javacSource=1.4
 javacTarget=1.4
 
 # Release Configuration
-releaseVersion=0.392
+releaseVersion=0.393
 releaseFile=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
 #releaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
 releaseDir=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}

htroot/EditProfile_p.java

@@ -96,7 +96,7 @@ public class EditProfile_p {
                 
                 // generate a news message
                 //HashMap map = new HashMap();
-                yacyCore.newsPool.enqueueMyNews(new yacyNewsRecord("updprfle", profile));
+                yacyCore.newsPool.publishMyNews(new yacyNewsRecord("updprfle", profile));
             }catch(IOException e){
             } finally {
                 if (fileOut != null) try { fileOut.close(); } catch (Exception e) {}

htroot/News.html

@@ -40,8 +40,12 @@ This is the news system (currently under testing).
 </p>
 ::
 <p>
+<form action="News.html" method="post" enctype="multipart/form-data">
+<input type="hidden" name="page" value="#[page]#">
+<input type="submit" name="delete" value="#(page)#::Process Selected News::Delete Selected News::Abort Publication of Selected News::Delete Selected News#(/page)#"><br><br>
 <table border="0" cellpadding="2" cellspacing="1">
 <tr class="TableHeader" valign="bottom">
+<td class="small"></td>
 <td class="small">Originator</td>
 <td class="small">Created</td>
 <td class="small">Category</td>
@@ -51,6 +55,7 @@ This is the news system (currently under testing).
 </tr>
 #{list}#
 <tr class="TableCell#(dark)#Light::Dark::Summary#(/dark)#">
+<td class="small" align="left"><input type="checkbox" name="del_#[id]#"></td>
 <td class="small" align="left">#[ori]#</td>
 <td class="small" align="left">#[cre]#</td>
 <td class="small" align="left">#[cat]#</td>
@@ -60,6 +65,7 @@ This is the news system (currently under testing).
 </tr>
 #{/list}#
 </table>
+</form>
 </p>
 #(/table)#
 #[footer]#

htroot/News.java

@@ -54,23 +54,50 @@ import de.anomic.server.serverDate;
 import de.anomic.yacy.yacyCore;
 import de.anomic.yacy.yacySeed;
 import de.anomic.yacy.yacyNewsRecord;
+import de.anomic.plasma.plasmaSwitchboard;
 
 public class News {
     
-    public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch sb) {
+    public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
-        // return variable that accumulates replacements
+        plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
         serverObjects prop = new serverObjects();
         boolean overview = (post == null) || (((String) post.get("page", "0")).equals("0"));
+        int tableID = (overview) ? -1 : Integer.parseInt((String) post.get("page", "0")) - 1;
+
+        // execute commands
+        if (post != null) {
+            if (switchboard.adminAuthenticated(header) < 2) {
+                // not authenticated, force log-in
+                prop.put("AUTHENTICATE", "admin log-in");
+                return prop;
+            }
+            
+            if ((post.containsKey("delete")) && (tableID >= 0)) {
+            Enumeration e = post.keys();
+            String check;
+            String id;
+            while (e.hasMoreElements()) {
+                check = (String) e.nextElement();
+                if ((check.startsWith("del_")) && (post.get(check, "off").equals("on"))) {
+                    id = check.substring(4);
+                    try {
+                        yacyCore.newsPool.moveOff(tableID, id);
+                    } catch (IOException ee) {ee.printStackTrace();}
+                }
+            }
+            }
+        }
         
+        // generate properties for output
         if (overview) {
             // show overview
             prop.put("table", 0);
             prop.put("page", 0);
         } else {
             // generate table
-            int tableID = Integer.parseInt((String) post.get("page", "1")) - 1;
             prop.put("table", 1);
             prop.put("page", tableID + 1);
+            prop.put("table_page", tableID + 1);
             
             if (yacyCore.seedDB == null) {
                 
@@ -84,6 +111,7 @@ public class News {
                     record = yacyCore.newsPool.get(tableID, i);
                     seed = yacyCore.seedDB.getConnected(record.originator());
                     if (seed == null) seed = yacyCore.seedDB.getDisconnected(record.originator());
+                    prop.put("table_list_" + i + "_id", record.id());
                     prop.put("table_list_" + i + "_ori", (seed == null) ? record.originator() : seed.getName());
                     prop.put("table_list_" + i + "_cre", yacyCore.universalDateShortString(record.created()));
                     prop.put("table_list_" + i + "_cat", record.category());

source/de/anomic/http/httpdFileHandler.java

@@ -113,10 +113,7 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
     private String[] defaultFiles = null;
     private File htDefaultPath = null;
     private File htLocalePath = null;
-    
     private serverSwitch switchboard;
-    private String adminAccountBase64MD5;
-    
     private MessageDigest md5Digest = null;
     
     public httpdFileHandler(serverSwitch switchboard) {
@@ -179,7 +176,6 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
              for (int i = 0; i < ps.length; i++) System.out.println("PACKAGE IN PROVIDER: " + ps[i].toString());
              */
         }
-        adminAccountBase64MD5 = null;
         
         // initialise an message digest for Content-MD5 support ...
         try {
@@ -247,17 +243,17 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
         }
         
         // check permission/granted access
-        if ((path.endsWith("_p.html")) &&
+        String authorization = (String) requestHeader.get(httpHeader.AUTHORIZATION);
-                ((adminAccountBase64MD5 = switchboard.getConfig("adminAccountBase64MD5", "")).length() != 0)) {
+        String adminAccountBase64MD5 = switchboard.getConfig("adminAccountBase64MD5", "");
+        if ((path.endsWith("_p.html")) && (adminAccountBase64MD5.length() != 0)) {
             // authentication required
-            String auth = (String) requestHeader.get(httpHeader.AUTHORIZATION);
+            if (authorization == null) {
-            if (auth == null) {
                 // no authorization given in response. Ask for that
                 httpHeader headers = getDefaultHeaders();
                 headers.put(httpHeader.WWW_AUTHENTICATE,"Basic realm=\"admin log-in\"");
                 httpd.sendRespondHeader(conProp,out,httpVersion,401,headers);
                 return;
-            } else if (adminAccountBase64MD5.equals(serverCodings.standardCoder.encodeMD5Hex(auth.trim().substring(6)))) {
+            } else if (adminAccountBase64MD5.equals(serverCodings.standardCoder.encodeMD5Hex(authorization.trim().substring(6)))) {
                 // remove brute-force flag
                 serverCore.bfHost.remove(conProp.getProperty("CLIENTIP"));
             } else {
@@ -277,6 +273,14 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
             }
         }
         
+        // handle bfHost in case we have authentified correctly
+        if ((authorization != null) &&
+            (adminAccountBase64MD5.length() != 0) &&
+            (adminAccountBase64MD5.equals(serverCodings.standardCoder.encodeMD5Hex(authorization.trim().substring(6))))) {
+            // remove brute-force flag
+            serverCore.bfHost.remove(conProp.getProperty("CLIENTIP"));
+        }
+        
         // parse arguments
         serverObjects args = new serverObjects();
         int argc;
@@ -409,6 +413,17 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
                             if (tp == null) tp = new serverObjects();
                             // check if the servlets requests authentification
                             if (tp.containsKey("AUTHENTICATE")) {
+                                // handle brute-force protection
+                                if (authorization != null) {
+                                    String clientIP = conProp.getProperty("CLIENTIP", "unknown-host");
+                                    serverLog.logInfo("HTTPD", "dynamic log-in for account 'admin' in http file handler for path '" + path + "' from host '" + clientIP + "'");
+                                    Integer attempts = (Integer) serverCore.bfHost.get(clientIP);
+                                    if (attempts == null)
+                                        serverCore.bfHost.put(clientIP, new Integer(1));
+                                    else
+                                        serverCore.bfHost.put(clientIP, new Integer(attempts.intValue() + 1));
+                                }
+                                // send authentication request to browser
                                 httpHeader headers = getDefaultHeaders();
                                 headers.put(httpHeader.WWW_AUTHENTICATE,"Basic realm=\"" + tp.get("AUTHENTICATE", "") + "\"");
                                 httpd.sendRespondHeader(conProp,out,httpVersion,401,headers);

source/de/anomic/yacy/yacyCore.java

@@ -389,8 +389,11 @@ public class yacyCore {
             
             // include a YaCyNews record to my seed
             try {
-                yacyNewsRecord record = newsPool.dequeueMyNews();
+                yacyNewsRecord record = newsPool.myPublication();
-                if (record != null) seedDB.mySeed.put("news", de.anomic.tools.crypt.simpleEncode(record.toString()));
+                if (record == null) 
+                    seedDB.mySeed.put("news", "");
+                else
+                    seedDB.mySeed.put("news", de.anomic.tools.crypt.simpleEncode(record.toString()));
             } catch (IOException e) {
                 e.printStackTrace();
             }

source/de/anomic/yacy/yacyNewsAction.java

@@ -53,12 +53,12 @@ public class yacyNewsAction implements yacyPeerAction {
     
     public void processPeerArrival(yacySeed peer, boolean direct) {
         String recordString = peer.get("news", null);
-        System.out.println("### triggered news arrival from peer " + peer.getName() + ", news " + ((recordString == null) ? "empty" : "attached"));
+        //System.out.println("### triggered news arrival from peer " + peer.getName() + ", news " + ((recordString == null) ? "empty" : "attached"));
-        if (recordString == null) return;
+        if ((recordString == null) || (recordString.length() == 0)) return;
         yacyNewsRecord record = new yacyNewsRecord(de.anomic.tools.crypt.simpleDecode(recordString, ""));
-        System.out.println("### news arrival from peer " + peer.getName() + ", news=" + record.toString());
+        //System.out.println("### news arrival from peer " + peer.getName() + ", news=" + record.toString());
         try {
-            this.pool.enqueueGlobalNews(record);
+            this.pool.enqueueIncomingNews(record);
         } catch (IOException e) {e.printStackTrace();}
     }
     

source/de/anomic/yacy/yacyNewsDB.java

@@ -101,6 +101,10 @@ public class yacyNewsDB {
     public int size() {
         return news.size();
     }
+    
+    public void remove(String id) throws IOException {
+        news.remove(id.getBytes());
+    }
 
     public synchronized yacyNewsRecord put(yacyNewsRecord record) throws IOException {
         try {

source/de/anomic/yacy/yacyNewsPool.java

@@ -64,11 +64,12 @@ public class yacyNewsPool {
         maxDistribution = 30;
     }
     
-    public void enqueueMyNews(yacyNewsRecord record) throws IOException {
+    public void publishMyNews(yacyNewsRecord record) throws IOException {
+        // this shall be called if our peer generated a new news record and wants to publish it
         if (newsDB.get(record.id()) == null) outgoingNews.push(record);
     }
     
-    public yacyNewsRecord dequeueMyNews() throws IOException {
+    public yacyNewsRecord myPublication() throws IOException {
         // generate a record for next peer-ping
         if (outgoingNews.size() == 0) return null;
         yacyNewsRecord record = outgoingNews.topInc();
@@ -79,15 +80,13 @@ public class yacyNewsPool {
         return record;
     }
     
-    public void enqueueGlobalNews(yacyNewsRecord record) throws IOException {
+    public void enqueueIncomingNews(yacyNewsRecord record) throws IOException {
+        // called if a news is attached to a seed
         if (newsDB.get(record.id()) == null) incomingNews.push(record);
     }
     
-    public yacyNewsRecord getGlobalNews(int job) throws IOException {
+    public synchronized boolean commitIncomingNews(String id) throws IOException {
-        return incomingNews.top(job);
+        // called if a incoming news was processed
-    }
-    
-    public synchronized boolean removeGlobalNews(String id) throws IOException {
         yacyNewsRecord record;
         for (int i = 0; i < incomingNews.size(); i++) {
             record = incomingNews.top(i);
@@ -100,24 +99,79 @@ public class yacyNewsPool {
         return false;
     }
     
+    public synchronized boolean deleteProcessedNews(String id) throws IOException {
+        // called if a processed news shall be removed
+        // the news stays in the news database to prevent
+        // that it is loaded again from the net
+        yacyNewsRecord record;
+        for (int i = 0; i < processedNews.size(); i++) {
+            record = processedNews.top(i);
+            if (record.id().equals(id)) {
+                processedNews.pop(i);
+                return true;
+            }
+        }
+        return false;
+    }
+    
+    public synchronized boolean interruptPublication(String id) throws IOException {
+        // called if a outgoing news shall not be published any more
+        yacyNewsRecord record;
+        for (int i = 0; i < outgoingNews.size(); i++) {
+            record = outgoingNews.top(i);
+            if (record.id().equals(id)) {
+                outgoingNews.pop(i);
+                publishedNews.push(record);
+                return true;
+            }
+        }
+        return false;
+    }
+    
+    public synchronized boolean deletePublishedNews(String id) throws IOException {
+        // called if a published news shall be removed
+        // the news is also removed from the news database
+        yacyNewsRecord record;
+        for (int i = 0; i < publishedNews.size(); i++) {
+            record = publishedNews.top(i);
+            if (record.id().equals(id)) {
+                publishedNews.pop(i);
+                newsDB.remove(id);
+                return true;
+            }
+        }
+        return false;
+    }
+    
     public int size(int dbKey) {
         switch (dbKey) {
-            case OUTGOING_DB:   return outgoingNews.size();
-            case PUBLISHED_DB:  return publishedNews.size();
             case INCOMING_DB:	return incomingNews.size();
             case PROCESSED_DB:  return processedNews.size();
+            case OUTGOING_DB:   return outgoingNews.size();
+            case PUBLISHED_DB:  return publishedNews.size();
             default: return -1;
         }
     }
     
     public yacyNewsRecord get(int dbKey, int element) throws IOException {
         switch (dbKey) {
-            case OUTGOING_DB:   return outgoingNews.top(element);
-            case PUBLISHED_DB:  return publishedNews.top(element);
             case INCOMING_DB:	return incomingNews.top(element);
             case PROCESSED_DB:  return processedNews.top(element);
+            case OUTGOING_DB:   return outgoingNews.top(element);
+            case PUBLISHED_DB:  return publishedNews.top(element);
             default: return null;
         }
     }
     
+    public void moveOff(int dbKey, String id) throws IOException {
+        // this is called if a queue element shall be moved to another queue or off the queue
+        // it depends on the dbKey how the record is handled
+        switch (dbKey) {
+            case INCOMING_DB:	commitIncomingNews(id); break;
+            case PROCESSED_DB:  deleteProcessedNews(id); break;
+            case OUTGOING_DB:   interruptPublication(id); break;
+            case PUBLISHED_DB:  deletePublishedNews(id); break;
+        }
+    }
+    
 }

source/de/anomic/yacy/yacyNewsRecord.java

@@ -58,9 +58,8 @@ public class yacyNewsRecord {
     private int    distributed; // counter that counts number of distributions of this news record
     private Map    attributes;  // elemets of the news for a special category
     
-    public yacyNewsRecord(String encodedNewsString) {
+    public yacyNewsRecord(String newsString) {
-        String decodedString = serverCodings.enhancedCoder.decodeBase64String(encodedNewsString);
+        this.attributes = serverCodings.string2map(newsString);
-        this.attributes = serverCodings.string2map(decodedString);
         this.received = (attributes.containsKey("rec")) ? yacyCore.parseUniversalDate((String) attributes.get("rec")) : new Date();
         this.created = (attributes.containsKey("cre")) ? yacyCore.parseUniversalDate((String) attributes.get("cre")) : new Date();
         this.category = (attributes.containsKey("cat")) ? (String) attributes.get("cat") : null;
@@ -101,11 +100,11 @@ public class yacyNewsRecord {
     public String toString() {
         // this creates the string that shall be distributed
         // attention: this has no additional encoding
-        if (originator != null) attributes.put("ori", originator);
+        if (this.originator != null) attributes.put("ori", this.originator);
-        if (category != null) attributes.put("cat", category);
+        if (this.category != null) attributes.put("cat", this.category);
-        attributes.put("cre", yacyCore.universalDateShortString(created));
+        attributes.put("cre", yacyCore.universalDateShortString(this.created));
-        attributes.put("rec", yacyCore.universalDateShortString(received));
+        attributes.put("rec", yacyCore.universalDateShortString(this.received));
-        attributes.put("dis", "" + distributed);
+        attributes.put("dis", "" +this. distributed);
         String theString = attributes.toString();
         removeStandards();
         return theString;

source/de/anomic/yacy/yacyPeerActions.java

@@ -323,7 +323,7 @@ public class yacyPeerActions {
 		// this is a return of a lost peer
 		yacyCore.log.logDebug("connect: returned KNOWN " + peerType + " peer '" + seed.getName() + "' from " + seed.getAddress());
 		seedDB.addConnected(seed);
-		return false;
+		return true;
 	    } else {
                 yacySeed connectedSeed = seedDB.getConnected(seed.hash);
                 if (connectedSeed != null) {
@@ -337,7 +337,7 @@ public class yacyPeerActions {
                     } catch (java.text.ParseException e) {}
                     yacyCore.log.logDebug("connect: updated KNOWN " + ((direct) ? "direct " : "") +  peerType + " peer '" + seed.getName() + "' from " + seed.getAddress());
 		    seedDB.addConnected(seed);
-                    return false;
+                    return true;
                 } else {
                     // the seed is new
                     if (((String) seed.get("IP", "127.0.0.1")).equals((String) seedDB.mySeed.get("IP", "127.0.0.1"))) {

yacy.init

@@ -482,7 +482,7 @@ maxWaitingWordFlush = 180
 isTransparentProxy=false
 
 # Specifies if yacy should use the http connection keep-alive feature
-connectionKeepAliveSupport=false
+connectionKeepAliveSupport=true
 
 # Configuration options needed to configure server port forwarding
 portForwardingEnabled=false