*) adding method to implement blacklist from file

- file transfer is done via soap attachments (see BlaclistSerivceTest for details) git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2855 6c8d7289-2bf4-0310-a012-ef5d649a1542
19 years ago · 9e8942a064
parent 4d1f933ea1
commit 9e8942a064
3 changed files with 155 additions and 10 deletions
--- a/source/de/anomic/soap/services/BlacklistService.java
+++ b/source/de/anomic/soap/services/BlacklistService.java
@ -1,14 +1,23 @@
 package de.anomic.soap.services;

+import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.io.PrintWriter;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashSet;

+import javax.activation.DataHandler;
+import javax.xml.soap.SOAPException;
+
 import org.apache.axis.AxisFault;
+import org.apache.axis.Message;
+import org.apache.axis.MessageContext;
+import org.apache.axis.attachments.AttachmentPart;
+import org.apache.axis.attachments.Attachments;
 import org.w3c.dom.Document;

 import de.anomic.data.listManager;
@ -62,7 +71,7 @@ public class BlacklistService extends AbstractService {
 		// extracting the message context
 		extractMessageContext(AUTHENTICATION_NEEDED);
 		
-        // initialize the list manager
+		// initialize the list manager
 		initBlacklistManager();
 		
 		// check if the blacklist already exists
@ -243,10 +252,84 @@ public class BlacklistService extends AbstractService {
 		removeBlacklistItemFromBlacklist(blacklistItem,blacklistName);
    }
    
+    public void importBlacklist(String blacklistName) throws IOException, SOAPException {
+    	// Check for errors
+    	if ((blacklistName == null)||(blacklistName.length() == 0)) 
+    		throw new IllegalArgumentException("Blacklist name must not be null or empty.");   
+    	
+		// extracting the message context
+		extractMessageContext(AUTHENTICATION_NEEDED);    
+		
+        // initialize the list manager
+		initBlacklistManager();		    	
+		
+		// check if the blacklist file exists
+		if (!blacklistExists(blacklistName)) {
+			// create blacklist
+			createBlacklistFile(blacklistName);
+		}
+		
+		// get attachment
+        MessageContext msgContext = MessageContext.getCurrentContext();
+
+        // getting the request message
+        Message reqMsg = msgContext.getRequestMessage();
+        
+        // getting the attachment implementation
+        Attachments messageAttachments = reqMsg.getAttachmentsImpl();
+        if (messageAttachments == null) {
+            throw new AxisFault("Attachments not supported");
+        }		
+        
+        int attachmentCount= messageAttachments.getAttachmentCount();
+        if (attachmentCount == 0) 
+            throw new AxisFault("No attachment found");
+        else if (attachmentCount != 1)
+            throw new AxisFault("Too many attachments as expected.");     
+        
+        // getting the attachments
+        AttachmentPart[] attachments = (AttachmentPart[])messageAttachments.getAttachments().toArray(new AttachmentPart[attachmentCount]);
+        
+        // getting the content of the attachment
+        DataHandler dh = attachments[0].getDataHandler();
+        
+        PrintWriter writer = null;
+        BufferedReader reader = null;
+        try {
+        	// getting a reader
+        	reader = new BufferedReader(new InputStreamReader(dh.getInputStream(),"UTF-8"));
+        	
+        	// getting blacklist file writer
+        	writer = getBlacklistFileWriter(blacklistName);
+        	
+        	// read new item
+        	String blacklistItem = null;
+        	while ((blacklistItem = reader.readLine()) != null) {
+        		// convert it into a proper format
+        		blacklistItem = prepareBlacklistItem(blacklistItem);
+        		
+        		// TODO: check if the item already exits
+        		
+        		// write item to blacklist file
+        		writer.println(blacklistItem);
+        		writer.flush();
+        		
+        		// inform blacklist engine about new item
+        		addBlacklistItemToBlacklist(blacklistItem, blacklistName);
+        	}
+        } finally {
+        	if (reader != null) try { reader.close(); } catch (Exception e) {/* */}
+        	if (writer != null) try { writer.close(); } catch (Exception e) {/* */}
+        }
+    }
+    
    public String[] getBlacklistTypes() throws AxisFault {
 		// extracting the message context
 		extractMessageContext(AUTHENTICATION_NEEDED);
 		
+		// initialize the list manager
+		initBlacklistManager();		
+		
 		// return supported types
 		return getSupportedBlacklistTypeArray();
    }
@ -271,8 +354,9 @@ public class BlacklistService extends AbstractService {
    private void addBlacklistItemToFile(String blacklistItem, String blacklistName) throws AxisFault {
        PrintWriter pw = null;
        try {
-            pw = new PrintWriter(new FileWriter(getBlacklistFile(blacklistName), true));
+            pw = getBlacklistFileWriter(blacklistName);
            pw.println(blacklistItem);
+            pw.flush();
            pw.close();
        } catch (IOException e) {
            throw new AxisFault("Unable to append blacklist entry.",e);
@ -281,6 +365,14 @@ public class BlacklistService extends AbstractService {
        }            	
    }
    
+    private PrintWriter getBlacklistFileWriter(String blacklistName) throws AxisFault {
+    	try {
+    	return new PrintWriter(new FileWriter(getBlacklistFile(blacklistName), true));
+    	} catch (IOException e) {
+    		throw new AxisFault("Unable to open blacklist file.",e);
+    	}
+    }
+    
    private void removeBlacklistItemFromBlacklistFile(String blacklistItem, String blacklistName) {
        // load blacklist data from file
        ArrayList list = listManager.getListArray(getBlacklistFile(blacklistName));
--- a/source/de/anomic/soap/services/blacklist.wsdl
+++ b/source/de/anomic/soap/services/blacklist.wsdl
@ -21,94 +21,104 @@ Built on Nov 16, 2004 (12:19:44 EST)--><wsdl:types><schema targetNamespace="http
    <wsdl:part name="getBlacklistTypesReturn" type="impl:ArrayOf_soapenc_string"/>

  </wsdl:message>
+  <wsdl:message name="importBlacklistRequest">
+    <wsdl:part name="blacklistName" type="soapenc:string"/>
+  </wsdl:message>
  <wsdl:message name="getBlacklistTypesRequest">
  </wsdl:message>
  <wsdl:message name="removeBlacklistItemRequest">
    <wsdl:part name="blacklistName" type="soapenc:string"/>
    <wsdl:part name="blacklistItem" type="soapenc:string"/>
+
  </wsdl:message>
  <wsdl:message name="addBlacklistItemResponse">
  </wsdl:message>
-
  <wsdl:message name="shareBlacklistResponse">
  </wsdl:message>
  <wsdl:message name="getBlacklistListResponse">
    <wsdl:part name="getBlacklistListReturn" type="apachesoap:Document"/>
  </wsdl:message>
  <wsdl:message name="createBlacklistResponse">
+
  </wsdl:message>
  <wsdl:message name="getBlacklistListRequest">
  </wsdl:message>
-
  <wsdl:message name="unshareBlacklistRequest">
    <wsdl:part name="blacklistName" type="soapenc:string"/>
  </wsdl:message>
  <wsdl:message name="shareBlacklistRequest">
    <wsdl:part name="blacklistName" type="soapenc:string"/>
  </wsdl:message>
+
  <wsdl:message name="deleteBlacklistResponse">
  </wsdl:message>
  <wsdl:message name="activateBlacklistRequest">
-
    <wsdl:part name="blacklistName" type="soapenc:string"/>
    <wsdl:part name="activateForBlacklistTypes" type="impl:ArrayOf_soapenc_string"/>
  </wsdl:message>
  <wsdl:message name="createBlacklistRequest">
    <wsdl:part name="blacklistName" type="soapenc:string"/>
    <wsdl:part name="shareBlacklist" type="xsd:boolean"/>
+
    <wsdl:part name="activateForBlacklistTypes" type="impl:ArrayOf_soapenc_string"/>
  </wsdl:message>
+  <wsdl:message name="importBlacklistResponse">
+  </wsdl:message>
  <wsdl:message name="deleteBlacklistRequest">
-
    <wsdl:part name="blacklistName" type="soapenc:string"/>
  </wsdl:message>
  <wsdl:message name="deactivateBlacklistRequest">
    <wsdl:part name="blacklistName" type="soapenc:string"/>
+
    <wsdl:part name="deactivateForBlacklistTypes" type="impl:ArrayOf_soapenc_string"/>
  </wsdl:message>
  <wsdl:message name="createNewXMLDocumentResponse">
    <wsdl:part name="createNewXMLDocumentReturn" type="apachesoap:Document"/>
  </wsdl:message>
-
  <wsdl:portType name="BlacklistService">
    <wsdl:operation name="getBlacklistList">
      <wsdl:input message="impl:getBlacklistListRequest" name="getBlacklistListRequest"/>
      <wsdl:output message="impl:getBlacklistListResponse" name="getBlacklistListResponse"/>
+
    </wsdl:operation>
    <wsdl:operation name="createBlacklist" parameterOrder="blacklistName shareBlacklist activateForBlacklistTypes">
      <wsdl:input message="impl:createBlacklistRequest" name="createBlacklistRequest"/>
      <wsdl:output message="impl:createBlacklistResponse" name="createBlacklistResponse"/>
    </wsdl:operation>
-
    <wsdl:operation name="shareBlacklist" parameterOrder="blacklistName">
      <wsdl:input message="impl:shareBlacklistRequest" name="shareBlacklistRequest"/>
      <wsdl:output message="impl:shareBlacklistResponse" name="shareBlacklistResponse"/>
    </wsdl:operation>
+
    <wsdl:operation name="deleteBlacklist" parameterOrder="blacklistName">
      <wsdl:input message="impl:deleteBlacklistRequest" name="deleteBlacklistRequest"/>
      <wsdl:output message="impl:deleteBlacklistResponse" name="deleteBlacklistResponse"/>
    </wsdl:operation>
    <wsdl:operation name="unshareBlacklist" parameterOrder="blacklistName">
-
      <wsdl:input message="impl:unshareBlacklistRequest" name="unshareBlacklistRequest"/>
      <wsdl:output message="impl:unshareBlacklistResponse" name="unshareBlacklistResponse"/>
    </wsdl:operation>
    <wsdl:operation name="activateBlacklist" parameterOrder="blacklistName activateForBlacklistTypes">
+
      <wsdl:input message="impl:activateBlacklistRequest" name="activateBlacklistRequest"/>
      <wsdl:output message="impl:activateBlacklistResponse" name="activateBlacklistResponse"/>
    </wsdl:operation>
    <wsdl:operation name="deactivateBlacklist" parameterOrder="blacklistName deactivateForBlacklistTypes">
      <wsdl:input message="impl:deactivateBlacklistRequest" name="deactivateBlacklistRequest"/>
-
      <wsdl:output message="impl:deactivateBlacklistResponse" name="deactivateBlacklistResponse"/>
    </wsdl:operation>
    <wsdl:operation name="addBlacklistItem" parameterOrder="blacklistName blacklistItem">
      <wsdl:input message="impl:addBlacklistItemRequest" name="addBlacklistItemRequest"/>
+
      <wsdl:output message="impl:addBlacklistItemResponse" name="addBlacklistItemResponse"/>
    </wsdl:operation>
    <wsdl:operation name="removeBlacklistItem" parameterOrder="blacklistName blacklistItem">
      <wsdl:input message="impl:removeBlacklistItemRequest" name="removeBlacklistItemRequest"/>
      <wsdl:output message="impl:removeBlacklistItemResponse" name="removeBlacklistItemResponse"/>
+    </wsdl:operation>
+    <wsdl:operation name="importBlacklist" parameterOrder="blacklistName">
+      <wsdl:input message="impl:importBlacklistRequest" name="importBlacklistRequest"/>
+      <wsdl:output message="impl:importBlacklistResponse" name="importBlacklistResponse"/>

    </wsdl:operation>
    <wsdl:operation name="getBlacklistTypes">
@ -213,6 +223,16 @@ Built on Nov 16, 2004 (12:19:44 EST)--><wsdl:types><schema targetNamespace="http
        <wsdlsoap:body encodingStyle="http://schemas.xmlsoap.org/soap/encoding/" namespace="http://yacy:8080/soap/blacklist" use="encoded"/>
      </wsdl:output>
    </wsdl:operation>
+    <wsdl:operation name="importBlacklist">
+      <wsdlsoap:operation soapAction=""/>
+      <wsdl:input name="importBlacklistRequest">
+        <wsdlsoap:body encodingStyle="http://schemas.xmlsoap.org/soap/encoding/" namespace="http://services.soap.anomic.de" use="encoded"/>
+      </wsdl:input>
+      <wsdl:output name="importBlacklistResponse">
+
+        <wsdlsoap:body encodingStyle="http://schemas.xmlsoap.org/soap/encoding/" namespace="http://yacy:8080/soap/blacklist" use="encoded"/>
+      </wsdl:output>
+    </wsdl:operation>
    <wsdl:operation name="getBlacklistTypes">
      <wsdlsoap:operation soapAction=""/>
      <wsdl:input name="getBlacklistTypesRequest">
--- a/test/de/anomic/soap/services/BlacklistServiceTest.java
+++ b/test/de/anomic/soap/services/BlacklistServiceTest.java
@ -1,9 +1,17 @@
 package de.anomic.soap.services;

+import java.io.IOException;
 import java.rmi.RemoteException;

+import javax.activation.DataHandler;
+import javax.activation.DataSource;
 import javax.xml.rpc.ServiceException;

+import org.apache.axis.attachments.AttachmentPart;
+import org.apache.axis.attachments.Attachments;
+import org.apache.axis.attachments.PlainTextDataSource;
+import org.apache.axis.client.Call;
+import org.apache.axis.client.Stub;
 import org.apache.axis.utils.XMLUtils;
 import org.w3c.dom.Document;

@ -61,5 +69,30 @@ public class BlacklistServiceTest extends AbstractServiceTest {
 		// delete blacklist
 		bl.deleteBlacklist(blacklistName);
 	}
+	
+	public void testBacklistImport() throws IOException {
+		BlacklistService bl = ((BlacklistService)service);
+		
+		// create datasource to hold the attachment content
+        DataSource data =  new PlainTextDataSource("import.txt","www.yacy.net/.*\r\n" +
+        														"www.yacy-websuche.de/.*");
+        DataHandler attachmentFile = new DataHandler(data);   
+        
+        // creating attachment part
+        AttachmentPart part = new AttachmentPart();
+        part.setDataHandler(attachmentFile);
+        part.setContentType("text/plain");
+
+        // setting the attachment format that should be used
+        ((Stub)service)._setProperty(org.apache.axis.client.Call.ATTACHMENT_ENCAPSULATION_FORMAT,org.apache.axis.client.Call.ATTACHMENT_ENCAPSULATION_FORMAT_MIME);
+        ((Stub)service).addAttachment(part);               
+        
+        // import it
+        String blacklistName = "junit_test_" + System.currentTimeMillis();
+        bl.importBlacklist(blacklistName);
+        
+        // delete blacklist
+        bl.deleteBlacklist(blacklistName);
+	}

 }