Fixed read/copy on input streams reading sometimes less than expected.

source/net/yacy/kelondro/util/FileUtils.java

@@ -110,10 +110,17 @@ public final class FileUtils {
 
         int c;
         long total = 0;
-        while ( (c = source.read(buffer, 0, chunkSize)) > 0 ) {
+        long remaining;
+        if(count > 0) {
+        	remaining = count;
+        } else {
+        	remaining = Long.MAX_VALUE;
+        }
+        while ( (c = source.read(buffer, 0, remaining < chunkSize ? (int)remaining : chunkSize)) > 0 ) {
             dest.write(buffer, 0, c);
             dest.flush();
             total += c;
+            remaining -= c;
 
             if ( count > 0 && count == total) {
                 break;
@@ -383,19 +390,11 @@ public final class FileUtils {
      * @throws NullPointerException when source parameter is null
      */
     public static byte[] read(final InputStream source, final int count) throws IOException {
-        if ( count > 0 ) {
-            final byte[] b = new byte[count];
-            final int c = source.read(b, 0, count);
-            assert c == count : "count = " + count + ", c = " + c;
-            if ( c != count ) {
-                final byte[] bb = new byte[c];
-                System.arraycopy(b, 0, bb, 0, c);
-                return bb;
-            }
-            return b;
+        if(count == 0) {
+        	return new byte[0];
         }
         final ByteArrayOutputStream baos = new ByteArrayOutputStream(512);
-        copy(source, baos);
+        copy(source, baos, count);
         baos.close();
         return baos.toByteArray();
     }

test/java/net/yacy/kelondro/util/FileUtilsTest.java

@@ -23,11 +23,13 @@ package net.yacy.kelondro.util;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.io.FilterInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 
 import org.junit.Assert;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 /**
@@ -35,6 +37,37 @@ import org.junit.Test;
  *
  */
 public class FileUtilsTest {
+	
+	@BeforeClass
+	public static void beforeAll() {
+		/* Disable assertions here : success/failure of these tests should not rely on assertions that are likely to be disabled at runtime */
+		FileUtilsTest.class.getClassLoader().setDefaultAssertionStatus(false);
+	}
+	
+	/**
+	 * A test stream reading each time less than desired bytes.
+	 * Simulates what can occur for example on real-world HTTP streams.
+	 */
+	private class LowerReadThanExpectedInputStream extends FilterInputStream {
+		
+		protected LowerReadThanExpectedInputStream(InputStream in) {
+			super(in);
+		}
+
+		/**
+		 * Reads less than specified len bytes
+		 */
+		@Override
+		public int read(byte[] b, int off, int len) throws IOException {
+			if(len > 10 ) {
+				return super.read(b, off, len - 10);
+			}
+			return super.read(b, off, len);
+		}
+		
+	}
+	
+	
 
 	/**
 	 * Copy stream : normal case
@@ -102,16 +135,71 @@ public class FileUtilsTest {
 		for(int i = 0; i < sourceBytes.length; i++) {
 			sourceBytes[i] = (byte)(i % Byte.MAX_VALUE);
 		}
-		InputStream source = new ByteArrayInputStream(sourceBytes);
-		ByteArrayOutputStream dest = new ByteArrayOutputStream();
-		final int COUNT = 8;
-		try {
+
+		final int COUNT = 1200;
+		try (InputStream source = new ByteArrayInputStream(sourceBytes);
+				ByteArrayOutputStream dest = new ByteArrayOutputStream();) {
 			FileUtils.copy(source, dest, COUNT);
-		} finally {
-			source.close();
-			dest.close();
+			Assert.assertEquals(COUNT, dest.size());
+		}
+		
+		/* Copy from a stream reading less than desired bytes (can occurs for example on real-world HTTP streams) */
+		try (InputStream bufferedSource = new ByteArrayInputStream(sourceBytes);
+				InputStream source = new LowerReadThanExpectedInputStream(bufferedSource);
+				ByteArrayOutputStream dest = new ByteArrayOutputStream();) {
+			FileUtils.copy(source, dest, COUNT);
+			Assert.assertEquals(COUNT, dest.size());
 		}
-		Assert.assertEquals(COUNT, dest.size());
 	}
+	
+	/**
+	 * Test reading n bytes in a stream
+	 * @throws IOException when a read/write error occurred
+	 */
+	@Test
+	public void testReadInputStream() throws IOException {
 
+
+		/* Fill an input stream with more bytes than FileUtils.DEFAULT_BUFFER_SIZE */
+		byte[] sourceBytes = new byte[2000];
+		for(int i = 0; i < sourceBytes.length; i++) {
+			sourceBytes[i] = (byte)(i % Byte.MAX_VALUE);
+		}
+		
+		/* Read all*/
+		try(InputStream source = new ByteArrayInputStream(sourceBytes);) {
+			Assert.assertEquals(sourceBytes.length, FileUtils.read(source, -1).length);
+		}
+		
+		/* Read zero */
+		try(InputStream source = new ByteArrayInputStream(sourceBytes);) {
+			Assert.assertEquals(0, FileUtils.read(source, 0).length);
+		}
+		
+		/* Read only one */
+		try(InputStream source = new ByteArrayInputStream(sourceBytes);) {
+			Assert.assertEquals(1, FileUtils.read(source, 1).length);
+		}
+		
+		/* Read half */
+		try(InputStream source = new ByteArrayInputStream(sourceBytes);) {
+			Assert.assertEquals(sourceBytes.length / 2, FileUtils.read(source, sourceBytes.length / 2).length);
+		}
+		
+		/* Read half on a stream reading each time less than desired bytes (can occurs for example on real-world HTTP streams) */
+		try(InputStream bufferedStream = new ByteArrayInputStream(sourceBytes);
+			InputStream source = new LowerReadThanExpectedInputStream(bufferedStream);) {
+			Assert.assertEquals(sourceBytes.length / 2, FileUtils.read(source, sourceBytes.length / 2).length);
+		}
+		
+		/* Read exactly source bytes count */
+		try(InputStream source = new ByteArrayInputStream(sourceBytes);) {
+			Assert.assertEquals(sourceBytes.length, FileUtils.read(source, sourceBytes.length).length);
+		}
+		
+		/* Trying to read more than source bytes count */
+		try(InputStream source = new ByteArrayInputStream(sourceBytes);) {
+			Assert.assertEquals(sourceBytes.length, FileUtils.read(source, sourceBytes.length + 10).length);
+		}
+	}
 }