From f369679d1cfc1cbd989fe7efbb3803c01c19f60d Mon Sep 17 00:00:00 2001 From: luccioman Date: Tue, 11 Jul 2017 09:00:27 +0200 Subject: [PATCH] Fixed read/copy on input streams reading sometimes less than expected. --- source/net/yacy/kelondro/util/FileUtils.java | 23 ++-- .../net/yacy/kelondro/util/FileUtilsTest.java | 104 ++++++++++++++++-- 2 files changed, 107 insertions(+), 20 deletions(-) diff --git a/source/net/yacy/kelondro/util/FileUtils.java b/source/net/yacy/kelondro/util/FileUtils.java index f1f892b2d..5b5936878 100644 --- a/source/net/yacy/kelondro/util/FileUtils.java +++ b/source/net/yacy/kelondro/util/FileUtils.java @@ -110,10 +110,17 @@ public final class FileUtils { int c; long total = 0; - while ( (c = source.read(buffer, 0, chunkSize)) > 0 ) { + long remaining; + if(count > 0) { + remaining = count; + } else { + remaining = Long.MAX_VALUE; + } + while ( (c = source.read(buffer, 0, remaining < chunkSize ? (int)remaining : chunkSize)) > 0 ) { dest.write(buffer, 0, c); dest.flush(); total += c; + remaining -= c; if ( count > 0 && count == total) { break; @@ -383,19 +390,11 @@ public final class FileUtils { * @throws NullPointerException when source parameter is null */ public static byte[] read(final InputStream source, final int count) throws IOException { - if ( count > 0 ) { - final byte[] b = new byte[count]; - final int c = source.read(b, 0, count); - assert c == count : "count = " + count + ", c = " + c; - if ( c != count ) { - final byte[] bb = new byte[c]; - System.arraycopy(b, 0, bb, 0, c); - return bb; - } - return b; + if(count == 0) { + return new byte[0]; } final ByteArrayOutputStream baos = new ByteArrayOutputStream(512); - copy(source, baos); + copy(source, baos, count); baos.close(); return baos.toByteArray(); } diff --git a/test/java/net/yacy/kelondro/util/FileUtilsTest.java b/test/java/net/yacy/kelondro/util/FileUtilsTest.java index d7c4747f2..a01341f1b 100644 --- a/test/java/net/yacy/kelondro/util/FileUtilsTest.java +++ b/test/java/net/yacy/kelondro/util/FileUtilsTest.java @@ -23,11 +23,13 @@ package net.yacy.kelondro.util; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Test; /** @@ -35,6 +37,37 @@ import org.junit.Test; * */ public class FileUtilsTest { + + @BeforeClass + public static void beforeAll() { + /* Disable assertions here : success/failure of these tests should not rely on assertions that are likely to be disabled at runtime */ + FileUtilsTest.class.getClassLoader().setDefaultAssertionStatus(false); + } + + /** + * A test stream reading each time less than desired bytes. + * Simulates what can occur for example on real-world HTTP streams. + */ + private class LowerReadThanExpectedInputStream extends FilterInputStream { + + protected LowerReadThanExpectedInputStream(InputStream in) { + super(in); + } + + /** + * Reads less than specified len bytes + */ + @Override + public int read(byte[] b, int off, int len) throws IOException { + if(len > 10 ) { + return super.read(b, off, len - 10); + } + return super.read(b, off, len); + } + + } + + /** * Copy stream : normal case @@ -102,16 +135,71 @@ public class FileUtilsTest { for(int i = 0; i < sourceBytes.length; i++) { sourceBytes[i] = (byte)(i % Byte.MAX_VALUE); } - InputStream source = new ByteArrayInputStream(sourceBytes); - ByteArrayOutputStream dest = new ByteArrayOutputStream(); - final int COUNT = 8; - try { + + final int COUNT = 1200; + try (InputStream source = new ByteArrayInputStream(sourceBytes); + ByteArrayOutputStream dest = new ByteArrayOutputStream();) { FileUtils.copy(source, dest, COUNT); - } finally { - source.close(); - dest.close(); + Assert.assertEquals(COUNT, dest.size()); + } + + /* Copy from a stream reading less than desired bytes (can occurs for example on real-world HTTP streams) */ + try (InputStream bufferedSource = new ByteArrayInputStream(sourceBytes); + InputStream source = new LowerReadThanExpectedInputStream(bufferedSource); + ByteArrayOutputStream dest = new ByteArrayOutputStream();) { + FileUtils.copy(source, dest, COUNT); + Assert.assertEquals(COUNT, dest.size()); } - Assert.assertEquals(COUNT, dest.size()); } + + /** + * Test reading n bytes in a stream + * @throws IOException when a read/write error occurred + */ + @Test + public void testReadInputStream() throws IOException { + + /* Fill an input stream with more bytes than FileUtils.DEFAULT_BUFFER_SIZE */ + byte[] sourceBytes = new byte[2000]; + for(int i = 0; i < sourceBytes.length; i++) { + sourceBytes[i] = (byte)(i % Byte.MAX_VALUE); + } + + /* Read all*/ + try(InputStream source = new ByteArrayInputStream(sourceBytes);) { + Assert.assertEquals(sourceBytes.length, FileUtils.read(source, -1).length); + } + + /* Read zero */ + try(InputStream source = new ByteArrayInputStream(sourceBytes);) { + Assert.assertEquals(0, FileUtils.read(source, 0).length); + } + + /* Read only one */ + try(InputStream source = new ByteArrayInputStream(sourceBytes);) { + Assert.assertEquals(1, FileUtils.read(source, 1).length); + } + + /* Read half */ + try(InputStream source = new ByteArrayInputStream(sourceBytes);) { + Assert.assertEquals(sourceBytes.length / 2, FileUtils.read(source, sourceBytes.length / 2).length); + } + + /* Read half on a stream reading each time less than desired bytes (can occurs for example on real-world HTTP streams) */ + try(InputStream bufferedStream = new ByteArrayInputStream(sourceBytes); + InputStream source = new LowerReadThanExpectedInputStream(bufferedStream);) { + Assert.assertEquals(sourceBytes.length / 2, FileUtils.read(source, sourceBytes.length / 2).length); + } + + /* Read exactly source bytes count */ + try(InputStream source = new ByteArrayInputStream(sourceBytes);) { + Assert.assertEquals(sourceBytes.length, FileUtils.read(source, sourceBytes.length).length); + } + + /* Trying to read more than source bytes count */ + try(InputStream source = new ByteArrayInputStream(sourceBytes);) { + Assert.assertEquals(sourceBytes.length, FileUtils.read(source, sourceBytes.length + 10).length); + } + } }