boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added Zstandard compressed data decompression for ZIM files type 5

Browse Source 
also: more generalization and performance enhancements
pull/607/head
Michael Peter Christen 1 year ago
parent ad8ee3a0b6
commit b0ae660790
4 changed files with 102 additions and 143 deletions
Show Stats Download Patch File Download Diff File

1
ivy.xml
Unescape View File

@ -98,5 +98,6 @@
<dependency org="org.hamcrest" name="hamcrest" rev="2.2" conf="test->default"/> <dependency org="org.hamcrest" name="hamcrest" rev="2.2" conf="test->default"/>
<dependency org="org.hamcrest" name="hamcrest-core" rev="2.2" conf="test->default"/> <dependency org="org.hamcrest" name="hamcrest-core" rev="2.2" conf="test->default"/>
<dependency org="org.hamcrest" name="hamcrest-library" rev="2.2" conf="test->default"/> <dependency org="org.hamcrest" name="hamcrest-library" rev="2.2" conf="test->default"/>
<dependency org="com.github.luben" name="zstd-jni" rev="1.5.5-6"></dependency>
</dependencies> </dependencies>
</ivy-module> </ivy-module>

14
source/org/openzim/RandomAccessFileZIMInputStream.java
Unescape View File

@ -18,6 +18,7 @@
package org.openzim; package org.openzim;
import java.io.ByteArrayOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.RandomAccessFile; import java.io.RandomAccessFile;
@ -70,7 +71,7 @@ public class RandomAccessFileZIMInputStream extends InputStream {
| ((buffer[2] & 0xFF) << 16) | ((buffer[3] & 0xFF) << 24)); | ((buffer[2] & 0xFF) << 16) | ((buffer[3] & 0xFF) << 24));
} }
private static long toEightLittleEndianLong(final byte[] buffer) { public static long toEightLittleEndianLong(final byte[] buffer) {
return // cast to long required otherwise this is again an integer return // cast to long required otherwise this is again an integer
((long)(buffer[0] & 0xFF) | ((long)(buffer[1] & 0xFF) << 8) ((long)(buffer[0] & 0xFF) | ((long)(buffer[1] & 0xFF) << 8)
| ((long)(buffer[2] & 0xFF) << 16) | ((long)(buffer[3] & 0xFF) << 24) | ((long)(buffer[2] & 0xFF) << 16) | ((long)(buffer[3] & 0xFF) << 24)
@ -85,13 +86,12 @@ public class RandomAccessFileZIMInputStream extends InputStream {
// Reads characters from the current position into a String and stops when a // Reads characters from the current position into a String and stops when a
// '\0' is encountered // '\0' is encountered
public String readZeroTerminatedString() throws IOException { public String readZeroTerminatedString() throws IOException {
final StringBuilder sb = new StringBuilder(); ByteArrayOutputStream buffer = new ByteArrayOutputStream();
int b = this.mRAFReader.read(); int b;
while (b != '\0') { while ((b = this.mRAFReader.read()) != '\0' && b != -1) {
sb.append((char) b); buffer.write(b);
b = this.mRAFReader.read();
} }
return sb.toString(); return buffer.toString("UTF-8");
} }
@Override @Override

177
source/org/openzim/ZIMReader.java
Unescape View File

@ -18,14 +18,15 @@
package org.openzim; package org.openzim;
import java.io.ByteArrayOutputStream;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream;
import java.io.RandomAccessFile; import java.io.RandomAccessFile;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import org.tukaani.xz.SingleXZInputStream; import org.tukaani.xz.SingleXZInputStream;
import com.github.luben.zstd.ZstdInputStream;
/** /**
* @author Arunesh Mathur * @author Arunesh Mathur
@ -36,7 +37,9 @@ import org.tukaani.xz.SingleXZInputStream;
* naming adoption to https://wiki.openzim.org/wiki/ZIM_file_format, * naming adoption to https://wiki.openzim.org/wiki/ZIM_file_format,
* change of Exception handling, * change of Exception handling,
* extension to more attributes as defined in spec (bugfix for mime type loading) * extension to more attributes as defined in spec (bugfix for mime type loading)
* bugfix to long parsing (prevented reading of large files) * bugfix to long parsing (prevented reading of large files),
* added extended cluster size parsing
* added ZStandard compression parsing (cluster type 5)
*/ */
public class ZIMReader { public class ZIMReader {
@ -110,91 +113,53 @@ public class ZIMReader {
return this.mFile; return this.mFile;
} }
// get a URL list that is sorted by the urls public String getURLByURLOrder(int entryNumber) throws IOException {
public List<String> getURLListByURL() throws IOException {
int i = 0, mimeType;
// The list that will eventually return the list of URL's
final ArrayList<String> returnList = new ArrayList<>();
// Move to the spot where URL's are listed // Move to the spot where URL's are listed
this.mReader.seek(this.mFile.header_urlPtrPos); this.mReader.seek(this.mFile.header_urlPtrPos + 8L * entryNumber);
for (i = 0; i < this.mFile.header_entryCount; i++) {
// The position of URL i // The position of URL i
long pos = this.mReader.readEightLittleEndianBytesLong(); long pos = this.mReader.readEightLittleEndianBytesLong();
// Mark the current position that we need to return to
this.mReader.mark();
// Move to the position of URL i // Move to the position of URL i
this.mReader.seek(pos); this.mReader.seek(pos);
// Article or Redirect entry? // Article or Redirect entry?
mimeType = this.mReader.readTwoLittleEndianBytesInt(); int mimeType = this.mReader.readTwoLittleEndianBytesInt();
if (mimeType == 65535) { if (mimeType == 65535) {
this.mReader.seek(pos + 12); this.mReader.seek(pos + 12);
returnList.add(this.mReader.readZeroTerminatedString()); return this.mReader.readZeroTerminatedString();
} else { } else {
this.mReader.seek(pos + 16); this.mReader.seek(pos + 16);
returnList.add(this.mReader.readZeroTerminatedString()); return this.mReader.readZeroTerminatedString();
}
this.mReader.reset();
} }
return returnList;
} }
// get a URL list that is sorted by the entry titles public String getURLByTitleOrder(int entryNumber) throws IOException {
public List<String> getURLListByTitle() throws IOException {
int i = 0, mimeType, articleNumber;
// The list that will eventually return the list of URL's
final ArrayList<String> returnList = new ArrayList<>();
// Get the UrlPtrPos or one time storage
long urlPtrPos = this.mFile.header_urlPtrPos;
// Move to the spot where URL's are listed // Move to the spot where URL's are listed
this.mReader.seek(this.mFile.header_titlePtrPos); this.mReader.seek(this.mFile.header_titlePtrPos + 8L * entryNumber);
for (i = 0; i < this.mFile.header_entryCount; i++) {
// The articleNumber of the position of URL i // The articleNumber of the position of URL i
articleNumber = this.mReader.readFourLittleEndianBytesInt(); int articleNumber = this.mReader.readFourLittleEndianBytesInt();
// Mark the current position that we need to return to
this.mReader.mark();
this.mReader.seek(urlPtrPos + (8L * (articleNumber))); this.mReader.seek(this.mFile.header_urlPtrPos + (8L * (articleNumber)));
// The position of URL i // The position of URL i
long pos = this.mReader.readEightLittleEndianBytesLong(); long pos = this.mReader.readEightLittleEndianBytesLong();
this.mReader.seek(pos); this.mReader.seek(pos);
// Article or Redirect entry? // Article or Redirect entry?
mimeType = this.mReader.readTwoLittleEndianBytesInt(); int mimeType = this.mReader.readTwoLittleEndianBytesInt();
if (mimeType == 65535) { if (mimeType == 65535) {
this.mReader.seek(pos + 12); this.mReader.seek(pos + 12);
final String url = this.mReader.readZeroTerminatedString(); return this.mReader.readZeroTerminatedString();
returnList.add(url);
} else { } else {
this.mReader.seek(pos + 16); this.mReader.seek(pos + 16);
final String url = this.mReader.readZeroTerminatedString(); return this.mReader.readZeroTerminatedString();
returnList.add(url);
} }
// Return to the marked position
this.mReader.reset();
}
return returnList;
} }
// position must be the seek position for the title in the Title Pointer List // position must be the seek position for the title in the Title Pointer List
@ -291,7 +256,7 @@ public class ZIMReader {
final ArticleEntry article = (ArticleEntry) directoryInfo; final ArticleEntry article = (ArticleEntry) directoryInfo;
// Move to the cluster entry in the clusterPtrPos // Move to the cluster entry in the clusterPtrPos
this.mReader.seek(this.mFile.header_clusterPtrPos + article.cluster_number * 8); this.mReader.seek(this.mFile.header_clusterPtrPos + article.cluster_number * 8L);
// Read the location of the cluster // Read the location of the cluster
final long clusterPos = this.mReader.readEightLittleEndianBytesLong(); final long clusterPos = this.mReader.readEightLittleEndianBytesLong();
@ -302,78 +267,74 @@ public class ZIMReader {
// Read the first byte, for compression information // Read the first byte, for compression information
final int compressionType = this.mReader.read(); final int compressionType = this.mReader.read();
// Reference declaration
int firstOffset, numberOfBlobs, offset1, offset2, location, differenceOffset;
// Check the compression type that was read // Check the compression type that was read
if (compressionType == 1) { // type = 1 uncompressed
if (compressionType <= 1 || compressionType == 8 || compressionType == 9) {
// The first four bytes are the offset of the zeroth blob boolean extended = compressionType > 1;
firstOffset = this.mReader.readFourLittleEndianBytesInt(); return readClusterEntry(this.mReader, article.blob_number, extended);
// The number of blobs
numberOfBlobs = firstOffset / 4;
// The blobNumber has to be lesser than the numberOfBlobs
assert article.blob_number < numberOfBlobs;
if (article.blob_number == 0) {
// The first offset is what we read earlier
offset1 = firstOffset;
} else {
location = (article.blob_number - 1) * 4;
RandomAccessFileZIMInputStream.skipFully(this.mReader, location);
offset1 = this.mReader.readFourLittleEndianBytesInt();
} }
// 2 for zlib and 3 for bzip2 (removed)
offset2 = this.mReader.readFourLittleEndianBytesInt(); // LZMA2 compressed data
differenceOffset = offset2 - offset1; if (compressionType == 4 || compressionType == 12) {
byte[] entry = new byte[differenceOffset]; boolean extended = compressionType == 12;
RandomAccessFileZIMInputStream.skipFully(this.mReader, (offset1 - 4 * (article.blob_number + 2))); // Create a dictionary with size 40MiB, the zimlib uses this size while creating
this.mReader.read(entry, 0, differenceOffset); SingleXZInputStream xzReader= new SingleXZInputStream(this.mReader, 41943040);
return readClusterEntry(xzReader, article.blob_number, extended);
}
return entry; // Zstandard compressed data
if (compressionType == 5 || compressionType == 13) {
boolean extended = compressionType == 13;
ZstdInputStream zReader = new ZstdInputStream(this.mReader);
return readClusterEntry(zReader, article.blob_number, extended);
} }
// 2 for zlib and 3 for bzip2 (removed)
// LZMA2 compressed data return null;
if (compressionType == 4) { }
// Read the first 4 bytes to find out the number of artciles private static byte[] readClusterEntry(InputStream is, int blob_number, boolean extended) throws IOException {
byte[] buffer = new byte[4];
// Create a dictionary with size 40MiB, the zimlib uses this size while creating // Read the first 4(8) bytes to find out the number of articles
SingleXZInputStream xzReader= new SingleXZInputStream(this.mReader, 4194304); byte[] buffer = new byte[extended ? 8 : 4];
// The first four bytes are the offset of the zeroth blob // The first four (eight) bytes are the offset of the zeroth blob
firstOffset = this.mReader.readFourLittleEndianBytesInt(); is.read(buffer);
long firstOffset = extended? RandomAccessFileZIMInputStream.toEightLittleEndianLong(buffer) : RandomAccessFileZIMInputStream.toFourLittleEndianInteger(buffer);
// The number of blobs // The number of blobs can be computed by the offset
numberOfBlobs = firstOffset / 4; // the actual number is one less because there is one more offset entry than the actual number
// to identify the end of the last blob.
long numberOfBlobs1 = extended ? firstOffset / 8 : firstOffset / 4;
// The blobNumber has to be lesser than the numberOfBlobs // The blobNumber has to be lesser than the numberOfBlobs - 1
assert article.blob_number < numberOfBlobs; // the blob numbers start with 0 even if the documentation states it is "the first blob".
if (article.blob_number == 0) { assert blob_number < numberOfBlobs1 - 1;
long offset1;
if (blob_number == 0) {
// The first offset is what we read earlier // The first offset is what we read earlier
offset1 = firstOffset; offset1 = firstOffset;
} else { } else {
location = (article.blob_number - 1) * 4; // skip one less than required to get to the offset entry because the first entry is already read
RandomAccessFileZIMInputStream.skipFully(xzReader, location); RandomAccessFileZIMInputStream.skipFully(is, (blob_number - 1) * (extended ? 8 : 4));
xzReader.read(buffer); is.read(buffer);
offset1 = RandomAccessFileZIMInputStream.toFourLittleEndianInteger(buffer); offset1 = extended? RandomAccessFileZIMInputStream.toEightLittleEndianLong(buffer) : RandomAccessFileZIMInputStream.toFourLittleEndianInteger(buffer);
} }
is.read(buffer);
xzReader.read(buffer); long offset2 = extended? RandomAccessFileZIMInputStream.toEightLittleEndianLong(buffer) : RandomAccessFileZIMInputStream.toFourLittleEndianInteger(buffer);
offset2 = RandomAccessFileZIMInputStream.toFourLittleEndianInteger(buffer); long blob_size = offset2 - offset1;
differenceOffset = offset2 - offset1; byte[] entry = new byte[(int) blob_size]; // TODO: we should be able to read blobs larger than MAXINT
byte[] entry = new byte[differenceOffset]; // we must do two skip steps: first to the end of the offset list and second to the start of the blob
RandomAccessFileZIMInputStream.skipFully(xzReader, (offset1 - 4 * (article.blob_number + 2))); // - the whole number of offset list entries is numberOfBlobs1, which includes the extra entry for the end offset
xzReader.read(entry, 0, differenceOffset); // - the number of offset entries that we alreay read now is article.blob_number + 2 (in any case at least 2)
// - the remaining number of offset entries to skip is therefore numberOfBlobs1 - (article.blob_number + 2)
// - the addon skip of number of bytes to the start of the entry is offset1 - firstoffset with firstoffset = 4 * numberOfBlobs1
// - the full skip length is 4 * (numberOfBlobs1 - (article.blob_number + 2)) + offset1 - 4 * numberOfBlobs1
// = offset1 - 4 * (article.blob_number + 2)
RandomAccessFileZIMInputStream.skipFully(is, (offset1 - (extended ? 8 : 4) * (blob_number + 2)));
is.read(entry, 0, entry.length);
return entry; return entry;
} }
// case 5: zstd compressed (missing!)
return null;
}
} }

9
source/org/openzim/ZIMTest.java
Unescape View File

@ -20,7 +20,6 @@ package org.openzim;
import java.io.IOException; import java.io.IOException;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.List;
import org.openzim.ZIMReader.DirectoryEntry; import org.openzim.ZIMReader.DirectoryEntry;
@ -40,12 +39,10 @@ public class ZIMTest {
final ZIMReader zReader = new ZIMReader(file); final ZIMReader zReader = new ZIMReader(file);
// print a list of urls and titles // print a list of urls and titles
final List<String> urls = zReader.getURLListByURL(); int c = Math.min(10, file.header_entryCount);
final List<String> titles = zReader.getURLListByTitle();
int c = Math.min(10, titles.size());
for (int i = 0; i < c; i++) { for (int i = 0; i < c; i++) {
System.out.println("URL by URL " + i + ": " + urls.get(i)); System.out.println("URL by URL " + i + ": " + zReader.getURLByURLOrder(i));
System.out.println("URL by Title " + i + ": " + titles.get(i)); System.out.println("URL by Title " + i + ": " + zReader.getURLByTitleOrder(i));
DirectoryEntry entry = zReader.getDirectoryInfo(i); DirectoryEntry entry = zReader.getDirectoryInfo(i);
System.out.println("URL by Pos " + i + ": " + entry.url); System.out.println("URL by Pos " + i + ": " + entry.url);
System.out.println("Title by Pos " + i + ": " + entry.title); System.out.println("Title by Pos " + i + ": " + entry.title);

Write Preview
Loading…
Cancel
Save