boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

gzip compression will perform more efficient and with better compression

Browse Source 
level
pull/8/head
Michael Peter Christen 10 years ago
parent 98be59ce9c
commit 34de1e8cbc
6 changed files with 14 additions and 9 deletions
Show Stats Download Patch File Download Diff File

3
source/net/yacy/cora/protocol/http/GzipCompressingEntity.java
Unescape View File

@ -27,6 +27,7 @@ package net.yacy.cora.protocol.http;
import java.io.IOException;
import java.io.OutputStream;
import java.util.zip.Deflater;
import java.util.zip.GZIPOutputStream;
import org.apache.http.Header;
@ -65,7 +66,7 @@ public class GzipCompressingEntity extends HttpEntityWrapper {
if (outstream == null) {
throw new IllegalArgumentException("Output stream may not be null");
}
GZIPOutputStream gzip = new GZIPOutputStream(outstream);
GZIPOutputStream gzip = new GZIPOutputStream(outstream, 65536){{def.setLevel(Deflater.BEST_SPEED);}};
wrappedEntity.writeTo(gzip);
gzip.finish();
}

3
source/net/yacy/kelondro/blob/Compressor.java
Unescape View File

@ -35,6 +35,7 @@ import java.io.OutputStream;
import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;
import java.util.zip.Deflater;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
@ -114,7 +115,7 @@ public class Compressor implements BLOB, Iterable<byte[]> {
//System.out.print("/(" + cdr + ")"); // DEBUG
final ByteArrayOutputStream baos = new ByteArrayOutputStream(b.length / 5);
baos.write(gzipMagic);
final OutputStream os = new GZIPOutputStream(baos, 512);
final OutputStream os = new GZIPOutputStream(baos, 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}};
os.write(b);
os.close();
baos.close();

3
source/net/yacy/kelondro/index/RowHandleMap.java
Unescape View File

@ -45,6 +45,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.zip.Deflater;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
@ -189,7 +190,7 @@ public final class RowHandleMap implements HandleMap, Iterable<Map.Entry<byte[],
} catch (final OutOfMemoryError e) {
os = new FileOutputStream(tmp);
}
if (file.getName().endsWith(".gz")) os = new GZIPOutputStream(os);
if (file.getName().endsWith(".gz")) os = new GZIPOutputStream(os, 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}};
int c = 0;
while (i.hasNext()) {
os.write(i.next().bytes());

4
source/net/yacy/search/Switchboard.java
Unescape View File

@ -76,6 +76,7 @@ import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import java.util.zip.Deflater;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import java.util.zip.ZipEntry;
@ -1959,8 +1960,7 @@ public final class Switchboard extends serverSwitch {
final String gzname = outfile.getName() + ".gz";
final File gzfile = new File(outfile.getParentFile(), gzname);
try {
final OutputStream os =
new BufferedOutputStream(new GZIPOutputStream(new FileOutputStream(gzfile)));
final OutputStream os = new BufferedOutputStream(new GZIPOutputStream(new FileOutputStream(gzfile), 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}});
BufferedInputStream bis = new BufferedInputStream(new FileInputStream(outfile));
FileUtils.copy(bis, os);
os.close();

3
source/net/yacy/search/index/Fulltext.java
Unescape View File

@ -41,6 +41,7 @@ import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;
import java.util.zip.Deflater;
import java.util.zip.GZIPOutputStream;
import net.yacy.cora.date.GenericFormatter;
@ -658,7 +659,7 @@ public final class Fulltext {
final File parentf = this.f.getParentFile();
if (parentf != null) parentf.mkdirs();
OutputStream os = new FileOutputStream(this.format == 3 ? new File(this.f.getAbsolutePath() + ".gz") : this.f);
if (this.format == 3) os = new GZIPOutputStream(os);
if (this.format == 3) os = new GZIPOutputStream(os, 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}};
final PrintWriter pw = new PrintWriter(new BufferedOutputStream(os));
if (this.format == 1) {
pw.println("<html><head></head><body>");

7
source/net/yacy/utils/gzip.java
Unescape View File

@ -35,6 +35,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.zip.Deflater;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
@ -49,7 +50,7 @@ public class gzip {
public static void gzipFile(final String inFile, final String outFile) {
try {
final InputStream fin = new BufferedInputStream(new FileInputStream(inFile));
final OutputStream fout = new GZIPOutputStream(new BufferedOutputStream(new FileOutputStream(outFile), 1024));
final OutputStream fout = new GZIPOutputStream(new BufferedOutputStream(new FileOutputStream(outFile)), 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}};
copy(fout, fin, 1024);
fin.close();
fout.close();
@ -87,7 +88,7 @@ public class gzip {
final InputStream fin = new ByteArrayInputStream(in.getBytes("UTF8"));
final int buffersize = Math.min(1024, in.length());
final ByteArrayOutputStream baos = new ByteArrayOutputStream(buffersize);
final OutputStream fout = new GZIPOutputStream(baos, Math.max(buffersize, 512));
final OutputStream fout = new GZIPOutputStream(baos, Math.max(buffersize, 65536)){{def.setLevel(Deflater.BEST_COMPRESSION);}};
copy(fout, fin, 1024);
fin.close();
fout.close();
@ -123,7 +124,7 @@ public class gzip {
java.util.zip.GZIPOutputStream gzipout = null;
try {
f.getParentFile().mkdirs();
gzipout = new java.util.zip.GZIPOutputStream(new FileOutputStream(f));
gzipout = new java.util.zip.GZIPOutputStream(new FileOutputStream(f), 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}};
gzipout.write(content, 0, content.length);
} finally {
if (gzipout!=null)try{gzipout.close();}catch(final Exception e){}

Write Preview
Loading…
Cancel
Save