From 046f3a7e8d45c48b6d942001030749652dac1126 Mon Sep 17 00:00:00 2001
From: Michael Peter Christen <mc@yacy.net>
Date: Mon, 16 Apr 2012 09:50:55 +0200
Subject: [PATCH] check if httpc has decompressed the release file and rename
 the file from .tar.gz to .tar if that happened

---
 source/de/anomic/tools/tarTools.java          |  5 ++---
 .../net/yacy/document/parser/tarParser.java   | 20 +++++++++++++++++++
 .../net/yacy/peers/operation/yacyRelease.java | 10 +++++++++-
 3 files changed, 31 insertions(+), 4 deletions(-)

diff --git a/source/de/anomic/tools/tarTools.java b/source/de/anomic/tools/tarTools.java
index cd9d82d48..043adf698 100644
--- a/source/de/anomic/tools/tarTools.java
+++ b/source/de/anomic/tools/tarTools.java
@@ -47,10 +47,9 @@ public class tarTools {
 		        return new GZIPInputStream(new FileInputStream(new File(tarFileName)));
 		    } catch (IOException e) {
 		        // this might happen if the stream is not in gzip format.
-		        // there may be a 'gz' extension, but it may stil be a raw tar file
+		        // there may be a 'gz' extension, but it may still be a raw tar file
 		        // this can be caused by 'one too much gzip-content header' that was attached
-		        // by a release file server
-		        // so just try to open is as normal stream
+		        // by a release file server, so just try to open is as normal stream
 		        return new FileInputStream(new File(tarFileName));
 		    }
 		}
diff --git a/source/net/yacy/document/parser/tarParser.java b/source/net/yacy/document/parser/tarParser.java
index de59b926c..df7f58d12 100644
--- a/source/net/yacy/document/parser/tarParser.java
+++ b/source/net/yacy/document/parser/tarParser.java
@@ -25,13 +25,16 @@
 package net.yacy.document.parser;
 
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.RandomAccessFile;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.zip.GZIPInputStream;
 
 import net.yacy.cora.document.MultiProtocolURI;
+import net.yacy.cora.document.UTF8;
 import net.yacy.document.AbstractParser;
 import net.yacy.document.Document;
 import net.yacy.document.Parser;
@@ -45,6 +48,8 @@ import org.apache.tools.tar.TarInputStream;
 
 public class tarParser extends AbstractParser implements Parser {
 
+    private final static String MAGIC = "ustar"; // A magic for a tar archive, may appear at #101h-#105
+    
     public tarParser() {
         super("Tape Archive File Parser");
         this.SUPPORTED_EXTENSIONS.add("tar");
@@ -98,4 +103,19 @@ public class tarParser extends AbstractParser implements Parser {
         }
         return docacc.toArray(new Document[docacc.size()]);
     }
+    
+    public final static boolean isTar(File f) {
+        if (!f.exists() || f.length() < 0x105) return false;
+        try {
+            RandomAccessFile raf = new RandomAccessFile(f, "r");
+            raf.seek(0x101);
+            byte[] b = new byte[5];
+            raf.read(b);
+            return MAGIC.equals(UTF8.String(b));
+        } catch (FileNotFoundException e) {
+            return false;
+        } catch (IOException e) {
+            return false;
+        }
+    }
 }
diff --git a/source/net/yacy/peers/operation/yacyRelease.java b/source/net/yacy/peers/operation/yacyRelease.java
index 80d132259..d4d88c40f 100644
--- a/source/net/yacy/peers/operation/yacyRelease.java
+++ b/source/net/yacy/peers/operation/yacyRelease.java
@@ -53,6 +53,7 @@ import net.yacy.cora.protocol.ResponseHeader;
 import net.yacy.cora.protocol.http.HTTPClient;
 import net.yacy.cora.services.federated.yacy.CacheStrategy;
 import net.yacy.document.Document;
+import net.yacy.document.parser.tarParser;
 import net.yacy.kelondro.data.meta.DigestURI;
 import net.yacy.kelondro.io.CharBuffer;
 import net.yacy.kelondro.logging.Log;
@@ -310,7 +311,7 @@ public final class yacyRelease extends yacyVersion {
             final ResponseHeader header = new ResponseHeader(client.getHttpResponse().getAllHeaders());
 
             final boolean unzipped = header.gzip() && (header.mime().toLowerCase().equals("application/x-tar")); // if true, then the httpc has unzipped the file
-            if ((unzipped) && (name.endsWith(".tar.gz"))) {
+            if (unzipped && name.endsWith(".tar.gz")) {
                 download = new File(storagePath, name.substring(0, name.length() - 3));
             } else {
                 download = new File(storagePath, name);
@@ -340,6 +341,13 @@ public final class yacyRelease extends yacyVersion {
                 client.writeTo(new BufferedOutputStream(new FileOutputStream(download)));
             }
             if ((!download.exists()) || (download.length() == 0)) throw new IOException("wget of url " + getUrl() + " failed");
+            // check again if this is actually a tar.gz or tar file since the httpc may have decompressed it
+            if (download.getName().endsWith("tar.gz") && tarParser.isTar(download)) {
+                String ts = download.getAbsoluteFile().toString();
+                File tar = new File(ts.substring(0, ts.length() - 3));
+                download.renameTo(tar);
+                download = tar;
+            }
         } catch (final IOException e) {
             // Saving file failed, abort download
             Log.logSevere("yacyVersion", "download of " + getName() + " failed: " + e.getMessage());