Add support for brotli compression

Build logs on cache.nixos.org are compressed using Brotli (since this allows them to be decompressed automatically by Chrome and Firefox), so it's handy if "nix log" can decompress them.
fricklerhandwerk · Mar 15, 2017 · e818608 · e818608
1 parent 73d7a51
commit e818608
Show file tree

Hide file tree

Showing 7 changed files with 38 additions and 5 deletions.
diff --git a/Makefile.config.in b/Makefile.config.in
@@ -14,6 +14,7 @@ LIBLZMA_LIBS = @LIBLZMA_LIBS@
 SQLITE3_LIBS = @SQLITE3_LIBS@
 bash = @bash@
 bindir = @bindir@
+bro = @bro@
 datadir = @datadir@
 datarootdir = @datarootdir@
 docdir = @docdir@

diff --git a/configure.ac b/configure.ac
@@ -128,6 +128,7 @@ NEED_PROG(gzip, gzip)
 NEED_PROG(xz, xz)
 AC_PATH_PROG(dot, dot)
 AC_PATH_PROG(pv, pv, pv)
+NEED_PROG(bro, bro)
 
 
 # Test that Perl has the open/fork feature (Perl 5.8.0 and beyond).

diff --git a/release.nix b/release.nix
@@ -24,7 +24,8 @@ let
         inherit officialRelease;
 
         buildInputs =
-          [ curl bison flex perl libxml2 libxslt bzip2 xz
+          [ curl bison flex perl libxml2 libxslt
+            bzip2 xz brotli
             pkgconfig sqlite libsodium boehmgc
             docbook5 docbook5_xsl
             autoconf-archive
@@ -73,7 +74,10 @@ let
         src = tarball;
 
         buildInputs =
-          [ curl perl bzip2 xz openssl pkgconfig sqlite boehmgc ]
+          [ curl perl
+            bzip2 xz brotli
+            openssl pkgconfig sqlite boehmgc
+          ]
           ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
           ++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
             (aws-sdk-cpp.override {

diff --git a/shell.nix b/shell.nix
@@ -6,7 +6,8 @@ with import <nixpkgs> {};
   name = "nix";
 
   buildInputs =
-    [ curl bison flex perl libxml2 libxslt bzip2 xz
+    [ curl bison flex perl libxml2 libxslt
+      bzip2 xz brotli
       pkgconfig sqlite libsodium boehmgc
       docbook5 docbook5_xsl
       autoconf-archive

diff --git a/src/libstore/download.cc b/src/libstore/download.cc
@@ -5,6 +5,8 @@
 #include "store-api.hh"
 #include "archive.hh"
 #include "s3.hh"
+#include "compression.hh"
+
 #ifdef ENABLE_S3
 #include <aws/core/client/ClientConfiguration.h>
 #endif
@@ -70,6 +72,8 @@ struct CurlDownloader : public Downloader
 
         struct curl_slist * requestHeaders = 0;
 
+        std::string encoding;
+
         DownloadItem(CurlDownloader & downloader, const DownloadRequest & request)
             : downloader(downloader), request(request)
         {
@@ -127,6 +131,7 @@ struct CurlDownloader : public Downloader
                 auto ss = tokenizeString<vector<string>>(line, " ");
                 status = ss.size() >= 2 ? ss[1] : "";
                 result.data = std::make_shared<std::string>();
+                encoding = "";
             } else {
                 auto i = line.find(':');
                 if (i != string::npos) {
@@ -142,7 +147,8 @@ struct CurlDownloader : public Downloader
                             debug(format("shutting down on 200 HTTP response with expected ETag"));
                             return 0;
                         }
-                    }
+                    } else if (name == "content-encoding")
+                        encoding = trim(string(line, i + 1));;
                 }
             }
             return realSize;
@@ -268,7 +274,18 @@ struct CurlDownloader : public Downloader
             {
                 result.cached = httpStatus == 304;
                 done = true;
-                callSuccess(success, failure, const_cast<const DownloadResult &>(result));
+
+                /* Ad hoc support for brotli, since curl doesn't do
+                   this yet. */
+                try {
+                    if (encoding == "br")
+                        result.data = decompress("br", *result.data);
+
+                    callSuccess(success, failure, const_cast<const DownloadResult &>(result));
+                } catch (...) {
+                    done = true;
+                    callFailure(failure, std::current_exception());
+                }
             } else {
                 Error err =
                     (httpStatus == 404 || code == CURLE_FILE_COULDNT_READ_FILE) ? NotFound :

diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
@@ -89,6 +89,11 @@ static ref<std::string> decompressBzip2(const std::string & in)
     }
 }
 
+static ref<std::string> decompressBrotli(const std::string & in)
+{
+    return make_ref<std::string>(runProgram(BRO, true, {"-d"}, in));
+}
+
 ref<std::string> compress(const std::string & method, const std::string & in)
 {
     StringSink ssink;
@@ -106,6 +111,8 @@ ref<std::string> decompress(const std::string & method, const std::string & in)
         return decompressXZ(in);
     else if (method == "bzip2")
         return decompressBzip2(in);
+    else if (method == "br")
+        return decompressBrotli(in);
     else
         throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method);
 }

diff --git a/src/libutil/local.mk b/src/libutil/local.mk
@@ -9,3 +9,5 @@ libutil_SOURCES := $(wildcard $(d)/*.cc)
 libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS)
 
 libutil_LIBS = libformat
+
+libutil_CXXFLAGS = -DBRO=\"$(bro)\"
Original file line number	Diff line number	Diff line change
Expand Up		@@ -9,3 +9,5 @@ libutil_SOURCES := $(wildcard $(d)/*.cc)
		libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS)

		libutil_LIBS = libformat

		libutil_CXXFLAGS = -DBRO=\"$(bro)\"