Add streaming/framed LZ4 compression.

* subversion/include/private/svn_subr_private.h
  (svn_lz4__compress_ctx_t,
   svn_lz4__decompress_ctx_t): New types.
  (svn_lz4__header_size_max,
   svn_lz4__compress_create,
   svn_lz4__compress_bound,
   svn_lz4__compress_update,
   svn_lz4__compress_flush,
   svn_lz4__compress_end,
   svn_lz4__decompress_create,
   svn_lz4__decompress): New prototypes.

* subversion/libsvn_subr/compress_lz4.c: Include svn_error.h,
   lz2frame.h and lz4hc.c.
  (free_lz4_cctx, free_lz4_dctx, check_compress_status): New helper functions.
  (svn_lz4__compress_ctx_t,
   svn_lz4__decompress_ctx_t): Define the new types.
  (svn_lz4__header_size_max,
   svn_lz4__compress_create,
   svn_lz4__compress_bound,
   svn_lz4__compress_update,
   svn_lz4__compress_flush,
   svn_lz4__compress_end,
   svn_lz4__decompress_create,
   svn_lz4__decompress): Implement the LZ4 compression functions.

* subversion/tests/svn_test.h
  (SVN_TEST_BUFFER_ASSERT): New test assertion for comparing binary buffers.

* subversion/tests/libsvn_subr/compress-test.c: Include apr_time.h,
   svn_error.h and svn_string.h, but not svn_pools.h which was not used.
  (uncompressed, uncompressed_size,
   compressed_block, compressed_block_size,
   compressed_frame, compressed_frame_size,
   empty_frame, empty_frame_size): New, constant data.
  (test_decompress_lz4,
   test_compress_lz4): Use the new constant data.
  (de_compress_lz4): New helper function.
  (test_compress_lz4_random): New test.
  (test_compress_lz4_empty): Use de_compress_lz4().
  (compress_lz4_stream,
   decompress_lz4_stream,
   decompress_lz4_frame,
   de_compress_lz4_stream,
   de_compress_lz4_frame_random): New helper functions.
  (test_decompress_lz4_frame,
   test_decompress_lz4_frame_stable,
   test_decompress_lz4_frame_empty,
   test_compress_lz4_frame_random,
   test_compress_lz4_frame_random_stable,
   test_compress_lz4_frame_no_header_space): New tests.
  (test_funcs): Register the new tests.


git-svn-id: https://svn.apache.org/repos/asf/subversion/branches/better-pristines@1931681 13f79535-47bb-0310-9956-ffa450edef68

Author: brainy

subversion/include/private/svn_subr_private.h

@@ -625,6 +625,100 @@ svn__decompress_lz4(const void *data, apr_size_t len,
                     svn_stringbuf_t *out,
                     apr_size_t limit);
 
+/*
+ * LZ4 stream compression with framing.
+ */
+
+/* The largest size of an LZ4 frame header; see: LZ4F_HEADER_SIZE_MAX */
+apr_size_t
+svn_lz4__header_size_max(void);
+
+/* LZ4 stream compression and decompression contexts. */
+typedef struct svn_lz4__compress_ctx_t svn_lz4__compress_ctx_t;
+typedef struct svn_lz4__decompress_ctx_t svn_lz4__decompress_ctx_t;
+
+/* Creates a new LZ4 compression context CCTX, allocated from POOL.
+ * The context will be freed when the pool is cleared.
+ *
+ * If STABLE_INPUT is TRUE, the source buffer must be contiguous for the whole
+ * compression; this means that all uncompressed source data is available to
+ * svn_lz4__compress_update() through negative offsets from the INPUT pointer.
+ * When this option can be set, the compressor can avoid copying and caching
+ * source data in CCTX.
+ */
+svn_error_t *
+svn_lz4__compress_create(svn_lz4__compress_ctx_t **cctx,
+                         svn_boolean_t stable_input,
+                         apr_pool_t *pool);
+
+/* Returns the minimum CAPACITY of the output buffer to guarantee that
+ * the next svn_lz4__compress_update() with CCTX and SIZE amount
+ * of input data will succeed. If SIZE is 0, the returned value is
+ * computed for svn_lz4__compress_flush()/_end().
+ */
+apr_size_t
+svn_lz4__compress_bound(svn_lz4__compress_ctx_t *cctx, apr_size_t size);
+
+/* Using the context CCTX, compresses SIZE bytes from INPUT into the OUTPUT
+ * buffer of size CAPACITY. *LENGTH will be the number of bytes actually
+ * written to OUTPUT and can be 0 if the input data was buffered in the
+ * context.
+ *
+ * At the first call of svn_lz4__compress_update(), CAPACITY must be
+ * at least svn_lz4__header_size_max() to accomodate the frame header.
+ */
+svn_error_t *
+svn_lz4__compress_update(apr_size_t *length,
+                         svn_lz4__compress_ctx_t *cctx,
+                         void *output, apr_size_t capacity,
+                         const void *input, apr_size_t size);
+
+/* Flushes any data buffered in CCTX to the OUTPUT buffer of size CAPACITY
+ * and returns the number of bytes written in *LENGTH.
+ */
+svn_error_t *
+svn_lz4__compress_flush(apr_size_t *length,
+                        svn_lz4__compress_ctx_t *cctx,
+                        void *output, apr_size_t capacity);
+
+/* Ends the compression stream, returning the number of bytes written
+ * to OUTPUT in *LENGTH. Afterwards, CCTX will be available for to start
+ * a new compression stream with svn_lz4__compress_update().
+ */
+svn_error_t *
+svn_lz4__compress_end(apr_size_t *length,
+                      svn_lz4__compress_ctx_t *cctx,
+                      void *output, apr_size_t capacity);
+
+/* Creates a new LZ4 decompression context DCTX, allocated from POOL.
+ * The context will be freed when the pool is cleared.
+ *
+ * If STABLE_OUTPUT is TRUE, the output buffer must be contiguous for the whole
+ * decompression; this means that previously decompressed content is available
+ * to svn_lz4__decompress() through negative offsets from the OUTPUT pointer.
+ * This is the case, for example, when we store the decompressed data to a
+ * stringbuf, which remains contiguous even when it's reallocated. When this
+ * option can be set, the decompressor can avoid copying and caching output
+ * data in DCTX.
+ */
+svn_error_t *
+svn_lz4__decompress_create(svn_lz4__decompress_ctx_t **dctx,
+                           svn_boolean_t stable_output,
+                           apr_pool_t *pool);
+
+/* Using DCTX, read at most *INPUT_SIZE compressed bytes from the INPUT buffer
+ * and decompress them to the OUTPUT buffer of size *OUTPUT_SIZE. Upon return,
+ * *INPUT_SIZE will be the number of bytes actally read from INPUT and
+ * *OUTPUT_SIZE will be the number of bytes written to *OUTPUT. The returned
+ * *SIZE_HINT is a hint for the expected OUTPUT_SIZE for the next call, or 0
+ * if the frame has been completely decoded.
+ */
+svn_error_t *
+svn_lz4__decompress(apr_size_t *size_hint,
+                    svn_lz4__decompress_ctx_t *dctx,
+                    void *output, apr_size_t *output_size,
+                    const void *input, apr_size_t *input_size);
+
 /** @} */
 
 /**

subversion/libsvn_subr/compress_lz4.c

@@ -23,16 +23,26 @@
 
 #include <assert.h>
 
+#include "svn_error.h"
 #include "private/svn_subr_private.h"
 
 #include "svn_private_config.h"
+#include "svn_types.h"
 
 #ifdef SVN_INTERNAL_LZ4
 #include "lz4/lz4internal.h"
+#include "lz4/lz4frame.h"
+#include "lz4/lz4hc.h"
 #else
 #include <lz4.h>
+#include <lz4frame.h>
+#include <lz4hc.h>
 #endif
 
+/*
+ * Simple compression and decompression
+ */
+
 svn_error_t *
 svn__compress_lz4(const void *data, apr_size_t len,
                   svn_stringbuf_t *out)
@@ -127,6 +137,212 @@ svn__decompress_lz4(const void *data, apr_size_t len,
   return SVN_NO_ERROR;
 }
 
+/*
+ * Streamy compression
+ */
+
+apr_size_t
+svn_lz4__header_size_max(void)
+{
+  return LZ4F_HEADER_SIZE_MAX;
+}
+
+struct svn_lz4__compress_ctx_t
+{
+  LZ4F_cctx *ctx;
+  unsigned started;
+  LZ4F_preferences_t prefs;
+  LZ4F_compressOptions_t options;
+};
+
+static apr_status_t free_lz4_cctx(void *data)
+{
+  svn_lz4__compress_ctx_t *const cctx = data;
+  const LZ4F_errorCode_t code = LZ4F_freeCompressionContext(cctx->ctx);
+  if (LZ4F_isError(code))
+    return SVN_ERR_LZ4_COMPRESSION_FAILED;
+  return APR_SUCCESS;
+}
+
+svn_error_t *
+svn_lz4__compress_create(svn_lz4__compress_ctx_t **cctx_out,
+                         svn_boolean_t stable_input,
+                         apr_pool_t *pool)
+{
+  static const LZ4F_preferences_t compression_prefs = {
+    {
+      LZ4F_max64KB,               /* frameInfo.blockSizeID */
+      LZ4F_blockLinked,           /* frameInfo.blockMode */
+      LZ4F_noContentChecksum,     /* frameInfo.contentChecksumFlag */
+      LZ4F_frame,                 /* frameInfo.frameType */
+      0,                          /* frameInfo.contentSize */
+      0,                          /* frameInfo.dictID */
+      LZ4F_blockChecksumEnabled   /* frameInfo.blockChecksumFlag */
+    },
+    /* NOTE: With LZ4 1.10+, the compression level will be 2, faster but less
+             compressed than with older versions of LZ4, where the value of
+             this constant was 3. This does not affect compression format
+             backward compatibility. */
+    LZ4HC_CLEVEL_MIN,           /* compressionLevel */
+    0,                          /* autoFlush */
+    1,                          /* favorDecSpeed */
+    { 0 }                       /* reserved */
+  };
+
+  LZ4F_cctx *ctx;
+  svn_lz4__compress_ctx_t *cctx;
+  LZ4F_errorCode_t code = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
+
+  if (LZ4F_isError(code))
+    return svn_error_createf(SVN_ERR_LZ4_COMPRESSION_FAILED, NULL,
+                             _("Create LZ4 compression context: %s"),
+                             LZ4F_getErrorName(code));
+
+  cctx = apr_pcalloc(pool, sizeof(*cctx));
+  cctx->ctx = ctx;
+  cctx->prefs = compression_prefs;
+  cctx->options.stableSrc = !!stable_input;
+  apr_pool_cleanup_register(pool, cctx, free_lz4_cctx, apr_pool_cleanup_null);
+  *cctx_out = cctx;
+  return SVN_NO_ERROR;
+}
+
+static SVN__FORCE_INLINE svn_error_t *
+check_compress_status(apr_size_t *length, apr_size_t status, apr_size_t extra)
+{
+  if (LZ4F_isError(status))
+    return svn_error_createf(SVN_ERR_LZ4_COMPRESSION_FAILED, NULL,
+                             _("LZ4 compress: %s"),
+                             LZ4F_getErrorName(status));
+
+  *length = status + extra;
+  return SVN_NO_ERROR;
+}
+
+apr_size_t
+svn_lz4__compress_bound(svn_lz4__compress_ctx_t *cctx,
+                        apr_size_t size)
+{
+  return LZ4F_compressBound(size, &cctx->prefs);
+}
+
+svn_error_t *
+svn_lz4__compress_update(apr_size_t *length,
+                         svn_lz4__compress_ctx_t *cctx,
+                         void *output, apr_size_t capacity,
+                         const void *input, apr_size_t size)
+{
+  apr_size_t header_size = 0;
+  apr_size_t status;
+
+  if (!cctx->started)
+    {
+      if (capacity < LZ4F_HEADER_SIZE_MAX)
+        return svn_error_create(SVN_ERR_LZ4_COMPRESSION_FAILED, NULL,
+                                _("LZ4 compress: no space for frame header"));
+
+      status = LZ4F_compressBegin(cctx->ctx, output, capacity, &cctx->prefs);
+      SVN_ERR(check_compress_status(&header_size, status, 0));
+      output = (char*)output + header_size;
+      capacity -= header_size;
+      cctx->started = TRUE;
+    }
+
+  status = LZ4F_compressUpdate(cctx->ctx, output, capacity,
+                               input, size, &cctx->options);
+  return svn_error_trace(check_compress_status(length, status, header_size));
+}
+
+svn_error_t *
+svn_lz4__compress_flush(apr_size_t *length,
+                        svn_lz4__compress_ctx_t *cctx,
+                        void *output, apr_size_t capacity)
+{
+  const apr_size_t status = LZ4F_flush(cctx->ctx,
+                                       output, capacity,
+                                       &cctx->options);
+  return svn_error_trace(check_compress_status(length, status, 0));
+}
+
+svn_error_t *
+svn_lz4__compress_end(apr_size_t *length,
+                      svn_lz4__compress_ctx_t *cctx,
+                      void *output, apr_size_t capacity)
+{
+  const apr_size_t status = LZ4F_compressEnd(cctx->ctx,
+                                             output, capacity,
+                                             &cctx->options);
+  svn_error_t *const err = check_compress_status(length, status, 0);
+  if (!err)
+    cctx->started = FALSE;
+  return svn_error_trace(err);
+}
+
+/*
+ * Streamy decompression
+ */
+
+struct svn_lz4__decompress_ctx_t
+{
+  LZ4F_dctx* ctx;
+  LZ4F_decompressOptions_t options;
+};
+
+static apr_status_t free_lz4_dctx(void *data)
+{
+  svn_lz4__decompress_ctx_t *const dctx = data;
+  const LZ4F_errorCode_t code = LZ4F_freeDecompressionContext(dctx->ctx);
+  if (LZ4F_isError(code))
+    return SVN_ERR_LZ4_DECOMPRESSION_FAILED;
+  return APR_SUCCESS;
+}
+
+svn_error_t *
+svn_lz4__decompress_create(svn_lz4__decompress_ctx_t **dctx_out,
+                           svn_boolean_t stable_output,
+                           apr_pool_t *pool)
+{
+  LZ4F_dctx* ctx;
+  svn_lz4__decompress_ctx_t *dctx;
+  LZ4F_errorCode_t code = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION);
+
+  if (LZ4F_isError(code))
+    return svn_error_createf(SVN_ERR_LZ4_DECOMPRESSION_FAILED, NULL,
+                             _("Create LZ4 decompression context: %s"),
+                             LZ4F_getErrorName(code));
+
+  dctx = apr_pcalloc(pool, sizeof(*dctx));
+  dctx->ctx = ctx;
+  dctx->options.stableDst = !!stable_output;
+  apr_pool_cleanup_register(pool, dctx, free_lz4_dctx, apr_pool_cleanup_null);
+  *dctx_out = dctx;
+  return SVN_NO_ERROR;
+}
+
+svn_error_t *
+svn_lz4__decompress(apr_size_t *size_hint,
+                    svn_lz4__decompress_ctx_t *dctx,
+                    void *output, apr_size_t *output_size,
+                    const void *input, apr_size_t *input_size)
+{
+  const apr_size_t status = LZ4F_decompress(dctx->ctx,
+                                            output, output_size,
+                                            input, input_size,
+                                            &dctx->options);
+
+  if (LZ4F_isError(status))
+    return svn_error_createf(SVN_ERR_LZ4_DECOMPRESSION_FAILED, NULL,
+                             _("LZ4 decompress: %s"),
+                             LZ4F_getErrorName(status));
+
+  *size_hint = status;
+  return SVN_NO_ERROR;
+}
+
+/*
+ * Library version
+ */
+
 const char *
 svn_lz4__compiled_version(void)
 {