Mercurial > repos > rliterman > csp2

annotate CSP2/CSP2_env/env-d9b9114564458d9d-741b3de822f2aaca6c6caa4325c4afce/include/zstd.h @ 69:33d812a61356

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit 2e9511a184a1ca667c7be0c6321a36dc4e3d116d
author jpayne
date Tue, 18 Mar 2025 17:55:14 -0400
parents
children
rev   line source
jpayne@69 1 /*
jpayne@69 2 * Copyright (c) Meta Platforms, Inc. and affiliates.
jpayne@69 3 * All rights reserved.
jpayne@69 4 *
jpayne@69 5 * This source code is licensed under both the BSD-style license (found in the
jpayne@69 6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
jpayne@69 7 * in the COPYING file in the root directory of this source tree).
jpayne@69 8 * You may select, at your option, one of the above-listed licenses.
jpayne@69 9 */
jpayne@69 10 #if defined (__cplusplus)
jpayne@69 11 extern "C" {
jpayne@69 12 #endif
jpayne@69 13
jpayne@69 14 #ifndef ZSTD_H_235446
jpayne@69 15 #define ZSTD_H_235446
jpayne@69 16
jpayne@69 17 /* ====== Dependencies ======*/
jpayne@69 18 #include <limits.h> /* INT_MAX */
jpayne@69 19 #include <stddef.h> /* size_t */
jpayne@69 20
jpayne@69 21
jpayne@69 22 /* ===== ZSTDLIB_API : control library symbols visibility ===== */
jpayne@69 23 #ifndef ZSTDLIB_VISIBLE
jpayne@69 24 /* Backwards compatibility with old macro name */
jpayne@69 25 # ifdef ZSTDLIB_VISIBILITY
jpayne@69 26 # define ZSTDLIB_VISIBLE ZSTDLIB_VISIBILITY
jpayne@69 27 # elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
jpayne@69 28 # define ZSTDLIB_VISIBLE __attribute__ ((visibility ("default")))
jpayne@69 29 # else
jpayne@69 30 # define ZSTDLIB_VISIBLE
jpayne@69 31 # endif
jpayne@69 32 #endif
jpayne@69 33
jpayne@69 34 #ifndef ZSTDLIB_HIDDEN
jpayne@69 35 # if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
jpayne@69 36 # define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden")))
jpayne@69 37 # else
jpayne@69 38 # define ZSTDLIB_HIDDEN
jpayne@69 39 # endif
jpayne@69 40 #endif
jpayne@69 41
jpayne@69 42 #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
jpayne@69 43 # define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBLE
jpayne@69 44 #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
jpayne@69 45 # define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
jpayne@69 46 #else
jpayne@69 47 # define ZSTDLIB_API ZSTDLIB_VISIBLE
jpayne@69 48 #endif
jpayne@69 49
jpayne@69 50 /* Deprecation warnings :
jpayne@69 51 * Should these warnings be a problem, it is generally possible to disable them,
jpayne@69 52 * typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual.
jpayne@69 53 * Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS.
jpayne@69 54 */
jpayne@69 55 #ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS
jpayne@69 56 # define ZSTD_DEPRECATED(message) /* disable deprecation warnings */
jpayne@69 57 #else
jpayne@69 58 # if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
jpayne@69 59 # define ZSTD_DEPRECATED(message) [[deprecated(message)]]
jpayne@69 60 # elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__)
jpayne@69 61 # define ZSTD_DEPRECATED(message) __attribute__((deprecated(message)))
jpayne@69 62 # elif defined(__GNUC__) && (__GNUC__ >= 3)
jpayne@69 63 # define ZSTD_DEPRECATED(message) __attribute__((deprecated))
jpayne@69 64 # elif defined(_MSC_VER)
jpayne@69 65 # define ZSTD_DEPRECATED(message) __declspec(deprecated(message))
jpayne@69 66 # else
jpayne@69 67 # pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler")
jpayne@69 68 # define ZSTD_DEPRECATED(message)
jpayne@69 69 # endif
jpayne@69 70 #endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */
jpayne@69 71
jpayne@69 72
jpayne@69 73 /*******************************************************************************
jpayne@69 74 Introduction
jpayne@69 75
jpayne@69 76 zstd, short for Zstandard, is a fast lossless compression algorithm, targeting
jpayne@69 77 real-time compression scenarios at zlib-level and better compression ratios.
jpayne@69 78 The zstd compression library provides in-memory compression and decompression
jpayne@69 79 functions.
jpayne@69 80
jpayne@69 81 The library supports regular compression levels from 1 up to ZSTD_maxCLevel(),
jpayne@69 82 which is currently 22. Levels >= 20, labeled `--ultra`, should be used with
jpayne@69 83 caution, as they require more memory. The library also offers negative
jpayne@69 84 compression levels, which extend the range of speed vs. ratio preferences.
jpayne@69 85 The lower the level, the faster the speed (at the cost of compression).
jpayne@69 86
jpayne@69 87 Compression can be done in:
jpayne@69 88 - a single step (described as Simple API)
jpayne@69 89 - a single step, reusing a context (described as Explicit context)
jpayne@69 90 - unbounded multiple steps (described as Streaming compression)
jpayne@69 91
jpayne@69 92 The compression ratio achievable on small data can be highly improved using
jpayne@69 93 a dictionary. Dictionary compression can be performed in:
jpayne@69 94 - a single step (described as Simple dictionary API)
jpayne@69 95 - a single step, reusing a dictionary (described as Bulk-processing
jpayne@69 96 dictionary API)
jpayne@69 97
jpayne@69 98 Advanced experimental functions can be accessed using
jpayne@69 99 `#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h.
jpayne@69 100
jpayne@69 101 Advanced experimental APIs should never be used with a dynamically-linked
jpayne@69 102 library. They are not "stable"; their definitions or signatures may change in
jpayne@69 103 the future. Only static linking is allowed.
jpayne@69 104 *******************************************************************************/
jpayne@69 105
jpayne@69 106 /*------ Version ------*/
jpayne@69 107 #define ZSTD_VERSION_MAJOR 1
jpayne@69 108 #define ZSTD_VERSION_MINOR 5
jpayne@69 109 #define ZSTD_VERSION_RELEASE 6
jpayne@69 110 #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
jpayne@69 111
jpayne@69 112 /*! ZSTD_versionNumber() :
jpayne@69 113 * Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE). */
jpayne@69 114 ZSTDLIB_API unsigned ZSTD_versionNumber(void);
jpayne@69 115
jpayne@69 116 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
jpayne@69 117 #define ZSTD_QUOTE(str) #str
jpayne@69 118 #define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str)
jpayne@69 119 #define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
jpayne@69 120
jpayne@69 121 /*! ZSTD_versionString() :
jpayne@69 122 * Return runtime library version, like "1.4.5". Requires v1.3.0+. */
jpayne@69 123 ZSTDLIB_API const char* ZSTD_versionString(void);
jpayne@69 124
jpayne@69 125 /* *************************************
jpayne@69 126 * Default constant
jpayne@69 127 ***************************************/
jpayne@69 128 #ifndef ZSTD_CLEVEL_DEFAULT
jpayne@69 129 # define ZSTD_CLEVEL_DEFAULT 3
jpayne@69 130 #endif
jpayne@69 131
jpayne@69 132 /* *************************************
jpayne@69 133 * Constants
jpayne@69 134 ***************************************/
jpayne@69 135
jpayne@69 136 /* All magic numbers are supposed read/written to/from files/memory using little-endian convention */
jpayne@69 137 #define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */
jpayne@69 138 #define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */
jpayne@69 139 #define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */
jpayne@69 140 #define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF0
jpayne@69 141
jpayne@69 142 #define ZSTD_BLOCKSIZELOG_MAX 17
jpayne@69 143 #define ZSTD_BLOCKSIZE_MAX (1<<ZSTD_BLOCKSIZELOG_MAX)
jpayne@69 144
jpayne@69 145
jpayne@69 146 /***************************************
jpayne@69 147 * Simple API
jpayne@69 148 ***************************************/
jpayne@69 149 /*! ZSTD_compress() :
jpayne@69 150 * Compresses `src` content as a single zstd compressed frame into already allocated `dst`.
jpayne@69 151 * NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have
jpayne@69 152 * enough space to successfully compress the data.
jpayne@69 153 * @return : compressed size written into `dst` (<= `dstCapacity),
jpayne@69 154 * or an error code if it fails (which can be tested using ZSTD_isError()). */
jpayne@69 155 ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
jpayne@69 156 const void* src, size_t srcSize,
jpayne@69 157 int compressionLevel);
jpayne@69 158
jpayne@69 159 /*! ZSTD_decompress() :
jpayne@69 160 * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
jpayne@69 161 * `dstCapacity` is an upper bound of originalSize to regenerate.
jpayne@69 162 * If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
jpayne@69 163 * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
jpayne@69 164 * or an errorCode if it fails (which can be tested using ZSTD_isError()). */
jpayne@69 165 ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
jpayne@69 166 const void* src, size_t compressedSize);
jpayne@69 167
jpayne@69 168 /*! ZSTD_getFrameContentSize() : requires v1.3.0+
jpayne@69 169 * `src` should point to the start of a ZSTD encoded frame.
jpayne@69 170 * `srcSize` must be at least as large as the frame header.
jpayne@69 171 * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough.
jpayne@69 172 * @return : - decompressed size of `src` frame content, if known
jpayne@69 173 * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
jpayne@69 174 * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small)
jpayne@69 175 * note 1 : a 0 return value means the frame is valid but "empty".
jpayne@69 176 * note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode.
jpayne@69 177 * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
jpayne@69 178 * In which case, it's necessary to use streaming mode to decompress data.
jpayne@69 179 * Optionally, application can rely on some implicit limit,
jpayne@69 180 * as ZSTD_decompress() only needs an upper bound of decompressed size.
jpayne@69 181 * (For example, data could be necessarily cut into blocks <= 16 KB).
jpayne@69 182 * note 3 : decompressed size is always present when compression is completed using single-pass functions,
jpayne@69 183 * such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict().
jpayne@69 184 * note 4 : decompressed size can be very large (64-bits value),
jpayne@69 185 * potentially larger than what local system can handle as a single memory segment.
jpayne@69 186 * In which case, it's necessary to use streaming mode to decompress data.
jpayne@69 187 * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified.
jpayne@69 188 * Always ensure return value fits within application's authorized limits.
jpayne@69 189 * Each application can set its own limits.
jpayne@69 190 * note 6 : This function replaces ZSTD_getDecompressedSize() */
jpayne@69 191 #define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
jpayne@69 192 #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
jpayne@69 193 ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
jpayne@69 194
jpayne@69 195 /*! ZSTD_getDecompressedSize() :
jpayne@69 196 * NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize().
jpayne@69 197 * Both functions work the same way, but ZSTD_getDecompressedSize() blends
jpayne@69 198 * "empty", "unknown" and "error" results to the same return value (0),
jpayne@69 199 * while ZSTD_getFrameContentSize() gives them separate return values.
jpayne@69 200 * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */
jpayne@69 201 ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize")
jpayne@69 202 ZSTDLIB_API
jpayne@69 203 unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
jpayne@69 204
jpayne@69 205 /*! ZSTD_findFrameCompressedSize() : Requires v1.4.0+
jpayne@69 206 * `src` should point to the start of a ZSTD frame or skippable frame.
jpayne@69 207 * `srcSize` must be >= first frame size
jpayne@69 208 * @return : the compressed size of the first frame starting at `src`,
jpayne@69 209 * suitable to pass as `srcSize` to `ZSTD_decompress` or similar,
jpayne@69 210 * or an error code if input is invalid */
jpayne@69 211 ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
jpayne@69 212
jpayne@69 213
jpayne@69 214 /*====== Helper functions ======*/
jpayne@69 215 /* ZSTD_compressBound() :
jpayne@69 216 * maximum compressed size in worst case single-pass scenario.
jpayne@69 217 * When invoking `ZSTD_compress()` or any other one-pass compression function,
jpayne@69 218 * it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize)
jpayne@69 219 * as it eliminates one potential failure scenario,
jpayne@69 220 * aka not enough room in dst buffer to write the compressed frame.
jpayne@69 221 * Note : ZSTD_compressBound() itself can fail, if @srcSize > ZSTD_MAX_INPUT_SIZE .
jpayne@69 222 * In which case, ZSTD_compressBound() will return an error code
jpayne@69 223 * which can be tested using ZSTD_isError().
jpayne@69 224 *
jpayne@69 225 * ZSTD_COMPRESSBOUND() :
jpayne@69 226 * same as ZSTD_compressBound(), but as a macro.
jpayne@69 227 * It can be used to produce constants, which can be useful for static allocation,
jpayne@69 228 * for example to size a static array on stack.
jpayne@69 229 * Will produce constant value 0 if srcSize too large.
jpayne@69 230 */
jpayne@69 231 #define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00ULL : 0xFF00FF00U)
jpayne@69 232 #define ZSTD_COMPRESSBOUND(srcSize) (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
jpayne@69 233 ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
jpayne@69 234 /* ZSTD_isError() :
jpayne@69 235 * Most ZSTD_* functions returning a size_t value can be tested for error,
jpayne@69 236 * using ZSTD_isError().
jpayne@69 237 * @return 1 if error, 0 otherwise
jpayne@69 238 */
jpayne@69 239 ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
jpayne@69 240 ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */
jpayne@69 241 ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed, requires v1.4.0+ */
jpayne@69 242 ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */
jpayne@69 243 ZSTDLIB_API int ZSTD_defaultCLevel(void); /*!< default compression level, specified by ZSTD_CLEVEL_DEFAULT, requires v1.5.0+ */
jpayne@69 244
jpayne@69 245
jpayne@69 246 /***************************************
jpayne@69 247 * Explicit context
jpayne@69 248 ***************************************/
jpayne@69 249 /*= Compression context
jpayne@69 250 * When compressing many times,
jpayne@69 251 * it is recommended to allocate a context just once,
jpayne@69 252 * and reuse it for each successive compression operation.
jpayne@69 253 * This will make workload friendlier for system's memory.
jpayne@69 254 * Note : re-using context is just a speed / resource optimization.
jpayne@69 255 * It doesn't change the compression ratio, which remains identical.
jpayne@69 256 * Note 2 : In multi-threaded environments,
jpayne@69 257 * use one different context per thread for parallel execution.
jpayne@69 258 */
jpayne@69 259 typedef struct ZSTD_CCtx_s ZSTD_CCtx;
jpayne@69 260 ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
jpayne@69 261 ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /* accept NULL pointer */
jpayne@69 262
jpayne@69 263 /*! ZSTD_compressCCtx() :
jpayne@69 264 * Same as ZSTD_compress(), using an explicit ZSTD_CCtx.
jpayne@69 265 * Important : in order to mirror `ZSTD_compress()` behavior,
jpayne@69 266 * this function compresses at the requested compression level,
jpayne@69 267 * __ignoring any other advanced parameter__ .
jpayne@69 268 * If any advanced parameter was set using the advanced API,
jpayne@69 269 * they will all be reset. Only `compressionLevel` remains.
jpayne@69 270 */
jpayne@69 271 ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
jpayne@69 272 void* dst, size_t dstCapacity,
jpayne@69 273 const void* src, size_t srcSize,
jpayne@69 274 int compressionLevel);
jpayne@69 275
jpayne@69 276 /*= Decompression context
jpayne@69 277 * When decompressing many times,
jpayne@69 278 * it is recommended to allocate a context only once,
jpayne@69 279 * and reuse it for each successive compression operation.
jpayne@69 280 * This will make workload friendlier for system's memory.
jpayne@69 281 * Use one context per thread for parallel execution. */
jpayne@69 282 typedef struct ZSTD_DCtx_s ZSTD_DCtx;
jpayne@69 283 ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
jpayne@69 284 ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /* accept NULL pointer */
jpayne@69 285
jpayne@69 286 /*! ZSTD_decompressDCtx() :
jpayne@69 287 * Same as ZSTD_decompress(),
jpayne@69 288 * requires an allocated ZSTD_DCtx.
jpayne@69 289 * Compatible with sticky parameters (see below).
jpayne@69 290 */
jpayne@69 291 ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx,
jpayne@69 292 void* dst, size_t dstCapacity,
jpayne@69 293 const void* src, size_t srcSize);
jpayne@69 294
jpayne@69 295
jpayne@69 296 /*********************************************
jpayne@69 297 * Advanced compression API (Requires v1.4.0+)
jpayne@69 298 **********************************************/
jpayne@69 299
jpayne@69 300 /* API design :
jpayne@69 301 * Parameters are pushed one by one into an existing context,
jpayne@69 302 * using ZSTD_CCtx_set*() functions.
jpayne@69 303 * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame.
jpayne@69 304 * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` !
jpayne@69 305 * __They do not apply to one-shot variants such as ZSTD_compressCCtx()__ .
jpayne@69 306 *
jpayne@69 307 * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset().
jpayne@69 308 *
jpayne@69 309 * This API supersedes all other "advanced" API entry points in the experimental section.
jpayne@69 310 * In the future, we expect to remove API entry points from experimental which are redundant with this API.
jpayne@69 311 */
jpayne@69 312
jpayne@69 313
jpayne@69 314 /* Compression strategies, listed from fastest to strongest */
jpayne@69 315 typedef enum { ZSTD_fast=1,
jpayne@69 316 ZSTD_dfast=2,
jpayne@69 317 ZSTD_greedy=3,
jpayne@69 318 ZSTD_lazy=4,
jpayne@69 319 ZSTD_lazy2=5,
jpayne@69 320 ZSTD_btlazy2=6,
jpayne@69 321 ZSTD_btopt=7,
jpayne@69 322 ZSTD_btultra=8,
jpayne@69 323 ZSTD_btultra2=9
jpayne@69 324 /* note : new strategies _might_ be added in the future.
jpayne@69 325 Only the order (from fast to strong) is guaranteed */
jpayne@69 326 } ZSTD_strategy;
jpayne@69 327
jpayne@69 328 typedef enum {
jpayne@69 329
jpayne@69 330 /* compression parameters
jpayne@69 331 * Note: When compressing with a ZSTD_CDict these parameters are superseded
jpayne@69 332 * by the parameters used to construct the ZSTD_CDict.
jpayne@69 333 * See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */
jpayne@69 334 ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table.
jpayne@69 335 * Note that exact compression parameters are dynamically determined,
jpayne@69 336 * depending on both compression level and srcSize (when known).
jpayne@69 337 * Default level is ZSTD_CLEVEL_DEFAULT==3.
jpayne@69 338 * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.
jpayne@69 339 * Note 1 : it's possible to pass a negative compression level.
jpayne@69 340 * Note 2 : setting a level does not automatically set all other compression parameters
jpayne@69 341 * to default. Setting this will however eventually dynamically impact the compression
jpayne@69 342 * parameters which have not been manually set. The manually set
jpayne@69 343 * ones will 'stick'. */
jpayne@69 344 /* Advanced compression parameters :
jpayne@69 345 * It's possible to pin down compression parameters to some specific values.
jpayne@69 346 * In which case, these values are no longer dynamically selected by the compressor */
jpayne@69 347 ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2.
jpayne@69 348 * This will set a memory budget for streaming decompression,
jpayne@69 349 * with larger values requiring more memory
jpayne@69 350 * and typically compressing more.
jpayne@69 351 * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
jpayne@69 352 * Special: value 0 means "use default windowLog".
jpayne@69 353 * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT
jpayne@69 354 * requires explicitly allowing such size at streaming decompression stage. */
jpayne@69 355 ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2.
jpayne@69 356 * Resulting memory usage is (1 << (hashLog+2)).
jpayne@69 357 * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
jpayne@69 358 * Larger tables improve compression ratio of strategies <= dFast,
jpayne@69 359 * and improve speed of strategies > dFast.
jpayne@69 360 * Special: value 0 means "use default hashLog". */
jpayne@69 361 ZSTD_c_chainLog=103, /* Size of the multi-probe search table, as a power of 2.
jpayne@69 362 * Resulting memory usage is (1 << (chainLog+2)).
jpayne@69 363 * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX.
jpayne@69 364 * Larger tables result in better and slower compression.
jpayne@69 365 * This parameter is useless for "fast" strategy.
jpayne@69 366 * It's still useful when using "dfast" strategy,
jpayne@69 367 * in which case it defines a secondary probe table.
jpayne@69 368 * Special: value 0 means "use default chainLog". */
jpayne@69 369 ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2.
jpayne@69 370 * More attempts result in better and slower compression.
jpayne@69 371 * This parameter is useless for "fast" and "dFast" strategies.
jpayne@69 372 * Special: value 0 means "use default searchLog". */
jpayne@69 373 ZSTD_c_minMatch=105, /* Minimum size of searched matches.
jpayne@69 374 * Note that Zstandard can still find matches of smaller size,
jpayne@69 375 * it just tweaks its search algorithm to look for this size and larger.
jpayne@69 376 * Larger values increase compression and decompression speed, but decrease ratio.
jpayne@69 377 * Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX.
jpayne@69 378 * Note that currently, for all strategies < btopt, effective minimum is 4.
jpayne@69 379 * , for all strategies > fast, effective maximum is 6.
jpayne@69 380 * Special: value 0 means "use default minMatchLength". */
jpayne@69 381 ZSTD_c_targetLength=106, /* Impact of this field depends on strategy.
jpayne@69 382 * For strategies btopt, btultra & btultra2:
jpayne@69 383 * Length of Match considered "good enough" to stop search.
jpayne@69 384 * Larger values make compression stronger, and slower.
jpayne@69 385 * For strategy fast:
jpayne@69 386 * Distance between match sampling.
jpayne@69 387 * Larger values make compression faster, and weaker.
jpayne@69 388 * Special: value 0 means "use default targetLength". */
jpayne@69 389 ZSTD_c_strategy=107, /* See ZSTD_strategy enum definition.
jpayne@69 390 * The higher the value of selected strategy, the more complex it is,
jpayne@69 391 * resulting in stronger and slower compression.
jpayne@69 392 * Special: value 0 means "use default strategy". */
jpayne@69 393
jpayne@69 394 ZSTD_c_targetCBlockSize=130, /* v1.5.6+
jpayne@69 395 * Attempts to fit compressed block size into approximatively targetCBlockSize.
jpayne@69 396 * Bound by ZSTD_TARGETCBLOCKSIZE_MIN and ZSTD_TARGETCBLOCKSIZE_MAX.
jpayne@69 397 * Note that it's not a guarantee, just a convergence target (default:0).
jpayne@69 398 * No target when targetCBlockSize == 0.
jpayne@69 399 * This is helpful in low bandwidth streaming environments to improve end-to-end latency,
jpayne@69 400 * when a client can make use of partial documents (a prominent example being Chrome).
jpayne@69 401 * Note: this parameter is stable since v1.5.6.
jpayne@69 402 * It was present as an experimental parameter in earlier versions,
jpayne@69 403 * but it's not recommended using it with earlier library versions
jpayne@69 404 * due to massive performance regressions.
jpayne@69 405 */
jpayne@69 406 /* LDM mode parameters */
jpayne@69 407 ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching.
jpayne@69 408 * This parameter is designed to improve compression ratio
jpayne@69 409 * for large inputs, by finding large matches at long distance.
jpayne@69 410 * It increases memory usage and window size.
jpayne@69 411 * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB
jpayne@69 412 * except when expressly set to a different value.
jpayne@69 413 * Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and
jpayne@69 414 * compression strategy >= ZSTD_btopt (== compression level 16+) */
jpayne@69 415 ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2.
jpayne@69 416 * Larger values increase memory usage and compression ratio,
jpayne@69 417 * but decrease compression speed.
jpayne@69 418 * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX
jpayne@69 419 * default: windowlog - 7.
jpayne@69 420 * Special: value 0 means "automatically determine hashlog". */
jpayne@69 421 ZSTD_c_ldmMinMatch=162, /* Minimum match size for long distance matcher.
jpayne@69 422 * Larger/too small values usually decrease compression ratio.
jpayne@69 423 * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX.
jpayne@69 424 * Special: value 0 means "use default value" (default: 64). */
jpayne@69 425 ZSTD_c_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution.
jpayne@69 426 * Larger values improve collision resolution but decrease compression speed.
jpayne@69 427 * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX.
jpayne@69 428 * Special: value 0 means "use default value" (default: 3). */
jpayne@69 429 ZSTD_c_ldmHashRateLog=164, /* Frequency of inserting/looking up entries into the LDM hash table.
jpayne@69 430 * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN).
jpayne@69 431 * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage.
jpayne@69 432 * Larger values improve compression speed.
jpayne@69 433 * Deviating far from default value will likely result in a compression ratio decrease.
jpayne@69 434 * Special: value 0 means "automatically determine hashRateLog". */
jpayne@69 435
jpayne@69 436 /* frame parameters */
jpayne@69 437 ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
jpayne@69 438 * Content size must be known at the beginning of compression.
jpayne@69 439 * This is automatically the case when using ZSTD_compress2(),
jpayne@69 440 * For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */
jpayne@69 441 ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */
jpayne@69 442 ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */
jpayne@69 443
jpayne@69 444 /* multi-threading parameters */
jpayne@69 445 /* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).
jpayne@69 446 * Otherwise, trying to set any other value than default (0) will be a no-op and return an error.
jpayne@69 447 * In a situation where it's unknown if the linked library supports multi-threading or not,
jpayne@69 448 * setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property.
jpayne@69 449 */
jpayne@69 450 ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel.
jpayne@69 451 * When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() :
jpayne@69 452 * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller,
jpayne@69 453 * while compression is performed in parallel, within worker thread(s).
jpayne@69 454 * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end :
jpayne@69 455 * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call).
jpayne@69 456 * More workers improve speed, but also increase memory usage.
jpayne@69 457 * Default value is `0`, aka "single-threaded mode" : no worker is spawned,
jpayne@69 458 * compression is performed inside Caller's thread, and all invocations are blocking */
jpayne@69 459 ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1.
jpayne@69 460 * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
jpayne@69 461 * 0 means default, which is dynamically determined based on compression parameters.
jpayne@69 462 * Job size must be a minimum of overlap size, or ZSTDMT_JOBSIZE_MIN (= 512 KB), whichever is largest.
jpayne@69 463 * The minimum size is automatically and transparently enforced. */
jpayne@69 464 ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size.
jpayne@69 465 * The overlap size is an amount of data reloaded from previous job at the beginning of a new job.
jpayne@69 466 * It helps preserve compression ratio, while each job is compressed in parallel.
jpayne@69 467 * This value is enforced only when nbWorkers >= 1.
jpayne@69 468 * Larger values increase compression ratio, but decrease speed.
jpayne@69 469 * Possible values range from 0 to 9 :
jpayne@69 470 * - 0 means "default" : value will be determined by the library, depending on strategy
jpayne@69 471 * - 1 means "no overlap"
jpayne@69 472 * - 9 means "full overlap", using a full window size.
jpayne@69 473 * Each intermediate rank increases/decreases load size by a factor 2 :
jpayne@69 474 * 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:no overlap; 0:default
jpayne@69 475 * default value varies between 6 and 9, depending on strategy */
jpayne@69 476
jpayne@69 477 /* note : additional experimental parameters are also available
jpayne@69 478 * within the experimental section of the API.
jpayne@69 479 * At the time of this writing, they include :
jpayne@69 480 * ZSTD_c_rsyncable
jpayne@69 481 * ZSTD_c_format
jpayne@69 482 * ZSTD_c_forceMaxWindow
jpayne@69 483 * ZSTD_c_forceAttachDict
jpayne@69 484 * ZSTD_c_literalCompressionMode
jpayne@69 485 * ZSTD_c_srcSizeHint
jpayne@69 486 * ZSTD_c_enableDedicatedDictSearch
jpayne@69 487 * ZSTD_c_stableInBuffer
jpayne@69 488 * ZSTD_c_stableOutBuffer
jpayne@69 489 * ZSTD_c_blockDelimiters
jpayne@69 490 * ZSTD_c_validateSequences
jpayne@69 491 * ZSTD_c_useBlockSplitter
jpayne@69 492 * ZSTD_c_useRowMatchFinder
jpayne@69 493 * ZSTD_c_prefetchCDictTables
jpayne@69 494 * ZSTD_c_enableSeqProducerFallback
jpayne@69 495 * ZSTD_c_maxBlockSize
jpayne@69 496 * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
jpayne@69 497 * note : never ever use experimentalParam? names directly;
jpayne@69 498 * also, the enums values themselves are unstable and can still change.
jpayne@69 499 */
jpayne@69 500 ZSTD_c_experimentalParam1=500,
jpayne@69 501 ZSTD_c_experimentalParam2=10,
jpayne@69 502 ZSTD_c_experimentalParam3=1000,
jpayne@69 503 ZSTD_c_experimentalParam4=1001,
jpayne@69 504 ZSTD_c_experimentalParam5=1002,
jpayne@69 505 /* was ZSTD_c_experimentalParam6=1003; is now ZSTD_c_targetCBlockSize */
jpayne@69 506 ZSTD_c_experimentalParam7=1004,
jpayne@69 507 ZSTD_c_experimentalParam8=1005,
jpayne@69 508 ZSTD_c_experimentalParam9=1006,
jpayne@69 509 ZSTD_c_experimentalParam10=1007,
jpayne@69 510 ZSTD_c_experimentalParam11=1008,
jpayne@69 511 ZSTD_c_experimentalParam12=1009,
jpayne@69 512 ZSTD_c_experimentalParam13=1010,
jpayne@69 513 ZSTD_c_experimentalParam14=1011,
jpayne@69 514 ZSTD_c_experimentalParam15=1012,
jpayne@69 515 ZSTD_c_experimentalParam16=1013,
jpayne@69 516 ZSTD_c_experimentalParam17=1014,
jpayne@69 517 ZSTD_c_experimentalParam18=1015,
jpayne@69 518 ZSTD_c_experimentalParam19=1016
jpayne@69 519 } ZSTD_cParameter;
jpayne@69 520
jpayne@69 521 typedef struct {
jpayne@69 522 size_t error;
jpayne@69 523 int lowerBound;
jpayne@69 524 int upperBound;
jpayne@69 525 } ZSTD_bounds;
jpayne@69 526
jpayne@69 527 /*! ZSTD_cParam_getBounds() :
jpayne@69 528 * All parameters must belong to an interval with lower and upper bounds,
jpayne@69 529 * otherwise they will either trigger an error or be automatically clamped.
jpayne@69 530 * @return : a structure, ZSTD_bounds, which contains
jpayne@69 531 * - an error status field, which must be tested using ZSTD_isError()
jpayne@69 532 * - lower and upper bounds, both inclusive
jpayne@69 533 */
jpayne@69 534 ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam);
jpayne@69 535
jpayne@69 536 /*! ZSTD_CCtx_setParameter() :
jpayne@69 537 * Set one compression parameter, selected by enum ZSTD_cParameter.
jpayne@69 538 * All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds().
jpayne@69 539 * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter).
jpayne@69 540 * Setting a parameter is generally only possible during frame initialization (before starting compression).
jpayne@69 541 * Exception : when using multi-threading mode (nbWorkers >= 1),
jpayne@69 542 * the following parameters can be updated _during_ compression (within same frame):
jpayne@69 543 * => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy.
jpayne@69 544 * new parameters will be active for next job only (after a flush()).
jpayne@69 545 * @return : an error code (which can be tested using ZSTD_isError()).
jpayne@69 546 */
jpayne@69 547 ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value);
jpayne@69 548
jpayne@69 549 /*! ZSTD_CCtx_setPledgedSrcSize() :
jpayne@69 550 * Total input data size to be compressed as a single frame.
jpayne@69 551 * Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag.
jpayne@69 552 * This value will also be controlled at end of frame, and trigger an error if not respected.
jpayne@69 553 * @result : 0, or an error code (which can be tested with ZSTD_isError()).
jpayne@69 554 * Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame.
jpayne@69 555 * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
jpayne@69 556 * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame.
jpayne@69 557 * Note 2 : pledgedSrcSize is only valid once, for the next frame.
jpayne@69 558 * It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN.
jpayne@69 559 * Note 3 : Whenever all input data is provided and consumed in a single round,
jpayne@69 560 * for example with ZSTD_compress2(),
jpayne@69 561 * or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end),
jpayne@69 562 * this value is automatically overridden by srcSize instead.
jpayne@69 563 */
jpayne@69 564 ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
jpayne@69 565
jpayne@69 566 typedef enum {
jpayne@69 567 ZSTD_reset_session_only = 1,
jpayne@69 568 ZSTD_reset_parameters = 2,
jpayne@69 569 ZSTD_reset_session_and_parameters = 3
jpayne@69 570 } ZSTD_ResetDirective;
jpayne@69 571
jpayne@69 572 /*! ZSTD_CCtx_reset() :
jpayne@69 573 * There are 2 different things that can be reset, independently or jointly :
jpayne@69 574 * - The session : will stop compressing current frame, and make CCtx ready to start a new one.
jpayne@69 575 * Useful after an error, or to interrupt any ongoing compression.
jpayne@69 576 * Any internal data not yet flushed is cancelled.
jpayne@69 577 * Compression parameters and dictionary remain unchanged.
jpayne@69 578 * They will be used to compress next frame.
jpayne@69 579 * Resetting session never fails.
jpayne@69 580 * - The parameters : changes all parameters back to "default".
jpayne@69 581 * This also removes any reference to any dictionary or external sequence producer.
jpayne@69 582 * Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing)
jpayne@69 583 * otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError())
jpayne@69 584 * - Both : similar to resetting the session, followed by resetting parameters.
jpayne@69 585 */
jpayne@69 586 ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset);
jpayne@69 587
jpayne@69 588 /*! ZSTD_compress2() :
jpayne@69 589 * Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API.
jpayne@69 590 * (note that this entry point doesn't even expose a compression level parameter).
jpayne@69 591 * ZSTD_compress2() always starts a new frame.
jpayne@69 592 * Should cctx hold data from a previously unfinished frame, everything about it is forgotten.
jpayne@69 593 * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
jpayne@69 594 * - The function is always blocking, returns when compression is completed.
jpayne@69 595 * NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have
jpayne@69 596 * enough space to successfully compress the data, though it is possible it fails for other reasons.
jpayne@69 597 * @return : compressed size written into `dst` (<= `dstCapacity),
jpayne@69 598 * or an error code if it fails (which can be tested using ZSTD_isError()).
jpayne@69 599 */
jpayne@69 600 ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx,
jpayne@69 601 void* dst, size_t dstCapacity,
jpayne@69 602 const void* src, size_t srcSize);
jpayne@69 603
jpayne@69 604
jpayne@69 605 /***********************************************
jpayne@69 606 * Advanced decompression API (Requires v1.4.0+)
jpayne@69 607 ************************************************/
jpayne@69 608
jpayne@69 609 /* The advanced API pushes parameters one by one into an existing DCtx context.
jpayne@69 610 * Parameters are sticky, and remain valid for all following frames
jpayne@69 611 * using the same DCtx context.
jpayne@69 612 * It's possible to reset parameters to default values using ZSTD_DCtx_reset().
jpayne@69 613 * Note : This API is compatible with existing ZSTD_decompressDCtx() and ZSTD_decompressStream().
jpayne@69 614 * Therefore, no new decompression function is necessary.
jpayne@69 615 */
jpayne@69 616
jpayne@69 617 typedef enum {
jpayne@69 618
jpayne@69 619 ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which
jpayne@69 620 * the streaming API will refuse to allocate memory buffer
jpayne@69 621 * in order to protect the host from unreasonable memory requirements.
jpayne@69 622 * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
jpayne@69 623 * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT).
jpayne@69 624 * Special: value 0 means "use default maximum windowLog". */
jpayne@69 625
jpayne@69 626 /* note : additional experimental parameters are also available
jpayne@69 627 * within the experimental section of the API.
jpayne@69 628 * At the time of this writing, they include :
jpayne@69 629 * ZSTD_d_format
jpayne@69 630 * ZSTD_d_stableOutBuffer
jpayne@69 631 * ZSTD_d_forceIgnoreChecksum
jpayne@69 632 * ZSTD_d_refMultipleDDicts
jpayne@69 633 * ZSTD_d_disableHuffmanAssembly
jpayne@69 634 * ZSTD_d_maxBlockSize
jpayne@69 635 * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
jpayne@69 636 * note : never ever use experimentalParam? names directly
jpayne@69 637 */
jpayne@69 638 ZSTD_d_experimentalParam1=1000,
jpayne@69 639 ZSTD_d_experimentalParam2=1001,
jpayne@69 640 ZSTD_d_experimentalParam3=1002,
jpayne@69 641 ZSTD_d_experimentalParam4=1003,
jpayne@69 642 ZSTD_d_experimentalParam5=1004,
jpayne@69 643 ZSTD_d_experimentalParam6=1005
jpayne@69 644
jpayne@69 645 } ZSTD_dParameter;
jpayne@69 646
jpayne@69 647 /*! ZSTD_dParam_getBounds() :
jpayne@69 648 * All parameters must belong to an interval with lower and upper bounds,
jpayne@69 649 * otherwise they will either trigger an error or be automatically clamped.
jpayne@69 650 * @return : a structure, ZSTD_bounds, which contains
jpayne@69 651 * - an error status field, which must be tested using ZSTD_isError()
jpayne@69 652 * - both lower and upper bounds, inclusive
jpayne@69 653 */
jpayne@69 654 ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam);
jpayne@69 655
jpayne@69 656 /*! ZSTD_DCtx_setParameter() :
jpayne@69 657 * Set one compression parameter, selected by enum ZSTD_dParameter.
jpayne@69 658 * All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds().
jpayne@69 659 * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter).
jpayne@69 660 * Setting a parameter is only possible during frame initialization (before starting decompression).
jpayne@69 661 * @return : 0, or an error code (which can be tested using ZSTD_isError()).
jpayne@69 662 */
jpayne@69 663 ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value);
jpayne@69 664
jpayne@69 665 /*! ZSTD_DCtx_reset() :
jpayne@69 666 * Return a DCtx to clean state.
jpayne@69 667 * Session and parameters can be reset jointly or separately.
jpayne@69 668 * Parameters can only be reset when no active frame is being decompressed.
jpayne@69 669 * @return : 0, or an error code, which can be tested with ZSTD_isError()
jpayne@69 670 */
jpayne@69 671 ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset);
jpayne@69 672
jpayne@69 673
jpayne@69 674 /****************************
jpayne@69 675 * Streaming
jpayne@69 676 ****************************/
jpayne@69 677
jpayne@69 678 typedef struct ZSTD_inBuffer_s {
jpayne@69 679 const void* src; /**< start of input buffer */
jpayne@69 680 size_t size; /**< size of input buffer */
jpayne@69 681 size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */
jpayne@69 682 } ZSTD_inBuffer;
jpayne@69 683
jpayne@69 684 typedef struct ZSTD_outBuffer_s {
jpayne@69 685 void* dst; /**< start of output buffer */
jpayne@69 686 size_t size; /**< size of output buffer */
jpayne@69 687 size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */
jpayne@69 688 } ZSTD_outBuffer;
jpayne@69 689
jpayne@69 690
jpayne@69 691
jpayne@69 692 /*-***********************************************************************
jpayne@69 693 * Streaming compression - HowTo
jpayne@69 694 *
jpayne@69 695 * A ZSTD_CStream object is required to track streaming operation.
jpayne@69 696 * Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
jpayne@69 697 * ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
jpayne@69 698 * It is recommended to reuse ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory.
jpayne@69 699 *
jpayne@69 700 * For parallel execution, use one separate ZSTD_CStream per thread.
jpayne@69 701 *
jpayne@69 702 * note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing.
jpayne@69 703 *
jpayne@69 704 * Parameters are sticky : when starting a new compression on the same context,
jpayne@69 705 * it will reuse the same sticky parameters as previous compression session.
jpayne@69 706 * When in doubt, it's recommended to fully initialize the context before usage.
jpayne@69 707 * Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(),
jpayne@69 708 * ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to
jpayne@69 709 * set more specific parameters, the pledged source size, or load a dictionary.
jpayne@69 710 *
jpayne@69 711 * Use ZSTD_compressStream2() with ZSTD_e_continue as many times as necessary to
jpayne@69 712 * consume input stream. The function will automatically update both `pos`
jpayne@69 713 * fields within `input` and `output`.
jpayne@69 714 * Note that the function may not consume the entire input, for example, because
jpayne@69 715 * the output buffer is already full, in which case `input.pos < input.size`.
jpayne@69 716 * The caller must check if input has been entirely consumed.
jpayne@69 717 * If not, the caller must make some room to receive more compressed data,
jpayne@69 718 * and then present again remaining input data.
jpayne@69 719 * note: ZSTD_e_continue is guaranteed to make some forward progress when called,
jpayne@69 720 * but doesn't guarantee maximal forward progress. This is especially relevant
jpayne@69 721 * when compressing with multiple threads. The call won't block if it can
jpayne@69 722 * consume some input, but if it can't it will wait for some, but not all,
jpayne@69 723 * output to be flushed.
jpayne@69 724 * @return : provides a minimum amount of data remaining to be flushed from internal buffers
jpayne@69 725 * or an error code, which can be tested using ZSTD_isError().
jpayne@69 726 *
jpayne@69 727 * At any moment, it's possible to flush whatever data might remain stuck within internal buffer,
jpayne@69 728 * using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated.
jpayne@69 729 * Note that, if `output->size` is too small, a single invocation with ZSTD_e_flush might not be enough (return code > 0).
jpayne@69 730 * In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush.
jpayne@69 731 * You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the
jpayne@69 732 * operation.
jpayne@69 733 * note: ZSTD_e_flush will flush as much output as possible, meaning when compressing with multiple threads, it will
jpayne@69 734 * block until the flush is complete or the output buffer is full.
jpayne@69 735 * @return : 0 if internal buffers are entirely flushed,
jpayne@69 736 * >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
jpayne@69 737 * or an error code, which can be tested using ZSTD_isError().
jpayne@69 738 *
jpayne@69 739 * Calling ZSTD_compressStream2() with ZSTD_e_end instructs to finish a frame.
jpayne@69 740 * It will perform a flush and write frame epilogue.
jpayne@69 741 * The epilogue is required for decoders to consider a frame completed.
jpayne@69 742 * flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush.
jpayne@69 743 * You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to
jpayne@69 744 * start a new frame.
jpayne@69 745 * note: ZSTD_e_end will flush as much output as possible, meaning when compressing with multiple threads, it will
jpayne@69 746 * block until the flush is complete or the output buffer is full.
jpayne@69 747 * @return : 0 if frame fully completed and fully flushed,
jpayne@69 748 * >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
jpayne@69 749 * or an error code, which can be tested using ZSTD_isError().
jpayne@69 750 *
jpayne@69 751 * *******************************************************************/
jpayne@69 752
jpayne@69 753 typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */
jpayne@69 754 /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */
jpayne@69 755 /*===== ZSTD_CStream management functions =====*/
jpayne@69 756 ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
jpayne@69 757 ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); /* accept NULL pointer */
jpayne@69 758
jpayne@69 759 /*===== Streaming compression functions =====*/
jpayne@69 760 typedef enum {
jpayne@69 761 ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */
jpayne@69 762 ZSTD_e_flush=1, /* flush any data provided so far,
jpayne@69 763 * it creates (at least) one new block, that can be decoded immediately on reception;
jpayne@69 764 * frame will continue: any future data can still reference previously compressed data, improving compression.
jpayne@69 765 * note : multithreaded compression will block to flush as much output as possible. */
jpayne@69 766 ZSTD_e_end=2 /* flush any remaining data _and_ close current frame.
jpayne@69 767 * note that frame is only closed after compressed data is fully flushed (return value == 0).
jpayne@69 768 * After that point, any additional data starts a new frame.
jpayne@69 769 * note : each frame is independent (does not reference any content from previous frame).
jpayne@69 770 : note : multithreaded compression will block to flush as much output as possible. */
jpayne@69 771 } ZSTD_EndDirective;
jpayne@69 772
jpayne@69 773 /*! ZSTD_compressStream2() : Requires v1.4.0+
jpayne@69 774 * Behaves about the same as ZSTD_compressStream, with additional control on end directive.
jpayne@69 775 * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
jpayne@69 776 * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
jpayne@69 777 * - output->pos must be <= dstCapacity, input->pos must be <= srcSize
jpayne@69 778 * - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
jpayne@69 779 * - endOp must be a valid directive
jpayne@69 780 * - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.
jpayne@69 781 * - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available,
jpayne@69 782 * and then immediately returns, just indicating that there is some data remaining to be flushed.
jpayne@69 783 * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
jpayne@69 784 * - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking.
jpayne@69 785 * - @return provides a minimum amount of data remaining to be flushed from internal buffers
jpayne@69 786 * or an error code, which can be tested using ZSTD_isError().
jpayne@69 787 * if @return != 0, flush is not fully completed, there is still some data left within internal buffers.
jpayne@69 788 * This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers.
jpayne@69 789 * For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed.
jpayne@69 790 * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0),
jpayne@69 791 * only ZSTD_e_end or ZSTD_e_flush operations are allowed.
jpayne@69 792 * Before starting a new compression job, or changing compression parameters,
jpayne@69 793 * it is required to fully flush internal buffers.
jpayne@69 794 * - note: if an operation ends with an error, it may leave @cctx in an undefined state.
jpayne@69 795 * Therefore, it's UB to invoke ZSTD_compressStream2() of ZSTD_compressStream() on such a state.
jpayne@69 796 * In order to be re-employed after an error, a state must be reset,
jpayne@69 797 * which can be done explicitly (ZSTD_CCtx_reset()),
jpayne@69 798 * or is sometimes implied by methods starting a new compression job (ZSTD_initCStream(), ZSTD_compressCCtx())
jpayne@69 799 */
jpayne@69 800 ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
jpayne@69 801 ZSTD_outBuffer* output,
jpayne@69 802 ZSTD_inBuffer* input,
jpayne@69 803 ZSTD_EndDirective endOp);
jpayne@69 804
jpayne@69 805
jpayne@69 806 /* These buffer sizes are softly recommended.
jpayne@69 807 * They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output.
jpayne@69 808 * Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(),
jpayne@69 809 * reducing the amount of memory shuffling and buffering, resulting in minor performance savings.
jpayne@69 810 *
jpayne@69 811 * However, note that these recommendations are from the perspective of a C caller program.
jpayne@69 812 * If the streaming interface is invoked from some other language,
jpayne@69 813 * especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo,
jpayne@69 814 * a major performance rule is to reduce crossing such interface to an absolute minimum.
jpayne@69 815 * It's not rare that performance ends being spent more into the interface, rather than compression itself.
jpayne@69 816 * In which cases, prefer using large buffers, as large as practical,
jpayne@69 817 * for both input and output, to reduce the nb of roundtrips.
jpayne@69 818 */
jpayne@69 819 ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */
jpayne@69 820 ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */
jpayne@69 821
jpayne@69 822
jpayne@69 823 /* *****************************************************************************
jpayne@69 824 * This following is a legacy streaming API, available since v1.0+ .
jpayne@69 825 * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2().
jpayne@69 826 * It is redundant, but remains fully supported.
jpayne@69 827 ******************************************************************************/
jpayne@69 828
jpayne@69 829 /*!
jpayne@69 830 * Equivalent to:
jpayne@69 831 *
jpayne@69 832 * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
jpayne@69 833 * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
jpayne@69 834 * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
jpayne@69 835 *
jpayne@69 836 * Note that ZSTD_initCStream() clears any previously set dictionary. Use the new API
jpayne@69 837 * to compress with a dictionary.
jpayne@69 838 */
jpayne@69 839 ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
jpayne@69 840 /*!
jpayne@69 841 * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).
jpayne@69 842 * NOTE: The return value is different. ZSTD_compressStream() returns a hint for
jpayne@69 843 * the next read size (if non-zero and not an error). ZSTD_compressStream2()
jpayne@69 844 * returns the minimum nb of bytes left to flush (if non-zero and not an error).
jpayne@69 845 */
jpayne@69 846 ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
jpayne@69 847 /*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */
jpayne@69 848 ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
jpayne@69 849 /*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */
jpayne@69 850 ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
jpayne@69 851
jpayne@69 852
jpayne@69 853 /*-***************************************************************************
jpayne@69 854 * Streaming decompression - HowTo
jpayne@69 855 *
jpayne@69 856 * A ZSTD_DStream object is required to track streaming operations.
jpayne@69 857 * Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
jpayne@69 858 * ZSTD_DStream objects can be reused multiple times.
jpayne@69 859 *
jpayne@69 860 * Use ZSTD_initDStream() to start a new decompression operation.
jpayne@69 861 * @return : recommended first input size
jpayne@69 862 * Alternatively, use advanced API to set specific properties.
jpayne@69 863 *
jpayne@69 864 * Use ZSTD_decompressStream() repetitively to consume your input.
jpayne@69 865 * The function will update both `pos` fields.
jpayne@69 866 * If `input.pos < input.size`, some input has not been consumed.
jpayne@69 867 * It's up to the caller to present again remaining data.
jpayne@69 868 * The function tries to flush all data decoded immediately, respecting output buffer size.
jpayne@69 869 * If `output.pos < output.size`, decoder has flushed everything it could.
jpayne@69 870 * But if `output.pos == output.size`, there might be some data left within internal buffers.,
jpayne@69 871 * In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer.
jpayne@69 872 * Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX.
jpayne@69 873 * @return : 0 when a frame is completely decoded and fully flushed,
jpayne@69 874 * or an error code, which can be tested using ZSTD_isError(),
jpayne@69 875 * or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
jpayne@69 876 * the return value is a suggested next input size (just a hint for better latency)
jpayne@69 877 * that will never request more than the remaining frame size.
jpayne@69 878 * *******************************************************************************/
jpayne@69 879
jpayne@69 880 typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */
jpayne@69 881 /* For compatibility with versions <= v1.2.0, prefer differentiating them. */
jpayne@69 882 /*===== ZSTD_DStream management functions =====*/
jpayne@69 883 ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
jpayne@69 884 ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); /* accept NULL pointer */
jpayne@69 885
jpayne@69 886 /*===== Streaming decompression functions =====*/
jpayne@69 887
jpayne@69 888 /*! ZSTD_initDStream() :
jpayne@69 889 * Initialize/reset DStream state for new decompression operation.
jpayne@69 890 * Call before new decompression operation using same DStream.
jpayne@69 891 *
jpayne@69 892 * Note : This function is redundant with the advanced API and equivalent to:
jpayne@69 893 * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
jpayne@69 894 * ZSTD_DCtx_refDDict(zds, NULL);
jpayne@69 895 */
jpayne@69 896 ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
jpayne@69 897
jpayne@69 898 /*! ZSTD_decompressStream() :
jpayne@69 899 * Streaming decompression function.
jpayne@69 900 * Call repetitively to consume full input updating it as necessary.
jpayne@69 901 * Function will update both input and output `pos` fields exposing current state via these fields:
jpayne@69 902 * - `input.pos < input.size`, some input remaining and caller should provide remaining input
jpayne@69 903 * on the next call.
jpayne@69 904 * - `output.pos < output.size`, decoder finished and flushed all remaining buffers.
jpayne@69 905 * - `output.pos == output.size`, potentially uncflushed data present in the internal buffers,
jpayne@69 906 * call ZSTD_decompressStream() again to flush remaining data to output.
jpayne@69 907 * Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX.
jpayne@69 908 *
jpayne@69 909 * @return : 0 when a frame is completely decoded and fully flushed,
jpayne@69 910 * or an error code, which can be tested using ZSTD_isError(),
jpayne@69 911 * or any other value > 0, which means there is some decoding or flushing to do to complete current frame.
jpayne@69 912 *
jpayne@69 913 * Note: when an operation returns with an error code, the @zds state may be left in undefined state.
jpayne@69 914 * It's UB to invoke `ZSTD_decompressStream()` on such a state.
jpayne@69 915 * In order to re-use such a state, it must be first reset,
jpayne@69 916 * which can be done explicitly (`ZSTD_DCtx_reset()`),
jpayne@69 917 * or is implied for operations starting some new decompression job (`ZSTD_initDStream`, `ZSTD_decompressDCtx()`, `ZSTD_decompress_usingDict()`)
jpayne@69 918 */
jpayne@69 919 ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
jpayne@69 920
jpayne@69 921 ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */
jpayne@69 922 ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */
jpayne@69 923
jpayne@69 924
jpayne@69 925 /**************************
jpayne@69 926 * Simple dictionary API
jpayne@69 927 ***************************/
jpayne@69 928 /*! ZSTD_compress_usingDict() :
jpayne@69 929 * Compression at an explicit compression level using a Dictionary.
jpayne@69 930 * A dictionary can be any arbitrary data segment (also called a prefix),
jpayne@69 931 * or a buffer with specified information (see zdict.h).
jpayne@69 932 * Note : This function loads the dictionary, resulting in significant startup delay.
jpayne@69 933 * It's intended for a dictionary used only once.
jpayne@69 934 * Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */
jpayne@69 935 ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
jpayne@69 936 void* dst, size_t dstCapacity,
jpayne@69 937 const void* src, size_t srcSize,
jpayne@69 938 const void* dict,size_t dictSize,
jpayne@69 939 int compressionLevel);
jpayne@69 940
jpayne@69 941 /*! ZSTD_decompress_usingDict() :
jpayne@69 942 * Decompression using a known Dictionary.
jpayne@69 943 * Dictionary must be identical to the one used during compression.
jpayne@69 944 * Note : This function loads the dictionary, resulting in significant startup delay.
jpayne@69 945 * It's intended for a dictionary used only once.
jpayne@69 946 * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
jpayne@69 947 ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
jpayne@69 948 void* dst, size_t dstCapacity,
jpayne@69 949 const void* src, size_t srcSize,
jpayne@69 950 const void* dict,size_t dictSize);
jpayne@69 951
jpayne@69 952
jpayne@69 953 /***********************************
jpayne@69 954 * Bulk processing dictionary API
jpayne@69 955 **********************************/
jpayne@69 956 typedef struct ZSTD_CDict_s ZSTD_CDict;
jpayne@69 957
jpayne@69 958 /*! ZSTD_createCDict() :
jpayne@69 959 * When compressing multiple messages or blocks using the same dictionary,
jpayne@69 960 * it's recommended to digest the dictionary only once, since it's a costly operation.
jpayne@69 961 * ZSTD_createCDict() will create a state from digesting a dictionary.
jpayne@69 962 * The resulting state can be used for future compression operations with very limited startup cost.
jpayne@69 963 * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
jpayne@69 964 * @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict.
jpayne@69 965 * Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content.
jpayne@69 966 * Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer,
jpayne@69 967 * in which case the only thing that it transports is the @compressionLevel.
jpayne@69 968 * This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively,
jpayne@69 969 * expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */
jpayne@69 970 ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
jpayne@69 971 int compressionLevel);
jpayne@69 972
jpayne@69 973 /*! ZSTD_freeCDict() :
jpayne@69 974 * Function frees memory allocated by ZSTD_createCDict().
jpayne@69 975 * If a NULL pointer is passed, no operation is performed. */
jpayne@69 976 ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
jpayne@69 977
jpayne@69 978 /*! ZSTD_compress_usingCDict() :
jpayne@69 979 * Compression using a digested Dictionary.
jpayne@69 980 * Recommended when same dictionary is used multiple times.
jpayne@69 981 * Note : compression level is _decided at dictionary creation time_,
jpayne@69 982 * and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */
jpayne@69 983 ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
jpayne@69 984 void* dst, size_t dstCapacity,
jpayne@69 985 const void* src, size_t srcSize,
jpayne@69 986 const ZSTD_CDict* cdict);
jpayne@69 987
jpayne@69 988
jpayne@69 989 typedef struct ZSTD_DDict_s ZSTD_DDict;
jpayne@69 990
jpayne@69 991 /*! ZSTD_createDDict() :
jpayne@69 992 * Create a digested dictionary, ready to start decompression operation without startup delay.
jpayne@69 993 * dictBuffer can be released after DDict creation, as its content is copied inside DDict. */
jpayne@69 994 ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
jpayne@69 995
jpayne@69 996 /*! ZSTD_freeDDict() :
jpayne@69 997 * Function frees memory allocated with ZSTD_createDDict()
jpayne@69 998 * If a NULL pointer is passed, no operation is performed. */
jpayne@69 999 ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict);
jpayne@69 1000
jpayne@69 1001 /*! ZSTD_decompress_usingDDict() :
jpayne@69 1002 * Decompression using a digested Dictionary.
jpayne@69 1003 * Recommended when same dictionary is used multiple times. */
jpayne@69 1004 ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
jpayne@69 1005 void* dst, size_t dstCapacity,
jpayne@69 1006 const void* src, size_t srcSize,
jpayne@69 1007 const ZSTD_DDict* ddict);
jpayne@69 1008
jpayne@69 1009
jpayne@69 1010 /********************************
jpayne@69 1011 * Dictionary helper functions
jpayne@69 1012 *******************************/
jpayne@69 1013
jpayne@69 1014 /*! ZSTD_getDictID_fromDict() : Requires v1.4.0+
jpayne@69 1015 * Provides the dictID stored within dictionary.
jpayne@69 1016 * if @return == 0, the dictionary is not conformant with Zstandard specification.
jpayne@69 1017 * It can still be loaded, but as a content-only dictionary. */
jpayne@69 1018 ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
jpayne@69 1019
jpayne@69 1020 /*! ZSTD_getDictID_fromCDict() : Requires v1.5.0+
jpayne@69 1021 * Provides the dictID of the dictionary loaded into `cdict`.
jpayne@69 1022 * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
jpayne@69 1023 * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
jpayne@69 1024 ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict);
jpayne@69 1025
jpayne@69 1026 /*! ZSTD_getDictID_fromDDict() : Requires v1.4.0+
jpayne@69 1027 * Provides the dictID of the dictionary loaded into `ddict`.
jpayne@69 1028 * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
jpayne@69 1029 * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
jpayne@69 1030 ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
jpayne@69 1031
jpayne@69 1032 /*! ZSTD_getDictID_fromFrame() : Requires v1.4.0+
jpayne@69 1033 * Provides the dictID required to decompressed the frame stored within `src`.
jpayne@69 1034 * If @return == 0, the dictID could not be decoded.
jpayne@69 1035 * This could for one of the following reasons :
jpayne@69 1036 * - The frame does not require a dictionary to be decoded (most common case).
jpayne@69 1037 * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden piece of information.
jpayne@69 1038 * Note : this use case also happens when using a non-conformant dictionary.
jpayne@69 1039 * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
jpayne@69 1040 * - This is not a Zstandard frame.
jpayne@69 1041 * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */
jpayne@69 1042 ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
jpayne@69 1043
jpayne@69 1044
jpayne@69 1045 /*******************************************************************************
jpayne@69 1046 * Advanced dictionary and prefix API (Requires v1.4.0+)
jpayne@69 1047 *
jpayne@69 1048 * This API allows dictionaries to be used with ZSTD_compress2(),
jpayne@69 1049 * ZSTD_compressStream2(), and ZSTD_decompressDCtx().
jpayne@69 1050 * Dictionaries are sticky, they remain valid when same context is reused,
jpayne@69 1051 * they only reset when the context is reset
jpayne@69 1052 * with ZSTD_reset_parameters or ZSTD_reset_session_and_parameters.
jpayne@69 1053 * In contrast, Prefixes are single-use.
jpayne@69 1054 ******************************************************************************/
jpayne@69 1055
jpayne@69 1056
jpayne@69 1057 /*! ZSTD_CCtx_loadDictionary() : Requires v1.4.0+
jpayne@69 1058 * Create an internal CDict from `dict` buffer.
jpayne@69 1059 * Decompression will have to use same dictionary.
jpayne@69 1060 * @result : 0, or an error code (which can be tested with ZSTD_isError()).
jpayne@69 1061 * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary,
jpayne@69 1062 * meaning "return to no-dictionary mode".
jpayne@69 1063 * Note 1 : Dictionary is sticky, it will be used for all future compressed frames,
jpayne@69 1064 * until parameters are reset, a new dictionary is loaded, or the dictionary
jpayne@69 1065 * is explicitly invalidated by loading a NULL dictionary.
jpayne@69 1066 * Note 2 : Loading a dictionary involves building tables.
jpayne@69 1067 * It's also a CPU consuming operation, with non-negligible impact on latency.
jpayne@69 1068 * Tables are dependent on compression parameters, and for this reason,
jpayne@69 1069 * compression parameters can no longer be changed after loading a dictionary.
jpayne@69 1070 * Note 3 :`dict` content will be copied internally.
jpayne@69 1071 * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead.
jpayne@69 1072 * In such a case, dictionary buffer must outlive its users.
jpayne@69 1073 * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced()
jpayne@69 1074 * to precisely select how dictionary content must be interpreted.
jpayne@69 1075 * Note 5 : This method does not benefit from LDM (long distance mode).
jpayne@69 1076 * If you want to employ LDM on some large dictionary content,
jpayne@69 1077 * prefer employing ZSTD_CCtx_refPrefix() described below.
jpayne@69 1078 */
jpayne@69 1079 ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
jpayne@69 1080
jpayne@69 1081 /*! ZSTD_CCtx_refCDict() : Requires v1.4.0+
jpayne@69 1082 * Reference a prepared dictionary, to be used for all future compressed frames.
jpayne@69 1083 * Note that compression parameters are enforced from within CDict,
jpayne@69 1084 * and supersede any compression parameter previously set within CCtx.
jpayne@69 1085 * The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
jpayne@69 1086 * The ignored parameters will be used again if the CCtx is returned to no-dictionary mode.
jpayne@69 1087 * The dictionary will remain valid for future compressed frames using same CCtx.
jpayne@69 1088 * @result : 0, or an error code (which can be tested with ZSTD_isError()).
jpayne@69 1089 * Special : Referencing a NULL CDict means "return to no-dictionary mode".
jpayne@69 1090 * Note 1 : Currently, only one dictionary can be managed.
jpayne@69 1091 * Referencing a new dictionary effectively "discards" any previous one.
jpayne@69 1092 * Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */
jpayne@69 1093 ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
jpayne@69 1094
jpayne@69 1095 /*! ZSTD_CCtx_refPrefix() : Requires v1.4.0+
jpayne@69 1096 * Reference a prefix (single-usage dictionary) for next compressed frame.
jpayne@69 1097 * A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end).
jpayne@69 1098 * Decompression will need same prefix to properly regenerate data.
jpayne@69 1099 * Compressing with a prefix is similar in outcome as performing a diff and compressing it,
jpayne@69 1100 * but performs much faster, especially during decompression (compression speed is tunable with compression level).
jpayne@69 1101 * This method is compatible with LDM (long distance mode).
jpayne@69 1102 * @result : 0, or an error code (which can be tested with ZSTD_isError()).
jpayne@69 1103 * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary
jpayne@69 1104 * Note 1 : Prefix buffer is referenced. It **must** outlive compression.
jpayne@69 1105 * Its content must remain unmodified during compression.
jpayne@69 1106 * Note 2 : If the intention is to diff some large src data blob with some prior version of itself,
jpayne@69 1107 * ensure that the window size is large enough to contain the entire source.
jpayne@69 1108 * See ZSTD_c_windowLog.
jpayne@69 1109 * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
jpayne@69 1110 * It's a CPU consuming operation, with non-negligible impact on latency.
jpayne@69 1111 * If there is a need to use the same prefix multiple times, consider loadDictionary instead.
jpayne@69 1112 * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent).
jpayne@69 1113 * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */
jpayne@69 1114 ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
jpayne@69 1115 const void* prefix, size_t prefixSize);
jpayne@69 1116
jpayne@69 1117 /*! ZSTD_DCtx_loadDictionary() : Requires v1.4.0+
jpayne@69 1118 * Create an internal DDict from dict buffer, to be used to decompress all future frames.
jpayne@69 1119 * The dictionary remains valid for all future frames, until explicitly invalidated, or
jpayne@69 1120 * a new dictionary is loaded.
jpayne@69 1121 * @result : 0, or an error code (which can be tested with ZSTD_isError()).
jpayne@69 1122 * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
jpayne@69 1123 * meaning "return to no-dictionary mode".
jpayne@69 1124 * Note 1 : Loading a dictionary involves building tables,
jpayne@69 1125 * which has a non-negligible impact on CPU usage and latency.
jpayne@69 1126 * It's recommended to "load once, use many times", to amortize the cost
jpayne@69 1127 * Note 2 :`dict` content will be copied internally, so `dict` can be released after loading.
jpayne@69 1128 * Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead.
jpayne@69 1129 * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of
jpayne@69 1130 * how dictionary content is loaded and interpreted.
jpayne@69 1131 */
jpayne@69 1132 ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
jpayne@69 1133
jpayne@69 1134 /*! ZSTD_DCtx_refDDict() : Requires v1.4.0+
jpayne@69 1135 * Reference a prepared dictionary, to be used to decompress next frames.
jpayne@69 1136 * The dictionary remains active for decompression of future frames using same DCtx.
jpayne@69 1137 *
jpayne@69 1138 * If called with ZSTD_d_refMultipleDDicts enabled, repeated calls of this function
jpayne@69 1139 * will store the DDict references in a table, and the DDict used for decompression
jpayne@69 1140 * will be determined at decompression time, as per the dict ID in the frame.
jpayne@69 1141 * The memory for the table is allocated on the first call to refDDict, and can be
jpayne@69 1142 * freed with ZSTD_freeDCtx().
jpayne@69 1143 *
jpayne@69 1144 * If called with ZSTD_d_refMultipleDDicts disabled (the default), only one dictionary
jpayne@69 1145 * will be managed, and referencing a dictionary effectively "discards" any previous one.
jpayne@69 1146 *
jpayne@69 1147 * @result : 0, or an error code (which can be tested with ZSTD_isError()).
jpayne@69 1148 * Special: referencing a NULL DDict means "return to no-dictionary mode".
jpayne@69 1149 * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
jpayne@69 1150 */
jpayne@69 1151 ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
jpayne@69 1152
jpayne@69 1153 /*! ZSTD_DCtx_refPrefix() : Requires v1.4.0+
jpayne@69 1154 * Reference a prefix (single-usage dictionary) to decompress next frame.
jpayne@69 1155 * This is the reverse operation of ZSTD_CCtx_refPrefix(),
jpayne@69 1156 * and must use the same prefix as the one used during compression.
jpayne@69 1157 * Prefix is **only used once**. Reference is discarded at end of frame.
jpayne@69 1158 * End of frame is reached when ZSTD_decompressStream() returns 0.
jpayne@69 1159 * @result : 0, or an error code (which can be tested with ZSTD_isError()).
jpayne@69 1160 * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary
jpayne@69 1161 * Note 2 : Prefix buffer is referenced. It **must** outlive decompression.
jpayne@69 1162 * Prefix buffer must remain unmodified up to the end of frame,
jpayne@69 1163 * reached when ZSTD_decompressStream() returns 0.
jpayne@69 1164 * Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent).
jpayne@69 1165 * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)
jpayne@69 1166 * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
jpayne@69 1167 * A full dictionary is more costly, as it requires building tables.
jpayne@69 1168 */
jpayne@69 1169 ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx,
jpayne@69 1170 const void* prefix, size_t prefixSize);
jpayne@69 1171
jpayne@69 1172 /* === Memory management === */
jpayne@69 1173
jpayne@69 1174 /*! ZSTD_sizeof_*() : Requires v1.4.0+
jpayne@69 1175 * These functions give the _current_ memory usage of selected object.
jpayne@69 1176 * Note that object memory usage can evolve (increase or decrease) over time. */
jpayne@69 1177 ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
jpayne@69 1178 ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
jpayne@69 1179 ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
jpayne@69 1180 ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
jpayne@69 1181 ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
jpayne@69 1182 ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
jpayne@69 1183
jpayne@69 1184 #endif /* ZSTD_H_235446 */
jpayne@69 1185
jpayne@69 1186
jpayne@69 1187 /* **************************************************************************************
jpayne@69 1188 * ADVANCED AND EXPERIMENTAL FUNCTIONS
jpayne@69 1189 ****************************************************************************************
jpayne@69 1190 * The definitions in the following section are considered experimental.
jpayne@69 1191 * They are provided for advanced scenarios.
jpayne@69 1192 * They should never be used with a dynamic library, as prototypes may change in the future.
jpayne@69 1193 * Use them only in association with static linking.
jpayne@69 1194 * ***************************************************************************************/
jpayne@69 1195
jpayne@69 1196 #if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
jpayne@69 1197 #define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
jpayne@69 1198
jpayne@69 1199 /* This can be overridden externally to hide static symbols. */
jpayne@69 1200 #ifndef ZSTDLIB_STATIC_API
jpayne@69 1201 # if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
jpayne@69 1202 # define ZSTDLIB_STATIC_API __declspec(dllexport) ZSTDLIB_VISIBLE
jpayne@69 1203 # elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
jpayne@69 1204 # define ZSTDLIB_STATIC_API __declspec(dllimport) ZSTDLIB_VISIBLE
jpayne@69 1205 # else
jpayne@69 1206 # define ZSTDLIB_STATIC_API ZSTDLIB_VISIBLE
jpayne@69 1207 # endif
jpayne@69 1208 #endif
jpayne@69 1209
jpayne@69 1210 /****************************************************************************************
jpayne@69 1211 * experimental API (static linking only)
jpayne@69 1212 ****************************************************************************************
jpayne@69 1213 * The following symbols and constants
jpayne@69 1214 * are not planned to join "stable API" status in the near future.
jpayne@69 1215 * They can still change in future versions.
jpayne@69 1216 * Some of them are planned to remain in the static_only section indefinitely.
jpayne@69 1217 * Some of them might be removed in the future (especially when redundant with existing stable functions)
jpayne@69 1218 * ***************************************************************************************/
jpayne@69 1219
jpayne@69 1220 #define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1) /* minimum input size required to query frame header size */
jpayne@69 1221 #define ZSTD_FRAMEHEADERSIZE_MIN(format) ((format) == ZSTD_f_zstd1 ? 6 : 2)
jpayne@69 1222 #define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */
jpayne@69 1223 #define ZSTD_SKIPPABLEHEADERSIZE 8
jpayne@69 1224
jpayne@69 1225 /* compression parameter bounds */
jpayne@69 1226 #define ZSTD_WINDOWLOG_MAX_32 30
jpayne@69 1227 #define ZSTD_WINDOWLOG_MAX_64 31
jpayne@69 1228 #define ZSTD_WINDOWLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
jpayne@69 1229 #define ZSTD_WINDOWLOG_MIN 10
jpayne@69 1230 #define ZSTD_HASHLOG_MAX ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30)
jpayne@69 1231 #define ZSTD_HASHLOG_MIN 6
jpayne@69 1232 #define ZSTD_CHAINLOG_MAX_32 29
jpayne@69 1233 #define ZSTD_CHAINLOG_MAX_64 30
jpayne@69 1234 #define ZSTD_CHAINLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64))
jpayne@69 1235 #define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN
jpayne@69 1236 #define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1)
jpayne@69 1237 #define ZSTD_SEARCHLOG_MIN 1
jpayne@69 1238 #define ZSTD_MINMATCH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */
jpayne@69 1239 #define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */
jpayne@69 1240 #define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX
jpayne@69 1241 #define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */
jpayne@69 1242 #define ZSTD_STRATEGY_MIN ZSTD_fast
jpayne@69 1243 #define ZSTD_STRATEGY_MAX ZSTD_btultra2
jpayne@69 1244 #define ZSTD_BLOCKSIZE_MAX_MIN (1 << 10) /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */
jpayne@69 1245
jpayne@69 1246
jpayne@69 1247 #define ZSTD_OVERLAPLOG_MIN 0
jpayne@69 1248 #define ZSTD_OVERLAPLOG_MAX 9
jpayne@69 1249
jpayne@69 1250 #define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame
jpayne@69 1251 * requiring larger than (1<<ZSTD_WINDOWLOG_LIMIT_DEFAULT) window size,
jpayne@69 1252 * to preserve host's memory from unreasonable requirements.
jpayne@69 1253 * This limit can be overridden using ZSTD_DCtx_setParameter(,ZSTD_d_windowLogMax,).
jpayne@69 1254 * The limit does not apply for one-pass decoders (such as ZSTD_decompress()), since no additional memory is allocated */
jpayne@69 1255
jpayne@69 1256
jpayne@69 1257 /* LDM parameter bounds */
jpayne@69 1258 #define ZSTD_LDM_HASHLOG_MIN ZSTD_HASHLOG_MIN
jpayne@69 1259 #define ZSTD_LDM_HASHLOG_MAX ZSTD_HASHLOG_MAX
jpayne@69 1260 #define ZSTD_LDM_MINMATCH_MIN 4
jpayne@69 1261 #define ZSTD_LDM_MINMATCH_MAX 4096
jpayne@69 1262 #define ZSTD_LDM_BUCKETSIZELOG_MIN 1
jpayne@69 1263 #define ZSTD_LDM_BUCKETSIZELOG_MAX 8
jpayne@69 1264 #define ZSTD_LDM_HASHRATELOG_MIN 0
jpayne@69 1265 #define ZSTD_LDM_HASHRATELOG_MAX (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN)
jpayne@69 1266
jpayne@69 1267 /* Advanced parameter bounds */
jpayne@69 1268 #define ZSTD_TARGETCBLOCKSIZE_MIN 1340 /* suitable to fit into an ethernet / wifi / 4G transport frame */
jpayne@69 1269 #define ZSTD_TARGETCBLOCKSIZE_MAX ZSTD_BLOCKSIZE_MAX
jpayne@69 1270 #define ZSTD_SRCSIZEHINT_MIN 0
jpayne@69 1271 #define ZSTD_SRCSIZEHINT_MAX INT_MAX
jpayne@69 1272
jpayne@69 1273
jpayne@69 1274 /* --- Advanced types --- */
jpayne@69 1275
jpayne@69 1276 typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params;
jpayne@69 1277
jpayne@69 1278 typedef struct {
jpayne@69 1279 unsigned int offset; /* The offset of the match. (NOT the same as the offset code)
jpayne@69 1280 * If offset == 0 and matchLength == 0, this sequence represents the last
jpayne@69 1281 * literals in the block of litLength size.
jpayne@69 1282 */
jpayne@69 1283
jpayne@69 1284 unsigned int litLength; /* Literal length of the sequence. */
jpayne@69 1285 unsigned int matchLength; /* Match length of the sequence. */
jpayne@69 1286
jpayne@69 1287 /* Note: Users of this API may provide a sequence with matchLength == litLength == offset == 0.
jpayne@69 1288 * In this case, we will treat the sequence as a marker for a block boundary.
jpayne@69 1289 */
jpayne@69 1290
jpayne@69 1291 unsigned int rep; /* Represents which repeat offset is represented by the field 'offset'.
jpayne@69 1292 * Ranges from [0, 3].
jpayne@69 1293 *
jpayne@69 1294 * Repeat offsets are essentially previous offsets from previous sequences sorted in
jpayne@69 1295 * recency order. For more detail, see doc/zstd_compression_format.md
jpayne@69 1296 *
jpayne@69 1297 * If rep == 0, then 'offset' does not contain a repeat offset.
jpayne@69 1298 * If rep > 0:
jpayne@69 1299 * If litLength != 0:
jpayne@69 1300 * rep == 1 --> offset == repeat_offset_1
jpayne@69 1301 * rep == 2 --> offset == repeat_offset_2
jpayne@69 1302 * rep == 3 --> offset == repeat_offset_3
jpayne@69 1303 * If litLength == 0:
jpayne@69 1304 * rep == 1 --> offset == repeat_offset_2
jpayne@69 1305 * rep == 2 --> offset == repeat_offset_3
jpayne@69 1306 * rep == 3 --> offset == repeat_offset_1 - 1
jpayne@69 1307 *
jpayne@69 1308 * Note: This field is optional. ZSTD_generateSequences() will calculate the value of
jpayne@69 1309 * 'rep', but repeat offsets do not necessarily need to be calculated from an external
jpayne@69 1310 * sequence provider's perspective. For example, ZSTD_compressSequences() does not
jpayne@69 1311 * use this 'rep' field at all (as of now).
jpayne@69 1312 */
jpayne@69 1313 } ZSTD_Sequence;
jpayne@69 1314
jpayne@69 1315 typedef struct {
jpayne@69 1316 unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */
jpayne@69 1317 unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
jpayne@69 1318 unsigned hashLog; /**< dispatch table : larger == faster, more memory */
jpayne@69 1319 unsigned searchLog; /**< nb of searches : larger == more compression, slower */
jpayne@69 1320 unsigned minMatch; /**< match length searched : larger == faster decompression, sometimes less compression */
jpayne@69 1321 unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */
jpayne@69 1322 ZSTD_strategy strategy; /**< see ZSTD_strategy definition above */
jpayne@69 1323 } ZSTD_compressionParameters;
jpayne@69 1324
jpayne@69 1325 typedef struct {
jpayne@69 1326 int contentSizeFlag; /**< 1: content size will be in frame header (when known) */
jpayne@69 1327 int checksumFlag; /**< 1: generate a 32-bits checksum using XXH64 algorithm at end of frame, for error detection */
jpayne@69 1328 int noDictIDFlag; /**< 1: no dictID will be saved into frame header (dictID is only useful for dictionary compression) */
jpayne@69 1329 } ZSTD_frameParameters;
jpayne@69 1330
jpayne@69 1331 typedef struct {
jpayne@69 1332 ZSTD_compressionParameters cParams;
jpayne@69 1333 ZSTD_frameParameters fParams;
jpayne@69 1334 } ZSTD_parameters;
jpayne@69 1335
jpayne@69 1336 typedef enum {
jpayne@69 1337 ZSTD_dct_auto = 0, /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */
jpayne@69 1338 ZSTD_dct_rawContent = 1, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */
jpayne@69 1339 ZSTD_dct_fullDict = 2 /* refuses to load a dictionary if it does not respect Zstandard's specification, starting with ZSTD_MAGIC_DICTIONARY */
jpayne@69 1340 } ZSTD_dictContentType_e;
jpayne@69 1341
jpayne@69 1342 typedef enum {
jpayne@69 1343 ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */
jpayne@69 1344 ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
jpayne@69 1345 } ZSTD_dictLoadMethod_e;
jpayne@69 1346
jpayne@69 1347 typedef enum {
jpayne@69 1348 ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */
jpayne@69 1349 ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number.
jpayne@69 1350 * Useful to save 4 bytes per generated frame.
jpayne@69 1351 * Decoder cannot recognise automatically this format, requiring this instruction. */
jpayne@69 1352 } ZSTD_format_e;
jpayne@69 1353
jpayne@69 1354 typedef enum {
jpayne@69 1355 /* Note: this enum controls ZSTD_d_forceIgnoreChecksum */
jpayne@69 1356 ZSTD_d_validateChecksum = 0,
jpayne@69 1357 ZSTD_d_ignoreChecksum = 1
jpayne@69 1358 } ZSTD_forceIgnoreChecksum_e;
jpayne@69 1359
jpayne@69 1360 typedef enum {
jpayne@69 1361 /* Note: this enum controls ZSTD_d_refMultipleDDicts */
jpayne@69 1362 ZSTD_rmd_refSingleDDict = 0,
jpayne@69 1363 ZSTD_rmd_refMultipleDDicts = 1
jpayne@69 1364 } ZSTD_refMultipleDDicts_e;
jpayne@69 1365
jpayne@69 1366 typedef enum {
jpayne@69 1367 /* Note: this enum and the behavior it controls are effectively internal
jpayne@69 1368 * implementation details of the compressor. They are expected to continue
jpayne@69 1369 * to evolve and should be considered only in the context of extremely
jpayne@69 1370 * advanced performance tuning.
jpayne@69 1371 *
jpayne@69 1372 * Zstd currently supports the use of a CDict in three ways:
jpayne@69 1373 *
jpayne@69 1374 * - The contents of the CDict can be copied into the working context. This
jpayne@69 1375 * means that the compression can search both the dictionary and input
jpayne@69 1376 * while operating on a single set of internal tables. This makes
jpayne@69 1377 * the compression faster per-byte of input. However, the initial copy of
jpayne@69 1378 * the CDict's tables incurs a fixed cost at the beginning of the
jpayne@69 1379 * compression. For small compressions (< 8 KB), that copy can dominate
jpayne@69 1380 * the cost of the compression.
jpayne@69 1381 *
jpayne@69 1382 * - The CDict's tables can be used in-place. In this model, compression is
jpayne@69 1383 * slower per input byte, because the compressor has to search two sets of
jpayne@69 1384 * tables. However, this model incurs no start-up cost (as long as the
jpayne@69 1385 * working context's tables can be reused). For small inputs, this can be
jpayne@69 1386 * faster than copying the CDict's tables.
jpayne@69 1387 *
jpayne@69 1388 * - The CDict's tables are not used at all, and instead we use the working
jpayne@69 1389 * context alone to reload the dictionary and use params based on the source
jpayne@69 1390 * size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict().
jpayne@69 1391 * This method is effective when the dictionary sizes are very small relative
jpayne@69 1392 * to the input size, and the input size is fairly large to begin with.
jpayne@69 1393 *
jpayne@69 1394 * Zstd has a simple internal heuristic that selects which strategy to use
jpayne@69 1395 * at the beginning of a compression. However, if experimentation shows that
jpayne@69 1396 * Zstd is making poor choices, it is possible to override that choice with
jpayne@69 1397 * this enum.
jpayne@69 1398 */
jpayne@69 1399 ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */
jpayne@69 1400 ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */
jpayne@69 1401 ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */
jpayne@69 1402 ZSTD_dictForceLoad = 3 /* Always reload the dictionary */
jpayne@69 1403 } ZSTD_dictAttachPref_e;
jpayne@69 1404
jpayne@69 1405 typedef enum {
jpayne@69 1406 ZSTD_lcm_auto = 0, /**< Automatically determine the compression mode based on the compression level.
jpayne@69 1407 * Negative compression levels will be uncompressed, and positive compression
jpayne@69 1408 * levels will be compressed. */
jpayne@69 1409 ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be
jpayne@69 1410 * emitted if Huffman compression is not profitable. */
jpayne@69 1411 ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */
jpayne@69 1412 } ZSTD_literalCompressionMode_e;
jpayne@69 1413
jpayne@69 1414 typedef enum {
jpayne@69 1415 /* Note: This enum controls features which are conditionally beneficial. Zstd typically will make a final
jpayne@69 1416 * decision on whether or not to enable the feature (ZSTD_ps_auto), but setting the switch to ZSTD_ps_enable
jpayne@69 1417 * or ZSTD_ps_disable allow for a force enable/disable the feature.
jpayne@69 1418 */
jpayne@69 1419 ZSTD_ps_auto = 0, /* Let the library automatically determine whether the feature shall be enabled */
jpayne@69 1420 ZSTD_ps_enable = 1, /* Force-enable the feature */
jpayne@69 1421 ZSTD_ps_disable = 2 /* Do not use the feature */
jpayne@69 1422 } ZSTD_paramSwitch_e;
jpayne@69 1423
jpayne@69 1424 /***************************************
jpayne@69 1425 * Frame header and size functions
jpayne@69 1426 ***************************************/
jpayne@69 1427
jpayne@69 1428 /*! ZSTD_findDecompressedSize() :
jpayne@69 1429 * `src` should point to the start of a series of ZSTD encoded and/or skippable frames
jpayne@69 1430 * `srcSize` must be the _exact_ size of this series
jpayne@69 1431 * (i.e. there should be a frame boundary at `src + srcSize`)
jpayne@69 1432 * @return : - decompressed size of all data in all successive frames
jpayne@69 1433 * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
jpayne@69 1434 * - if an error occurred: ZSTD_CONTENTSIZE_ERROR
jpayne@69 1435 *
jpayne@69 1436 * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
jpayne@69 1437 * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
jpayne@69 1438 * In which case, it's necessary to use streaming mode to decompress data.
jpayne@69 1439 * note 2 : decompressed size is always present when compression is done with ZSTD_compress()
jpayne@69 1440 * note 3 : decompressed size can be very large (64-bits value),
jpayne@69 1441 * potentially larger than what local system can handle as a single memory segment.
jpayne@69 1442 * In which case, it's necessary to use streaming mode to decompress data.
jpayne@69 1443 * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
jpayne@69 1444 * Always ensure result fits within application's authorized limits.
jpayne@69 1445 * Each application can set its own limits.
jpayne@69 1446 * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to
jpayne@69 1447 * read each contained frame header. This is fast as most of the data is skipped,
jpayne@69 1448 * however it does mean that all frame data must be present and valid. */
jpayne@69 1449 ZSTDLIB_STATIC_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
jpayne@69 1450
jpayne@69 1451 /*! ZSTD_decompressBound() :
jpayne@69 1452 * `src` should point to the start of a series of ZSTD encoded and/or skippable frames
jpayne@69 1453 * `srcSize` must be the _exact_ size of this series
jpayne@69 1454 * (i.e. there should be a frame boundary at `src + srcSize`)
jpayne@69 1455 * @return : - upper-bound for the decompressed size of all data in all successive frames
jpayne@69 1456 * - if an error occurred: ZSTD_CONTENTSIZE_ERROR
jpayne@69 1457 *
jpayne@69 1458 * note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame.
jpayne@69 1459 * note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`.
jpayne@69 1460 * in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value.
jpayne@69 1461 * note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by:
jpayne@69 1462 * upper-bound = # blocks * min(128 KB, Window_Size)
jpayne@69 1463 */
jpayne@69 1464 ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize);
jpayne@69 1465
jpayne@69 1466 /*! ZSTD_frameHeaderSize() :
jpayne@69 1467 * srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX.
jpayne@69 1468 * @return : size of the Frame Header,
jpayne@69 1469 * or an error code (if srcSize is too small) */
jpayne@69 1470 ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
jpayne@69 1471
jpayne@69 1472 typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e;
jpayne@69 1473 typedef struct {
jpayne@69 1474 unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */
jpayne@69 1475 unsigned long long windowSize; /* can be very large, up to <= frameContentSize */
jpayne@69 1476 unsigned blockSizeMax;
jpayne@69 1477 ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */
jpayne@69 1478 unsigned headerSize;
jpayne@69 1479 unsigned dictID;
jpayne@69 1480 unsigned checksumFlag;
jpayne@69 1481 unsigned _reserved1;
jpayne@69 1482 unsigned _reserved2;
jpayne@69 1483 } ZSTD_frameHeader;
jpayne@69 1484
jpayne@69 1485 /*! ZSTD_getFrameHeader() :
jpayne@69 1486 * decode Frame Header, or requires larger `srcSize`.
jpayne@69 1487 * @return : 0, `zfhPtr` is correctly filled,
jpayne@69 1488 * >0, `srcSize` is too small, value is wanted `srcSize` amount,
jpayne@69 1489 * or an error code, which can be tested using ZSTD_isError() */
jpayne@69 1490 ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */
jpayne@69 1491 /*! ZSTD_getFrameHeader_advanced() :
jpayne@69 1492 * same as ZSTD_getFrameHeader(),
jpayne@69 1493 * with added capability to select a format (like ZSTD_f_zstd1_magicless) */
jpayne@69 1494 ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
jpayne@69 1495
jpayne@69 1496 /*! ZSTD_decompressionMargin() :
jpayne@69 1497 * Zstd supports in-place decompression, where the input and output buffers overlap.
jpayne@69 1498 * In this case, the output buffer must be at least (Margin + Output_Size) bytes large,
jpayne@69 1499 * and the input buffer must be at the end of the output buffer.
jpayne@69 1500 *
jpayne@69 1501 * _______________________ Output Buffer ________________________
jpayne@69 1502 * | |
jpayne@69 1503 * | ____ Input Buffer ____|
jpayne@69 1504 * | | |
jpayne@69 1505 * v v v
jpayne@69 1506 * |---------------------------------------|-----------|----------|
jpayne@69 1507 * ^ ^ ^
jpayne@69 1508 * |___________________ Output_Size ___________________|_ Margin _|
jpayne@69 1509 *
jpayne@69 1510 * NOTE: See also ZSTD_DECOMPRESSION_MARGIN().
jpayne@69 1511 * NOTE: This applies only to single-pass decompression through ZSTD_decompress() or
jpayne@69 1512 * ZSTD_decompressDCtx().
jpayne@69 1513 * NOTE: This function supports multi-frame input.
jpayne@69 1514 *
jpayne@69 1515 * @param src The compressed frame(s)
jpayne@69 1516 * @param srcSize The size of the compressed frame(s)
jpayne@69 1517 * @returns The decompression margin or an error that can be checked with ZSTD_isError().
jpayne@69 1518 */
jpayne@69 1519 ZSTDLIB_STATIC_API size_t ZSTD_decompressionMargin(const void* src, size_t srcSize);
jpayne@69 1520
jpayne@69 1521 /*! ZSTD_DECOMPRESS_MARGIN() :
jpayne@69 1522 * Similar to ZSTD_decompressionMargin(), but instead of computing the margin from
jpayne@69 1523 * the compressed frame, compute it from the original size and the blockSizeLog.
jpayne@69 1524 * See ZSTD_decompressionMargin() for details.
jpayne@69 1525 *
jpayne@69 1526 * WARNING: This macro does not support multi-frame input, the input must be a single
jpayne@69 1527 * zstd frame. If you need that support use the function, or implement it yourself.
jpayne@69 1528 *
jpayne@69 1529 * @param originalSize The original uncompressed size of the data.
jpayne@69 1530 * @param blockSize The block size == MIN(windowSize, ZSTD_BLOCKSIZE_MAX).
jpayne@69 1531 * Unless you explicitly set the windowLog smaller than
jpayne@69 1532 * ZSTD_BLOCKSIZELOG_MAX you can just use ZSTD_BLOCKSIZE_MAX.
jpayne@69 1533 */
jpayne@69 1534 #define ZSTD_DECOMPRESSION_MARGIN(originalSize, blockSize) ((size_t)( \
jpayne@69 1535 ZSTD_FRAMEHEADERSIZE_MAX /* Frame header */ + \
jpayne@69 1536 4 /* checksum */ + \
jpayne@69 1537 ((originalSize) == 0 ? 0 : 3 * (((originalSize) + (blockSize) - 1) / blockSize)) /* 3 bytes per block */ + \
jpayne@69 1538 (blockSize) /* One block of margin */ \
jpayne@69 1539 ))
jpayne@69 1540
jpayne@69 1541 typedef enum {
jpayne@69 1542 ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */
jpayne@69 1543 ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */
jpayne@69 1544 } ZSTD_sequenceFormat_e;
jpayne@69 1545
jpayne@69 1546 /*! ZSTD_sequenceBound() :
jpayne@69 1547 * `srcSize` : size of the input buffer
jpayne@69 1548 * @return : upper-bound for the number of sequences that can be generated
jpayne@69 1549 * from a buffer of srcSize bytes
jpayne@69 1550 *
jpayne@69 1551 * note : returns number of sequences - to get bytes, multiply by sizeof(ZSTD_Sequence).
jpayne@69 1552 */
jpayne@69 1553 ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize);
jpayne@69 1554
jpayne@69 1555 /*! ZSTD_generateSequences() :
jpayne@69 1556 * WARNING: This function is meant for debugging and informational purposes ONLY!
jpayne@69 1557 * Its implementation is flawed, and it will be deleted in a future version.
jpayne@69 1558 * It is not guaranteed to succeed, as there are several cases where it will give
jpayne@69 1559 * up and fail. You should NOT use this function in production code.
jpayne@69 1560 *
jpayne@69 1561 * This function is deprecated, and will be removed in a future version.
jpayne@69 1562 *
jpayne@69 1563 * Generate sequences using ZSTD_compress2(), given a source buffer.
jpayne@69 1564 *
jpayne@69 1565 * @param zc The compression context to be used for ZSTD_compress2(). Set any
jpayne@69 1566 * compression parameters you need on this context.
jpayne@69 1567 * @param outSeqs The output sequences buffer of size @p outSeqsSize
jpayne@69 1568 * @param outSeqsSize The size of the output sequences buffer.
jpayne@69 1569 * ZSTD_sequenceBound(srcSize) is an upper bound on the number
jpayne@69 1570 * of sequences that can be generated.
jpayne@69 1571 * @param src The source buffer to generate sequences from of size @p srcSize.
jpayne@69 1572 * @param srcSize The size of the source buffer.
jpayne@69 1573 *
jpayne@69 1574 * Each block will end with a dummy sequence
jpayne@69 1575 * with offset == 0, matchLength == 0, and litLength == length of last literals.
jpayne@69 1576 * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0)
jpayne@69 1577 * simply acts as a block delimiter.
jpayne@69 1578 *
jpayne@69 1579 * @returns The number of sequences generated, necessarily less than
jpayne@69 1580 * ZSTD_sequenceBound(srcSize), or an error code that can be checked
jpayne@69 1581 * with ZSTD_isError().
jpayne@69 1582 */
jpayne@69 1583 ZSTD_DEPRECATED("For debugging only, will be replaced by ZSTD_extractSequences()")
jpayne@69 1584 ZSTDLIB_STATIC_API size_t
jpayne@69 1585 ZSTD_generateSequences(ZSTD_CCtx* zc,
jpayne@69 1586 ZSTD_Sequence* outSeqs, size_t outSeqsSize,
jpayne@69 1587 const void* src, size_t srcSize);
jpayne@69 1588
jpayne@69 1589 /*! ZSTD_mergeBlockDelimiters() :
jpayne@69 1590 * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals
jpayne@69 1591 * by merging them into the literals of the next sequence.
jpayne@69 1592 *
jpayne@69 1593 * As such, the final generated result has no explicit representation of block boundaries,
jpayne@69 1594 * and the final last literals segment is not represented in the sequences.
jpayne@69 1595 *
jpayne@69 1596 * The output of this function can be fed into ZSTD_compressSequences() with CCtx
jpayne@69 1597 * setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters
jpayne@69 1598 * @return : number of sequences left after merging
jpayne@69 1599 */
jpayne@69 1600 ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize);
jpayne@69 1601
jpayne@69 1602 /*! ZSTD_compressSequences() :
jpayne@69 1603 * Compress an array of ZSTD_Sequence, associated with @src buffer, into dst.
jpayne@69 1604 * @src contains the entire input (not just the literals).
jpayne@69 1605 * If @srcSize > sum(sequence.length), the remaining bytes are considered all literals
jpayne@69 1606 * If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.)
jpayne@69 1607 * The entire source is compressed into a single frame.
jpayne@69 1608 *
jpayne@69 1609 * The compression behavior changes based on cctx params. In particular:
jpayne@69 1610 * If ZSTD_c_blockDelimiters == ZSTD_sf_noBlockDelimiters, the array of ZSTD_Sequence is expected to contain
jpayne@69 1611 * no block delimiters (defined in ZSTD_Sequence). Block boundaries are roughly determined based on
jpayne@69 1612 * the block size derived from the cctx, and sequences may be split. This is the default setting.
jpayne@69 1613 *
jpayne@69 1614 * If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain
jpayne@69 1615 * block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided.
jpayne@69 1616 *
jpayne@69 1617 * If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined
jpayne@69 1618 * behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for
jpayne@69 1619 * specifics regarding offset/matchlength requirements) then the function will bail out and return an error.
jpayne@69 1620 *
jpayne@69 1621 * In addition to the two adjustable experimental params, there are other important cctx params.
jpayne@69 1622 * - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN.
jpayne@69 1623 * - ZSTD_c_compressionLevel accordingly adjusts the strength of the entropy coder, as it would in typical compression.
jpayne@69 1624 * - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset
jpayne@69 1625 * is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md
jpayne@69 1626 *
jpayne@69 1627 * Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused.
jpayne@69 1628 * Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly,
jpayne@69 1629 * and cannot emit an RLE block that disagrees with the repcode history
jpayne@69 1630 * @return : final compressed size, or a ZSTD error code.
jpayne@69 1631 */
jpayne@69 1632 ZSTDLIB_STATIC_API size_t
jpayne@69 1633 ZSTD_compressSequences( ZSTD_CCtx* cctx, void* dst, size_t dstSize,
jpayne@69 1634 const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
jpayne@69 1635 const void* src, size_t srcSize);
jpayne@69 1636
jpayne@69 1637
jpayne@69 1638 /*! ZSTD_writeSkippableFrame() :
jpayne@69 1639 * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer.
jpayne@69 1640 *
jpayne@69 1641 * Skippable frames begin with a 4-byte magic number. There are 16 possible choices of magic number,
jpayne@69 1642 * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15.
jpayne@69 1643 * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so
jpayne@69 1644 * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.
jpayne@69 1645 *
jpayne@69 1646 * Returns an error if destination buffer is not large enough, if the source size is not representable
jpayne@69 1647 * with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid).
jpayne@69 1648 *
jpayne@69 1649 * @return : number of bytes written or a ZSTD error.
jpayne@69 1650 */
jpayne@69 1651 ZSTDLIB_STATIC_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
jpayne@69 1652 const void* src, size_t srcSize, unsigned magicVariant);
jpayne@69 1653
jpayne@69 1654 /*! ZSTD_readSkippableFrame() :
jpayne@69 1655 * Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer.
jpayne@69 1656 *
jpayne@69 1657 * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written,
jpayne@69 1658 * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested
jpayne@69 1659 * in the magicVariant.
jpayne@69 1660 *
jpayne@69 1661 * Returns an error if destination buffer is not large enough, or if the frame is not skippable.
jpayne@69 1662 *
jpayne@69 1663 * @return : number of bytes written or a ZSTD error.
jpayne@69 1664 */
jpayne@69 1665 ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant,
jpayne@69 1666 const void* src, size_t srcSize);
jpayne@69 1667
jpayne@69 1668 /*! ZSTD_isSkippableFrame() :
jpayne@69 1669 * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame.
jpayne@69 1670 */
jpayne@69 1671 ZSTDLIB_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size);
jpayne@69 1672
jpayne@69 1673
jpayne@69 1674
jpayne@69 1675 /***************************************
jpayne@69 1676 * Memory management
jpayne@69 1677 ***************************************/
jpayne@69 1678
jpayne@69 1679 /*! ZSTD_estimate*() :
jpayne@69 1680 * These functions make it possible to estimate memory usage
jpayne@69 1681 * of a future {D,C}Ctx, before its creation.
jpayne@69 1682 * This is useful in combination with ZSTD_initStatic(),
jpayne@69 1683 * which makes it possible to employ a static buffer for ZSTD_CCtx* state.
jpayne@69 1684 *
jpayne@69 1685 * ZSTD_estimateCCtxSize() will provide a memory budget large enough
jpayne@69 1686 * to compress data of any size using one-shot compression ZSTD_compressCCtx() or ZSTD_compress2()
jpayne@69 1687 * associated with any compression level up to max specified one.
jpayne@69 1688 * The estimate will assume the input may be arbitrarily large,
jpayne@69 1689 * which is the worst case.
jpayne@69 1690 *
jpayne@69 1691 * Note that the size estimation is specific for one-shot compression,
jpayne@69 1692 * it is not valid for streaming (see ZSTD_estimateCStreamSize*())
jpayne@69 1693 * nor other potential ways of using a ZSTD_CCtx* state.
jpayne@69 1694 *
jpayne@69 1695 * When srcSize can be bound by a known and rather "small" value,
jpayne@69 1696 * this knowledge can be used to provide a tighter budget estimation
jpayne@69 1697 * because the ZSTD_CCtx* state will need less memory for small inputs.
jpayne@69 1698 * This tighter estimation can be provided by employing more advanced functions
jpayne@69 1699 * ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(),
jpayne@69 1700 * and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter().
jpayne@69 1701 * Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits.
jpayne@69 1702 *
jpayne@69 1703 * Note : only single-threaded compression is supported.
jpayne@69 1704 * ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
jpayne@69 1705 */
jpayne@69 1706 ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int maxCompressionLevel);
jpayne@69 1707 ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
jpayne@69 1708 ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params);
jpayne@69 1709 ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void);
jpayne@69 1710
jpayne@69 1711 /*! ZSTD_estimateCStreamSize() :
jpayne@69 1712 * ZSTD_estimateCStreamSize() will provide a memory budget large enough for streaming compression
jpayne@69 1713 * using any compression level up to the max specified one.
jpayne@69 1714 * It will also consider src size to be arbitrarily "large", which is a worst case scenario.
jpayne@69 1715 * If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation.
jpayne@69 1716 * ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
jpayne@69 1717 * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
jpayne@69 1718 * Note : CStream size estimation is only correct for single-threaded compression.
jpayne@69 1719 * ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
jpayne@69 1720 * Note 2 : ZSTD_estimateCStreamSize* functions are not compatible with the Block-Level Sequence Producer API at this time.
jpayne@69 1721 * Size estimates assume that no external sequence producer is registered.
jpayne@69 1722 *
jpayne@69 1723 * ZSTD_DStream memory budget depends on frame's window Size.
jpayne@69 1724 * This information can be passed manually, using ZSTD_estimateDStreamSize,
jpayne@69 1725 * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
jpayne@69 1726 * Any frame requesting a window size larger than max specified one will be rejected.
jpayne@69 1727 * Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
jpayne@69 1728 * an internal ?Dict will be created, which additional size is not estimated here.
jpayne@69 1729 * In this case, get total size by adding ZSTD_estimate?DictSize
jpayne@69 1730 */
jpayne@69 1731 ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int maxCompressionLevel);
jpayne@69 1732 ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams);
jpayne@69 1733 ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params);
jpayne@69 1734 ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize(size_t maxWindowSize);
jpayne@69 1735 ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
jpayne@69 1736
jpayne@69 1737 /*! ZSTD_estimate?DictSize() :
jpayne@69 1738 * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict().
jpayne@69 1739 * ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced().
jpayne@69 1740 * Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller.
jpayne@69 1741 */
jpayne@69 1742 ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
jpayne@69 1743 ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod);
jpayne@69 1744 ZSTDLIB_STATIC_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod);
jpayne@69 1745
jpayne@69 1746 /*! ZSTD_initStatic*() :
jpayne@69 1747 * Initialize an object using a pre-allocated fixed-size buffer.
jpayne@69 1748 * workspace: The memory area to emplace the object into.
jpayne@69 1749 * Provided pointer *must be 8-bytes aligned*.
jpayne@69 1750 * Buffer must outlive object.
jpayne@69 1751 * workspaceSize: Use ZSTD_estimate*Size() to determine
jpayne@69 1752 * how large workspace must be to support target scenario.
jpayne@69 1753 * @return : pointer to object (same address as workspace, just different type),
jpayne@69 1754 * or NULL if error (size too small, incorrect alignment, etc.)
jpayne@69 1755 * Note : zstd will never resize nor malloc() when using a static buffer.
jpayne@69 1756 * If the object requires more memory than available,
jpayne@69 1757 * zstd will just error out (typically ZSTD_error_memory_allocation).
jpayne@69 1758 * Note 2 : there is no corresponding "free" function.
jpayne@69 1759 * Since workspace is allocated externally, it must be freed externally too.
jpayne@69 1760 * Note 3 : cParams : use ZSTD_getCParams() to convert a compression level
jpayne@69 1761 * into its associated cParams.
jpayne@69 1762 * Limitation 1 : currently not compatible with internal dictionary creation, triggered by
jpayne@69 1763 * ZSTD_CCtx_loadDictionary(), ZSTD_initCStream_usingDict() or ZSTD_initDStream_usingDict().
jpayne@69 1764 * Limitation 2 : static cctx currently not compatible with multi-threading.
jpayne@69 1765 * Limitation 3 : static dctx is incompatible with legacy support.
jpayne@69 1766 */
jpayne@69 1767 ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize);
jpayne@69 1768 ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */
jpayne@69 1769
jpayne@69 1770 ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize);
jpayne@69 1771 ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */
jpayne@69 1772
jpayne@69 1773 ZSTDLIB_STATIC_API const ZSTD_CDict* ZSTD_initStaticCDict(
jpayne@69 1774 void* workspace, size_t workspaceSize,
jpayne@69 1775 const void* dict, size_t dictSize,
jpayne@69 1776 ZSTD_dictLoadMethod_e dictLoadMethod,
jpayne@69 1777 ZSTD_dictContentType_e dictContentType,
jpayne@69 1778 ZSTD_compressionParameters cParams);
jpayne@69 1779
jpayne@69 1780 ZSTDLIB_STATIC_API const ZSTD_DDict* ZSTD_initStaticDDict(
jpayne@69 1781 void* workspace, size_t workspaceSize,
jpayne@69 1782 const void* dict, size_t dictSize,
jpayne@69 1783 ZSTD_dictLoadMethod_e dictLoadMethod,
jpayne@69 1784 ZSTD_dictContentType_e dictContentType);
jpayne@69 1785
jpayne@69 1786
jpayne@69 1787 /*! Custom memory allocation :
jpayne@69 1788 * These prototypes make it possible to pass your own allocation/free functions.
jpayne@69 1789 * ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below.
jpayne@69 1790 * All allocation/free operations will be completed using these custom variants instead of regular <stdlib.h> ones.
jpayne@69 1791 */
jpayne@69 1792 typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
jpayne@69 1793 typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
jpayne@69 1794 typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
jpayne@69 1795 static
jpayne@69 1796 #ifdef __GNUC__
jpayne@69 1797 __attribute__((__unused__))
jpayne@69 1798 #endif
jpayne@69 1799 ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */
jpayne@69 1800
jpayne@69 1801 ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
jpayne@69 1802 ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
jpayne@69 1803 ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
jpayne@69 1804 ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
jpayne@69 1805
jpayne@69 1806 ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
jpayne@69 1807 ZSTD_dictLoadMethod_e dictLoadMethod,
jpayne@69 1808 ZSTD_dictContentType_e dictContentType,
jpayne@69 1809 ZSTD_compressionParameters cParams,
jpayne@69 1810 ZSTD_customMem customMem);
jpayne@69 1811
jpayne@69 1812 /*! Thread pool :
jpayne@69 1813 * These prototypes make it possible to share a thread pool among multiple compression contexts.
jpayne@69 1814 * This can limit resources for applications with multiple threads where each one uses
jpayne@69 1815 * a threaded compression mode (via ZSTD_c_nbWorkers parameter).
jpayne@69 1816 * ZSTD_createThreadPool creates a new thread pool with a given number of threads.
jpayne@69 1817 * Note that the lifetime of such pool must exist while being used.
jpayne@69 1818 * ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value
jpayne@69 1819 * to use an internal thread pool).
jpayne@69 1820 * ZSTD_freeThreadPool frees a thread pool, accepts NULL pointer.
jpayne@69 1821 */
jpayne@69 1822 typedef struct POOL_ctx_s ZSTD_threadPool;
jpayne@69 1823 ZSTDLIB_STATIC_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads);
jpayne@69 1824 ZSTDLIB_STATIC_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool); /* accept NULL pointer */
jpayne@69 1825 ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool);
jpayne@69 1826
jpayne@69 1827
jpayne@69 1828 /*
jpayne@69 1829 * This API is temporary and is expected to change or disappear in the future!
jpayne@69 1830 */
jpayne@69 1831 ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced2(
jpayne@69 1832 const void* dict, size_t dictSize,
jpayne@69 1833 ZSTD_dictLoadMethod_e dictLoadMethod,
jpayne@69 1834 ZSTD_dictContentType_e dictContentType,
jpayne@69 1835 const ZSTD_CCtx_params* cctxParams,
jpayne@69 1836 ZSTD_customMem customMem);
jpayne@69 1837
jpayne@69 1838 ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_advanced(
jpayne@69 1839 const void* dict, size_t dictSize,
jpayne@69 1840 ZSTD_dictLoadMethod_e dictLoadMethod,
jpayne@69 1841 ZSTD_dictContentType_e dictContentType,
jpayne@69 1842 ZSTD_customMem customMem);
jpayne@69 1843
jpayne@69 1844
jpayne@69 1845 /***************************************
jpayne@69 1846 * Advanced compression functions
jpayne@69 1847 ***************************************/
jpayne@69 1848
jpayne@69 1849 /*! ZSTD_createCDict_byReference() :
jpayne@69 1850 * Create a digested dictionary for compression
jpayne@69 1851 * Dictionary content is just referenced, not duplicated.
jpayne@69 1852 * As a consequence, `dictBuffer` **must** outlive CDict,
jpayne@69 1853 * and its content must remain unmodified throughout the lifetime of CDict.
jpayne@69 1854 * note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */
jpayne@69 1855 ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
jpayne@69 1856
jpayne@69 1857 /*! ZSTD_getCParams() :
jpayne@69 1858 * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
jpayne@69 1859 * `estimatedSrcSize` value is optional, select 0 if not known */
jpayne@69 1860 ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
jpayne@69 1861
jpayne@69 1862 /*! ZSTD_getParams() :
jpayne@69 1863 * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`.
jpayne@69 1864 * All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */
jpayne@69 1865 ZSTDLIB_STATIC_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
jpayne@69 1866
jpayne@69 1867 /*! ZSTD_checkCParams() :
jpayne@69 1868 * Ensure param values remain within authorized range.
jpayne@69 1869 * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */
jpayne@69 1870 ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
jpayne@69 1871
jpayne@69 1872 /*! ZSTD_adjustCParams() :
jpayne@69 1873 * optimize params for a given `srcSize` and `dictSize`.
jpayne@69 1874 * `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN.
jpayne@69 1875 * `dictSize` must be `0` when there is no dictionary.
jpayne@69 1876 * cPar can be invalid : all parameters will be clamped within valid range in the @return struct.
jpayne@69 1877 * This function never fails (wide contract) */
jpayne@69 1878 ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
jpayne@69 1879
jpayne@69 1880 /*! ZSTD_CCtx_setCParams() :
jpayne@69 1881 * Set all parameters provided within @p cparams into the working @p cctx.
jpayne@69 1882 * Note : if modifying parameters during compression (MT mode only),
jpayne@69 1883 * note that changes to the .windowLog parameter will be ignored.
jpayne@69 1884 * @return 0 on success, or an error code (can be checked with ZSTD_isError()).
jpayne@69 1885 * On failure, no parameters are updated.
jpayne@69 1886 */
jpayne@69 1887 ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams);
jpayne@69 1888
jpayne@69 1889 /*! ZSTD_CCtx_setFParams() :
jpayne@69 1890 * Set all parameters provided within @p fparams into the working @p cctx.
jpayne@69 1891 * @return 0 on success, or an error code (can be checked with ZSTD_isError()).
jpayne@69 1892 */
jpayne@69 1893 ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams);
jpayne@69 1894
jpayne@69 1895 /*! ZSTD_CCtx_setParams() :
jpayne@69 1896 * Set all parameters provided within @p params into the working @p cctx.
jpayne@69 1897 * @return 0 on success, or an error code (can be checked with ZSTD_isError()).
jpayne@69 1898 */
jpayne@69 1899 ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params);
jpayne@69 1900
jpayne@69 1901 /*! ZSTD_compress_advanced() :
jpayne@69 1902 * Note : this function is now DEPRECATED.
jpayne@69 1903 * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
jpayne@69 1904 * This prototype will generate compilation warnings. */
jpayne@69 1905 ZSTD_DEPRECATED("use ZSTD_compress2")
jpayne@69 1906 ZSTDLIB_STATIC_API
jpayne@69 1907 size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
jpayne@69 1908 void* dst, size_t dstCapacity,
jpayne@69 1909 const void* src, size_t srcSize,
jpayne@69 1910 const void* dict,size_t dictSize,
jpayne@69 1911 ZSTD_parameters params);
jpayne@69 1912
jpayne@69 1913 /*! ZSTD_compress_usingCDict_advanced() :
jpayne@69 1914 * Note : this function is now DEPRECATED.
jpayne@69 1915 * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters.
jpayne@69 1916 * This prototype will generate compilation warnings. */
jpayne@69 1917 ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary")
jpayne@69 1918 ZSTDLIB_STATIC_API
jpayne@69 1919 size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
jpayne@69 1920 void* dst, size_t dstCapacity,
jpayne@69 1921 const void* src, size_t srcSize,
jpayne@69 1922 const ZSTD_CDict* cdict,
jpayne@69 1923 ZSTD_frameParameters fParams);
jpayne@69 1924
jpayne@69 1925
jpayne@69 1926 /*! ZSTD_CCtx_loadDictionary_byReference() :
jpayne@69 1927 * Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx.
jpayne@69 1928 * It saves some memory, but also requires that `dict` outlives its usage within `cctx` */
jpayne@69 1929 ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
jpayne@69 1930
jpayne@69 1931 /*! ZSTD_CCtx_loadDictionary_advanced() :
jpayne@69 1932 * Same as ZSTD_CCtx_loadDictionary(), but gives finer control over
jpayne@69 1933 * how to load the dictionary (by copy ? by reference ?)
jpayne@69 1934 * and how to interpret it (automatic ? force raw mode ? full mode only ?) */
jpayne@69 1935 ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
jpayne@69 1936
jpayne@69 1937 /*! ZSTD_CCtx_refPrefix_advanced() :
jpayne@69 1938 * Same as ZSTD_CCtx_refPrefix(), but gives finer control over
jpayne@69 1939 * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */
jpayne@69 1940 ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
jpayne@69 1941
jpayne@69 1942 /* === experimental parameters === */
jpayne@69 1943 /* these parameters can be used with ZSTD_setParameter()
jpayne@69 1944 * they are not guaranteed to remain supported in the future */
jpayne@69 1945
jpayne@69 1946 /* Enables rsyncable mode,
jpayne@69 1947 * which makes compressed files more rsync friendly
jpayne@69 1948 * by adding periodic synchronization points to the compressed data.
jpayne@69 1949 * The target average block size is ZSTD_c_jobSize / 2.
jpayne@69 1950 * It's possible to modify the job size to increase or decrease
jpayne@69 1951 * the granularity of the synchronization point.
jpayne@69 1952 * Once the jobSize is smaller than the window size,
jpayne@69 1953 * it will result in compression ratio degradation.
jpayne@69 1954 * NOTE 1: rsyncable mode only works when multithreading is enabled.
jpayne@69 1955 * NOTE 2: rsyncable performs poorly in combination with long range mode,
jpayne@69 1956 * since it will decrease the effectiveness of synchronization points,
jpayne@69 1957 * though mileage may vary.
jpayne@69 1958 * NOTE 3: Rsyncable mode limits maximum compression speed to ~400 MB/s.
jpayne@69 1959 * If the selected compression level is already running significantly slower,
jpayne@69 1960 * the overall speed won't be significantly impacted.
jpayne@69 1961 */
jpayne@69 1962 #define ZSTD_c_rsyncable ZSTD_c_experimentalParam1
jpayne@69 1963
jpayne@69 1964 /* Select a compression format.
jpayne@69 1965 * The value must be of type ZSTD_format_e.
jpayne@69 1966 * See ZSTD_format_e enum definition for details */
jpayne@69 1967 #define ZSTD_c_format ZSTD_c_experimentalParam2
jpayne@69 1968
jpayne@69 1969 /* Force back-reference distances to remain < windowSize,
jpayne@69 1970 * even when referencing into Dictionary content (default:0) */
jpayne@69 1971 #define ZSTD_c_forceMaxWindow ZSTD_c_experimentalParam3
jpayne@69 1972
jpayne@69 1973 /* Controls whether the contents of a CDict
jpayne@69 1974 * are used in place, or copied into the working context.
jpayne@69 1975 * Accepts values from the ZSTD_dictAttachPref_e enum.
jpayne@69 1976 * See the comments on that enum for an explanation of the feature. */
jpayne@69 1977 #define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4
jpayne@69 1978
jpayne@69 1979 /* Controlled with ZSTD_paramSwitch_e enum.
jpayne@69 1980 * Default is ZSTD_ps_auto.
jpayne@69 1981 * Set to ZSTD_ps_disable to never compress literals.
jpayne@69 1982 * Set to ZSTD_ps_enable to always compress literals. (Note: uncompressed literals
jpayne@69 1983 * may still be emitted if huffman is not beneficial to use.)
jpayne@69 1984 *
jpayne@69 1985 * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
jpayne@69 1986 * literals compression based on the compression parameters - specifically,
jpayne@69 1987 * negative compression levels do not use literal compression.
jpayne@69 1988 */
jpayne@69 1989 #define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5
jpayne@69 1990
jpayne@69 1991 /* User's best guess of source size.
jpayne@69 1992 * Hint is not valid when srcSizeHint == 0.
jpayne@69 1993 * There is no guarantee that hint is close to actual source size,
jpayne@69 1994 * but compression ratio may regress significantly if guess considerably underestimates */
jpayne@69 1995 #define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam7
jpayne@69 1996
jpayne@69 1997 /* Controls whether the new and experimental "dedicated dictionary search
jpayne@69 1998 * structure" can be used. This feature is still rough around the edges, be
jpayne@69 1999 * prepared for surprising behavior!
jpayne@69 2000 *
jpayne@69 2001 * How to use it:
jpayne@69 2002 *
jpayne@69 2003 * When using a CDict, whether to use this feature or not is controlled at
jpayne@69 2004 * CDict creation, and it must be set in a CCtxParams set passed into that
jpayne@69 2005 * construction (via ZSTD_createCDict_advanced2()). A compression will then
jpayne@69 2006 * use the feature or not based on how the CDict was constructed; the value of
jpayne@69 2007 * this param, set in the CCtx, will have no effect.
jpayne@69 2008 *
jpayne@69 2009 * However, when a dictionary buffer is passed into a CCtx, such as via
jpayne@69 2010 * ZSTD_CCtx_loadDictionary(), this param can be set on the CCtx to control
jpayne@69 2011 * whether the CDict that is created internally can use the feature or not.
jpayne@69 2012 *
jpayne@69 2013 * What it does:
jpayne@69 2014 *
jpayne@69 2015 * Normally, the internal data structures of the CDict are analogous to what
jpayne@69 2016 * would be stored in a CCtx after compressing the contents of a dictionary.
jpayne@69 2017 * To an approximation, a compression using a dictionary can then use those
jpayne@69 2018 * data structures to simply continue what is effectively a streaming
jpayne@69 2019 * compression where the simulated compression of the dictionary left off.
jpayne@69 2020 * Which is to say, the search structures in the CDict are normally the same
jpayne@69 2021 * format as in the CCtx.
jpayne@69 2022 *
jpayne@69 2023 * It is possible to do better, since the CDict is not like a CCtx: the search
jpayne@69 2024 * structures are written once during CDict creation, and then are only read
jpayne@69 2025 * after that, while the search structures in the CCtx are both read and
jpayne@69 2026 * written as the compression goes along. This means we can choose a search
jpayne@69 2027 * structure for the dictionary that is read-optimized.
jpayne@69 2028 *
jpayne@69 2029 * This feature enables the use of that different structure.
jpayne@69 2030 *
jpayne@69 2031 * Note that some of the members of the ZSTD_compressionParameters struct have
jpayne@69 2032 * different semantics and constraints in the dedicated search structure. It is
jpayne@69 2033 * highly recommended that you simply set a compression level in the CCtxParams
jpayne@69 2034 * you pass into the CDict creation call, and avoid messing with the cParams
jpayne@69 2035 * directly.
jpayne@69 2036 *
jpayne@69 2037 * Effects:
jpayne@69 2038 *
jpayne@69 2039 * This will only have any effect when the selected ZSTD_strategy
jpayne@69 2040 * implementation supports this feature. Currently, that's limited to
jpayne@69 2041 * ZSTD_greedy, ZSTD_lazy, and ZSTD_lazy2.
jpayne@69 2042 *
jpayne@69 2043 * Note that this means that the CDict tables can no longer be copied into the
jpayne@69 2044 * CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be
jpayne@69 2045 * usable. The dictionary can only be attached or reloaded.
jpayne@69 2046 *
jpayne@69 2047 * In general, you should expect compression to be faster--sometimes very much
jpayne@69 2048 * so--and CDict creation to be slightly slower. Eventually, we will probably
jpayne@69 2049 * make this mode the default.
jpayne@69 2050 */
jpayne@69 2051 #define ZSTD_c_enableDedicatedDictSearch ZSTD_c_experimentalParam8
jpayne@69 2052
jpayne@69 2053 /* ZSTD_c_stableInBuffer
jpayne@69 2054 * Experimental parameter.
jpayne@69 2055 * Default is 0 == disabled. Set to 1 to enable.
jpayne@69 2056 *
jpayne@69 2057 * Tells the compressor that input data presented with ZSTD_inBuffer
jpayne@69 2058 * will ALWAYS be the same between calls.
jpayne@69 2059 * Technically, the @src pointer must never be changed,
jpayne@69 2060 * and the @pos field can only be updated by zstd.
jpayne@69 2061 * However, it's possible to increase the @size field,
jpayne@69 2062 * allowing scenarios where more data can be appended after compressions starts.
jpayne@69 2063 * These conditions are checked by the compressor,
jpayne@69 2064 * and compression will fail if they are not respected.
jpayne@69 2065 * Also, data in the ZSTD_inBuffer within the range [src, src + pos)
jpayne@69 2066 * MUST not be modified during compression or it will result in data corruption.
jpayne@69 2067 *
jpayne@69 2068 * When this flag is enabled zstd won't allocate an input window buffer,
jpayne@69 2069 * because the user guarantees it can reference the ZSTD_inBuffer until
jpayne@69 2070 * the frame is complete. But, it will still allocate an output buffer
jpayne@69 2071 * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also
jpayne@69 2072 * avoid the memcpy() from the input buffer to the input window buffer.
jpayne@69 2073 *
jpayne@69 2074 * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using
jpayne@69 2075 * this flag is ALWAYS memory safe, and will never access out-of-bounds
jpayne@69 2076 * memory. However, compression WILL fail if conditions are not respected.
jpayne@69 2077 *
jpayne@69 2078 * WARNING: The data in the ZSTD_inBuffer in the range [src, src + pos) MUST
jpayne@69 2079 * not be modified during compression or it will result in data corruption.
jpayne@69 2080 * This is because zstd needs to reference data in the ZSTD_inBuffer to find
jpayne@69 2081 * matches. Normally zstd maintains its own window buffer for this purpose,
jpayne@69 2082 * but passing this flag tells zstd to rely on user provided buffer instead.
jpayne@69 2083 */
jpayne@69 2084 #define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9
jpayne@69 2085
jpayne@69 2086 /* ZSTD_c_stableOutBuffer
jpayne@69 2087 * Experimental parameter.
jpayne@69 2088 * Default is 0 == disabled. Set to 1 to enable.
jpayne@69 2089 *
jpayne@69 2090 * Tells he compressor that the ZSTD_outBuffer will not be resized between
jpayne@69 2091 * calls. Specifically: (out.size - out.pos) will never grow. This gives the
jpayne@69 2092 * compressor the freedom to say: If the compressed data doesn't fit in the
jpayne@69 2093 * output buffer then return ZSTD_error_dstSizeTooSmall. This allows us to
jpayne@69 2094 * always decompress directly into the output buffer, instead of decompressing
jpayne@69 2095 * into an internal buffer and copying to the output buffer.
jpayne@69 2096 *
jpayne@69 2097 * When this flag is enabled zstd won't allocate an output buffer, because
jpayne@69 2098 * it can write directly to the ZSTD_outBuffer. It will still allocate the
jpayne@69 2099 * input window buffer (see ZSTD_c_stableInBuffer).
jpayne@69 2100 *
jpayne@69 2101 * Zstd will check that (out.size - out.pos) never grows and return an error
jpayne@69 2102 * if it does. While not strictly necessary, this should prevent surprises.
jpayne@69 2103 */
jpayne@69 2104 #define ZSTD_c_stableOutBuffer ZSTD_c_experimentalParam10
jpayne@69 2105
jpayne@69 2106 /* ZSTD_c_blockDelimiters
jpayne@69 2107 * Default is 0 == ZSTD_sf_noBlockDelimiters.
jpayne@69 2108 *
jpayne@69 2109 * For use with sequence compression API: ZSTD_compressSequences().
jpayne@69 2110 *
jpayne@69 2111 * Designates whether or not the given array of ZSTD_Sequence contains block delimiters
jpayne@69 2112 * and last literals, which are defined as sequences with offset == 0 and matchLength == 0.
jpayne@69 2113 * See the definition of ZSTD_Sequence for more specifics.
jpayne@69 2114 */
jpayne@69 2115 #define ZSTD_c_blockDelimiters ZSTD_c_experimentalParam11
jpayne@69 2116
jpayne@69 2117 /* ZSTD_c_validateSequences
jpayne@69 2118 * Default is 0 == disabled. Set to 1 to enable sequence validation.
jpayne@69 2119 *
jpayne@69 2120 * For use with sequence compression API: ZSTD_compressSequences().
jpayne@69 2121 * Designates whether or not we validate sequences provided to ZSTD_compressSequences()
jpayne@69 2122 * during function execution.
jpayne@69 2123 *
jpayne@69 2124 * Without validation, providing a sequence that does not conform to the zstd spec will cause
jpayne@69 2125 * undefined behavior, and may produce a corrupted block.
jpayne@69 2126 *
jpayne@69 2127 * With validation enabled, if sequence is invalid (see doc/zstd_compression_format.md for
jpayne@69 2128 * specifics regarding offset/matchlength requirements) then the function will bail out and
jpayne@69 2129 * return an error.
jpayne@69 2130 *
jpayne@69 2131 */
jpayne@69 2132 #define ZSTD_c_validateSequences ZSTD_c_experimentalParam12
jpayne@69 2133
jpayne@69 2134 /* ZSTD_c_useBlockSplitter
jpayne@69 2135 * Controlled with ZSTD_paramSwitch_e enum.
jpayne@69 2136 * Default is ZSTD_ps_auto.
jpayne@69 2137 * Set to ZSTD_ps_disable to never use block splitter.
jpayne@69 2138 * Set to ZSTD_ps_enable to always use block splitter.
jpayne@69 2139 *
jpayne@69 2140 * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
jpayne@69 2141 * block splitting based on the compression parameters.
jpayne@69 2142 */
jpayne@69 2143 #define ZSTD_c_useBlockSplitter ZSTD_c_experimentalParam13
jpayne@69 2144
jpayne@69 2145 /* ZSTD_c_useRowMatchFinder
jpayne@69 2146 * Controlled with ZSTD_paramSwitch_e enum.
jpayne@69 2147 * Default is ZSTD_ps_auto.
jpayne@69 2148 * Set to ZSTD_ps_disable to never use row-based matchfinder.
jpayne@69 2149 * Set to ZSTD_ps_enable to force usage of row-based matchfinder.
jpayne@69 2150 *
jpayne@69 2151 * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
jpayne@69 2152 * the row-based matchfinder based on support for SIMD instructions and the window log.
jpayne@69 2153 * Note that this only pertains to compression strategies: greedy, lazy, and lazy2
jpayne@69 2154 */
jpayne@69 2155 #define ZSTD_c_useRowMatchFinder ZSTD_c_experimentalParam14
jpayne@69 2156
jpayne@69 2157 /* ZSTD_c_deterministicRefPrefix
jpayne@69 2158 * Default is 0 == disabled. Set to 1 to enable.
jpayne@69 2159 *
jpayne@69 2160 * Zstd produces different results for prefix compression when the prefix is
jpayne@69 2161 * directly adjacent to the data about to be compressed vs. when it isn't.
jpayne@69 2162 * This is because zstd detects that the two buffers are contiguous and it can
jpayne@69 2163 * use a more efficient match finding algorithm. However, this produces different
jpayne@69 2164 * results than when the two buffers are non-contiguous. This flag forces zstd
jpayne@69 2165 * to always load the prefix in non-contiguous mode, even if it happens to be
jpayne@69 2166 * adjacent to the data, to guarantee determinism.
jpayne@69 2167 *
jpayne@69 2168 * If you really care about determinism when using a dictionary or prefix,
jpayne@69 2169 * like when doing delta compression, you should select this option. It comes
jpayne@69 2170 * at a speed penalty of about ~2.5% if the dictionary and data happened to be
jpayne@69 2171 * contiguous, and is free if they weren't contiguous. We don't expect that
jpayne@69 2172 * intentionally making the dictionary and data contiguous will be worth the
jpayne@69 2173 * cost to memcpy() the data.
jpayne@69 2174 */
jpayne@69 2175 #define ZSTD_c_deterministicRefPrefix ZSTD_c_experimentalParam15
jpayne@69 2176
jpayne@69 2177 /* ZSTD_c_prefetchCDictTables
jpayne@69 2178 * Controlled with ZSTD_paramSwitch_e enum. Default is ZSTD_ps_auto.
jpayne@69 2179 *
jpayne@69 2180 * In some situations, zstd uses CDict tables in-place rather than copying them
jpayne@69 2181 * into the working context. (See docs on ZSTD_dictAttachPref_e above for details).
jpayne@69 2182 * In such situations, compression speed is seriously impacted when CDict tables are
jpayne@69 2183 * "cold" (outside CPU cache). This parameter instructs zstd to prefetch CDict tables
jpayne@69 2184 * when they are used in-place.
jpayne@69 2185 *
jpayne@69 2186 * For sufficiently small inputs, the cost of the prefetch will outweigh the benefit.
jpayne@69 2187 * For sufficiently large inputs, zstd will by default memcpy() CDict tables
jpayne@69 2188 * into the working context, so there is no need to prefetch. This parameter is
jpayne@69 2189 * targeted at a middle range of input sizes, where a prefetch is cheap enough to be
jpayne@69 2190 * useful but memcpy() is too expensive. The exact range of input sizes where this
jpayne@69 2191 * makes sense is best determined by careful experimentation.
jpayne@69 2192 *
jpayne@69 2193 * Note: for this parameter, ZSTD_ps_auto is currently equivalent to ZSTD_ps_disable,
jpayne@69 2194 * but in the future zstd may conditionally enable this feature via an auto-detection
jpayne@69 2195 * heuristic for cold CDicts.
jpayne@69 2196 * Use ZSTD_ps_disable to opt out of prefetching under any circumstances.
jpayne@69 2197 */
jpayne@69 2198 #define ZSTD_c_prefetchCDictTables ZSTD_c_experimentalParam16
jpayne@69 2199
jpayne@69 2200 /* ZSTD_c_enableSeqProducerFallback
jpayne@69 2201 * Allowed values are 0 (disable) and 1 (enable). The default setting is 0.
jpayne@69 2202 *
jpayne@69 2203 * Controls whether zstd will fall back to an internal sequence producer if an
jpayne@69 2204 * external sequence producer is registered and returns an error code. This fallback
jpayne@69 2205 * is block-by-block: the internal sequence producer will only be called for blocks
jpayne@69 2206 * where the external sequence producer returns an error code. Fallback parsing will
jpayne@69 2207 * follow any other cParam settings, such as compression level, the same as in a
jpayne@69 2208 * normal (fully-internal) compression operation.
jpayne@69 2209 *
jpayne@69 2210 * The user is strongly encouraged to read the full Block-Level Sequence Producer API
jpayne@69 2211 * documentation (below) before setting this parameter. */
jpayne@69 2212 #define ZSTD_c_enableSeqProducerFallback ZSTD_c_experimentalParam17
jpayne@69 2213
jpayne@69 2214 /* ZSTD_c_maxBlockSize
jpayne@69 2215 * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB).
jpayne@69 2216 * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default.
jpayne@69 2217 *
jpayne@69 2218 * This parameter can be used to set an upper bound on the blocksize
jpayne@69 2219 * that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper
jpayne@69 2220 * bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make
jpayne@69 2221 * compressBound() inaccurate). Only currently meant to be used for testing.
jpayne@69 2222 *
jpayne@69 2223 */
jpayne@69 2224 #define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18
jpayne@69 2225
jpayne@69 2226 /* ZSTD_c_searchForExternalRepcodes
jpayne@69 2227 * This parameter affects how zstd parses external sequences, such as sequences
jpayne@69 2228 * provided through the compressSequences() API or from an external block-level
jpayne@69 2229 * sequence producer.
jpayne@69 2230 *
jpayne@69 2231 * If set to ZSTD_ps_enable, the library will check for repeated offsets in
jpayne@69 2232 * external sequences, even if those repcodes are not explicitly indicated in
jpayne@69 2233 * the "rep" field. Note that this is the only way to exploit repcode matches
jpayne@69 2234 * while using compressSequences() or an external sequence producer, since zstd
jpayne@69 2235 * currently ignores the "rep" field of external sequences.
jpayne@69 2236 *
jpayne@69 2237 * If set to ZSTD_ps_disable, the library will not exploit repeated offsets in
jpayne@69 2238 * external sequences, regardless of whether the "rep" field has been set. This
jpayne@69 2239 * reduces sequence compression overhead by about 25% while sacrificing some
jpayne@69 2240 * compression ratio.
jpayne@69 2241 *
jpayne@69 2242 * The default value is ZSTD_ps_auto, for which the library will enable/disable
jpayne@69 2243 * based on compression level.
jpayne@69 2244 *
jpayne@69 2245 * Note: for now, this param only has an effect if ZSTD_c_blockDelimiters is
jpayne@69 2246 * set to ZSTD_sf_explicitBlockDelimiters. That may change in the future.
jpayne@69 2247 */
jpayne@69 2248 #define ZSTD_c_searchForExternalRepcodes ZSTD_c_experimentalParam19
jpayne@69 2249
jpayne@69 2250 /*! ZSTD_CCtx_getParameter() :
jpayne@69 2251 * Get the requested compression parameter value, selected by enum ZSTD_cParameter,
jpayne@69 2252 * and store it into int* value.
jpayne@69 2253 * @return : 0, or an error code (which can be tested with ZSTD_isError()).
jpayne@69 2254 */
jpayne@69 2255 ZSTDLIB_STATIC_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
jpayne@69 2256
jpayne@69 2257
jpayne@69 2258 /*! ZSTD_CCtx_params :
jpayne@69 2259 * Quick howto :
jpayne@69 2260 * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure
jpayne@69 2261 * - ZSTD_CCtxParams_setParameter() : Push parameters one by one into
jpayne@69 2262 * an existing ZSTD_CCtx_params structure.
jpayne@69 2263 * This is similar to
jpayne@69 2264 * ZSTD_CCtx_setParameter().
jpayne@69 2265 * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to
jpayne@69 2266 * an existing CCtx.
jpayne@69 2267 * These parameters will be applied to
jpayne@69 2268 * all subsequent frames.
jpayne@69 2269 * - ZSTD_compressStream2() : Do compression using the CCtx.
jpayne@69 2270 * - ZSTD_freeCCtxParams() : Free the memory, accept NULL pointer.
jpayne@69 2271 *
jpayne@69 2272 * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams()
jpayne@69 2273 * for static allocation of CCtx for single-threaded compression.
jpayne@69 2274 */
jpayne@69 2275 ZSTDLIB_STATIC_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
jpayne@69 2276 ZSTDLIB_STATIC_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); /* accept NULL pointer */
jpayne@69 2277
jpayne@69 2278 /*! ZSTD_CCtxParams_reset() :
jpayne@69 2279 * Reset params to default values.
jpayne@69 2280 */
jpayne@69 2281 ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params);
jpayne@69 2282
jpayne@69 2283 /*! ZSTD_CCtxParams_init() :
jpayne@69 2284 * Initializes the compression parameters of cctxParams according to
jpayne@69 2285 * compression level. All other parameters are reset to their default values.
jpayne@69 2286 */
jpayne@69 2287 ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel);
jpayne@69 2288
jpayne@69 2289 /*! ZSTD_CCtxParams_init_advanced() :
jpayne@69 2290 * Initializes the compression and frame parameters of cctxParams according to
jpayne@69 2291 * params. All other parameters are reset to their default values.
jpayne@69 2292 */
jpayne@69 2293 ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params);
jpayne@69 2294
jpayne@69 2295 /*! ZSTD_CCtxParams_setParameter() : Requires v1.4.0+
jpayne@69 2296 * Similar to ZSTD_CCtx_setParameter.
jpayne@69 2297 * Set one compression parameter, selected by enum ZSTD_cParameter.
jpayne@69 2298 * Parameters must be applied to a ZSTD_CCtx using
jpayne@69 2299 * ZSTD_CCtx_setParametersUsingCCtxParams().
jpayne@69 2300 * @result : a code representing success or failure (which can be tested with
jpayne@69 2301 * ZSTD_isError()).
jpayne@69 2302 */
jpayne@69 2303 ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
jpayne@69 2304
jpayne@69 2305 /*! ZSTD_CCtxParams_getParameter() :
jpayne@69 2306 * Similar to ZSTD_CCtx_getParameter.
jpayne@69 2307 * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
jpayne@69 2308 * @result : 0, or an error code (which can be tested with ZSTD_isError()).
jpayne@69 2309 */
jpayne@69 2310 ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
jpayne@69 2311
jpayne@69 2312 /*! ZSTD_CCtx_setParametersUsingCCtxParams() :
jpayne@69 2313 * Apply a set of ZSTD_CCtx_params to the compression context.
jpayne@69 2314 * This can be done even after compression is started,
jpayne@69 2315 * if nbWorkers==0, this will have no impact until a new compression is started.
jpayne@69 2316 * if nbWorkers>=1, new parameters will be picked up at next job,
jpayne@69 2317 * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated).
jpayne@69 2318 */
jpayne@69 2319 ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParametersUsingCCtxParams(
jpayne@69 2320 ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params);
jpayne@69 2321
jpayne@69 2322 /*! ZSTD_compressStream2_simpleArgs() :
jpayne@69 2323 * Same as ZSTD_compressStream2(),
jpayne@69 2324 * but using only integral types as arguments.
jpayne@69 2325 * This variant might be helpful for binders from dynamic languages
jpayne@69 2326 * which have troubles handling structures containing memory pointers.
jpayne@69 2327 */
jpayne@69 2328 ZSTDLIB_STATIC_API size_t ZSTD_compressStream2_simpleArgs (
jpayne@69 2329 ZSTD_CCtx* cctx,
jpayne@69 2330 void* dst, size_t dstCapacity, size_t* dstPos,
jpayne@69 2331 const void* src, size_t srcSize, size_t* srcPos,
jpayne@69 2332 ZSTD_EndDirective endOp);
jpayne@69 2333
jpayne@69 2334
jpayne@69 2335 /***************************************
jpayne@69 2336 * Advanced decompression functions
jpayne@69 2337 ***************************************/
jpayne@69 2338
jpayne@69 2339 /*! ZSTD_isFrame() :
jpayne@69 2340 * Tells if the content of `buffer` starts with a valid Frame Identifier.
jpayne@69 2341 * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
jpayne@69 2342 * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled.
jpayne@69 2343 * Note 3 : Skippable Frame Identifiers are considered valid. */
jpayne@69 2344 ZSTDLIB_STATIC_API unsigned ZSTD_isFrame(const void* buffer, size_t size);
jpayne@69 2345
jpayne@69 2346 /*! ZSTD_createDDict_byReference() :
jpayne@69 2347 * Create a digested dictionary, ready to start decompression operation without startup delay.
jpayne@69 2348 * Dictionary content is referenced, and therefore stays in dictBuffer.
jpayne@69 2349 * It is important that dictBuffer outlives DDict,
jpayne@69 2350 * it must remain read accessible throughout the lifetime of DDict */
jpayne@69 2351 ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
jpayne@69 2352
jpayne@69 2353 /*! ZSTD_DCtx_loadDictionary_byReference() :
jpayne@69 2354 * Same as ZSTD_DCtx_loadDictionary(),
jpayne@69 2355 * but references `dict` content instead of copying it into `dctx`.
jpayne@69 2356 * This saves memory if `dict` remains around.,
jpayne@69 2357 * However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */
jpayne@69 2358 ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
jpayne@69 2359
jpayne@69 2360 /*! ZSTD_DCtx_loadDictionary_advanced() :
jpayne@69 2361 * Same as ZSTD_DCtx_loadDictionary(),
jpayne@69 2362 * but gives direct control over
jpayne@69 2363 * how to load the dictionary (by copy ? by reference ?)
jpayne@69 2364 * and how to interpret it (automatic ? force raw mode ? full mode only ?). */
jpayne@69 2365 ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
jpayne@69 2366
jpayne@69 2367 /*! ZSTD_DCtx_refPrefix_advanced() :
jpayne@69 2368 * Same as ZSTD_DCtx_refPrefix(), but gives finer control over
jpayne@69 2369 * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */
jpayne@69 2370 ZSTDLIB_STATIC_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
jpayne@69 2371
jpayne@69 2372 /*! ZSTD_DCtx_setMaxWindowSize() :
jpayne@69 2373 * Refuses allocating internal buffers for frames requiring a window size larger than provided limit.
jpayne@69 2374 * This protects a decoder context from reserving too much memory for itself (potential attack scenario).
jpayne@69 2375 * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
jpayne@69 2376 * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT)
jpayne@69 2377 * @return : 0, or an error code (which can be tested using ZSTD_isError()).
jpayne@69 2378 */
jpayne@69 2379 ZSTDLIB_STATIC_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
jpayne@69 2380
jpayne@69 2381 /*! ZSTD_DCtx_getParameter() :
jpayne@69 2382 * Get the requested decompression parameter value, selected by enum ZSTD_dParameter,
jpayne@69 2383 * and store it into int* value.
jpayne@69 2384 * @return : 0, or an error code (which can be tested with ZSTD_isError()).
jpayne@69 2385 */
jpayne@69 2386 ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value);
jpayne@69 2387
jpayne@69 2388 /* ZSTD_d_format
jpayne@69 2389 * experimental parameter,
jpayne@69 2390 * allowing selection between ZSTD_format_e input compression formats
jpayne@69 2391 */
jpayne@69 2392 #define ZSTD_d_format ZSTD_d_experimentalParam1
jpayne@69 2393 /* ZSTD_d_stableOutBuffer
jpayne@69 2394 * Experimental parameter.
jpayne@69 2395 * Default is 0 == disabled. Set to 1 to enable.
jpayne@69 2396 *
jpayne@69 2397 * Tells the decompressor that the ZSTD_outBuffer will ALWAYS be the same
jpayne@69 2398 * between calls, except for the modifications that zstd makes to pos (the
jpayne@69 2399 * caller must not modify pos). This is checked by the decompressor, and
jpayne@69 2400 * decompression will fail if it ever changes. Therefore the ZSTD_outBuffer
jpayne@69 2401 * MUST be large enough to fit the entire decompressed frame. This will be
jpayne@69 2402 * checked when the frame content size is known. The data in the ZSTD_outBuffer
jpayne@69 2403 * in the range [dst, dst + pos) MUST not be modified during decompression
jpayne@69 2404 * or you will get data corruption.
jpayne@69 2405 *
jpayne@69 2406 * When this flag is enabled zstd won't allocate an output buffer, because
jpayne@69 2407 * it can write directly to the ZSTD_outBuffer, but it will still allocate
jpayne@69 2408 * an input buffer large enough to fit any compressed block. This will also
jpayne@69 2409 * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer.
jpayne@69 2410 * If you need to avoid the input buffer allocation use the buffer-less
jpayne@69 2411 * streaming API.
jpayne@69 2412 *
jpayne@69 2413 * NOTE: So long as the ZSTD_outBuffer always points to valid memory, using
jpayne@69 2414 * this flag is ALWAYS memory safe, and will never access out-of-bounds
jpayne@69 2415 * memory. However, decompression WILL fail if you violate the preconditions.
jpayne@69 2416 *
jpayne@69 2417 * WARNING: The data in the ZSTD_outBuffer in the range [dst, dst + pos) MUST
jpayne@69 2418 * not be modified during decompression or you will get data corruption. This
jpayne@69 2419 * is because zstd needs to reference data in the ZSTD_outBuffer to regenerate
jpayne@69 2420 * matches. Normally zstd maintains its own buffer for this purpose, but passing
jpayne@69 2421 * this flag tells zstd to use the user provided buffer.
jpayne@69 2422 */
jpayne@69 2423 #define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam2
jpayne@69 2424
jpayne@69 2425 /* ZSTD_d_forceIgnoreChecksum
jpayne@69 2426 * Experimental parameter.
jpayne@69 2427 * Default is 0 == disabled. Set to 1 to enable
jpayne@69 2428 *
jpayne@69 2429 * Tells the decompressor to skip checksum validation during decompression, regardless
jpayne@69 2430 * of whether checksumming was specified during compression. This offers some
jpayne@69 2431 * slight performance benefits, and may be useful for debugging.
jpayne@69 2432 * Param has values of type ZSTD_forceIgnoreChecksum_e
jpayne@69 2433 */
jpayne@69 2434 #define ZSTD_d_forceIgnoreChecksum ZSTD_d_experimentalParam3
jpayne@69 2435
jpayne@69 2436 /* ZSTD_d_refMultipleDDicts
jpayne@69 2437 * Experimental parameter.
jpayne@69 2438 * Default is 0 == disabled. Set to 1 to enable
jpayne@69 2439 *
jpayne@69 2440 * If enabled and dctx is allocated on the heap, then additional memory will be allocated
jpayne@69 2441 * to store references to multiple ZSTD_DDict. That is, multiple calls of ZSTD_refDDict()
jpayne@69 2442 * using a given ZSTD_DCtx, rather than overwriting the previous DDict reference, will instead
jpayne@69 2443 * store all references. At decompression time, the appropriate dictID is selected
jpayne@69 2444 * from the set of DDicts based on the dictID in the frame.
jpayne@69 2445 *
jpayne@69 2446 * Usage is simply calling ZSTD_refDDict() on multiple dict buffers.
jpayne@69 2447 *
jpayne@69 2448 * Param has values of byte ZSTD_refMultipleDDicts_e
jpayne@69 2449 *
jpayne@69 2450 * WARNING: Enabling this parameter and calling ZSTD_DCtx_refDDict(), will trigger memory
jpayne@69 2451 * allocation for the hash table. ZSTD_freeDCtx() also frees this memory.
jpayne@69 2452 * Memory is allocated as per ZSTD_DCtx::customMem.
jpayne@69 2453 *
jpayne@69 2454 * Although this function allocates memory for the table, the user is still responsible for
jpayne@69 2455 * memory management of the underlying ZSTD_DDict* themselves.
jpayne@69 2456 */
jpayne@69 2457 #define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4
jpayne@69 2458
jpayne@69 2459 /* ZSTD_d_disableHuffmanAssembly
jpayne@69 2460 * Set to 1 to disable the Huffman assembly implementation.
jpayne@69 2461 * The default value is 0, which allows zstd to use the Huffman assembly
jpayne@69 2462 * implementation if available.
jpayne@69 2463 *
jpayne@69 2464 * This parameter can be used to disable Huffman assembly at runtime.
jpayne@69 2465 * If you want to disable it at compile time you can define the macro
jpayne@69 2466 * ZSTD_DISABLE_ASM.
jpayne@69 2467 */
jpayne@69 2468 #define ZSTD_d_disableHuffmanAssembly ZSTD_d_experimentalParam5
jpayne@69 2469
jpayne@69 2470 /* ZSTD_d_maxBlockSize
jpayne@69 2471 * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB).
jpayne@69 2472 * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default.
jpayne@69 2473 *
jpayne@69 2474 * Forces the decompressor to reject blocks whose content size is
jpayne@69 2475 * larger than the configured maxBlockSize. When maxBlockSize is
jpayne@69 2476 * larger than the windowSize, the windowSize is used instead.
jpayne@69 2477 * This saves memory on the decoder when you know all blocks are small.
jpayne@69 2478 *
jpayne@69 2479 * This option is typically used in conjunction with ZSTD_c_maxBlockSize.
jpayne@69 2480 *
jpayne@69 2481 * WARNING: This causes the decoder to reject otherwise valid frames
jpayne@69 2482 * that have block sizes larger than the configured maxBlockSize.
jpayne@69 2483 */
jpayne@69 2484 #define ZSTD_d_maxBlockSize ZSTD_d_experimentalParam6
jpayne@69 2485
jpayne@69 2486
jpayne@69 2487 /*! ZSTD_DCtx_setFormat() :
jpayne@69 2488 * This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter().
jpayne@69 2489 * Instruct the decoder context about what kind of data to decode next.
jpayne@69 2490 * This instruction is mandatory to decode data without a fully-formed header,
jpayne@69 2491 * such ZSTD_f_zstd1_magicless for example.
jpayne@69 2492 * @return : 0, or an error code (which can be tested using ZSTD_isError()). */
jpayne@69 2493 ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead")
jpayne@69 2494 ZSTDLIB_STATIC_API
jpayne@69 2495 size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
jpayne@69 2496
jpayne@69 2497 /*! ZSTD_decompressStream_simpleArgs() :
jpayne@69 2498 * Same as ZSTD_decompressStream(),
jpayne@69 2499 * but using only integral types as arguments.
jpayne@69 2500 * This can be helpful for binders from dynamic languages
jpayne@69 2501 * which have troubles handling structures containing memory pointers.
jpayne@69 2502 */
jpayne@69 2503 ZSTDLIB_STATIC_API size_t ZSTD_decompressStream_simpleArgs (
jpayne@69 2504 ZSTD_DCtx* dctx,
jpayne@69 2505 void* dst, size_t dstCapacity, size_t* dstPos,
jpayne@69 2506 const void* src, size_t srcSize, size_t* srcPos);
jpayne@69 2507
jpayne@69 2508
jpayne@69 2509 /********************************************************************
jpayne@69 2510 * Advanced streaming functions
jpayne@69 2511 * Warning : most of these functions are now redundant with the Advanced API.
jpayne@69 2512 * Once Advanced API reaches "stable" status,
jpayne@69 2513 * redundant functions will be deprecated, and then at some point removed.
jpayne@69 2514 ********************************************************************/
jpayne@69 2515
jpayne@69 2516 /*===== Advanced Streaming compression functions =====*/
jpayne@69 2517
jpayne@69 2518 /*! ZSTD_initCStream_srcSize() :
jpayne@69 2519 * This function is DEPRECATED, and equivalent to:
jpayne@69 2520 * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
jpayne@69 2521 * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
jpayne@69 2522 * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
jpayne@69 2523 * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
jpayne@69 2524 *
jpayne@69 2525 * pledgedSrcSize must be correct. If it is not known at init time, use
jpayne@69 2526 * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs,
jpayne@69 2527 * "0" also disables frame content size field. It may be enabled in the future.
jpayne@69 2528 * This prototype will generate compilation warnings.
jpayne@69 2529 */
jpayne@69 2530 ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
jpayne@69 2531 ZSTDLIB_STATIC_API
jpayne@69 2532 size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
jpayne@69 2533 int compressionLevel,
jpayne@69 2534 unsigned long long pledgedSrcSize);
jpayne@69 2535
jpayne@69 2536 /*! ZSTD_initCStream_usingDict() :
jpayne@69 2537 * This function is DEPRECATED, and is equivalent to:
jpayne@69 2538 * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
jpayne@69 2539 * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
jpayne@69 2540 * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
jpayne@69 2541 *
jpayne@69 2542 * Creates of an internal CDict (incompatible with static CCtx), except if
jpayne@69 2543 * dict == NULL or dictSize < 8, in which case no dict is used.
jpayne@69 2544 * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
jpayne@69 2545 * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
jpayne@69 2546 * This prototype will generate compilation warnings.
jpayne@69 2547 */
jpayne@69 2548 ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
jpayne@69 2549 ZSTDLIB_STATIC_API
jpayne@69 2550 size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
jpayne@69 2551 const void* dict, size_t dictSize,
jpayne@69 2552 int compressionLevel);
jpayne@69 2553
jpayne@69 2554 /*! ZSTD_initCStream_advanced() :
jpayne@69 2555 * This function is DEPRECATED, and is equivalent to:
jpayne@69 2556 * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
jpayne@69 2557 * ZSTD_CCtx_setParams(zcs, params);
jpayne@69 2558 * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
jpayne@69 2559 * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
jpayne@69 2560 *
jpayne@69 2561 * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
jpayne@69 2562 * pledgedSrcSize must be correct.
jpayne@69 2563 * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
jpayne@69 2564 * This prototype will generate compilation warnings.
jpayne@69 2565 */
jpayne@69 2566 ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
jpayne@69 2567 ZSTDLIB_STATIC_API
jpayne@69 2568 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
jpayne@69 2569 const void* dict, size_t dictSize,
jpayne@69 2570 ZSTD_parameters params,
jpayne@69 2571 unsigned long long pledgedSrcSize);
jpayne@69 2572
jpayne@69 2573 /*! ZSTD_initCStream_usingCDict() :
jpayne@69 2574 * This function is DEPRECATED, and equivalent to:
jpayne@69 2575 * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
jpayne@69 2576 * ZSTD_CCtx_refCDict(zcs, cdict);
jpayne@69 2577 *
jpayne@69 2578 * note : cdict will just be referenced, and must outlive compression session
jpayne@69 2579 * This prototype will generate compilation warnings.
jpayne@69 2580 */
jpayne@69 2581 ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
jpayne@69 2582 ZSTDLIB_STATIC_API
jpayne@69 2583 size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
jpayne@69 2584
jpayne@69 2585 /*! ZSTD_initCStream_usingCDict_advanced() :
jpayne@69 2586 * This function is DEPRECATED, and is equivalent to:
jpayne@69 2587 * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
jpayne@69 2588 * ZSTD_CCtx_setFParams(zcs, fParams);
jpayne@69 2589 * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
jpayne@69 2590 * ZSTD_CCtx_refCDict(zcs, cdict);
jpayne@69 2591 *
jpayne@69 2592 * same as ZSTD_initCStream_usingCDict(), with control over frame parameters.
jpayne@69 2593 * pledgedSrcSize must be correct. If srcSize is not known at init time, use
jpayne@69 2594 * value ZSTD_CONTENTSIZE_UNKNOWN.
jpayne@69 2595 * This prototype will generate compilation warnings.
jpayne@69 2596 */
jpayne@69 2597 ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
jpayne@69 2598 ZSTDLIB_STATIC_API
jpayne@69 2599 size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
jpayne@69 2600 const ZSTD_CDict* cdict,
jpayne@69 2601 ZSTD_frameParameters fParams,
jpayne@69 2602 unsigned long long pledgedSrcSize);
jpayne@69 2603
jpayne@69 2604 /*! ZSTD_resetCStream() :
jpayne@69 2605 * This function is DEPRECATED, and is equivalent to:
jpayne@69 2606 * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
jpayne@69 2607 * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
jpayne@69 2608 * Note: ZSTD_resetCStream() interprets pledgedSrcSize == 0 as ZSTD_CONTENTSIZE_UNKNOWN, but
jpayne@69 2609 * ZSTD_CCtx_setPledgedSrcSize() does not do the same, so ZSTD_CONTENTSIZE_UNKNOWN must be
jpayne@69 2610 * explicitly specified.
jpayne@69 2611 *
jpayne@69 2612 * start a new frame, using same parameters from previous frame.
jpayne@69 2613 * This is typically useful to skip dictionary loading stage, since it will reuse it in-place.
jpayne@69 2614 * Note that zcs must be init at least once before using ZSTD_resetCStream().
jpayne@69 2615 * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.
jpayne@69 2616 * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end.
jpayne@69 2617 * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
jpayne@69 2618 * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead.
jpayne@69 2619 * @return : 0, or an error code (which can be tested using ZSTD_isError())
jpayne@69 2620 * This prototype will generate compilation warnings.
jpayne@69 2621 */
jpayne@69 2622 ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
jpayne@69 2623 ZSTDLIB_STATIC_API
jpayne@69 2624 size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
jpayne@69 2625
jpayne@69 2626
jpayne@69 2627 typedef struct {
jpayne@69 2628 unsigned long long ingested; /* nb input bytes read and buffered */
jpayne@69 2629 unsigned long long consumed; /* nb input bytes actually compressed */
jpayne@69 2630 unsigned long long produced; /* nb of compressed bytes generated and buffered */
jpayne@69 2631 unsigned long long flushed; /* nb of compressed bytes flushed : not provided; can be tracked from caller side */
jpayne@69 2632 unsigned currentJobID; /* MT only : latest started job nb */
jpayne@69 2633 unsigned nbActiveWorkers; /* MT only : nb of workers actively compressing at probe time */
jpayne@69 2634 } ZSTD_frameProgression;
jpayne@69 2635
jpayne@69 2636 /* ZSTD_getFrameProgression() :
jpayne@69 2637 * tells how much data has been ingested (read from input)
jpayne@69 2638 * consumed (input actually compressed) and produced (output) for current frame.
jpayne@69 2639 * Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed.
jpayne@69 2640 * Aggregates progression inside active worker threads.
jpayne@69 2641 */
jpayne@69 2642 ZSTDLIB_STATIC_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx);
jpayne@69 2643
jpayne@69 2644 /*! ZSTD_toFlushNow() :
jpayne@69 2645 * Tell how many bytes are ready to be flushed immediately.
jpayne@69 2646 * Useful for multithreading scenarios (nbWorkers >= 1).
jpayne@69 2647 * Probe the oldest active job, defined as oldest job not yet entirely flushed,
jpayne@69 2648 * and check its output buffer.
jpayne@69 2649 * @return : amount of data stored in oldest job and ready to be flushed immediately.
jpayne@69 2650 * if @return == 0, it means either :
jpayne@69 2651 * + there is no active job (could be checked with ZSTD_frameProgression()), or
jpayne@69 2652 * + oldest job is still actively compressing data,
jpayne@69 2653 * but everything it has produced has also been flushed so far,
jpayne@69 2654 * therefore flush speed is limited by production speed of oldest job
jpayne@69 2655 * irrespective of the speed of concurrent (and newer) jobs.
jpayne@69 2656 */
jpayne@69 2657 ZSTDLIB_STATIC_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
jpayne@69 2658
jpayne@69 2659
jpayne@69 2660 /*===== Advanced Streaming decompression functions =====*/
jpayne@69 2661
jpayne@69 2662 /*!
jpayne@69 2663 * This function is deprecated, and is equivalent to:
jpayne@69 2664 *
jpayne@69 2665 * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
jpayne@69 2666 * ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
jpayne@69 2667 *
jpayne@69 2668 * note: no dictionary will be used if dict == NULL or dictSize < 8
jpayne@69 2669 */
jpayne@69 2670 ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_loadDictionary, see zstd.h for detailed instructions")
jpayne@69 2671 ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
jpayne@69 2672
jpayne@69 2673 /*!
jpayne@69 2674 * This function is deprecated, and is equivalent to:
jpayne@69 2675 *
jpayne@69 2676 * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
jpayne@69 2677 * ZSTD_DCtx_refDDict(zds, ddict);
jpayne@69 2678 *
jpayne@69 2679 * note : ddict is referenced, it must outlive decompression session
jpayne@69 2680 */
jpayne@69 2681 ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_refDDict, see zstd.h for detailed instructions")
jpayne@69 2682 ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
jpayne@69 2683
jpayne@69 2684 /*!
jpayne@69 2685 * This function is deprecated, and is equivalent to:
jpayne@69 2686 *
jpayne@69 2687 * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
jpayne@69 2688 *
jpayne@69 2689 * reuse decompression parameters from previous init; saves dictionary loading
jpayne@69 2690 */
jpayne@69 2691 ZSTD_DEPRECATED("use ZSTD_DCtx_reset, see zstd.h for detailed instructions")
jpayne@69 2692 ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
jpayne@69 2693
jpayne@69 2694
jpayne@69 2695 /* ********************* BLOCK-LEVEL SEQUENCE PRODUCER API *********************
jpayne@69 2696 *
jpayne@69 2697 * *** OVERVIEW ***
jpayne@69 2698 * The Block-Level Sequence Producer API allows users to provide their own custom
jpayne@69 2699 * sequence producer which libzstd invokes to process each block. The produced list
jpayne@69 2700 * of sequences (literals and matches) is then post-processed by libzstd to produce
jpayne@69 2701 * valid compressed blocks.
jpayne@69 2702 *
jpayne@69 2703 * This block-level offload API is a more granular complement of the existing
jpayne@69 2704 * frame-level offload API compressSequences() (introduced in v1.5.1). It offers
jpayne@69 2705 * an easier migration story for applications already integrated with libzstd: the
jpayne@69 2706 * user application continues to invoke the same compression functions
jpayne@69 2707 * ZSTD_compress2() or ZSTD_compressStream2() as usual, and transparently benefits
jpayne@69 2708 * from the specific advantages of the external sequence producer. For example,
jpayne@69 2709 * the sequence producer could be tuned to take advantage of known characteristics
jpayne@69 2710 * of the input, to offer better speed / ratio, or could leverage hardware
jpayne@69 2711 * acceleration not available within libzstd itself.
jpayne@69 2712 *
jpayne@69 2713 * See contrib/externalSequenceProducer for an example program employing the
jpayne@69 2714 * Block-Level Sequence Producer API.
jpayne@69 2715 *
jpayne@69 2716 * *** USAGE ***
jpayne@69 2717 * The user is responsible for implementing a function of type
jpayne@69 2718 * ZSTD_sequenceProducer_F. For each block, zstd will pass the following
jpayne@69 2719 * arguments to the user-provided function:
jpayne@69 2720 *
jpayne@69 2721 * - sequenceProducerState: a pointer to a user-managed state for the sequence
jpayne@69 2722 * producer.
jpayne@69 2723 *
jpayne@69 2724 * - outSeqs, outSeqsCapacity: an output buffer for the sequence producer.
jpayne@69 2725 * outSeqsCapacity is guaranteed >= ZSTD_sequenceBound(srcSize). The memory
jpayne@69 2726 * backing outSeqs is managed by the CCtx.
jpayne@69 2727 *
jpayne@69 2728 * - src, srcSize: an input buffer for the sequence producer to parse.
jpayne@69 2729 * srcSize is guaranteed to be <= ZSTD_BLOCKSIZE_MAX.
jpayne@69 2730 *
jpayne@69 2731 * - dict, dictSize: a history buffer, which may be empty, which the sequence
jpayne@69 2732 * producer may reference as it parses the src buffer. Currently, zstd will
jpayne@69 2733 * always pass dictSize == 0 into external sequence producers, but this will
jpayne@69 2734 * change in the future.
jpayne@69 2735 *
jpayne@69 2736 * - compressionLevel: a signed integer representing the zstd compression level
jpayne@69 2737 * set by the user for the current operation. The sequence producer may choose
jpayne@69 2738 * to use this information to change its compression strategy and speed/ratio
jpayne@69 2739 * tradeoff. Note: the compression level does not reflect zstd parameters set
jpayne@69 2740 * through the advanced API.
jpayne@69 2741 *
jpayne@69 2742 * - windowSize: a size_t representing the maximum allowed offset for external
jpayne@69 2743 * sequences. Note that sequence offsets are sometimes allowed to exceed the
jpayne@69 2744 * windowSize if a dictionary is present, see doc/zstd_compression_format.md
jpayne@69 2745 * for details.
jpayne@69 2746 *
jpayne@69 2747 * The user-provided function shall return a size_t representing the number of
jpayne@69 2748 * sequences written to outSeqs. This return value will be treated as an error
jpayne@69 2749 * code if it is greater than outSeqsCapacity. The return value must be non-zero
jpayne@69 2750 * if srcSize is non-zero. The ZSTD_SEQUENCE_PRODUCER_ERROR macro is provided
jpayne@69 2751 * for convenience, but any value greater than outSeqsCapacity will be treated as
jpayne@69 2752 * an error code.
jpayne@69 2753 *
jpayne@69 2754 * If the user-provided function does not return an error code, the sequences
jpayne@69 2755 * written to outSeqs must be a valid parse of the src buffer. Data corruption may
jpayne@69 2756 * occur if the parse is not valid. A parse is defined to be valid if the
jpayne@69 2757 * following conditions hold:
jpayne@69 2758 * - The sum of matchLengths and literalLengths must equal srcSize.
jpayne@69 2759 * - All sequences in the parse, except for the final sequence, must have
jpayne@69 2760 * matchLength >= ZSTD_MINMATCH_MIN. The final sequence must have
jpayne@69 2761 * matchLength >= ZSTD_MINMATCH_MIN or matchLength == 0.
jpayne@69 2762 * - All offsets must respect the windowSize parameter as specified in
jpayne@69 2763 * doc/zstd_compression_format.md.
jpayne@69 2764 * - If the final sequence has matchLength == 0, it must also have offset == 0.
jpayne@69 2765 *
jpayne@69 2766 * zstd will only validate these conditions (and fail compression if they do not
jpayne@69 2767 * hold) if the ZSTD_c_validateSequences cParam is enabled. Note that sequence
jpayne@69 2768 * validation has a performance cost.
jpayne@69 2769 *
jpayne@69 2770 * If the user-provided function returns an error, zstd will either fall back
jpayne@69 2771 * to an internal sequence producer or fail the compression operation. The user can
jpayne@69 2772 * choose between the two behaviors by setting the ZSTD_c_enableSeqProducerFallback
jpayne@69 2773 * cParam. Fallback compression will follow any other cParam settings, such as
jpayne@69 2774 * compression level, the same as in a normal compression operation.
jpayne@69 2775 *
jpayne@69 2776 * The user shall instruct zstd to use a particular ZSTD_sequenceProducer_F
jpayne@69 2777 * function by calling
jpayne@69 2778 * ZSTD_registerSequenceProducer(cctx,
jpayne@69 2779 * sequenceProducerState,
jpayne@69 2780 * sequenceProducer)
jpayne@69 2781 * This setting will persist until the next parameter reset of the CCtx.
jpayne@69 2782 *
jpayne@69 2783 * The sequenceProducerState must be initialized by the user before calling
jpayne@69 2784 * ZSTD_registerSequenceProducer(). The user is responsible for destroying the
jpayne@69 2785 * sequenceProducerState.
jpayne@69 2786 *
jpayne@69 2787 * *** LIMITATIONS ***
jpayne@69 2788 * This API is compatible with all zstd compression APIs which respect advanced parameters.
jpayne@69 2789 * However, there are three limitations:
jpayne@69 2790 *
jpayne@69 2791 * First, the ZSTD_c_enableLongDistanceMatching cParam is not currently supported.
jpayne@69 2792 * COMPRESSION WILL FAIL if it is enabled and the user tries to compress with a block-level
jpayne@69 2793 * external sequence producer.
jpayne@69 2794 * - Note that ZSTD_c_enableLongDistanceMatching is auto-enabled by default in some
jpayne@69 2795 * cases (see its documentation for details). Users must explicitly set
jpayne@69 2796 * ZSTD_c_enableLongDistanceMatching to ZSTD_ps_disable in such cases if an external
jpayne@69 2797 * sequence producer is registered.
jpayne@69 2798 * - As of this writing, ZSTD_c_enableLongDistanceMatching is disabled by default
jpayne@69 2799 * whenever ZSTD_c_windowLog < 128MB, but that's subject to change. Users should
jpayne@69 2800 * check the docs on ZSTD_c_enableLongDistanceMatching whenever the Block-Level Sequence
jpayne@69 2801 * Producer API is used in conjunction with advanced settings (like ZSTD_c_windowLog).
jpayne@69 2802 *
jpayne@69 2803 * Second, history buffers are not currently supported. Concretely, zstd will always pass
jpayne@69 2804 * dictSize == 0 to the external sequence producer (for now). This has two implications:
jpayne@69 2805 * - Dictionaries are not currently supported. Compression will *not* fail if the user
jpayne@69 2806 * references a dictionary, but the dictionary won't have any effect.
jpayne@69 2807 * - Stream history is not currently supported. All advanced compression APIs, including
jpayne@69 2808 * streaming APIs, work with external sequence producers, but each block is treated as
jpayne@69 2809 * an independent chunk without history from previous blocks.
jpayne@69 2810 *
jpayne@69 2811 * Third, multi-threading within a single compression is not currently supported. In other words,
jpayne@69 2812 * COMPRESSION WILL FAIL if ZSTD_c_nbWorkers > 0 and an external sequence producer is registered.
jpayne@69 2813 * Multi-threading across compressions is fine: simply create one CCtx per thread.
jpayne@69 2814 *
jpayne@69 2815 * Long-term, we plan to overcome all three limitations. There is no technical blocker to
jpayne@69 2816 * overcoming them. It is purely a question of engineering effort.
jpayne@69 2817 */
jpayne@69 2818
jpayne@69 2819 #define ZSTD_SEQUENCE_PRODUCER_ERROR ((size_t)(-1))
jpayne@69 2820
jpayne@69 2821 typedef size_t (*ZSTD_sequenceProducer_F) (
jpayne@69 2822 void* sequenceProducerState,
jpayne@69 2823 ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
jpayne@69 2824 const void* src, size_t srcSize,
jpayne@69 2825 const void* dict, size_t dictSize,
jpayne@69 2826 int compressionLevel,
jpayne@69 2827 size_t windowSize
jpayne@69 2828 );
jpayne@69 2829
jpayne@69 2830 /*! ZSTD_registerSequenceProducer() :
jpayne@69 2831 * Instruct zstd to use a block-level external sequence producer function.
jpayne@69 2832 *
jpayne@69 2833 * The sequenceProducerState must be initialized by the caller, and the caller is
jpayne@69 2834 * responsible for managing its lifetime. This parameter is sticky across
jpayne@69 2835 * compressions. It will remain set until the user explicitly resets compression
jpayne@69 2836 * parameters.
jpayne@69 2837 *
jpayne@69 2838 * Sequence producer registration is considered to be an "advanced parameter",
jpayne@69 2839 * part of the "advanced API". This means it will only have an effect on compression
jpayne@69 2840 * APIs which respect advanced parameters, such as compress2() and compressStream2().
jpayne@69 2841 * Older compression APIs such as compressCCtx(), which predate the introduction of
jpayne@69 2842 * "advanced parameters", will ignore any external sequence producer setting.
jpayne@69 2843 *
jpayne@69 2844 * The sequence producer can be "cleared" by registering a NULL function pointer. This
jpayne@69 2845 * removes all limitations described above in the "LIMITATIONS" section of the API docs.
jpayne@69 2846 *
jpayne@69 2847 * The user is strongly encouraged to read the full API documentation (above) before
jpayne@69 2848 * calling this function. */
jpayne@69 2849 ZSTDLIB_STATIC_API void
jpayne@69 2850 ZSTD_registerSequenceProducer(
jpayne@69 2851 ZSTD_CCtx* cctx,
jpayne@69 2852 void* sequenceProducerState,
jpayne@69 2853 ZSTD_sequenceProducer_F sequenceProducer
jpayne@69 2854 );
jpayne@69 2855
jpayne@69 2856 /*! ZSTD_CCtxParams_registerSequenceProducer() :
jpayne@69 2857 * Same as ZSTD_registerSequenceProducer(), but operates on ZSTD_CCtx_params.
jpayne@69 2858 * This is used for accurate size estimation with ZSTD_estimateCCtxSize_usingCCtxParams(),
jpayne@69 2859 * which is needed when creating a ZSTD_CCtx with ZSTD_initStaticCCtx().
jpayne@69 2860 *
jpayne@69 2861 * If you are using the external sequence producer API in a scenario where ZSTD_initStaticCCtx()
jpayne@69 2862 * is required, then this function is for you. Otherwise, you probably don't need it.
jpayne@69 2863 *
jpayne@69 2864 * See tests/zstreamtest.c for example usage. */
jpayne@69 2865 ZSTDLIB_STATIC_API void
jpayne@69 2866 ZSTD_CCtxParams_registerSequenceProducer(
jpayne@69 2867 ZSTD_CCtx_params* params,
jpayne@69 2868 void* sequenceProducerState,
jpayne@69 2869 ZSTD_sequenceProducer_F sequenceProducer
jpayne@69 2870 );
jpayne@69 2871
jpayne@69 2872
jpayne@69 2873 /*********************************************************************
jpayne@69 2874 * Buffer-less and synchronous inner streaming functions (DEPRECATED)
jpayne@69 2875 *
jpayne@69 2876 * This API is deprecated, and will be removed in a future version.
jpayne@69 2877 * It allows streaming (de)compression with user allocated buffers.
jpayne@69 2878 * However, it is hard to use, and not as well tested as the rest of
jpayne@69 2879 * our API.
jpayne@69 2880 *
jpayne@69 2881 * Please use the normal streaming API instead: ZSTD_compressStream2,
jpayne@69 2882 * and ZSTD_decompressStream.
jpayne@69 2883 * If there is functionality that you need, but it doesn't provide,
jpayne@69 2884 * please open an issue on our GitHub.
jpayne@69 2885 ********************************************************************* */
jpayne@69 2886
jpayne@69 2887 /**
jpayne@69 2888 Buffer-less streaming compression (synchronous mode)
jpayne@69 2889
jpayne@69 2890 A ZSTD_CCtx object is required to track streaming operations.
jpayne@69 2891 Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
jpayne@69 2892 ZSTD_CCtx object can be reused multiple times within successive compression operations.
jpayne@69 2893
jpayne@69 2894 Start by initializing a context.
jpayne@69 2895 Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression.
jpayne@69 2896
jpayne@69 2897 Then, consume your input using ZSTD_compressContinue().
jpayne@69 2898 There are some important considerations to keep in mind when using this advanced function :
jpayne@69 2899 - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only.
jpayne@69 2900 - Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks.
jpayne@69 2901 - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario.
jpayne@69 2902 Worst case evaluation is provided by ZSTD_compressBound().
jpayne@69 2903 ZSTD_compressContinue() doesn't guarantee recover after a failed compression.
jpayne@69 2904 - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog).
jpayne@69 2905 It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks)
jpayne@69 2906 - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps.
jpayne@69 2907 In which case, it will "discard" the relevant memory section from its history.
jpayne@69 2908
jpayne@69 2909 Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum.
jpayne@69 2910 It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame.
jpayne@69 2911 Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders.
jpayne@69 2912
jpayne@69 2913 `ZSTD_CCtx` object can be reused (ZSTD_compressBegin()) to compress again.
jpayne@69 2914 */
jpayne@69 2915
jpayne@69 2916 /*===== Buffer-less streaming compression functions =====*/
jpayne@69 2917 ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
jpayne@69 2918 ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
jpayne@69 2919 ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
jpayne@69 2920 ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
jpayne@69 2921 ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
jpayne@69 2922 ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */
jpayne@69 2923
jpayne@69 2924 ZSTD_DEPRECATED("This function will likely be removed in a future release. It is misleading and has very limited utility.")
jpayne@69 2925 ZSTDLIB_STATIC_API
jpayne@69 2926 size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
jpayne@69 2927
jpayne@69 2928 ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
jpayne@69 2929 ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
jpayne@69 2930 ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
jpayne@69 2931 ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
jpayne@69 2932
jpayne@69 2933 /* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */
jpayne@69 2934 ZSTD_DEPRECATED("use advanced API to access custom parameters")
jpayne@69 2935 ZSTDLIB_STATIC_API
jpayne@69 2936 size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */
jpayne@69 2937 ZSTD_DEPRECATED("use advanced API to access custom parameters")
jpayne@69 2938 ZSTDLIB_STATIC_API
jpayne@69 2939 size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
jpayne@69 2940 /**
jpayne@69 2941 Buffer-less streaming decompression (synchronous mode)
jpayne@69 2942
jpayne@69 2943 A ZSTD_DCtx object is required to track streaming operations.
jpayne@69 2944 Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
jpayne@69 2945 A ZSTD_DCtx object can be reused multiple times.
jpayne@69 2946
jpayne@69 2947 First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader().
jpayne@69 2948 Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
jpayne@69 2949 Data fragment must be large enough to ensure successful decoding.
jpayne@69 2950 `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
jpayne@69 2951 result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
jpayne@69 2952 >0 : `srcSize` is too small, please provide at least result bytes on next attempt.
jpayne@69 2953 errorCode, which can be tested using ZSTD_isError().
jpayne@69 2954
jpayne@69 2955 It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
jpayne@69 2956 such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`).
jpayne@69 2957 Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information.
jpayne@69 2958 As a consequence, check that values remain within valid application range.
jpayne@69 2959 For example, do not allocate memory blindly, check that `windowSize` is within expectation.
jpayne@69 2960 Each application can set its own limits, depending on local restrictions.
jpayne@69 2961 For extended interoperability, it is recommended to support `windowSize` of at least 8 MB.
jpayne@69 2962
jpayne@69 2963 ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes.
jpayne@69 2964 ZSTD_decompressContinue() is very sensitive to contiguity,
jpayne@69 2965 if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place,
jpayne@69 2966 or that previous contiguous segment is large enough to properly handle maximum back-reference distance.
jpayne@69 2967 There are multiple ways to guarantee this condition.
jpayne@69 2968
jpayne@69 2969 The most memory efficient way is to use a round buffer of sufficient size.
jpayne@69 2970 Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(),
jpayne@69 2971 which can return an error code if required value is too large for current system (in 32-bits mode).
jpayne@69 2972 In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one,
jpayne@69 2973 up to the moment there is not enough room left in the buffer to guarantee decoding another full block,
jpayne@69 2974 which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`.
jpayne@69 2975 At which point, decoding can resume from the beginning of the buffer.
jpayne@69 2976 Note that already decoded data stored in the buffer should be flushed before being overwritten.
jpayne@69 2977
jpayne@69 2978 There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory.
jpayne@69 2979
jpayne@69 2980 Finally, if you control the compression process, you can also ignore all buffer size rules,
jpayne@69 2981 as long as the encoder and decoder progress in "lock-step",
jpayne@69 2982 aka use exactly the same buffer sizes, break contiguity at the same place, etc.
jpayne@69 2983
jpayne@69 2984 Once buffers are setup, start decompression, with ZSTD_decompressBegin().
jpayne@69 2985 If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict().
jpayne@69 2986
jpayne@69 2987 Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
jpayne@69 2988 ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue().
jpayne@69 2989 ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail.
jpayne@69 2990
jpayne@69 2991 result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
jpayne@69 2992 It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item.
jpayne@69 2993 It can also be an error code, which can be tested with ZSTD_isError().
jpayne@69 2994
jpayne@69 2995 A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
jpayne@69 2996 Context can then be reset to start a new decompression.
jpayne@69 2997
jpayne@69 2998 Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType().
jpayne@69 2999 This information is not required to properly decode a frame.
jpayne@69 3000
jpayne@69 3001 == Special case : skippable frames ==
jpayne@69 3002
jpayne@69 3003 Skippable frames allow integration of user-defined data into a flow of concatenated frames.
jpayne@69 3004 Skippable frames will be ignored (skipped) by decompressor.
jpayne@69 3005 The format of skippable frames is as follows :
jpayne@69 3006 a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F
jpayne@69 3007 b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits
jpayne@69 3008 c) Frame Content - any content (User Data) of length equal to Frame Size
jpayne@69 3009 For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame.
jpayne@69 3010 For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content.
jpayne@69 3011 */
jpayne@69 3012
jpayne@69 3013 /*===== Buffer-less streaming decompression functions =====*/
jpayne@69 3014
jpayne@69 3015 ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */
jpayne@69 3016
jpayne@69 3017 ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
jpayne@69 3018 ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
jpayne@69 3019 ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
jpayne@69 3020
jpayne@69 3021 ZSTDLIB_STATIC_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
jpayne@69 3022 ZSTDLIB_STATIC_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
jpayne@69 3023
jpayne@69 3024 /* misc */
jpayne@69 3025 ZSTD_DEPRECATED("This function will likely be removed in the next minor release. It is misleading and has very limited utility.")
jpayne@69 3026 ZSTDLIB_STATIC_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
jpayne@69 3027 typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
jpayne@69 3028 ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
jpayne@69 3029
jpayne@69 3030
jpayne@69 3031
jpayne@69 3032
jpayne@69 3033 /* ========================================= */
jpayne@69 3034 /** Block level API (DEPRECATED) */
jpayne@69 3035 /* ========================================= */
jpayne@69 3036
jpayne@69 3037 /*!
jpayne@69 3038
jpayne@69 3039 This API is deprecated in favor of the regular compression API.
jpayne@69 3040 You can get the frame header down to 2 bytes by setting:
jpayne@69 3041 - ZSTD_c_format = ZSTD_f_zstd1_magicless
jpayne@69 3042 - ZSTD_c_contentSizeFlag = 0
jpayne@69 3043 - ZSTD_c_checksumFlag = 0
jpayne@69 3044 - ZSTD_c_dictIDFlag = 0
jpayne@69 3045
jpayne@69 3046 This API is not as well tested as our normal API, so we recommend not using it.
jpayne@69 3047 We will be removing it in a future version. If the normal API doesn't provide
jpayne@69 3048 the functionality you need, please open a GitHub issue.
jpayne@69 3049
jpayne@69 3050 Block functions produce and decode raw zstd blocks, without frame metadata.
jpayne@69 3051 Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
jpayne@69 3052 But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.
jpayne@69 3053
jpayne@69 3054 A few rules to respect :
jpayne@69 3055 - Compressing and decompressing require a context structure
jpayne@69 3056 + Use ZSTD_createCCtx() and ZSTD_createDCtx()
jpayne@69 3057 - It is necessary to init context before starting
jpayne@69 3058 + compression : any ZSTD_compressBegin*() variant, including with dictionary
jpayne@69 3059 + decompression : any ZSTD_decompressBegin*() variant, including with dictionary
jpayne@69 3060 - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
jpayne@69 3061 + If input is larger than a block size, it's necessary to split input data into multiple blocks
jpayne@69 3062 + For inputs larger than a single block, consider using regular ZSTD_compress() instead.
jpayne@69 3063 Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block.
jpayne@69 3064 - When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) !
jpayne@69 3065 ===> In which case, nothing is produced into `dst` !
jpayne@69 3066 + User __must__ test for such outcome and deal directly with uncompressed data
jpayne@69 3067 + A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0.
jpayne@69 3068 Doing so would mess up with statistics history, leading to potential data corruption.
jpayne@69 3069 + ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !!
jpayne@69 3070 + In case of multiple successive blocks, should some of them be uncompressed,
jpayne@69 3071 decoder must be informed of their existence in order to follow proper history.
jpayne@69 3072 Use ZSTD_insertBlock() for such a case.
jpayne@69 3073 */
jpayne@69 3074
jpayne@69 3075 /*===== Raw zstd block functions =====*/
jpayne@69 3076 ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
jpayne@69 3077 ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx);
jpayne@69 3078 ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
jpayne@69 3079 ZSTDLIB_STATIC_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
jpayne@69 3080 ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
jpayne@69 3081 ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
jpayne@69 3082 ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
jpayne@69 3083 ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */
jpayne@69 3084
jpayne@69 3085 #endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */
jpayne@69 3086
jpayne@69 3087 #if defined (__cplusplus)
jpayne@69 3088 }
jpayne@69 3089 #endif