diff --git a/Configurations/00-base-templates.conf b/Configurations/00-base-templates.conf index 0e5d38d5a6..4bc1915906 100644 --- a/Configurations/00-base-templates.conf +++ b/Configurations/00-base-templates.conf @@ -48,6 +48,8 @@ my %targets=( defines => sub { my @defs = ( 'OPENSSL_BUILDING_OPENSSL' ); + push @defs, "BROTLI" unless $disabled{brotli}; + push @defs, "BROTLI_SHARED" unless $disabled{"brotli-dynamic"}; push @defs, "ZLIB" unless $disabled{zlib}; push @defs, "ZLIB_SHARED" unless $disabled{"zlib-dynamic"}; return [ @defs ]; @@ -55,6 +57,8 @@ my %targets=( includes => sub { my @incs = (); + push @incs, $withargs{brotli_include} + if !$disabled{brotli} && $withargs{brotli_include}; push @incs, $withargs{zlib_include} if !$disabled{zlib} && $withargs{zlib_include}; return [ @incs ]; @@ -69,11 +73,24 @@ my %targets=( ARFLAGS => "qc", CC => "cc", lflags => - sub { $withargs{zlib_lib} ? "-L".$withargs{zlib_lib} : () }, + sub { + my @libs = (); + push(@libs, "-L".$withargs{zlib_lib}) if $withargs{zlib_lib}; + push(@libs, "-L".$withargs{brotli_lib}) if $withargs{brotli_lib}; + return join(" ", @libs); + }, ex_libs => - sub { !defined($disabled{zlib}) - && defined($disabled{"zlib-dynamic"}) - ? "-lz" : () }, + sub { + my @libs = (); + push(@libs, "-lz") if !defined($disabled{zlib}) && defined($disabled{"zlib-dynamic"}); + if (!defined($disabled{brotli}) && defined($disabled{"brotli-dynamic"})) { + push(@libs, "-lbrotlienc"); + push(@libs, "-lbrotlidec"); + push(@libs, "-lbrotlicommon"); + push(@libs, "-lm"); + } + return join(" ", @libs); + }, HASHBANGPERL => "/usr/bin/env perl", # Only Unix actually cares RANLIB => sub { which("$config{cross_compile_prefix}ranlib") ? "ranlib" : "" }, @@ -100,12 +117,24 @@ my %targets=( }, ex_libs => sub { + my @libs = (); unless ($disabled{zlib}) { if (defined($disabled{"zlib-dynamic"})) { - return $withargs{zlib_lib} // "ZLIB1"; + push(@libs, $withargs{zlib_lib} // "ZLIB1"); } } - return (); + unless ($disabled{brotli}) { + if (defined($disabled{"brotli-dynamic"})) { + my $path = ""; + if (defined($withargs{brotli_lib})) { + $path = $withargs{brotli_lib} . "\\"; + } + push(@libs, $path . "brotlicommon.lib"); + push(@libs, $path . "brotlidec.lib"); + push(@libs, $path . "brotlienc.lib"); + } + } + return join(" ", @libs); }, MT => "mt", diff --git a/Configure b/Configure index fbafe0e867..7f7c889227 100755 --- a/Configure +++ b/Configure @@ -27,7 +27,7 @@ use OpenSSL::config; my $orig_death_handler = $SIG{__DIE__}; $SIG{__DIE__} = \&death_handler; -my $usage="Usage: Configure [no- ...] [enable- ...] [-Dxxx] [-lxxx] [-Lxxx] [-fxxx] [-Kxxx] [no-hw-xxx|no-hw] [[no-]threads] [[no-]thread-pool] [[no-]default-thread-pool] [[no-]shared] [[no-]zlib|zlib-dynamic] [no-asm] [no-egd] [sctp] [386] [--prefix=DIR] [--openssldir=OPENSSLDIR] [--with-xxx[=vvv]] [--config=FILE] os/compiler[:flags]\n"; +my $usage="Usage: Configure [no- ...] [enable- ...] [-Dxxx] [-lxxx] [-Lxxx] [-fxxx] [-Kxxx] [no-hw-xxx|no-hw] [[no-]threads] [[no-]thread-pool] [[no-]default-thread-pool] [[no-]shared] [[no-]zlib|zlib-dynamic] [no-asm] [no-egd] [sctp] [386] [--prefix=DIR] [--openssldir=OPENSSLDIR] [--with-xxx[=vvv]] [--config=FILE] os/compiler[:flags]\n"; my $banner = <<"EOF"; @@ -92,7 +92,7 @@ EOF # no-egd do not compile support for the entropy-gathering daemon APIs # [no-]zlib [don't] compile support for zlib compression. # zlib-dynamic Like "zlib", but the zlib library is expected to be a shared -# library and will be loaded in run-time by the OpenSSL library. +# library and will be loaded at run-time by the OpenSSL library. # sctp include SCTP support # enable-quic include QUIC support (currently just for developers as the # implementation is by no means complete and usable) @@ -416,6 +416,8 @@ my @disablables = ( "autoload-config", "bf", "blake2", + "brotli", + "brotli-dynamic", "buildtest-c++", "bulk", "cached-fetch", @@ -546,6 +548,8 @@ my %deprecated_disablables = ( our %disabled = ( # "what" => "comment" "fips" => "default", "asan" => "default", + "brotli" => "default", + "brotli-dynamic" => "default", "buildtest-c++" => "default", "crypto-mdebug" => "default", "crypto-mdebug-backtrace" => "default", @@ -597,6 +601,7 @@ my @disable_cascades = ( "ssl" => [ "ssl3" ], "ssl3-method" => [ "ssl3" ], "zlib" => [ "zlib-dynamic" ], + "brotli" => [ "brotli-dynamic" ], "des" => [ "mdc2" ], "ec" => [ "ec2m", "ecdsa", "ecdh", "sm2", "gost" ], "dgram" => [ "dtls", "quic", "sctp" ], @@ -642,7 +647,7 @@ my @disable_cascades = ( "stdio" => [ "apps", "capieng", "egd" ], "apps" => [ "tests" ], "tests" => [ "external-tests" ], - "comp" => [ "zlib" ], + "comp" => [ "zlib", "brotli" ], "sm3" => [ "sm2" ], sub { !$disabled{"unit-test"} } => [ "heartbeats" ], @@ -903,6 +908,10 @@ while (@argvcopy) { delete $disabled{"zlib"}; } + elsif ($1 eq "brotli-dynamic") + { + delete $disabled{"brotli"}; + } my $algo = $1; delete $disabled{$algo}; @@ -979,6 +988,14 @@ while (@argvcopy) { $withargs{zlib_include}=$1; } + elsif (/^--with-brotli-lib=(.*)$/) + { + $withargs{brotli_lib}=$1; + } + elsif (/^--with-brotli-include=(.*)$/) + { + $withargs{brotli_include}=$1; + } elsif (/^--with-fuzzer-lib=(.*)$/) { $withargs{fuzzer_lib}=$1; diff --git a/INSTALL.md b/INSTALL.md index f16ecf9c89..234f39c201 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -19,7 +19,7 @@ Table of Contents - [Build Type](#build-type) - [Directories](#directories) - [Compiler Warnings](#compiler-warnings) - - [ZLib Flags](#zlib-flags) + - [Compression Algorithm Flags](#compression-algorithm-flags) - [Seeding the Random Generator](#seeding-the-random-generator) - [Setting the FIPS HMAC key](#setting-the-FIPS-HMAC-key) - [Enable and Disable Features](#enable-and-disable-features) @@ -382,8 +382,39 @@ for OpenSSL development. It only works when using gcc or clang as the compiler. If you are developing a patch for OpenSSL then it is recommended that you use this option where possible. -ZLib Flags ----------- +Compression Algorithm Flags +--------------------------- + +### with-brotli-include + + --with-brotli-include=DIR + +The directory for the location of the brotli include files (i.e. the location +of the **brotli** include directory). This option is only necessary if +[enable-brotli](#enable-brotli) is used and the include files are not already +on the system include path. + +### with-brotli-lib + + --with-brotli-lib=LIB + +**On Unix**: this is the directory containing the brotli libraries. +If not provided, the system library path will be used. + +The names of the libraries are: + +* libbrotlicommon.a or libbrotlicommon.so +* libbrotlidec.a or libbrotlidec.so +* libbrotlienc.a or libbrotlienc.so + +**On Windows:** this is the directory containing the brotli libraries. +If not provided, the system library path will be used. + +The names of the libraries are: + +* brotlicommon.lib +* brotlidec.lib +* brotlienc.lib ### with-zlib-include @@ -556,6 +587,17 @@ Typically OpenSSL will automatically load human readable error strings. For a statically linked application this may be undesirable if small executable size is an objective. +### enable-brotli + +Build with support for brotli compression/decompression. + +### enable-brotli-dynamic + +Like the enable-brotli option, but has OpenSSL load the brotli library dynamically +when needed. + +This is only supported on systems where loading of shared libraries is supported. + ### no-autoload-config Don't automatically load the default `openssl.cnf` file. diff --git a/apps/enc.c b/apps/enc.c index 26ad3deb9a..4da2342791 100644 --- a/apps/enc.c +++ b/apps/enc.c @@ -134,6 +134,8 @@ int enc_main(int argc, char **argv) int do_zlib = 0; BIO *bzl = NULL; #endif + int do_brotli = 0; + BIO *bbrot = NULL; /* first check the command name */ if (strcmp(argv[0], "base64") == 0) @@ -141,6 +143,10 @@ int enc_main(int argc, char **argv) #ifdef ZLIB else if (strcmp(argv[0], "zlib") == 0) do_zlib = 1; +#endif +#ifndef OPENSSL_NO_BROTLI + else if (strcmp(argv[0], "brotli") == 0) + do_brotli = 1; #endif else if (strcmp(argv[0], "enc") != 0) ciphername = argv[0]; @@ -321,14 +327,18 @@ int enc_main(int argc, char **argv) BIO_printf(bio_err, "bufsize=%d\n", bsize); #ifdef ZLIB - if (!do_zlib) + if (do_zlib) + base64 = 0; #endif - if (base64) { - if (enc) - outformat = FORMAT_BASE64; - else - informat = FORMAT_BASE64; - } + if (do_brotli) + base64 = 0; + + if (base64) { + if (enc) + outformat = FORMAT_BASE64; + else + informat = FORMAT_BASE64; + } strbuf = app_malloc(SIZE, "strbuf"); buff = app_malloc(EVP_ENCODE_LENGTH(bsize), "evp buffer"); @@ -398,7 +408,8 @@ int enc_main(int argc, char **argv) rbio = in; wbio = out; -#ifdef ZLIB +#ifndef OPENSSL_NO_COMP +# ifdef ZLIB if (do_zlib) { if ((bzl = BIO_new(BIO_f_zlib())) == NULL) goto end; @@ -411,6 +422,20 @@ int enc_main(int argc, char **argv) else rbio = BIO_push(bzl, rbio); } +# endif + + if (do_brotli) { + if ((bbrot = BIO_new(BIO_f_brotli())) == NULL) + goto end; + if (debug) { + BIO_set_callback_ex(bbrot, BIO_debug_callback_ex); + BIO_set_callback_arg(bbrot, (char *)bio_err); + } + if (enc) + wbio = BIO_push(bbrot, wbio); + else + rbio = BIO_push(bbrot, rbio); + } #endif if (base64) { @@ -656,6 +681,7 @@ int enc_main(int argc, char **argv) #ifdef ZLIB BIO_free(bzl); #endif + BIO_free(bbrot); release_engine(e); OPENSSL_free(pass); return ret; diff --git a/apps/list.c b/apps/list.c index adcfaa4260..f198c1cda7 100644 --- a/apps/list.c +++ b/apps/list.c @@ -1424,6 +1424,9 @@ static void list_disabled(void) #ifndef ZLIB BIO_puts(bio_out, "ZLIB\n"); #endif +#ifdef OPENSSL_NO_BROTLI + BIO_puts(bio_out, "BROTLI\n"); +#endif } /* Unified enum for help and list commands. */ diff --git a/apps/progs.pl b/apps/progs.pl index 29f9be13ca..c4e7ae59cf 100644 --- a/apps/progs.pl +++ b/apps/progs.pl @@ -188,7 +188,7 @@ EOF "camellia-128-cbc", "camellia-128-ecb", "camellia-192-cbc", "camellia-192-ecb", "camellia-256-cbc", "camellia-256-ecb", - "base64", "zlib", + "base64", "zlib", "brotli", "des", "des3", "desx", "idea", "seed", "rc4", "rc4-40", "rc2", "bf", "cast", "rc5", "des-ecb", "des-ede", "des-ede3", diff --git a/crypto/comp/build.info b/crypto/comp/build.info index 65df46a175..014628e45d 100644 --- a/crypto/comp/build.info +++ b/crypto/comp/build.info @@ -1,4 +1,5 @@ LIBS=../../libcrypto SOURCE[../../libcrypto]= \ comp_lib.c comp_err.c \ + c_brotli.c \ c_zlib.c diff --git a/crypto/comp/c_brotli.c b/crypto/comp/c_brotli.c new file mode 100644 index 0000000000..ace6f221b8 --- /dev/null +++ b/crypto/comp/c_brotli.c @@ -0,0 +1,770 @@ +/* + * Copyright 1998-2021 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + * + * Uses brotli compression library from https://github.com/google/brotli + */ + +#include +#include +#include +#include +#include "internal/comp.h" +#include +#include "crypto/cryptlib.h" +#include "internal/bio.h" +#include "internal/thread_once.h" +#include "comp_local.h" + +COMP_METHOD *COMP_brotli(void); + +static COMP_METHOD brotli_method_nobrotli = { + NID_undef, + "(undef)", + NULL, + NULL, + NULL, + NULL, +}; + +#ifdef OPENSSL_NO_BROTLI +# undef BROTLI_SHARED +#else + +# include +# include + +/* memory allocations functions for brotli initialisation */ +static void *brotli_alloc(void *opaque, size_t size) +{ + return OPENSSL_zalloc(size); +} + +static void brotli_free(void *opaque, void *address) +{ + OPENSSL_free(address); +} + +/* + * When OpenSSL is built on Windows, we do not want to require that + * the BROTLI.DLL be available in order for the OpenSSL DLLs to + * work. Therefore, all BROTLI routines are loaded at run time + * and we do not link to a .LIB file when BROTLI_SHARED is set. + */ +# if defined(OPENSSL_SYS_WINDOWS) || defined(OPENSSL_SYS_WIN32) +# include +# endif + +# ifdef BROTLI_SHARED +# include "internal/dso.h" + +/* Function pointers */ +typedef BrotliEncoderState *(*encode_init_ft)(brotli_alloc_func, brotli_free_func, void *); +typedef BROTLI_BOOL (*encode_stream_ft)(BrotliEncoderState *, BrotliEncoderOperation, size_t *, const uint8_t **, size_t *, uint8_t **, size_t *); +typedef BROTLI_BOOL (*encode_has_more_ft)(BrotliEncoderState *); +typedef void (*encode_end_ft)(BrotliEncoderState *); +typedef BROTLI_BOOL (*encode_oneshot_ft)(int, int, BrotliEncoderMode, size_t, const uint8_t in[], size_t *, uint8_t out[]); + +typedef BrotliDecoderState *(*decode_init_ft)(brotli_alloc_func, brotli_free_func, void *); +typedef BROTLI_BOOL (*decode_stream_ft)(BrotliDecoderState *, size_t *, const uint8_t **, size_t *, uint8_t **, size_t *); +typedef BROTLI_BOOL (*decode_has_more_ft)(BrotliDecoderState *); +typedef void (*decode_end_ft)(BrotliDecoderState *); +typedef BrotliDecoderErrorCode (*decode_error_ft)(BrotliDecoderState *); +typedef const char *(*decode_error_string_ft)(BrotliDecoderErrorCode); +typedef BROTLI_BOOL (*decode_is_finished_ft)(BrotliDecoderState *); +typedef BrotliDecoderResult (*decode_oneshot_ft)(size_t, const uint8_t in[], size_t *, uint8_t out[]); + +static encode_init_ft p_encode_init = NULL; +static encode_stream_ft p_encode_stream = NULL; +static encode_has_more_ft p_encode_has_more = NULL; +static encode_end_ft p_encode_end = NULL; +static encode_oneshot_ft p_encode_oneshot = NULL; + +static decode_init_ft p_decode_init = NULL; +static decode_stream_ft p_decode_stream = NULL; +static decode_has_more_ft p_decode_has_more = NULL; +static decode_end_ft p_decode_end = NULL; +static decode_error_ft p_decode_error = NULL; +static decode_error_string_ft p_decode_error_string = NULL; +static decode_is_finished_ft p_decode_is_finished = NULL; +static decode_oneshot_ft p_decode_oneshot = NULL; + +static DSO *brotli_encode_dso = NULL; +static DSO *brotli_decode_dso = NULL; + +# define BrotliEncoderCreateInstance p_encode_init +# define BrotliEncoderCompressStream p_encode_stream +# define BrotliEncoderHasMoreOutput p_encode_has_more +# define BrotliEncoderDestroyInstance p_encode_end +# define BrotliEncoderCompress p_encode_oneshot + +# define BrotliDecoderCreateInstance p_decode_init +# define BrotliDecoderDecompressStream p_decode_stream +# define BrotliDecoderHasMoreOutput p_decode_has_more +# define BrotliDecoderDestroyInstance p_decode_end +# define BrotliDecoderGetErrorCode p_decode_error +# define BrotliDecoderErrorString p_decode_error_string +# define BrotliDecoderIsFinished p_decode_is_finished +# define BrotliDecoderDecompress p_decode_oneshot + +# endif /* ifdef BROTLI_SHARED */ + + +struct brotli_state { + BrotliEncoderState *encoder; + BrotliDecoderState *decoder; +}; + +static int brotli_stateful_init(COMP_CTX *ctx) +{ + struct brotli_state *state = OPENSSL_zalloc(sizeof(*state)); + + if (state == NULL) + return 0; + + state->encoder = BrotliEncoderCreateInstance(brotli_alloc, brotli_free, NULL); + if (state->encoder == NULL) + goto err; + + state->decoder = BrotliDecoderCreateInstance(brotli_alloc, brotli_free, NULL); + if (state->decoder == NULL) + goto err; + + ctx->data = state; + return 1; + err: + BrotliDecoderDestroyInstance(state->decoder); + BrotliEncoderDestroyInstance(state->encoder); + OPENSSL_free(state); + return 0; +} + +static void brotli_stateful_finish(COMP_CTX *ctx) +{ + struct brotli_state *state = ctx->data; + + if (state != NULL) { + BrotliDecoderDestroyInstance(state->decoder); + BrotliEncoderDestroyInstance(state->encoder); + OPENSSL_free(state); + ctx->data = NULL; + } +} + +static int brotli_stateful_compress_block(COMP_CTX *ctx, unsigned char *out, + unsigned int olen, unsigned char *in, + unsigned int ilen) +{ + BROTLI_BOOL done; + struct brotli_state *state = ctx->data; + size_t in_avail = ilen; + size_t out_avail = olen; + + if (state == NULL) + return -1; + + if (ilen == 0) + return 0; + + /* + * The finish API does not provide a final output buffer, + * so each compress operation has to be flushed, if all + * the input data can't be accepted, or there is more output, + * this has to be considered an error, since there is no more + * output buffer space + */ + done = BrotliEncoderCompressStream(state->encoder, BROTLI_OPERATION_FLUSH, + &in_avail, (const uint8_t**)&in, &out_avail, &out, NULL); + if (done == BROTLI_FALSE || in_avail != 0 + || BrotliEncoderHasMoreOutput(state->encoder)) + return -1; + + return (int)(olen - out_avail); +} + +static int brotli_stateful_expand_block(COMP_CTX *ctx, unsigned char *out, + unsigned int olen, unsigned char *in, + unsigned int ilen) +{ + BrotliDecoderResult result; + struct brotli_state *state = ctx->data; + size_t in_avail = ilen; + size_t out_avail = olen; + + if (state == NULL) + return -1; + + if (ilen == 0) + return 0; + + result = BrotliDecoderDecompressStream(state->decoder, &in_avail, (const uint8_t**)&in, &out_avail, &out, NULL); + if (result == BROTLI_DECODER_RESULT_ERROR || in_avail != 0 + || BrotliDecoderHasMoreOutput(state->decoder)) + return -1; + + return (int)(olen - out_avail); +} + +static COMP_METHOD brotli_stateful_method = { + NID_brotli, + LN_brotli, + brotli_stateful_init, + brotli_stateful_finish, + brotli_stateful_compress_block, + brotli_stateful_expand_block +}; + +static int brotli_oneshot_init(COMP_CTX *ctx) +{ + return 1; +} + +static void brotli_oneshot_finish(COMP_CTX *ctx) +{ +} + +static int brotli_oneshot_compress_block(COMP_CTX *ctx, unsigned char *out, + unsigned int olen, unsigned char *in, + unsigned int ilen) +{ + size_t out_size = olen; + + if (ilen == 0) + return 0; + + if (BrotliEncoderCompress(BROTLI_DEFAULT_QUALITY, BROTLI_DEFAULT_WINDOW, + BROTLI_DEFAULT_MODE, ilen, in, + &out_size, out) == BROTLI_FALSE) + return -1; + + return (int)out_size; +} + +static int brotli_oneshot_expand_block(COMP_CTX *ctx, unsigned char *out, + unsigned int olen, unsigned char *in, + unsigned int ilen) +{ + size_t out_size = olen; + + if (ilen == 0) + return 0; + + if (BrotliDecoderDecompress(ilen, in, &out_size, out) != BROTLI_DECODER_RESULT_SUCCESS) + return -1; + + return (int)out_size; +} + +static COMP_METHOD brotli_oneshot_method = { + NID_brotli, + LN_brotli, + brotli_oneshot_init, + brotli_oneshot_finish, + brotli_oneshot_compress_block, + brotli_oneshot_expand_block +}; + +static CRYPTO_ONCE brotli_once = CRYPTO_ONCE_STATIC_INIT; +DEFINE_RUN_ONCE_STATIC(ossl_comp_brotli_init) +{ +# ifdef BROTLI_SHARED +# if defined(OPENSSL_SYS_WINDOWS) || defined(OPENSSL_SYS_WIN32) +# define LIBBROTLIENC "BROTLIENC" +# define LIBBROTLIDEC "BROTLIDEC" +# else +# define LIBBROTLIENC "brotlienc" +# define LIBBROTLIDEC "brotlidec" +# endif + + brotli_encode_dso = DSO_load(NULL, LIBBROTLIENC, NULL, 0); + if (brotli_encode_dso != NULL) { + p_encode_init = (encode_init_ft)DSO_bind_func(brotli_encode_dso, "BrotliEncoderCreateInstance"); + p_encode_stream = (encode_stream_ft)DSO_bind_func(brotli_encode_dso, "BrotliEncoderCompressStream"); + p_encode_has_more = (encode_has_more_ft)DSO_bind_func(brotli_encode_dso, "BrotliEncoderHasMoreOutput"); + p_encode_end = (encode_end_ft)DSO_bind_func(brotli_encode_dso, "BrotliEncoderDestroyInstance"); + p_encode_oneshot = (encode_oneshot_ft)DSO_bind_func(brotli_encode_dso, "BrotliEncoderCompress"); + } + + brotli_decode_dso = DSO_load(NULL, LIBBROTLIDEC, NULL, 0); + if (brotli_decode_dso != NULL) { + p_decode_init = (decode_init_ft)DSO_bind_func(brotli_decode_dso, "BrotliDecoderCreateInstance"); + p_decode_stream = (decode_stream_ft)DSO_bind_func(brotli_decode_dso, "BrotliDecoderDecompressStream"); + p_decode_has_more = (decode_has_more_ft)DSO_bind_func(brotli_decode_dso, "BrotliDecoderHasMoreOutput"); + p_decode_end = (decode_end_ft)DSO_bind_func(brotli_decode_dso, "BrotliDecoderDestroyInstance"); + p_decode_error = (decode_error_ft)DSO_bind_func(brotli_decode_dso, "BrotliDecoderGetErrorCode"); + p_decode_error_string = (decode_error_string_ft)DSO_bind_func(brotli_decode_dso, "BrotliDecoderErrorString"); + p_decode_is_finished = (decode_is_finished_ft)DSO_bind_func(brotli_decode_dso, "BrotliDecoderIsFinished"); + p_decode_oneshot = (decode_oneshot_ft)DSO_bind_func(brotli_decode_dso, "BrotliDecoderDecompress"); + } + + if (p_encode_init == NULL || p_encode_stream == NULL || p_encode_has_more == NULL + || p_encode_end == NULL || p_encode_oneshot == NULL || p_decode_init == NULL + || p_decode_stream == NULL || p_decode_has_more == NULL || p_decode_end == NULL + || p_decode_error == NULL || p_decode_error_string == NULL || p_decode_is_finished == NULL + || p_decode_oneshot == NULL) { + ossl_comp_brotli_cleanup(); + return 0; + } +# endif + return 1; +} +#endif /* ifndef BROTLI / else */ + +COMP_METHOD *COMP_brotli(void) +{ + COMP_METHOD *meth = &brotli_method_nobrotli; + +#ifndef OPENSSL_NO_BROTLI + if (RUN_ONCE(&brotli_once, ossl_comp_brotli_init)) + meth = &brotli_stateful_method; +#endif + return meth; +} + +COMP_METHOD *COMP_brotli_oneshot(void) +{ + COMP_METHOD *meth = &brotli_method_nobrotli; + +#ifndef OPENSSL_NO_BROTLI + if (RUN_ONCE(&brotli_once, ossl_comp_brotli_init)) + meth = &brotli_oneshot_method; +#endif + return meth; +} + +/* Also called from OPENSSL_cleanup() */ +void ossl_comp_brotli_cleanup(void) +{ +#ifdef BROTLI_SHARED + DSO_free(brotli_encode_dso); + brotli_encode_dso = NULL; + DSO_free(brotli_decode_dso); + brotli_decode_dso = NULL; + p_encode_init = NULL; + p_encode_stream = NULL; + p_encode_has_more = NULL; + p_encode_end = NULL; + p_encode_oneshot = NULL; + p_decode_init = NULL; + p_decode_stream = NULL; + p_decode_has_more = NULL; + p_decode_end = NULL; + p_decode_error = NULL; + p_decode_error_string = NULL; + p_decode_is_finished = NULL; + p_decode_oneshot = NULL; +#endif +} + +#ifndef OPENSSL_NO_BROTLI + +/* Brotli-based compression/decompression filter BIO */ + +typedef struct { + struct { /* input structure */ + size_t avail_in; + unsigned char *next_in; + size_t avail_out; + unsigned char *next_out; + unsigned char *buf; + size_t bufsize; + BrotliDecoderState *state; + } decode; + struct { /* output structure */ + size_t avail_in; + unsigned char *next_in; + size_t avail_out; + unsigned char *next_out; + unsigned char *buf; + size_t bufsize; + BrotliEncoderState *state; + int mode; /* Encoder mode to use */ + int done; + unsigned char *ptr; + size_t count; + } encode; +} BIO_BROTLI_CTX; + +# define BROTLI_DEFAULT_BUFSIZE 1024 + +static int bio_brotli_new(BIO *bi); +static int bio_brotli_free(BIO *bi); +static int bio_brotli_read(BIO *b, char *out, int outl); +static int bio_brotli_write(BIO *b, const char *in, int inl); +static long bio_brotli_ctrl(BIO *b, int cmd, long num, void *ptr); +static long bio_brotli_callback_ctrl(BIO *b, int cmd, BIO_info_cb *fp); + +static const BIO_METHOD bio_meth_brotli = { + BIO_TYPE_COMP, + "brotli", + /* TODO: Convert to new style write function */ + bwrite_conv, + bio_brotli_write, + /* TODO: Convert to new style read function */ + bread_conv, + bio_brotli_read, + NULL, /* bio_brotli_puts, */ + NULL, /* bio_brotli_gets, */ + bio_brotli_ctrl, + bio_brotli_new, + bio_brotli_free, + bio_brotli_callback_ctrl +}; +#endif + +const BIO_METHOD *BIO_f_brotli(void) +{ +#ifndef OPENSSL_NO_BROTLI + return &bio_meth_brotli; +#else + return NULL; +#endif +} + +#ifndef OPENSSL_NO_BROTLI + +static int bio_brotli_new(BIO *bi) +{ + BIO_BROTLI_CTX *ctx; + +# ifdef BROTLI_SHARED + if (!RUN_ONCE(&brotli_once, ossl_comp_brotli_init)) { + ERR_raise(ERR_LIB_COMP, COMP_R_BROTLI_NOT_SUPPORTED); + return 0; + } +# endif + ctx = OPENSSL_zalloc(sizeof(*ctx)); + if (ctx == NULL) { + ERR_raise(ERR_LIB_COMP, ERR_R_MALLOC_FAILURE); + return 0; + } + ctx->decode.bufsize = BROTLI_DEFAULT_BUFSIZE; + ctx->decode.state = BrotliDecoderCreateInstance(brotli_alloc, brotli_free, NULL); + if (ctx->decode.state == NULL) + goto err; + ctx->encode.bufsize = BROTLI_DEFAULT_BUFSIZE; + ctx->encode.state = BrotliEncoderCreateInstance(brotli_alloc, brotli_free, NULL); + if (ctx->encode.state == NULL) + goto err; + ctx->encode.mode = BROTLI_DEFAULT_MODE; + BIO_set_init(bi, 1); + BIO_set_data(bi, ctx); + + return 1; + + err: + ERR_raise(ERR_LIB_COMP, ERR_R_MALLOC_FAILURE); + BrotliDecoderDestroyInstance(ctx->decode.state); + BrotliEncoderDestroyInstance(ctx->encode.state); + OPENSSL_free(ctx); + return 0; +} + +static int bio_brotli_free(BIO *bi) +{ + BIO_BROTLI_CTX *ctx; + + if (bi == NULL) + return 0; + + ctx = BIO_get_data(bi); + if (ctx != NULL) { + BrotliDecoderDestroyInstance(ctx->decode.state); + OPENSSL_free(ctx->decode.buf); + BrotliEncoderDestroyInstance(ctx->encode.state); + OPENSSL_free(ctx->encode.buf); + OPENSSL_free(ctx); + } + BIO_set_data(bi, NULL); + BIO_set_init(bi, 0); + + return 1; +} + +static int bio_brotli_read(BIO *b, char *out, int outl) +{ + BIO_BROTLI_CTX *ctx; + BrotliDecoderResult bret; + int ret; + BIO *next = BIO_next(b); + + if (out == NULL || outl <= 0) + return 0; + + ctx = BIO_get_data(b); + BIO_clear_retry_flags(b); + if (ctx->decode.buf == NULL) { + ctx->decode.buf = OPENSSL_malloc(ctx->decode.bufsize); + if (ctx->decode.buf == NULL) { + ERR_raise(ERR_LIB_COMP, ERR_R_MALLOC_FAILURE); + return 0; + } + ctx->decode.next_in = ctx->decode.buf; + ctx->decode.avail_in = 0; + } + + /* Copy output data directly to supplied buffer */ + ctx->decode.next_out = (unsigned char *)out; + ctx->decode.avail_out = (size_t)outl; + for (;;) { + /* Decompress while data available */ + while (ctx->decode.avail_in > 0 || BrotliDecoderHasMoreOutput(ctx->decode.state)) { + bret = BrotliDecoderDecompressStream(ctx->decode.state, &ctx->decode.avail_in, (const uint8_t**)&ctx->decode.next_in, + &ctx->decode.avail_out, &ctx->decode.next_out, NULL); + if (bret == BROTLI_DECODER_RESULT_ERROR) { + ERR_raise(ERR_LIB_COMP, COMP_R_BROTLI_DECODE_ERROR); + ERR_add_error_data(1, BrotliDecoderErrorString(BrotliDecoderGetErrorCode(ctx->decode.state))); + return 0; + } + /* If EOF or we've read everything then return */ + if (BrotliDecoderIsFinished(ctx->decode.state) || ctx->decode.avail_out == 0) + return (int)(outl - ctx->decode.avail_out); + } + + /* If EOF */ + if (BrotliDecoderIsFinished(ctx->decode.state)) + return 0; + + /* + * No data in input buffer try to read some in, if an error then + * return the total data read. + */ + ret = BIO_read(next, ctx->decode.buf, ctx->decode.bufsize); + if (ret <= 0) { + /* Total data read */ + int tot = outl - ctx->decode.avail_out; + + BIO_copy_next_retry(b); + if (ret < 0) + return (tot > 0) ? tot : ret; + return tot; + } + ctx->decode.avail_in = ret; + ctx->decode.next_in = ctx->decode.buf; + } +} + +static int bio_brotli_write(BIO *b, const char *in, int inl) +{ + BIO_BROTLI_CTX *ctx; + BROTLI_BOOL brret; + int ret; + BIO *next = BIO_next(b); + + if (in == NULL || inl <= 0) + return 0; + + ctx = BIO_get_data(b); + if (ctx->encode.done) + return 0; + + BIO_clear_retry_flags(b); + if (ctx->encode.buf == NULL) { + ctx->encode.buf = OPENSSL_malloc(ctx->encode.bufsize); + if (ctx->encode.buf == NULL) { + ERR_raise(ERR_LIB_COMP, ERR_R_MALLOC_FAILURE); + return 0; + } + ctx->encode.ptr = ctx->encode.buf; + ctx->encode.count = 0; + ctx->encode.next_out = ctx->encode.buf; + ctx->encode.avail_out = ctx->encode.bufsize; + } + /* Obtain input data directly from supplied buffer */ + ctx->encode.next_in = (unsigned char *)in; + ctx->encode.avail_in = inl; + for (;;) { + /* If data in output buffer write it first */ + while (ctx->encode.count > 0) { + ret = BIO_write(next, ctx->encode.ptr, ctx->encode.count); + if (ret <= 0) { + /* Total data written */ + int tot = inl - ctx->encode.avail_in; + + BIO_copy_next_retry(b); + if (ret < 0) + return (tot > 0) ? tot : ret; + return tot; + } + ctx->encode.ptr += ret; + ctx->encode.count -= ret; + } + + /* Have we consumed all supplied data? */ + if (ctx->encode.avail_in == 0 && !BrotliEncoderHasMoreOutput(ctx->encode.state)) + return inl; + + /* Compress some more */ + + /* Reset buffer */ + ctx->encode.ptr = ctx->encode.buf; + ctx->encode.next_out = ctx->encode.buf; + ctx->encode.avail_out = ctx->encode.bufsize; + /* Compress some more */ + brret = BrotliEncoderCompressStream(ctx->encode.state, BROTLI_OPERATION_FLUSH, &ctx->encode.avail_in, (const uint8_t**)&ctx->encode.next_in, + &ctx->encode.avail_out, &ctx->encode.next_out, NULL); + if (brret != BROTLI_TRUE) { + ERR_raise(ERR_LIB_COMP, COMP_R_BROTLI_ENCODE_ERROR); + ERR_add_error_data(1, "brotli encoder error"); + return 0; + } + ctx->encode.count = ctx->encode.bufsize - ctx->encode.avail_out; + } +} + +static int bio_brotli_flush(BIO *b) +{ + BIO_BROTLI_CTX *ctx; + BROTLI_BOOL brret; + int ret; + BIO *next = BIO_next(b); + + ctx = BIO_get_data(b); + + /* If no data written or already flush show success */ + if (ctx->encode.buf == NULL || (ctx->encode.done && ctx->encode.count == 0)) + return 1; + + BIO_clear_retry_flags(b); + /* No more input data */ + ctx->encode.next_in = NULL; + ctx->encode.avail_in = 0; + for (;;) { + /* If data in output buffer write it first */ + while (ctx->encode.count > 0) { + ret = BIO_write(next, ctx->encode.ptr, ctx->encode.count); + if (ret <= 0) { + BIO_copy_next_retry(b); + return ret; + } + ctx->encode.ptr += ret; + ctx->encode.count -= ret; + } + if (ctx->encode.done) + return 1; + + /* Compress some more */ + + /* Reset buffer */ + ctx->encode.ptr = ctx->encode.buf; + ctx->encode.next_out = ctx->encode.buf; + ctx->encode.avail_out = ctx->encode.bufsize; + /* Compress some more */ + brret = BrotliEncoderCompressStream(ctx->encode.state, BROTLI_OPERATION_FINISH, &ctx->encode.avail_in, + (const uint8_t**)&ctx->encode.next_in, &ctx->encode.avail_out, &ctx->encode.next_out, NULL); + if (brret != BROTLI_TRUE) { + ERR_raise(ERR_LIB_COMP, COMP_R_BROTLI_DECODE_ERROR); + ERR_add_error_data(1, "brotli encoder error"); + return 0; + } + if (!BrotliEncoderHasMoreOutput(ctx->encode.state) && ctx->encode.avail_in == 0) + ctx->encode.done = 1; + ctx->encode.count = ctx->encode.bufsize - ctx->encode.avail_out; + } +} + +static long bio_brotli_ctrl(BIO *b, int cmd, long num, void *ptr) +{ + BIO_BROTLI_CTX *ctx; + unsigned char *tmp; + int ret = 0, *ip; + size_t ibs, obs; + BIO *next = BIO_next(b); + + if (next == NULL) + return 0; + ctx = BIO_get_data(b); + switch (cmd) { + + case BIO_CTRL_RESET: + ctx->encode.count = 0; + ctx->encode.done = 0; + ret = 1; + break; + + case BIO_CTRL_FLUSH: + ret = bio_brotli_flush(b); + if (ret > 0) + ret = BIO_flush(next); + break; + + case BIO_C_SET_BUFF_SIZE: + ibs = ctx->decode.bufsize; + obs = ctx->encode.bufsize; + if (ptr != NULL) { + ip = ptr; + if (*ip == 0) + ibs = (size_t)num; + else + obs = (size_t)num; + } else { + ibs = (size_t)num; + obs = ibs; + } + + if (ibs > 0 && ibs != ctx->decode.bufsize) { + /* Do not free/alloc, only reallocate */ + if (ctx->decode.buf != NULL) { + tmp = OPENSSL_realloc(ctx->decode.buf, ibs); + if (tmp == NULL) + return 0; + ctx->decode.buf = tmp; + } + ctx->decode.bufsize = ibs; + } + + if (obs > 0 && obs != ctx->encode.bufsize) { + /* Do not free/alloc, only reallocate */ + if (ctx->encode.buf != NULL) { + tmp = OPENSSL_realloc(ctx->encode.buf, obs); + if (tmp == NULL) + return 0; + ctx->encode.buf = tmp; + } + ctx->encode.bufsize = obs; + } + ret = 1; + break; + + case BIO_C_DO_STATE_MACHINE: + BIO_clear_retry_flags(b); + ret = BIO_ctrl(next, cmd, num, ptr); + BIO_copy_next_retry(b); + break; + + case BIO_CTRL_WPENDING: + if (BrotliEncoderHasMoreOutput(ctx->encode.state)) + ret = 1; + else + ret = BIO_ctrl(next, cmd, num, ptr); + break; + + case BIO_CTRL_PENDING: + if (!BrotliDecoderIsFinished(ctx->decode.state)) + ret = 1; + else + ret = BIO_ctrl(next, cmd, num, ptr); + break; + + default: + ret = BIO_ctrl(next, cmd, num, ptr); + break; + + } + + return ret; +} + +static long bio_brotli_callback_ctrl(BIO *b, int cmd, BIO_info_cb *fp) +{ + BIO *next = BIO_next(b); + if (next == NULL) + return 0; + return BIO_callback_ctrl(next, cmd, fp); +} + +#endif diff --git a/crypto/comp/comp_err.c b/crypto/comp/comp_err.c index 70a6eea0f0..4f55f820da 100644 --- a/crypto/comp/comp_err.c +++ b/crypto/comp/comp_err.c @@ -17,6 +17,16 @@ # ifndef OPENSSL_NO_ERR static const ERR_STRING_DATA COMP_str_reasons[] = { + {ERR_PACK(ERR_LIB_COMP, 0, COMP_R_BROTLI_DECODE_ERROR), + "brotli decode error"}, + {ERR_PACK(ERR_LIB_COMP, 0, COMP_R_BROTLI_DEFLATE_ERROR), + "brotli deflate error"}, + {ERR_PACK(ERR_LIB_COMP, 0, COMP_R_BROTLI_ENCODE_ERROR), + "brotli encode error"}, + {ERR_PACK(ERR_LIB_COMP, 0, COMP_R_BROTLI_INFLATE_ERROR), + "brotli inflate error"}, + {ERR_PACK(ERR_LIB_COMP, 0, COMP_R_BROTLI_NOT_SUPPORTED), + "brotli not supported"}, {ERR_PACK(ERR_LIB_COMP, 0, COMP_R_ZLIB_DEFLATE_ERROR), "zlib deflate error"}, {ERR_PACK(ERR_LIB_COMP, 0, COMP_R_ZLIB_INFLATE_ERROR), diff --git a/crypto/err/openssl.txt b/crypto/err/openssl.txt index c2978074a6..67179fa9ae 100644 --- a/crypto/err/openssl.txt +++ b/crypto/err/openssl.txt @@ -382,6 +382,11 @@ CMS_R_UNWRAP_ERROR:157:unwrap error CMS_R_UNWRAP_FAILURE:180:unwrap failure CMS_R_VERIFICATION_FAILURE:158:verification failure CMS_R_WRAP_ERROR:159:wrap error +COMP_R_BROTLI_DECODE_ERROR:102:brotli decode error +COMP_R_BROTLI_DEFLATE_ERROR:103:brotli deflate error +COMP_R_BROTLI_ENCODE_ERROR:106:brotli encode error +COMP_R_BROTLI_INFLATE_ERROR:104:brotli inflate error +COMP_R_BROTLI_NOT_SUPPORTED:105:brotli not supported COMP_R_ZLIB_DEFLATE_ERROR:99:zlib deflate error COMP_R_ZLIB_INFLATE_ERROR:100:zlib inflate error COMP_R_ZLIB_NOT_SUPPORTED:101:zlib not supported diff --git a/crypto/init.c b/crypto/init.c index a224542e03..fa8f0d694a 100644 --- a/crypto/init.c +++ b/crypto/init.c @@ -389,6 +389,8 @@ void OPENSSL_cleanup(void) #ifndef OPENSSL_NO_COMP OSSL_TRACE(INIT, "OPENSSL_cleanup: ossl_comp_zlib_cleanup()\n"); ossl_comp_zlib_cleanup(); + OSSL_TRACE(INIT, "OPENSSL_cleanup: ossl_comp_brotli_cleanup()\n"); + ossl_comp_brotli_cleanup(); #endif if (async_inited) { diff --git a/crypto/objects/obj_dat.h b/crypto/objects/obj_dat.h index b97118922c..115c707cd1 100644 --- a/crypto/objects/obj_dat.h +++ b/crypto/objects/obj_dat.h @@ -1154,7 +1154,7 @@ static const unsigned char so[8356] = { 0x2A,0x86,0x48,0x86,0xF7,0x0D,0x01,0x09,0x10,0x01,0x32, /* [ 8344] OBJ_id_ct_signedTAL */ }; -#define NUM_NID 1288 +#define NUM_NID 1289 static const ASN1_OBJECT nid_objs[NUM_NID] = { {"UNDEF", "undefined", NID_undef}, {"rsadsi", "RSA Data Security, Inc.", NID_rsadsi, 6, &so[0]}, @@ -2444,9 +2444,10 @@ static const ASN1_OBJECT nid_objs[NUM_NID] = { {"brainpoolP256r1tls13", "brainpoolP256r1tls13", NID_brainpoolP256r1tls13}, {"brainpoolP384r1tls13", "brainpoolP384r1tls13", NID_brainpoolP384r1tls13}, {"brainpoolP512r1tls13", "brainpoolP512r1tls13", NID_brainpoolP512r1tls13}, + {"brotli", "Brotli compression", NID_brotli}, }; -#define NUM_SN 1279 +#define NUM_SN 1280 static const unsigned int sn_objs[NUM_SN] = { 364, /* "AD_DVCS" */ 419, /* "AES-128-CBC" */ @@ -2794,6 +2795,7 @@ static const unsigned int sn_objs[NUM_SN] = { 933, /* "brainpoolP512r1" */ 1287, /* "brainpoolP512r1tls13" */ 934, /* "brainpoolP512t1" */ + 1288, /* "brotli" */ 494, /* "buildingName" */ 860, /* "businessCategory" */ 691, /* "c2onb191v4" */ @@ -3729,7 +3731,7 @@ static const unsigned int sn_objs[NUM_SN] = { 1093, /* "x509ExtAdmission" */ }; -#define NUM_LN 1279 +#define NUM_LN 1280 static const unsigned int ln_objs[NUM_LN] = { 363, /* "AD Time Stamping" */ 405, /* "ANSI X9.62" */ @@ -3741,6 +3743,7 @@ static const unsigned int ln_objs[NUM_LN] = { 365, /* "Basic OCSP Response" */ 285, /* "Biometric Info" */ 1221, /* "Brand Indicator for Message Identification" */ + 1288, /* "Brotli compression" */ 179, /* "CA Issuers" */ 785, /* "CA Repository" */ 1219, /* "CMC Archive Server" */ diff --git a/crypto/objects/obj_mac.num b/crypto/objects/obj_mac.num index 64dffcb7c1..5940f6911b 100644 --- a/crypto/objects/obj_mac.num +++ b/crypto/objects/obj_mac.num @@ -1285,3 +1285,4 @@ id_ct_signedTAL 1284 brainpoolP256r1tls13 1285 brainpoolP384r1tls13 1286 brainpoolP512r1tls13 1287 +brotli 1288 diff --git a/crypto/objects/objects.txt b/crypto/objects/objects.txt index b627cfdfd1..a0991529b9 100644 --- a/crypto/objects/objects.txt +++ b/crypto/objects/objects.txt @@ -1802,3 +1802,6 @@ dstu4145le 2 9 : uacurve9 : DSTU curve 9 joint-iso-itu-t 16 840 1 113894 : oracle-organization : Oracle organization # Jdk trustedKeyUsage attribute oracle 746875 1 1 : oracle-jdk-trustedkeyusage : Trusted key usage (Oracle) + +# NID for brotli + : brotli : Brotli compression diff --git a/doc/build.info b/doc/build.info index 45886a4f27..8884b0dd5f 100644 --- a/doc/build.info +++ b/doc/build.info @@ -859,6 +859,10 @@ DEPEND[html/man3/CMS_verify_receipt.html]=man3/CMS_verify_receipt.pod GENERATE[html/man3/CMS_verify_receipt.html]=man3/CMS_verify_receipt.pod DEPEND[man/man3/CMS_verify_receipt.3]=man3/CMS_verify_receipt.pod GENERATE[man/man3/CMS_verify_receipt.3]=man3/CMS_verify_receipt.pod +DEPEND[html/man3/COMP_CTX_new.html]=man3/COMP_CTX_new.pod +GENERATE[html/man3/COMP_CTX_new.html]=man3/COMP_CTX_new.pod +DEPEND[man/man3/COMP_CTX_new.3]=man3/COMP_CTX_new.pod +GENERATE[man/man3/COMP_CTX_new.3]=man3/COMP_CTX_new.pod DEPEND[html/man3/CONF_modules_free.html]=man3/CONF_modules_free.pod GENERATE[html/man3/CONF_modules_free.html]=man3/CONF_modules_free.pod DEPEND[man/man3/CONF_modules_free.3]=man3/CONF_modules_free.pod @@ -2982,6 +2986,7 @@ html/man3/CMS_sign_receipt.html \ html/man3/CMS_uncompress.html \ html/man3/CMS_verify.html \ html/man3/CMS_verify_receipt.html \ +html/man3/COMP_CTX_new.html \ html/man3/CONF_modules_free.html \ html/man3/CONF_modules_load_file.html \ html/man3/CRYPTO_THREAD_run_once.html \ @@ -3586,6 +3591,7 @@ man/man3/CMS_sign_receipt.3 \ man/man3/CMS_uncompress.3 \ man/man3/CMS_verify.3 \ man/man3/CMS_verify_receipt.3 \ +man/man3/COMP_CTX_new.3 \ man/man3/CONF_modules_free.3 \ man/man3/CONF_modules_load_file.3 \ man/man3/CRYPTO_THREAD_run_once.3 \ diff --git a/doc/man3/COMP_CTX_new.pod b/doc/man3/COMP_CTX_new.pod new file mode 100644 index 0000000000..826fbb02d3 --- /dev/null +++ b/doc/man3/COMP_CTX_new.pod @@ -0,0 +1,156 @@ +=pod + +=head1 NAME + +COMP_CTX_new, +COMP_CTX_get_method, +COMP_CTX_get_type, +COMP_get_type, +COMP_get_name, +COMP_CTX_free, +COMP_compress_block, +COMP_expand_block, +COMP_zlib, +COMP_brotli, +COMP_brotli_oneshot, +BIO_f_zlib, +BIO_f_brotli +- Compression support + +=head1 SYNOPSIS + + #include + + COMP_CTX *COMP_CTX_new(COMP_METHOD *meth); + void COMP_CTX_free(COMP_CTX *ctx); + const COMP_METHOD *COMP_CTX_get_method(const COMP_CTX *ctx); + int COMP_CTX_get_type(const COMP_CTX* comp); + int COMP_get_type(const COMP_METHOD *meth); + const char *COMP_get_name(const COMP_METHOD *meth); + + int COMP_compress_block(COMP_CTX *ctx, unsigned char *out, int olen, + unsigned char *in, int ilen); + int COMP_expand_block(COMP_CTX *ctx, unsigned char *out, int olen, + unsigned char *in, int ilen); + + COMP_METHOD *COMP_zlib(void); + COMP_METHOD *COMP_brotli(void); + COMP_METHOD *COMP_brotli_oneshot(void); + + const BIO_METHOD *BIO_f_zlib(void); + const BIO_METHOD *BIO_f_brotli(void); + +=head1 DESCRIPTION + +These functions provide compression support for OpenSSL. Compression is used within +the OpenSSL library to support TLS record and certificate compression. + +COMP_CTX_new() is used to create a new B structure used to compress data. +COMP_CTX_free() is used to free the returned B. + +COMP_CTX_get_method() returns the B of the given I. + +COMP_CTX_get_type() and COMP_get_type() return the NID for the B and +B, respectively. COMP_get_name() returns the name of the algorithm +of the given B. + +COMP_compress_block() compresses b bytes from the buffer I into the +buffer b of size I using the algorithm specified by I. + +COMP_expand_block() expands I bytes from the buffer I into the +buffer I of size I using the lgorithm specified by I. + +Methods (B) may be specified by one of these functions. These functions +will be available even if their corresponding compression algorithm is not configured +into the OpenSSL library. In such a case, a non-operative method will be returned. +Any compression operations using a non-operative method will fail. + +=over 4 + +=item * + +COMP_zlib() returns a B for stream-based ZLIB compression. + +=item * + +COMP_brotli() returns a B for stream-based Brotli compression. + +=item * + +COMP_brotli_oneshot() returns a B for one-shot Brotli compression. + +=back + +BIO_f_zlib() and BIO_f_brotli() each return a B that may be used to +create a B via L to read and write compressed files or streams. +The functions are only available if the corresponding algorithm is compiled into +the OpenSSL library. + +=head1 NOTES + +While compressing non-compressible data, the output may be larger than the +input. Care should be taken to size output buffers appropriate for both +compression and expansion. + +Compression support and compression algorithms must be enabled and built into +the library before use. Refer to the INSTALL.md file when configuring OpenSSL. + +ZLIB may be found at L + +Brotli may be found at L. + +Compression of SSL/TLS records is not recommended, as it has been +shown to lead to the CRIME attack L. +It is disabled by default, and may be enabled by clearing the +SSL_OP_NO_COMPRESSION options of the L or +L functions. + +Compression is also used to support certificate compression as described +in RFC8879 L. +It may be disabled via the SSL_OP_NO_CERTIFICATE_COMPRESSION option of +the L or L functions. + +COMP_zlib() and COMP_brotli() are both stream-based compression methods. +Internal state (including compression dictionary) is maintained between calls. +If an error is returned, the stream is corrupted, and should be closed. + +COMP_brotli_oneshot() is not stream-based, it does not maintain state +between calls. An error in one call does not affect future calls. + +=head1 RETURN VALUES + +COMP_CTX_new() returns a B on success, or NULL on failure. + +COMP_CTX_get_method(), COMP_zlib(), COMP_brotli(), and COMP_brotli_oneshot() +return a B on success, or NULL on failure. + +COMP_CTX_get_type() and COMP_get_type() return a NID value. On failure, +NID_undef is returned. + +COMP_compress_block() and COMP_expand_block() return the number of +bytes stored in the output buffer I. This may be 0. On failure, +-1 is returned. + +COMP_get_name() returns a B that must not be freed +on success, or NULL on failure. + +BIO_f_zlib() and BIO_f_brotli() return a B. + +=head1 SEE ALSO + +L, L, L + +=head1 HISTORY + +Brotli functions were added in OpenSSL 3.1.0. + +=head1 COPYRIGHT + +Copyright 2022 The OpenSSL Project Authors. All Rights Reserved. + +Licensed under the Apache License 2.0 (the "License"). You may not use +this file except in compliance with the License. You can obtain a copy +in the file LICENSE in the source distribution or at +L. + +=cut diff --git a/doc/man3/SSL_COMP_add_compression_method.pod b/doc/man3/SSL_COMP_add_compression_method.pod index 9e3dc560c4..a5aa674cdb 100644 --- a/doc/man3/SSL_COMP_add_compression_method.pod +++ b/doc/man3/SSL_COMP_add_compression_method.pod @@ -60,7 +60,17 @@ when a matching identifier is found. There is no way to restrict the list of compression methods supported on a per connection basis. If enabled during compilation, the OpenSSL library will have the -COMP_zlib() compression method available. +following compression methods available: + +=over 4 + +=item COMP_zlib() + +=item COMP_brotli() + +=item COMP_brotli_oneshot() + +=back =head1 RETURN VALUES diff --git a/include/internal/comp.h b/include/internal/comp.h index 3ad86fc7b1..45cab6c720 100644 --- a/include/internal/comp.h +++ b/include/internal/comp.h @@ -10,3 +10,4 @@ #include void ossl_comp_zlib_cleanup(void); +void ossl_comp_brotli_cleanup(void); diff --git a/include/openssl/comp.h b/include/openssl/comp.h index 06ff58100f..28f674de4d 100644 --- a/include/openssl/comp.h +++ b/include/openssl/comp.h @@ -40,6 +40,8 @@ int COMP_expand_block(COMP_CTX *ctx, unsigned char *out, int olen, unsigned char *in, int ilen); COMP_METHOD *COMP_zlib(void); +COMP_METHOD *COMP_brotli(void); +COMP_METHOD *COMP_brotli_oneshot(void); #ifndef OPENSSL_NO_DEPRECATED_1_1_0 # define COMP_zlib_cleanup() while(0) continue @@ -49,6 +51,7 @@ COMP_METHOD *COMP_zlib(void); # ifdef ZLIB const BIO_METHOD *BIO_f_zlib(void); # endif +const BIO_METHOD *BIO_f_brotli(void); # endif diff --git a/include/openssl/comperr.h b/include/openssl/comperr.h index 01dd3e6bc6..31dcda8957 100644 --- a/include/openssl/comperr.h +++ b/include/openssl/comperr.h @@ -23,6 +23,11 @@ /* * COMP reason codes. */ +# define COMP_R_BROTLI_DECODE_ERROR 102 +# define COMP_R_BROTLI_DEFLATE_ERROR 103 +# define COMP_R_BROTLI_ENCODE_ERROR 106 +# define COMP_R_BROTLI_INFLATE_ERROR 104 +# define COMP_R_BROTLI_NOT_SUPPORTED 105 # define COMP_R_ZLIB_DEFLATE_ERROR 99 # define COMP_R_ZLIB_INFLATE_ERROR 100 # define COMP_R_ZLIB_NOT_SUPPORTED 101 diff --git a/include/openssl/obj_mac.h b/include/openssl/obj_mac.h index 8ad445259d..daa5233294 100644 --- a/include/openssl/obj_mac.h +++ b/include/openssl/obj_mac.h @@ -5593,6 +5593,10 @@ #define NID_oracle_jdk_trustedkeyusage 1283 #define OBJ_oracle_jdk_trustedkeyusage OBJ_oracle,746875L,1L,1L +#define SN_brotli "brotli" +#define LN_brotli "Brotli compression" +#define NID_brotli 1288 + #endif /* OPENSSL_OBJ_MAC_H */ #ifndef OPENSSL_NO_DEPRECATED_3_0 diff --git a/test/bio_comp_test.c b/test/bio_comp_test.c new file mode 100644 index 0000000000..b148d02dd3 --- /dev/null +++ b/test/bio_comp_test.c @@ -0,0 +1,143 @@ +/* + * Copyright 2022 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ +#include +#include +#include +#include +#include +#include + +#include "testutil.h" +#include "testutil/output.h" +#include "testutil/tu_local.h" + +#define COMPRESS 1 +#define EXPAND 0 + +#define BUFFER_SIZE 32 * 1024 +#define NUM_SIZES 4 +static int sizes[NUM_SIZES] = { 64, 512, 2048, 16 * 1024 }; + +/* using global buffers */ +uint8_t *original = NULL; +uint8_t *result = NULL; + +/* + * For compression: + * the write operation compresses + * the read operation decompresses + */ + +static int do_bio_comp_test(const BIO_METHOD *meth, size_t size) +{ + BIO *bcomp = NULL; + BIO *bmem = NULL; + BIO *bexp = NULL; + int osize; + int rsize; + int ret = 0; + + /* Compress */ + if (!TEST_ptr(bcomp = BIO_new(meth))) + goto err; + if (!TEST_ptr(bmem = BIO_new(BIO_s_mem()))) + goto err; + BIO_push(bcomp, bmem); + osize = BIO_write(bcomp, original, size); + if (!TEST_int_eq(osize, size) + || !TEST_true(BIO_flush(bcomp))) + goto err; + BIO_free(bcomp); + bcomp = NULL; + + /* decompress */ + if (!TEST_ptr(bexp = BIO_new(meth))) + goto err; + BIO_push(bexp, bmem); + rsize = BIO_read(bexp, result, size); + + if (!TEST_int_eq(size, osize) + || !TEST_int_eq(size, rsize) + || !TEST_mem_eq(original, osize, result, rsize)) + goto err; + + ret = 1; + err: + BIO_free(bexp); + BIO_free(bcomp); + BIO_free(bmem); + return ret; +} + +static int do_bio_comp(const BIO_METHOD *meth, int n) +{ + int i; + int success = 0; + int size = sizes[n % 4]; + int type = n / 4; + + if (!TEST_ptr(original = OPENSSL_malloc(BUFFER_SIZE)) + || !TEST_ptr(result = OPENSSL_malloc(BUFFER_SIZE))) + goto err; + + switch (type) { + case 0: + test_printf_stdout("# zeros of size %d\n", size); + memset(original, 0, BUFFER_SIZE); + break; + case 1: + test_printf_stdout("# ones of size %d\n", size); + memset(original, 0, BUFFER_SIZE); + break; + case 2: + test_printf_stdout("# sequential of size %d\n", size); + for (i = 0; i < BUFFER_SIZE; i++) + original[i] = i & 0xFF; + break; + case 3: + test_printf_stdout("# random of size %d\n", size); + if (!TEST_int_gt(RAND_bytes(original, BUFFER_SIZE), 0)) + goto err; + break; + default: + goto err; + } + + if (!TEST_true(do_bio_comp_test(meth, size))) + goto err; + success = 1; + err: + OPENSSL_free(original); + OPENSSL_free(result); + return success; +} + +#ifndef OPENSSL_NO_BROTLI +static int test_brotli(int n) +{ + return do_bio_comp(BIO_f_brotli(), n); +} +#endif +#ifdef ZLIB +static int test_zlib(int n) +{ + return do_bio_comp(BIO_f_zlib(), n); +} +#endif + +int setup_tests(void) +{ +#ifdef ZLIB + ADD_ALL_TESTS(test_zlib, NUM_SIZES * 4); +#endif +#ifndef OPENSSL_NO_BROTLI + ADD_ALL_TESTS(test_brotli, NUM_SIZES * 4); +#endif + return 1; +} diff --git a/test/build.info b/test/build.info index e2cfddb222..1fffaa15eb 100644 --- a/test/build.info +++ b/test/build.info @@ -899,6 +899,13 @@ IF[{- !$disabled{tests} -}] INCLUDE[context_internal_test]=.. ../include ../apps/include DEPEND[context_internal_test]=../libcrypto.a libtestutil.a + IF[{- !$disabled{zlib} || !$disabled{brotli} -}] + PROGRAMS{noinst}=bio_comp_test + SOURCE[bio_comp_test]=bio_comp_test.c + INCLUDE[bio_comp_test]=../include ../apps/include + DEPEND[bio_comp_test]=../libcrypto.a libtestutil.a + ENDIF + PROGRAMS{noinst}=provider_internal_test DEFINE[provider_internal_test]=PROVIDER_INIT_FUNCTION_NAME=p_test_init SOURCE[provider_internal_test]=provider_internal_test.c p_test.c diff --git a/test/recipes/07-test_bio_comp.t b/test/recipes/07-test_bio_comp.t new file mode 100644 index 0000000000..e43fc49a52 --- /dev/null +++ b/test/recipes/07-test_bio_comp.t @@ -0,0 +1,19 @@ +#! /usr/bin/env perl +# Copyright 2022 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +use strict; +use OpenSSL::Test; +use OpenSSL::Test::Simple; +use OpenSSL::Test::Utils; + +setup("test_bio_comp"); + +plan skip_all => "No compression algorithms" + if disabled("zlib") && disabled("brotli"); + +simple_test("test_bio_comp", "bio_comp_test"); diff --git a/util/libcrypto.num b/util/libcrypto.num index f5951d59e5..245b02ff2e 100644 --- a/util/libcrypto.num +++ b/util/libcrypto.num @@ -5469,3 +5469,6 @@ OSSL_sleep ? 3_2_0 EXIST::FUNCTION: OSSL_get_thread_support_flags ? 3_2_0 EXIST::FUNCTION: OSSL_set_max_threads ? 3_2_0 EXIST::FUNCTION: OSSL_get_max_threads ? 3_2_0 EXIST::FUNCTION: +COMP_brotli ? 3_2_0 EXIST::FUNCTION:COMP +COMP_brotli_oneshot ? 3_2_0 EXIST::FUNCTION:COMP +BIO_f_brotli ? 3_2_0 EXIST::FUNCTION:COMP diff --git a/util/missingcrypto.txt b/util/missingcrypto.txt index be292c1b51..56ee90c8a1 100644 --- a/util/missingcrypto.txt +++ b/util/missingcrypto.txt @@ -190,7 +190,6 @@ BIO_f_asn1(3) BIO_f_linebuffer(3) BIO_f_nbio_test(3) BIO_f_reliable(3) -BIO_f_zlib(3) BIO_fd_non_fatal_error(3) BIO_fd_should_retry(3) BIO_get_accept_socket(3) @@ -353,15 +352,6 @@ CMS_unsigned_get_attr(3) CMS_unsigned_get_attr_by_NID(3) CMS_unsigned_get_attr_by_OBJ(3) CMS_unsigned_get_attr_count(3) -COMP_CTX_free(3) -COMP_CTX_get_method(3) -COMP_CTX_get_type(3) -COMP_CTX_new(3) -COMP_compress_block(3) -COMP_expand_block(3) -COMP_get_name(3) -COMP_get_type(3) -COMP_zlib(3) CONF_dump_bio(3) CONF_dump_fp(3) CONF_free(3) diff --git a/util/perl/OpenSSL/Ordinals.pm b/util/perl/OpenSSL/Ordinals.pm index f6c63d14c4..66914254e0 100644 --- a/util/perl/OpenSSL/Ordinals.pm +++ b/util/perl/OpenSSL/Ordinals.pm @@ -414,6 +414,7 @@ sub _parse_features { my $def = $'; if ($def =~ m{^ZLIB$}) { $features{$&} = $op; } + if ($def =~ m{^BROTLI$}) { $features{$&} = $op; } if ($def =~ m{^OPENSSL_USE_}) { $features{$'} = $op; } if ($def =~ m{^OPENSSL_NO_}) { $features{$'} = !$op; } }