From 829902879eb7ba1260a9444f6b6b91d84ca61037 Mon Sep 17 00:00:00 2001 From: "Dr. David von Oheimb" Date: Mon, 3 May 2021 16:33:10 +0200 Subject: [PATCH] HTTP client API: Generalize to arbitrary request and response contents Reviewed-by: Tomas Mraz (Merged from https://github.com/openssl/openssl/pull/15053) --- NEWS.md | 6 ++++-- apps/include/apps.h | 1 + apps/lib/apps.c | 3 ++- apps/ocsp.c | 1 + crypto/http/http_client.c | 15 ++++++++++++--- doc/man3/OSSL_HTTP_REQ_CTX.pod | 3 ++- doc/man3/OSSL_HTTP_transfer.pod | 34 +++++++++++++++++++++------------ include/openssl/http.h | 2 +- 8 files changed, 45 insertions(+), 20 deletions(-) diff --git a/NEWS.md b/NEWS.md index 3193ce6149..78d0772b9a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -53,8 +53,10 @@ OpenSSL 3.0 also covering CRMF (RFC 4211) and HTTP transfer (RFC 6712). It is part of the crypto lib and adds a 'cmp' app with a demo configuration. All widely used CMP features are supported for both clients and servers. - * Added a proper HTTP(S) client to libcrypto supporting GET and POST, - redirection, plain and ASN.1-encoded contents, proxies, and timeouts. + * Added a proper HTTP client supporting GET with optional redirection, POST, + arbitrary request and response content types, TLS, persistent connections, + connections via HTTP(s) proxies, connections and exchange via user-defined + BIOs (allowing implicit connections), and timeout checks. * Added util/check-format.pl for checking adherence to the coding guidelines. * Added OSSL_ENCODER, a generic encoder API. * Added OSSL_PARAM_BLD, an easier to use API to OSSL_PARAM. diff --git a/apps/include/apps.h b/apps/include/apps.h index 207ed41bc7..41178a6e22 100644 --- a/apps/include/apps.h +++ b/apps/include/apps.h @@ -285,6 +285,7 @@ ASN1_VALUE *app_http_post_asn1(const char *host, const char *port, const STACK_OF(CONF_VALUE) *headers, const char *content_type, ASN1_VALUE *req, const ASN1_ITEM *req_it, + const char *expected_content_type, long timeout, const ASN1_ITEM *rsp_it); # endif diff --git a/apps/lib/apps.c b/apps/lib/apps.c index dafcf419bf..d32f6c5490 100644 --- a/apps/lib/apps.c +++ b/apps/lib/apps.c @@ -2521,6 +2521,7 @@ ASN1_VALUE *app_http_post_asn1(const char *host, const char *port, const STACK_OF(CONF_VALUE) *headers, const char *content_type, ASN1_VALUE *req, const ASN1_ITEM *req_it, + const char *expected_content_type, long timeout, const ASN1_ITEM *rsp_it) { APP_HTTP_TLS_INFO info; @@ -2538,7 +2539,7 @@ ASN1_VALUE *app_http_post_asn1(const char *host, const char *port, proxy, no_proxy, NULL /* bio */, NULL /* rbio */, app_http_tls_cb, &info, 0 /* buf_size */, headers, content_type, req_mem, - NULL /* expected_ct */, 1 /* expect_asn1 */, + expected_content_type, 1 /* expect_asn1 */, HTTP_DEFAULT_MAX_RESP_LEN, timeout, 0 /* keep_alive */); BIO_free(req_mem); diff --git a/apps/ocsp.c b/apps/ocsp.c index 694855fe09..dd816c4221 100644 --- a/apps/ocsp.c +++ b/apps/ocsp.c @@ -1214,6 +1214,7 @@ OCSP_RESPONSE *process_responder(OCSP_REQUEST *req, app_http_post_asn1(host, port, path, NULL, NULL /* no proxy used */, ctx, headers, "application/ocsp-request", (ASN1_VALUE *)req, ASN1_ITEM_rptr(OCSP_REQUEST), + "application/ocsp-response", req_timeout, ASN1_ITEM_rptr(OCSP_RESPONSE)); if (resp == NULL) diff --git a/crypto/http/http_client.c b/crypto/http/http_client.c index ee97f64ef6..077159cab6 100644 --- a/crypto/http/http_client.c +++ b/crypto/http/http_client.c @@ -796,6 +796,7 @@ static BIO *HTTP_new_bio(const char *server /* optionally includes ":port" */, } #endif /* OPENSSL_NO_SOCK */ +/* Exchange request and response via HTTP on (non-)blocking BIO */ BIO *OSSL_HTTP_REQ_CTX_exchange(OSSL_HTTP_REQ_CTX *rctx) { int rv; @@ -856,6 +857,10 @@ OSSL_HTTP_REQ_CTX *OSSL_HTTP_open(const char *server, const char *port, if (bio != NULL) { cbio = bio; + if (proxy != NULL || no_proxy != NULL) { + ERR_raise(ERR_LIB_HTTP, ERR_R_PASSED_INVALID_ARGUMENT); + return NULL; + } } else { #ifndef OPENSSL_NO_SOCK char *proxy_host = NULL, *proxy_port = NULL; @@ -1064,7 +1069,7 @@ BIO *OSSL_HTTP_get(const char *url, const char *proxy, const char *no_proxy, NULL /* content_type */, NULL /* req_mem */, expected_ct, expect_asn1, - -1 /* use same max time */, + -1 /* use same max time (timeout) */, 0 /* no keep_alive */)) OSSL_HTTP_REQ_CTX_free(rctx); else @@ -1100,6 +1105,7 @@ BIO *OSSL_HTTP_get(const char *url, const char *proxy, const char *no_proxy, return resp; } +/* Exchange request and response over a connection managed via |prctx| */ BIO *OSSL_HTTP_transfer(OSSL_HTTP_REQ_CTX **prctx, const char *server, const char *port, const char *path, int use_ssl, @@ -1122,11 +1128,14 @@ BIO *OSSL_HTTP_transfer(OSSL_HTTP_REQ_CTX **prctx, } if (rctx != NULL) { if (OSSL_HTTP_set_request(rctx, path, headers, content_type, req, - expected_ct, expect_asn1, timeout, keep_alive)) + expected_ct, expect_asn1, + timeout, keep_alive)) resp = OSSL_HTTP_exchange(rctx, NULL); if (resp == NULL || !OSSL_HTTP_is_alive(rctx)) { - if (!OSSL_HTTP_close(rctx, resp != NULL)) + if (!OSSL_HTTP_close(rctx, resp != NULL)) { + BIO_free(resp); resp = NULL; + } rctx = NULL; } } diff --git a/doc/man3/OSSL_HTTP_REQ_CTX.pod b/doc/man3/OSSL_HTTP_REQ_CTX.pod index 52e5d441f3..f5f70584df 100644 --- a/doc/man3/OSSL_HTTP_REQ_CTX.pod +++ b/doc/man3/OSSL_HTTP_REQ_CTX.pod @@ -123,6 +123,7 @@ and to gather the response via HTTP, using the I and I that were given when calling OSSL_HTTP_REQ_CTX_new(). If successful, the contents of the internal memory B contains the contents of the HTTP response, without the response headers. +This does not support streaming. The function may need to be called again if its result is -1, which indicates L. In such a case it is advisable to sleep a little in between, using L on the read BIO to prevent a busy loop. @@ -221,7 +222,7 @@ OSSL_HTTP_REQ_CTX_nbio() and OSSL_HTTP_REQ_CTX_nbio_d2i() return 1 for success, 0 on error or redirection, -1 if retry is needed. OSSL_HTTP_REQ_CTX_exchange() and OSSL_HTTP_REQ_CTX_get0_mem_bio() -returns a pointer to a B on success and NULL on failure. +return a pointer to a B on success and NULL on failure. OSSL_HTTP_REQ_CTX_get_resp_len() returns the size of the response contents or 0 if not available or an error occurred. diff --git a/doc/man3/OSSL_HTTP_transfer.pod b/doc/man3/OSSL_HTTP_transfer.pod index 9745932e37..71294913e5 100644 --- a/doc/man3/OSSL_HTTP_transfer.pod +++ b/doc/man3/OSSL_HTTP_transfer.pod @@ -30,7 +30,7 @@ OSSL_HTTP_close const STACK_OF(CONF_VALUE) *headers, const char *content_type, BIO *req, const char *expected_content_type, int expect_asn1, - size_t max_resp_len, int timeout, int keep_alive); + int timeout, int keep_alive); BIO *OSSL_HTTP_exchange(OSSL_HTTP_REQ_CTX *rctx, char **redirection_url); BIO *OSSL_HTTP_get(const char *url, const char *proxy, const char *no_proxy, BIO *bio, BIO *rbio, @@ -52,8 +52,8 @@ OSSL_HTTP_close =head1 DESCRIPTION -OSSL_HTTP_open() initiates an HTTP session using I if not NULL, else -by connecting to a given I optionally via a I. +OSSL_HTTP_open() initiates an HTTP session using the I argument if not +NULL, else by connecting to a given I optionally via a I. Typically the OpenSSL build supports sockets and the I parameter is NULL. In this case I must be NULL as well, and the @@ -148,12 +148,12 @@ Since this function is typically called by applications such as L it uses the I and I parameters (unless NULL) to print additional diagnostic information in a user-oriented way. - OSSL_HTTP_set_request() sets up in I the request header and content data and expectations on the response using the following parameters. If I is NULL it defaults to "/". -If I is NULL the HTTP GET method will be used to send the request -else HTTP POST with the contents of I and optional I. +If I is NULL the HTTP GET method will be used to send the request +else HTTP POST with the contents of I and optional I, where +I must contain data with determined length; streaming is not supported. The optional list I may contain additional custom HTTP header lines. If the parameter I is not NULL then the client will check that the given content type string @@ -172,15 +172,15 @@ If the value is 1 or 2 then a persistent connection is requested. If the value is 2 then a persistent connection is required, i.e., an error occurs in case the server does not grant it. -OSSL_HTTP_transfer() exchanges any form of HTTP request and response +OSSL_HTTP_exchange() exchanges any form of HTTP request and response as specified by I, which must include both connection and request data, typically set up using OSSL_HTTP_open() and OSSL_HTTP_set_request(). It implements the core of the functions described below. If the HTTP method is GET and I is not NULL the latter pointer is used to provide any new location that the server may return with HTTP code 301 (MOVED_PERMANENTLY) or 302 (FOUND). -In this case the caller is responsible for deallocating this URL with -L. +In this case the function returns NULL and the caller is +responsible for deallocating the URL with L. If the response header contains one or more "Content-Length" header lines and/or an ASN.1-encoded response is expected, which should include a total length, the length indications received are checked for consistency @@ -203,6 +203,17 @@ and the I, as described for OSSL_HTTP_open(), must be provided. Also the remaining parameters are interpreted as described for OSSL_HTTP_open() and OSSL_HTTP_set_request(), respectively. +OSSL_HTTP_transfer() exchanges an HTTP request and response +over a connection managed via I without supporting redirection. +It combines OSSL_HTTP_open(), OSSL_HTTP_set_request(), OSSL_HTTP_exchange(), +and OSSL_HTTP_close(). +If I is not NULL it reuses any open connection represented by a non-NULL +I<*prctx>. It keeps the connection open if a persistent connection is requested +or required and this was granted by the server, else it closes the connection +and assigns NULL to I<*prctx>. +The remaining parameters are interpreted as described for OSSL_HTTP_open() +and OSSL_HTTP_set_request(), respectively. + OSSL_HTTP_close() closes the connection and releases I. The I parameter is passed to any BIO update function given during setup as described above for OSSL_HTTP_open(). @@ -222,9 +233,8 @@ OSSL_HTTP_proxy_connect() and OSSL_HTTP_set_request() return 1 on success, 0 on error. On success, OSSL_HTTP_exchange(), OSSL_HTTP_get(), and OSSL_HTTP_transfer() -return a memory BIO containing the data received if an ASN.1-encoded response -is expected, else a BIO that may support streaming. -The BIO must be freed by the caller. +return a memory BIO containing the data received. +This must be freed by the caller. On failure, they return NULL. Failure conditions include connection/transfer timeout, parse errors, etc. diff --git a/include/openssl/http.h b/include/openssl/http.h index e3bd9d7579..7552b2f42e 100644 --- a/include/openssl/http.h +++ b/include/openssl/http.h @@ -72,7 +72,7 @@ int OSSL_HTTP_proxy_connect(BIO *bio, const char *server, const char *port, int timeout, BIO *bio_err, const char *prog); int OSSL_HTTP_set_request(OSSL_HTTP_REQ_CTX *rctx, const char *path, const STACK_OF(CONF_VALUE) *headers, - const char *content_type, BIO *req_mem, + const char *content_type, BIO *req, const char *expected_content_type, int expect_asn1, int timeout, int keep_alive); BIO *OSSL_HTTP_exchange(OSSL_HTTP_REQ_CTX *rctx, char **redirection_url);