demos/http3/ossl-nghttp3.c - third_party/openssl - Git at Google

 /*
  * Copyright 2023-2024 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
  * in the file LICENSE in the source distribution or at
  * https://www.openssl.org/source/license.html
  */
 #include "ossl-nghttp3.h"
 #include <openssl/err.h>
 #include <assert.h>

 #include <sys/socket.h>
 #include <netinet/in.h>

 #define ARRAY_LEN(x) (sizeof(x) / sizeof((x)[0]))

 enum {
     OSSL_DEMO_H3_STREAM_TYPE_CTRL_SEND,
     OSSL_DEMO_H3_STREAM_TYPE_QPACK_ENC_SEND,
     OSSL_DEMO_H3_STREAM_TYPE_QPACK_DEC_SEND,
     OSSL_DEMO_H3_STREAM_TYPE_REQ,
 };

 #define BUF_SIZE 4096

 struct ossl_demo_h3_stream_st {
     uint64_t id; /* QUIC stream ID */
     SSL *s; /* QUIC stream SSL object */
     int done_recv_fin; /* Received FIN */
     void *user_data;

     uint8_t buf[BUF_SIZE];
     size_t buf_cur, buf_total;
 };

 DEFINE_LHASH_OF_EX(OSSL_DEMO_H3_STREAM);

 static void h3_stream_free(OSSL_DEMO_H3_STREAM *s)
 {
     if (s == NULL)
         return;

     SSL_free(s->s);
     OPENSSL_free(s);
 }

 static unsigned long h3_stream_hash(const OSSL_DEMO_H3_STREAM *s)
 {
     return (unsigned long)s->id;
 }

 static int h3_stream_eq(const OSSL_DEMO_H3_STREAM *a, const OSSL_DEMO_H3_STREAM *b)
 {
     if (a->id < b->id)
         return -1;
     if (a->id > b->id)
         return 1;
     return 0;
 }

 void *OSSL_DEMO_H3_STREAM_get_user_data(const OSSL_DEMO_H3_STREAM *s)
 {
     return s->user_data;
 }

 struct ossl_demo_h3_conn_st {
     /* QUIC connection SSL object */
     SSL *qconn;
     /* BIO wrapping QCSO */
     BIO *qconn_bio;
     /* HTTP/3 connection object */
     nghttp3_conn *h3conn;
     /* map of stream IDs to OSSL_DEMO_H3_STREAMs */
     LHASH_OF(OSSL_DEMO_H3_STREAM) *streams;
     /* opaque user data pointer */
     void *user_data;

     int pump_res;
     size_t consumed_app_data;

     /* Forwarding callbacks */
     nghttp3_recv_data recv_data_cb;
     nghttp3_stream_close stream_close_cb;
     nghttp3_stop_sending stop_sending_cb;
     nghttp3_reset_stream reset_stream_cb;
     nghttp3_deferred_consume deferred_consume_cb;
 };

 void OSSL_DEMO_H3_CONN_free(OSSL_DEMO_H3_CONN *conn)
 {
     if (conn == NULL)
         return;

     lh_OSSL_DEMO_H3_STREAM_doall(conn->streams, h3_stream_free);

     nghttp3_conn_del(conn->h3conn);
     BIO_free_all(conn->qconn_bio);
     lh_OSSL_DEMO_H3_STREAM_free(conn->streams);
     OPENSSL_free(conn);
 }

 static OSSL_DEMO_H3_STREAM *h3_conn_create_stream(OSSL_DEMO_H3_CONN *conn, int type)
 {
     OSSL_DEMO_H3_STREAM *s;
     uint64_t flags = SSL_STREAM_FLAG_ADVANCE;

     if ((s = OPENSSL_zalloc(sizeof(OSSL_DEMO_H3_STREAM))) == NULL)
         return NULL;

     if (type != OSSL_DEMO_H3_STREAM_TYPE_REQ)
         flags |= SSL_STREAM_FLAG_UNI;

     if ((s->s = SSL_new_stream(conn->qconn, flags)) == NULL) {
         ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
             "could not create QUIC stream object");
         goto err;
     }

     s->id = SSL_get_stream_id(s->s);
     lh_OSSL_DEMO_H3_STREAM_insert(conn->streams, s);
     return s;

 err:
     OPENSSL_free(s);
     return NULL;
 }

 static void h3_conn_remove_stream(OSSL_DEMO_H3_CONN *conn, OSSL_DEMO_H3_STREAM *s)
 {
     if (s == NULL)
         return;

     lh_OSSL_DEMO_H3_STREAM_delete(conn->streams, s);
     h3_stream_free(s);
 }

 static int h3_conn_recv_data(nghttp3_conn *h3conn, int64_t stream_id,
     const uint8_t *data, size_t datalen,
     void *conn_user_data, void *stream_user_data)
 {
     OSSL_DEMO_H3_CONN *conn = conn_user_data;

     conn->consumed_app_data += datalen;
     if (conn->recv_data_cb == NULL)
         return 0;

     return conn->recv_data_cb(h3conn, stream_id, data, datalen,
         conn_user_data, stream_user_data);
 }

 static int h3_conn_stream_close(nghttp3_conn *h3conn, int64_t stream_id,
     uint64_t app_error_code,
     void *conn_user_data, void *stream_user_data)
 {
     int ret = 0;
     OSSL_DEMO_H3_CONN *conn = conn_user_data;
     OSSL_DEMO_H3_STREAM *stream = stream_user_data;

     if (conn->stream_close_cb != NULL)
         ret = conn->stream_close_cb(h3conn, stream_id, app_error_code,
             conn_user_data, stream_user_data);

     h3_conn_remove_stream(conn, stream);
     return ret;
 }

 static int h3_conn_stop_sending(nghttp3_conn *h3conn, int64_t stream_id,
     uint64_t app_error_code,
     void *conn_user_data, void *stream_user_data)
 {
     int ret = 0;
     OSSL_DEMO_H3_CONN *conn = conn_user_data;
     OSSL_DEMO_H3_STREAM *stream = stream_user_data;

     if (conn->stop_sending_cb != NULL)
         ret = conn->stop_sending_cb(h3conn, stream_id, app_error_code,
             conn_user_data, stream_user_data);

     SSL_free(stream->s);
     stream->s = NULL;
     return ret;
 }

 static int h3_conn_reset_stream(nghttp3_conn *h3conn, int64_t stream_id,
     uint64_t app_error_code,
     void *conn_user_data, void *stream_user_data)
 {
     int ret = 0;
     OSSL_DEMO_H3_CONN *conn = conn_user_data;
     OSSL_DEMO_H3_STREAM *stream = stream_user_data;
     SSL_STREAM_RESET_ARGS args = { 0 };

     if (conn->reset_stream_cb != NULL)
         ret = conn->reset_stream_cb(h3conn, stream_id, app_error_code,
             conn_user_data, stream_user_data);

     if (stream->s != NULL) {
         args.quic_error_code = app_error_code;

         if (!SSL_stream_reset(stream->s, &args, sizeof(args)))
             return 1;
     }

     return ret;
 }

 static int h3_conn_deferred_consume(nghttp3_conn *h3conn, int64_t stream_id,
     size_t consumed,
     void *conn_user_data, void *stream_user_data)
 {
     int ret = 0;
     OSSL_DEMO_H3_CONN *conn = conn_user_data;

     if (conn->deferred_consume_cb != NULL)
         ret = conn->deferred_consume_cb(h3conn, stream_id, consumed,
             conn_user_data, stream_user_data);

     conn->consumed_app_data += consumed;
     return ret;
 }

 OSSL_DEMO_H3_CONN *OSSL_DEMO_H3_CONN_new_for_conn(SSL *qconn,
     const nghttp3_callbacks *callbacks,
     const nghttp3_settings *settings,
     void *user_data)
 {
     int ec;
     OSSL_DEMO_H3_CONN *conn;
     OSSL_DEMO_H3_STREAM *s_ctl_send = NULL;
     OSSL_DEMO_H3_STREAM *s_qpenc_send = NULL;
     OSSL_DEMO_H3_STREAM *s_qpdec_send = NULL;
     nghttp3_settings dsettings = { 0 };
     nghttp3_callbacks intl_callbacks = { 0 };
     static const unsigned char alpn[] = { 2, 'h', '3' };

     if (qconn == NULL) {
         ERR_raise_data(ERR_LIB_USER, ERR_R_PASSED_NULL_PARAMETER,
             "QUIC connection BIO must be provided");
         return NULL;
     }

     if ((conn = OPENSSL_zalloc(sizeof(OSSL_DEMO_H3_CONN))) == NULL)
         return NULL;

     conn->qconn = qconn;
     conn->user_data = user_data;

     /* Create the map of stream IDs to OSSL_DEMO_H3_STREAM structures. */
     if ((conn->streams = lh_OSSL_DEMO_H3_STREAM_new(h3_stream_hash, h3_stream_eq)) == NULL)
         goto err;

     if (SSL_set_alpn_protos(conn->qconn, alpn, sizeof(alpn))) {
         /* SSL_set_alpn_protos returns 1 on failure */
         ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
             "failed to configure ALPN");
         goto err;
     }

     /*
      * Disable default stream mode and create all streams explicitly. Each QUIC
      * stream will be represented by its own QUIC stream SSL object (QSSO). This
      * also automatically enables us to accept incoming streams (see
      * SSL_set_incoming_stream_policy(3)).
      */
     if (!SSL_set_default_stream_mode(conn->qconn, SSL_DEFAULT_STREAM_MODE_NONE)) {
         ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
             "failed to configure default stream mode");
         goto err;
     }

     /*
      * HTTP/3 requires a couple of unidirectional management streams: a control
      * stream and some QPACK state management streams for each side of a
      * connection. These are the instances on our side (with us sending); the
      * server will also create its own equivalent unidirectional streams on its
      * side, which we handle subsequently as they come in (see SSL_accept_stream
      * in the event handling code below).
      */
     if ((s_ctl_send
             = h3_conn_create_stream(conn, OSSL_DEMO_H3_STREAM_TYPE_CTRL_SEND))
         == NULL)
         goto err;

     if ((s_qpenc_send
             = h3_conn_create_stream(conn, OSSL_DEMO_H3_STREAM_TYPE_QPACK_ENC_SEND))
         == NULL)
         goto err;

     if ((s_qpdec_send
             = h3_conn_create_stream(conn, OSSL_DEMO_H3_STREAM_TYPE_QPACK_DEC_SEND))
         == NULL)
         goto err;

     if (settings == NULL) {
         nghttp3_settings_default(&dsettings);
         settings = &dsettings;
     }

     if (callbacks != NULL)
         intl_callbacks = *callbacks;

     /*
      * We need to do some of our own processing when many of these events occur,
      * so we note the original callback functions and forward appropriately.
      */
     conn->recv_data_cb = intl_callbacks.recv_data;
     conn->stream_close_cb = intl_callbacks.stream_close;
     conn->stop_sending_cb = intl_callbacks.stop_sending;
     conn->reset_stream_cb = intl_callbacks.reset_stream;
     conn->deferred_consume_cb = intl_callbacks.deferred_consume;

     intl_callbacks.recv_data = h3_conn_recv_data;
     intl_callbacks.stream_close = h3_conn_stream_close;
     intl_callbacks.stop_sending = h3_conn_stop_sending;
     intl_callbacks.reset_stream = h3_conn_reset_stream;
     intl_callbacks.deferred_consume = h3_conn_deferred_consume;

     /* Create the HTTP/3 client state. */
     ec = nghttp3_conn_client_new(&conn->h3conn, &intl_callbacks, settings,
         NULL, conn);
     if (ec < 0) {
         ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
             "cannot create nghttp3 connection: %s (%d)",
             nghttp3_strerror(ec), ec);
         goto err;
     }

     /*
      * Tell the HTTP/3 stack which stream IDs are used for our outgoing control
      * and QPACK streams. Note that we don't have to tell the HTTP/3 stack what
      * IDs are used for incoming streams as this is inferred automatically from
      * the stream type byte which starts every incoming unidirectional stream,
      * so it will autodetect the correct stream IDs for the incoming control and
      * QPACK streams initiated by the server.
      */
     ec = nghttp3_conn_bind_control_stream(conn->h3conn, s_ctl_send->id);
     if (ec < 0) {
         ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
             "cannot bind nghttp3 control stream: %s (%d)",
             nghttp3_strerror(ec), ec);
         goto err;
     }

     ec = nghttp3_conn_bind_qpack_streams(conn->h3conn,
         s_qpenc_send->id,
         s_qpdec_send->id);
     if (ec < 0) {
         ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
             "cannot bind nghttp3 QPACK streams: %s (%d)",
             nghttp3_strerror(ec), ec);
         goto err;
     }

     return conn;

 err:
     nghttp3_conn_del(conn->h3conn);
     h3_stream_free(s_ctl_send);
     h3_stream_free(s_qpenc_send);
     h3_stream_free(s_qpdec_send);
     lh_OSSL_DEMO_H3_STREAM_free(conn->streams);
     OPENSSL_free(conn);
     return NULL;
 }

 /*
  * Create BIO with UDP connected socket for one of the address we got from
  * resolver.
  */
 static BIO *
 create_socket_bio(const BIO_ADDRINFO *bai)
 {
     int sock;
     BIO *bio;

     sock = BIO_socket(BIO_ADDRINFO_family(bai), SOCK_DGRAM, 0, 0);
     if (sock == -1)
         return NULL;

     if (!BIO_connect(sock, BIO_ADDRINFO_address(bai), 0))
         goto err;

     if (!BIO_socket_nbio(sock, 1))
         goto err;

     if ((bio = BIO_new(BIO_s_datagram())) == NULL)
         goto err;
     BIO_set_fd(bio, sock, BIO_CLOSE);

     return bio;
 err:
     BIO_closesocket(sock);

     return NULL;
 }

 OSSL_DEMO_H3_CONN *OSSL_DEMO_H3_CONN_new_for_addr(SSL_CTX *ctx, const BIO_ADDRINFO *bai,
     const char *bare_hostname,
     const nghttp3_callbacks *callbacks,
     const nghttp3_settings *settings,
     void *user_data)
 {
     BIO *qconn_bio = NULL;
     SSL *qconn = NULL;
     OSSL_DEMO_H3_CONN *conn = NULL;

     if ((qconn_bio = create_socket_bio(bai)) == NULL)
         return NULL;

     qconn = SSL_new(ctx);
     if (qconn == NULL)
         goto err;

     SSL_set_bio(qconn, qconn_bio, qconn_bio);
     qconn_bio = NULL;

     /* Set the hostname we will validate the X.509 certificate against. */
     if (SSL_set1_host(qconn, bare_hostname) <= 0)
         goto err;

     /* Configure SNI */
     if (!SSL_set_tlsext_host_name(qconn, bare_hostname))
         goto err;

     conn = OSSL_DEMO_H3_CONN_new_for_conn(qconn, callbacks,
         settings, user_data);
     if (conn == NULL)
         goto err;

     return conn;

 err:
     SSL_free(qconn);
     BIO_free_all(qconn_bio);
     return NULL;
 }

 void *OSSL_DEMO_H3_CONN_get_user_data(const OSSL_DEMO_H3_CONN *conn)
 {
     return conn->user_data;
 }

 SSL *OSSL_DEMO_H3_CONN_get0_connection(const OSSL_DEMO_H3_CONN *conn)
 {
     return conn->qconn;
 }

 typedef struct ossl_demo_h3_poll_list {
     SSL_POLL_ITEM *poll_list;
     OSSL_DEMO_H3_STREAM **h3_streams;
     OSSL_DEMO_H3_CONN *conn;
     size_t poll_count;
     size_t idx;
 } OSSL_DEMO_H3_POLL_LIST;

 static void h3_conn_collect_streams(OSSL_DEMO_H3_STREAM *s, void *list)
 {
     OSSL_DEMO_H3_POLL_LIST *pollist = list;

     pollist->poll_list[pollist->idx].desc = SSL_as_poll_descriptor(s->s);
     pollist->poll_list[pollist->idx].revents = 0;
     pollist->poll_list[pollist->idx].events = SSL_POLL_EVENT_R;
     pollist->h3_streams[pollist->idx] = s;
     pollist->idx++;
 }

 /* Pumps received data to the HTTP/3 stack for a single stream. */
 static void h3_conn_pump_stream(OSSL_DEMO_H3_STREAM *s, void *conn_)
 {
     int ec;
     OSSL_DEMO_H3_CONN *conn = conn_;
     size_t num_bytes, consumed;
     uint64_t aec;

     if (!conn->pump_res)
         /*
          * Handling of a previous stream in the iteration over all streams
          * failed, so just do nothing.
          */
         return;

     for (;;) {
         if (s->s == NULL /* If we already did STOP_SENDING, ignore this stream. */
             /* If this is a write-only stream, there is no read data to check. */
             || SSL_get_stream_read_state(s->s) == SSL_STREAM_STATE_WRONG_DIR
             /*
              * If we already got a FIN for this stream, there is nothing more to
              * do for it.
              */
             || s->done_recv_fin)
             break;

         /*
          * Pump data from OpenSSL QUIC to the HTTP/3 stack by calling SSL_peek
          * to get received data and passing it to nghttp3 using
          * nghttp3_conn_read_stream. Note that this function is confusingly
          * named and inputs data to the HTTP/3 stack.
          */
         if (s->buf_cur == s->buf_total) {
             /* Need more data. */
             ec = SSL_read_ex(s->s, s->buf, sizeof(s->buf), &num_bytes);
             if (ec <= 0) {
                 num_bytes = 0;
                 if (SSL_get_error(s->s, ec) == SSL_ERROR_ZERO_RETURN) {
                     /* Stream concluded normally. Pass FIN to HTTP/3 stack. */
                     ec = nghttp3_conn_read_stream(conn->h3conn, s->id, NULL, 0,
                         /*fin=*/1);
                     if (ec < 0) {
                         ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
                             "cannot pass FIN to nghttp3: %s (%d)",
                             nghttp3_strerror(ec), ec);
                         goto err;
                     }

                     s->done_recv_fin = 1;
                 } else if (SSL_get_stream_read_state(s->s)
                     == SSL_STREAM_STATE_RESET_REMOTE) {
                     /* Stream was reset by peer. */
                     if (!SSL_get_stream_read_error_code(s->s, &aec))
                         goto err;

                     ec = nghttp3_conn_close_stream(conn->h3conn, s->id, aec);
                     if (ec < 0) {
                         ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
                             "cannot mark stream as reset: %s (%d)",
                             nghttp3_strerror(ec), ec);
                         goto err;
                     }

                     s->done_recv_fin = 1;
                 } else {
                     /* Other error. */
                     goto err;
                 }
             }

             s->buf_cur = 0;
             s->buf_total = num_bytes;
         }

         if (s->buf_cur == s->buf_total)
             break;

         /*
          * This function is confusingly named as it is is named from nghttp3's
          * 'perspective'; it is used to pass data *into* the HTTP/3 stack which
          * has been received from the network.
          */
         assert(conn->consumed_app_data == 0);
         ec = nghttp3_conn_read_stream(conn->h3conn, s->id, s->buf + s->buf_cur,
             s->buf_total - s->buf_cur, /*fin=*/0);
         if (ec < 0) {
             ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
                 "nghttp3 failed to process incoming data: %s (%d)",
                 nghttp3_strerror(ec), ec);
             goto err;
         }

         /*
          * read_stream reports the data it consumes from us in two different
          * ways; the non-application data is returned as a number of bytes 'ec'
          * above, but the number of bytes of application data has to be recorded
          * by our callback. We sum the two to determine the total number of
          * bytes which nghttp3 consumed.
          */
         consumed = ec + conn->consumed_app_data;
         assert(consumed <= s->buf_total - s->buf_cur);
         s->buf_cur += consumed;
         conn->consumed_app_data = 0;
     }

     return;
 err:
     conn->pump_res = 0;
 }

 int OSSL_DEMO_H3_CONN_handle_events(OSSL_DEMO_H3_CONN *conn)
 {
     int ec, fin;
     size_t i, num_vecs, written, total_written, total_len;
     int64_t stream_id;
     uint64_t flags;
     nghttp3_vec vecs[8] = { 0 };
     OSSL_DEMO_H3_STREAM key, *s;
     SSL *snew;
     OSSL_DEMO_H3_POLL_LIST *pollist = NULL;
     size_t poll_num;
     struct timeval poll_timeout;
     size_t result_count;

     if (conn == NULL)
         return 0;

     /*
      * We handle events by doing three things:
      *
      * 1. Handle new incoming streams
      * 2. Pump outgoing data from the HTTP/3 stack to the QUIC engine
      * 3. Pump incoming data from the QUIC engine to the HTTP/3 stack
      */

     /* 1. Check for new incoming streams */
     for (;;) {
         if ((snew = SSL_accept_stream(conn->qconn, SSL_ACCEPT_STREAM_NO_BLOCK)) == NULL)
             break;
     }

     /* 2. Pump outgoing data from HTTP/3 engine to QUIC. */
     for (;;) {
         /*
          * Get a number of send vectors from the HTTP/3 engine.
          *
          * Note that this function is confusingly named as it is named from
          * nghttp3's 'perspective': this outputs pointers to data which nghttp3
          * wants to *write* to the network.
          */
         ec = nghttp3_conn_writev_stream(conn->h3conn, &stream_id, &fin,
             vecs, ARRAY_LEN(vecs));
         if (ec < 0)
             return 0;
         if (ec == 0)
             break;

         /*
          * we let SSL_write_ex2(3) to conclude the stream for us (send FIN)
          * after all data are written.
          */
         flags = (fin == 0) ? 0 : SSL_WRITE_FLAG_CONCLUDE;

         /* For each of the vectors returned, pass it to OpenSSL QUIC. */
         key.id = stream_id;
         if ((s = lh_OSSL_DEMO_H3_STREAM_retrieve(conn->streams, &key)) == NULL) {
             ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
                 "no stream for ID %zd", stream_id);
             return 0;
         }

         num_vecs = ec;
         total_len = nghttp3_vec_len(vecs, num_vecs);
         total_written = 0;
         for (i = 0; i < num_vecs; ++i) {
             if (vecs[i].len == 0)
                 continue;

             if (s->s == NULL) {
                 /* Already did STOP_SENDING and threw away stream, ignore */
                 written = vecs[i].len;
             } else if (!SSL_write_ex2(s->s, vecs[i].base, vecs[i].len, flags, &written)) {
                 if (SSL_get_error(s->s, 0) == SSL_ERROR_WANT_WRITE) {
                     /*
                      * We have filled our send buffer so tell nghttp3 to stop
                      * generating more data; we have to do this explicitly.
                      */
                     written = 0;
                     nghttp3_conn_block_stream(conn->h3conn, stream_id);
                 } else {
                     ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
                         "writing HTTP/3 data to network failed");
                     return 0;
                 }
             } else {
                 /*
                  * Tell nghttp3 it can resume generating more data in case we
                  * previously called block_stream.
                  */
                 nghttp3_conn_unblock_stream(conn->h3conn, stream_id);
             }

             total_written += written;
             if (written > 0) {
                 /*
                  * Tell nghttp3 we have consumed the data it output when we
                  * called writev_stream, otherwise subsequent calls to
                  * writev_stream will output the same data.
                  */
                 ec = nghttp3_conn_add_write_offset(conn->h3conn, stream_id, written);
                 if (ec < 0)
                     return 0;

                 /*
                  * Tell nghttp3 it can free the buffered data because we will
                  * not need it again. In our case we can always do this right
                  * away because we copy the data into our QUIC send buffers
                  * rather than simply storing a reference to it.
                  */
                 ec = nghttp3_conn_add_ack_offset(conn->h3conn, stream_id, written);
                 if (ec < 0)
                     return 0;
             }
         }

         if (fin && total_written == total_len) {

             if (total_len == 0) {
                 /*
                  * As a special case, if nghttp3 requested to write a
                  * zero-length stream with a FIN, we have to tell it we did this
                  * by calling add_write_offset(0).
                  */
                 ec = nghttp3_conn_add_write_offset(conn->h3conn, stream_id, 0);
                 if (ec < 0)
                     return 0;
             }
         }
     }

     /* 3. Build a list of streams to poll on */
     conn->pump_res = 1; /* cleared in below call if an error occurs */
     poll_num = lh_OSSL_DEMO_H3_STREAM_num_items(conn->streams);
     pollist = OPENSSL_malloc(sizeof(OSSL_DEMO_H3_POLL_LIST) + (sizeof(SSL_POLL_ITEM) * poll_num) + (sizeof(OSSL_DEMO_H3_STREAM *) * poll_num));
     pollist->poll_count = poll_num;
     pollist->poll_list = (SSL_POLL_ITEM *)(pollist + 1);
     pollist->h3_streams = (OSSL_DEMO_H3_STREAM **)(pollist->poll_list + poll_num);
     pollist->conn = conn;
     pollist->idx = 0;
     lh_OSSL_DEMO_H3_STREAM_doall_arg(conn->streams, h3_conn_collect_streams, pollist);
     poll_timeout.tv_sec = 0;
     poll_timeout.tv_usec = 0;
     result_count = 0;

     /* 4. poll the list built above, looking for streams that are ready to read */
     if (!SSL_poll(pollist->poll_list, pollist->idx, sizeof(SSL_POLL_ITEM),
             &poll_timeout, 0, &result_count)) {
         fprintf(stderr, "Failed to poll\n");
         goto end;
     }

     /* 5. Pump incoming data from QUIC to HTTP/3 engine. */
     for (i = 0; result_count != 0; i++) {
         if (pollist->poll_list[i].revents == SSL_POLL_EVENT_R) {
             result_count--;
             h3_conn_pump_stream(pollist->h3_streams[i], pollist->conn);
         }
     }
 end:
     OPENSSL_free(pollist);
     if (!conn->pump_res)
         return 0;

     return 1;
 }

 int OSSL_DEMO_H3_CONN_submit_request(OSSL_DEMO_H3_CONN *conn,
     const nghttp3_nv *nva, size_t nvlen,
     const nghttp3_data_reader *dr,
     void *user_data)
 {
     int ec;
     OSSL_DEMO_H3_STREAM *s_req = NULL;

     if (conn == NULL) {
         ERR_raise_data(ERR_LIB_USER, ERR_R_PASSED_NULL_PARAMETER,
             "connection must be specified");
         return 0;
     }

     /* Each HTTP/3 request is represented by a stream. */
     if ((s_req = h3_conn_create_stream(conn, OSSL_DEMO_H3_STREAM_TYPE_REQ)) == NULL)
         goto err;

     s_req->user_data = user_data;

     ec = nghttp3_conn_submit_request(conn->h3conn, s_req->id, nva, nvlen,
         dr, s_req);
     if (ec < 0) {
         ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
             "cannot submit HTTP/3 request: %s (%d)",
             nghttp3_strerror(ec), ec);
         goto err;
     }

     return 1;

 err:
     h3_conn_remove_stream(conn, s_req);
     return 0;
 }
	/*
	* Copyright 2023-2024 The OpenSSL Project Authors. All Rights Reserved.
	*
	* Licensed under the Apache License 2.0 (the "License"). You may not use
	* this file except in compliance with the License. You can obtain a copy
	* in the file LICENSE in the source distribution or at
	* https://www.openssl.org/source/license.html
	*/
	#include "ossl-nghttp3.h"
	#include <openssl/err.h>
	#include <assert.h>

	#include <sys/socket.h>
	#include <netinet/in.h>

	#define ARRAY_LEN(x) (sizeof(x) / sizeof((x)[0]))

	enum {
	OSSL_DEMO_H3_STREAM_TYPE_CTRL_SEND,
	OSSL_DEMO_H3_STREAM_TYPE_QPACK_ENC_SEND,
	OSSL_DEMO_H3_STREAM_TYPE_QPACK_DEC_SEND,
	OSSL_DEMO_H3_STREAM_TYPE_REQ,
	};

	#define BUF_SIZE 4096

	struct ossl_demo_h3_stream_st {
	uint64_t id; /* QUIC stream ID */
	SSL s; / QUIC stream SSL object */
	int done_recv_fin; /* Received FIN */
	void *user_data;

	uint8_t buf[BUF_SIZE];
	size_t buf_cur, buf_total;
	};

	DEFINE_LHASH_OF_EX(OSSL_DEMO_H3_STREAM);

	static void h3_stream_free(OSSL_DEMO_H3_STREAM *s)
	{
	if (s == NULL)
	return;

	SSL_free(s->s);
	OPENSSL_free(s);
	}

	static unsigned long h3_stream_hash(const OSSL_DEMO_H3_STREAM *s)
	{
	return (unsigned long)s->id;
	}

	static int h3_stream_eq(const OSSL_DEMO_H3_STREAM a, const OSSL_DEMO_H3_STREAM b)
	{
	if (a->id < b->id)
	return -1;
	if (a->id > b->id)
	return 1;
	return 0;
	}

	void OSSL_DEMO_H3_STREAM_get_user_data(const OSSL_DEMO_H3_STREAM s)
	{
	return s->user_data;
	}

	struct ossl_demo_h3_conn_st {
	/* QUIC connection SSL object */
	SSL *qconn;
	/* BIO wrapping QCSO */
	BIO *qconn_bio;
	/* HTTP/3 connection object */
	nghttp3_conn *h3conn;
	/* map of stream IDs to OSSL_DEMO_H3_STREAMs */
	LHASH_OF(OSSL_DEMO_H3_STREAM) *streams;
	/* opaque user data pointer */
	void *user_data;

	int pump_res;
	size_t consumed_app_data;

	/* Forwarding callbacks */
	nghttp3_recv_data recv_data_cb;
	nghttp3_stream_close stream_close_cb;
	nghttp3_stop_sending stop_sending_cb;
	nghttp3_reset_stream reset_stream_cb;
	nghttp3_deferred_consume deferred_consume_cb;
	};

	void OSSL_DEMO_H3_CONN_free(OSSL_DEMO_H3_CONN *conn)
	{
	if (conn == NULL)
	return;

	lh_OSSL_DEMO_H3_STREAM_doall(conn->streams, h3_stream_free);

	nghttp3_conn_del(conn->h3conn);
	BIO_free_all(conn->qconn_bio);
	lh_OSSL_DEMO_H3_STREAM_free(conn->streams);
	OPENSSL_free(conn);
	}

	static OSSL_DEMO_H3_STREAM h3_conn_create_stream(OSSL_DEMO_H3_CONN conn, int type)
	{
	OSSL_DEMO_H3_STREAM *s;
	uint64_t flags = SSL_STREAM_FLAG_ADVANCE;

	if ((s = OPENSSL_zalloc(sizeof(OSSL_DEMO_H3_STREAM))) == NULL)
	return NULL;

	if (type != OSSL_DEMO_H3_STREAM_TYPE_REQ)
	flags \|= SSL_STREAM_FLAG_UNI;

	if ((s->s = SSL_new_stream(conn->qconn, flags)) == NULL) {
	ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
	"could not create QUIC stream object");
	goto err;
	}

	s->id = SSL_get_stream_id(s->s);
	lh_OSSL_DEMO_H3_STREAM_insert(conn->streams, s);
	return s;

	err:
	OPENSSL_free(s);
	return NULL;
	}

	static void h3_conn_remove_stream(OSSL_DEMO_H3_CONN conn, OSSL_DEMO_H3_STREAM s)
	{
	if (s == NULL)
	return;

	lh_OSSL_DEMO_H3_STREAM_delete(conn->streams, s);
	h3_stream_free(s);
	}

	static int h3_conn_recv_data(nghttp3_conn *h3conn, int64_t stream_id,
	const uint8_t *data, size_t datalen,
	void conn_user_data, void stream_user_data)
	{
	OSSL_DEMO_H3_CONN *conn = conn_user_data;

	conn->consumed_app_data += datalen;
	if (conn->recv_data_cb == NULL)
	return 0;

	return conn->recv_data_cb(h3conn, stream_id, data, datalen,
	conn_user_data, stream_user_data);
	}

	static int h3_conn_stream_close(nghttp3_conn *h3conn, int64_t stream_id,
	uint64_t app_error_code,
	void conn_user_data, void stream_user_data)
	{
	int ret = 0;
	OSSL_DEMO_H3_CONN *conn = conn_user_data;
	OSSL_DEMO_H3_STREAM *stream = stream_user_data;

	if (conn->stream_close_cb != NULL)
	ret = conn->stream_close_cb(h3conn, stream_id, app_error_code,
	conn_user_data, stream_user_data);

	h3_conn_remove_stream(conn, stream);
	return ret;
	}

	static int h3_conn_stop_sending(nghttp3_conn *h3conn, int64_t stream_id,
	uint64_t app_error_code,
	void conn_user_data, void stream_user_data)
	{
	int ret = 0;
	OSSL_DEMO_H3_CONN *conn = conn_user_data;
	OSSL_DEMO_H3_STREAM *stream = stream_user_data;

	if (conn->stop_sending_cb != NULL)
	ret = conn->stop_sending_cb(h3conn, stream_id, app_error_code,
	conn_user_data, stream_user_data);

	SSL_free(stream->s);
	stream->s = NULL;
	return ret;
	}

	static int h3_conn_reset_stream(nghttp3_conn *h3conn, int64_t stream_id,
	uint64_t app_error_code,
	void conn_user_data, void stream_user_data)
	{
	int ret = 0;
	OSSL_DEMO_H3_CONN *conn = conn_user_data;
	OSSL_DEMO_H3_STREAM *stream = stream_user_data;
	SSL_STREAM_RESET_ARGS args = { 0 };

	if (conn->reset_stream_cb != NULL)
	ret = conn->reset_stream_cb(h3conn, stream_id, app_error_code,
	conn_user_data, stream_user_data);

	if (stream->s != NULL) {
	args.quic_error_code = app_error_code;

	if (!SSL_stream_reset(stream->s, &args, sizeof(args)))
	return 1;
	}

	return ret;
	}

	static int h3_conn_deferred_consume(nghttp3_conn *h3conn, int64_t stream_id,
	size_t consumed,
	void conn_user_data, void stream_user_data)
	{
	int ret = 0;
	OSSL_DEMO_H3_CONN *conn = conn_user_data;

	if (conn->deferred_consume_cb != NULL)
	ret = conn->deferred_consume_cb(h3conn, stream_id, consumed,
	conn_user_data, stream_user_data);

	conn->consumed_app_data += consumed;
	return ret;
	}

	OSSL_DEMO_H3_CONN OSSL_DEMO_H3_CONN_new_for_conn(SSL qconn,
	const nghttp3_callbacks *callbacks,
	const nghttp3_settings *settings,
	void *user_data)
	{
	int ec;
	OSSL_DEMO_H3_CONN *conn;
	OSSL_DEMO_H3_STREAM *s_ctl_send = NULL;
	OSSL_DEMO_H3_STREAM *s_qpenc_send = NULL;
	OSSL_DEMO_H3_STREAM *s_qpdec_send = NULL;
	nghttp3_settings dsettings = { 0 };
	nghttp3_callbacks intl_callbacks = { 0 };
	static const unsigned char alpn[] = { 2, 'h', '3' };

	if (qconn == NULL) {
	ERR_raise_data(ERR_LIB_USER, ERR_R_PASSED_NULL_PARAMETER,
	"QUIC connection BIO must be provided");
	return NULL;
	}

	if ((conn = OPENSSL_zalloc(sizeof(OSSL_DEMO_H3_CONN))) == NULL)
	return NULL;

	conn->qconn = qconn;
	conn->user_data = user_data;

	/* Create the map of stream IDs to OSSL_DEMO_H3_STREAM structures. */
	if ((conn->streams = lh_OSSL_DEMO_H3_STREAM_new(h3_stream_hash, h3_stream_eq)) == NULL)
	goto err;

	if (SSL_set_alpn_protos(conn->qconn, alpn, sizeof(alpn))) {
	/* SSL_set_alpn_protos returns 1 on failure */
	ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
	"failed to configure ALPN");
	goto err;
	}

	/*
	* Disable default stream mode and create all streams explicitly. Each QUIC
	* stream will be represented by its own QUIC stream SSL object (QSSO). This
	* also automatically enables us to accept incoming streams (see
	* SSL_set_incoming_stream_policy(3)).
	*/
	if (!SSL_set_default_stream_mode(conn->qconn, SSL_DEFAULT_STREAM_MODE_NONE)) {
	ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
	"failed to configure default stream mode");
	goto err;
	}

	/*
	* HTTP/3 requires a couple of unidirectional management streams: a control
	* stream and some QPACK state management streams for each side of a
	* connection. These are the instances on our side (with us sending); the
	* server will also create its own equivalent unidirectional streams on its
	* side, which we handle subsequently as they come in (see SSL_accept_stream
	* in the event handling code below).
	*/
	if ((s_ctl_send
	= h3_conn_create_stream(conn, OSSL_DEMO_H3_STREAM_TYPE_CTRL_SEND))
	== NULL)
	goto err;

	if ((s_qpenc_send
	= h3_conn_create_stream(conn, OSSL_DEMO_H3_STREAM_TYPE_QPACK_ENC_SEND))
	== NULL)
	goto err;

	if ((s_qpdec_send
	= h3_conn_create_stream(conn, OSSL_DEMO_H3_STREAM_TYPE_QPACK_DEC_SEND))
	== NULL)
	goto err;

	if (settings == NULL) {
	nghttp3_settings_default(&dsettings);
	settings = &dsettings;
	}

	if (callbacks != NULL)
	intl_callbacks = *callbacks;

	/*
	* We need to do some of our own processing when many of these events occur,
	* so we note the original callback functions and forward appropriately.
	*/
	conn->recv_data_cb = intl_callbacks.recv_data;
	conn->stream_close_cb = intl_callbacks.stream_close;
	conn->stop_sending_cb = intl_callbacks.stop_sending;
	conn->reset_stream_cb = intl_callbacks.reset_stream;
	conn->deferred_consume_cb = intl_callbacks.deferred_consume;

	intl_callbacks.recv_data = h3_conn_recv_data;
	intl_callbacks.stream_close = h3_conn_stream_close;
	intl_callbacks.stop_sending = h3_conn_stop_sending;
	intl_callbacks.reset_stream = h3_conn_reset_stream;
	intl_callbacks.deferred_consume = h3_conn_deferred_consume;

	/* Create the HTTP/3 client state. */
	ec = nghttp3_conn_client_new(&conn->h3conn, &intl_callbacks, settings,
	NULL, conn);
	if (ec < 0) {
	ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
	"cannot create nghttp3 connection: %s (%d)",
	nghttp3_strerror(ec), ec);
	goto err;
	}

	/*
	* Tell the HTTP/3 stack which stream IDs are used for our outgoing control
	* and QPACK streams. Note that we don't have to tell the HTTP/3 stack what
	* IDs are used for incoming streams as this is inferred automatically from
	* the stream type byte which starts every incoming unidirectional stream,
	* so it will autodetect the correct stream IDs for the incoming control and
	* QPACK streams initiated by the server.
	*/
	ec = nghttp3_conn_bind_control_stream(conn->h3conn, s_ctl_send->id);
	if (ec < 0) {
	ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
	"cannot bind nghttp3 control stream: %s (%d)",
	nghttp3_strerror(ec), ec);
	goto err;
	}

	ec = nghttp3_conn_bind_qpack_streams(conn->h3conn,
	s_qpenc_send->id,
	s_qpdec_send->id);
	if (ec < 0) {
	ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
	"cannot bind nghttp3 QPACK streams: %s (%d)",
	nghttp3_strerror(ec), ec);
	goto err;
	}

	return conn;

	err:
	nghttp3_conn_del(conn->h3conn);
	h3_stream_free(s_ctl_send);
	h3_stream_free(s_qpenc_send);
	h3_stream_free(s_qpdec_send);
	lh_OSSL_DEMO_H3_STREAM_free(conn->streams);
	OPENSSL_free(conn);
	return NULL;
	}

	/*
	* Create BIO with UDP connected socket for one of the address we got from
	* resolver.
	*/
	static BIO *
	create_socket_bio(const BIO_ADDRINFO *bai)
	{
	int sock;
	BIO *bio;

	sock = BIO_socket(BIO_ADDRINFO_family(bai), SOCK_DGRAM, 0, 0);
	if (sock == -1)
	return NULL;

	if (!BIO_connect(sock, BIO_ADDRINFO_address(bai), 0))
	goto err;

	if (!BIO_socket_nbio(sock, 1))
	goto err;

	if ((bio = BIO_new(BIO_s_datagram())) == NULL)
	goto err;
	BIO_set_fd(bio, sock, BIO_CLOSE);

	return bio;
	err:
	BIO_closesocket(sock);

	return NULL;
	}

	OSSL_DEMO_H3_CONN OSSL_DEMO_H3_CONN_new_for_addr(SSL_CTX ctx, const BIO_ADDRINFO *bai,
	const char *bare_hostname,
	const nghttp3_callbacks *callbacks,
	const nghttp3_settings *settings,
	void *user_data)
	{
	BIO *qconn_bio = NULL;
	SSL *qconn = NULL;
	OSSL_DEMO_H3_CONN *conn = NULL;

	if ((qconn_bio = create_socket_bio(bai)) == NULL)
	return NULL;

	qconn = SSL_new(ctx);
	if (qconn == NULL)
	goto err;

	SSL_set_bio(qconn, qconn_bio, qconn_bio);
	qconn_bio = NULL;

	/* Set the hostname we will validate the X.509 certificate against. */
	if (SSL_set1_host(qconn, bare_hostname) <= 0)
	goto err;

	/* Configure SNI */
	if (!SSL_set_tlsext_host_name(qconn, bare_hostname))
	goto err;

	conn = OSSL_DEMO_H3_CONN_new_for_conn(qconn, callbacks,
	settings, user_data);
	if (conn == NULL)
	goto err;

	return conn;

	err:
	SSL_free(qconn);
	BIO_free_all(qconn_bio);
	return NULL;
	}

	void OSSL_DEMO_H3_CONN_get_user_data(const OSSL_DEMO_H3_CONN conn)
	{
	return conn->user_data;
	}

	SSL OSSL_DEMO_H3_CONN_get0_connection(const OSSL_DEMO_H3_CONN conn)
	{
	return conn->qconn;
	}

	typedef struct ossl_demo_h3_poll_list {
	SSL_POLL_ITEM *poll_list;
	OSSL_DEMO_H3_STREAM **h3_streams;
	OSSL_DEMO_H3_CONN *conn;
	size_t poll_count;
	size_t idx;
	} OSSL_DEMO_H3_POLL_LIST;

	static void h3_conn_collect_streams(OSSL_DEMO_H3_STREAM s, void list)
	{
	OSSL_DEMO_H3_POLL_LIST *pollist = list;

	pollist->poll_list[pollist->idx].desc = SSL_as_poll_descriptor(s->s);
	pollist->poll_list[pollist->idx].revents = 0;
	pollist->poll_list[pollist->idx].events = SSL_POLL_EVENT_R;
	pollist->h3_streams[pollist->idx] = s;
	pollist->idx++;
	}

	/* Pumps received data to the HTTP/3 stack for a single stream. */
	static void h3_conn_pump_stream(OSSL_DEMO_H3_STREAM s, void conn_)
	{
	int ec;
	OSSL_DEMO_H3_CONN *conn = conn_;
	size_t num_bytes, consumed;
	uint64_t aec;

	if (!conn->pump_res)
	/*
	* Handling of a previous stream in the iteration over all streams
	* failed, so just do nothing.
	*/
	return;

	for (;;) {
	if (s->s == NULL /* If we already did STOP_SENDING, ignore this stream. */
	/* If this is a write-only stream, there is no read data to check. */
	\|\| SSL_get_stream_read_state(s->s) == SSL_STREAM_STATE_WRONG_DIR
	/*
	* If we already got a FIN for this stream, there is nothing more to
	* do for it.
	*/
	\|\| s->done_recv_fin)
	break;

	/*
	* Pump data from OpenSSL QUIC to the HTTP/3 stack by calling SSL_peek
	* to get received data and passing it to nghttp3 using
	* nghttp3_conn_read_stream. Note that this function is confusingly
	* named and inputs data to the HTTP/3 stack.
	*/
	if (s->buf_cur == s->buf_total) {
	/* Need more data. */
	ec = SSL_read_ex(s->s, s->buf, sizeof(s->buf), &num_bytes);
	if (ec <= 0) {
	num_bytes = 0;
	if (SSL_get_error(s->s, ec) == SSL_ERROR_ZERO_RETURN) {
	/* Stream concluded normally. Pass FIN to HTTP/3 stack. */
	ec = nghttp3_conn_read_stream(conn->h3conn, s->id, NULL, 0,
	/fin=/1);
	if (ec < 0) {
	ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
	"cannot pass FIN to nghttp3: %s (%d)",
	nghttp3_strerror(ec), ec);
	goto err;
	}

	s->done_recv_fin = 1;
	} else if (SSL_get_stream_read_state(s->s)
	== SSL_STREAM_STATE_RESET_REMOTE) {
	/* Stream was reset by peer. */
	if (!SSL_get_stream_read_error_code(s->s, &aec))
	goto err;

	ec = nghttp3_conn_close_stream(conn->h3conn, s->id, aec);
	if (ec < 0) {
	ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
	"cannot mark stream as reset: %s (%d)",
	nghttp3_strerror(ec), ec);
	goto err;
	}

	s->done_recv_fin = 1;
	} else {
	/* Other error. */
	goto err;
	}
	}

	s->buf_cur = 0;
	s->buf_total = num_bytes;
	}

	if (s->buf_cur == s->buf_total)
	break;

	/*
	* This function is confusingly named as it is is named from nghttp3's
	* 'perspective'; it is used to pass data into the HTTP/3 stack which
	* has been received from the network.
	*/
	assert(conn->consumed_app_data == 0);
	ec = nghttp3_conn_read_stream(conn->h3conn, s->id, s->buf + s->buf_cur,
	s->buf_total - s->buf_cur, /fin=/0);
	if (ec < 0) {
	ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
	"nghttp3 failed to process incoming data: %s (%d)",
	nghttp3_strerror(ec), ec);
	goto err;
	}

	/*
	* read_stream reports the data it consumes from us in two different
	* ways; the non-application data is returned as a number of bytes 'ec'
	* above, but the number of bytes of application data has to be recorded
	* by our callback. We sum the two to determine the total number of
	* bytes which nghttp3 consumed.
	*/
	consumed = ec + conn->consumed_app_data;
	assert(consumed <= s->buf_total - s->buf_cur);
	s->buf_cur += consumed;
	conn->consumed_app_data = 0;
	}

	return;
	err:
	conn->pump_res = 0;
	}

	int OSSL_DEMO_H3_CONN_handle_events(OSSL_DEMO_H3_CONN *conn)
	{
	int ec, fin;
	size_t i, num_vecs, written, total_written, total_len;
	int64_t stream_id;
	uint64_t flags;
	nghttp3_vec vecs[8] = { 0 };
	OSSL_DEMO_H3_STREAM key, *s;
	SSL *snew;
	OSSL_DEMO_H3_POLL_LIST *pollist = NULL;
	size_t poll_num;
	struct timeval poll_timeout;
	size_t result_count;

	if (conn == NULL)
	return 0;

	/*
	* We handle events by doing three things:
	*
	* 1. Handle new incoming streams
	* 2. Pump outgoing data from the HTTP/3 stack to the QUIC engine
	* 3. Pump incoming data from the QUIC engine to the HTTP/3 stack
	*/

	/* 1. Check for new incoming streams */
	for (;;) {
	if ((snew = SSL_accept_stream(conn->qconn, SSL_ACCEPT_STREAM_NO_BLOCK)) == NULL)
	break;
	}

	/* 2. Pump outgoing data from HTTP/3 engine to QUIC. */
	for (;;) {
	/*
	* Get a number of send vectors from the HTTP/3 engine.
	*
	* Note that this function is confusingly named as it is named from
	* nghttp3's 'perspective': this outputs pointers to data which nghttp3
	* wants to write to the network.
	*/
	ec = nghttp3_conn_writev_stream(conn->h3conn, &stream_id, &fin,
	vecs, ARRAY_LEN(vecs));
	if (ec < 0)
	return 0;
	if (ec == 0)
	break;

	/*
	* we let SSL_write_ex2(3) to conclude the stream for us (send FIN)
	* after all data are written.
	*/
	flags = (fin == 0) ? 0 : SSL_WRITE_FLAG_CONCLUDE;

	/* For each of the vectors returned, pass it to OpenSSL QUIC. */
	key.id = stream_id;
	if ((s = lh_OSSL_DEMO_H3_STREAM_retrieve(conn->streams, &key)) == NULL) {
	ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
	"no stream for ID %zd", stream_id);
	return 0;
	}

	num_vecs = ec;
	total_len = nghttp3_vec_len(vecs, num_vecs);
	total_written = 0;
	for (i = 0; i < num_vecs; ++i) {
	if (vecs[i].len == 0)
	continue;

	if (s->s == NULL) {
	/* Already did STOP_SENDING and threw away stream, ignore */
	written = vecs[i].len;
	} else if (!SSL_write_ex2(s->s, vecs[i].base, vecs[i].len, flags, &written)) {
	if (SSL_get_error(s->s, 0) == SSL_ERROR_WANT_WRITE) {
	/*
	* We have filled our send buffer so tell nghttp3 to stop
	* generating more data; we have to do this explicitly.
	*/
	written = 0;
	nghttp3_conn_block_stream(conn->h3conn, stream_id);
	} else {
	ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
	"writing HTTP/3 data to network failed");
	return 0;
	}
	} else {
	/*
	* Tell nghttp3 it can resume generating more data in case we
	* previously called block_stream.
	*/
	nghttp3_conn_unblock_stream(conn->h3conn, stream_id);
	}

	total_written += written;
	if (written > 0) {
	/*
	* Tell nghttp3 we have consumed the data it output when we
	* called writev_stream, otherwise subsequent calls to
	* writev_stream will output the same data.
	*/
	ec = nghttp3_conn_add_write_offset(conn->h3conn, stream_id, written);
	if (ec < 0)
	return 0;

	/*
	* Tell nghttp3 it can free the buffered data because we will
	* not need it again. In our case we can always do this right
	* away because we copy the data into our QUIC send buffers
	* rather than simply storing a reference to it.
	*/
	ec = nghttp3_conn_add_ack_offset(conn->h3conn, stream_id, written);
	if (ec < 0)
	return 0;
	}
	}

	if (fin && total_written == total_len) {

	if (total_len == 0) {
	/*
	* As a special case, if nghttp3 requested to write a
	* zero-length stream with a FIN, we have to tell it we did this
	* by calling add_write_offset(0).
	*/
	ec = nghttp3_conn_add_write_offset(conn->h3conn, stream_id, 0);
	if (ec < 0)
	return 0;
	}
	}
	}

	/* 3. Build a list of streams to poll on */
	conn->pump_res = 1; /* cleared in below call if an error occurs */
	poll_num = lh_OSSL_DEMO_H3_STREAM_num_items(conn->streams);
	pollist = OPENSSL_malloc(sizeof(OSSL_DEMO_H3_POLL_LIST) + (sizeof(SSL_POLL_ITEM) * poll_num) + (sizeof(OSSL_DEMO_H3_STREAM ) poll_num));
	pollist->poll_count = poll_num;
	pollist->poll_list = (SSL_POLL_ITEM *)(pollist + 1);
	pollist->h3_streams = (OSSL_DEMO_H3_STREAM **)(pollist->poll_list + poll_num);
	pollist->conn = conn;
	pollist->idx = 0;
	lh_OSSL_DEMO_H3_STREAM_doall_arg(conn->streams, h3_conn_collect_streams, pollist);
	poll_timeout.tv_sec = 0;
	poll_timeout.tv_usec = 0;
	result_count = 0;

	/* 4. poll the list built above, looking for streams that are ready to read */
	if (!SSL_poll(pollist->poll_list, pollist->idx, sizeof(SSL_POLL_ITEM),
	&poll_timeout, 0, &result_count)) {
	fprintf(stderr, "Failed to poll\n");
	goto end;
	}

	/* 5. Pump incoming data from QUIC to HTTP/3 engine. */
	for (i = 0; result_count != 0; i++) {
	if (pollist->poll_list[i].revents == SSL_POLL_EVENT_R) {
	result_count--;
	h3_conn_pump_stream(pollist->h3_streams[i], pollist->conn);
	}
	}
	end:
	OPENSSL_free(pollist);
	if (!conn->pump_res)
	return 0;

	return 1;
	}

	int OSSL_DEMO_H3_CONN_submit_request(OSSL_DEMO_H3_CONN *conn,
	const nghttp3_nv *nva, size_t nvlen,
	const nghttp3_data_reader *dr,
	void *user_data)
	{
	int ec;
	OSSL_DEMO_H3_STREAM *s_req = NULL;

	if (conn == NULL) {
	ERR_raise_data(ERR_LIB_USER, ERR_R_PASSED_NULL_PARAMETER,
	"connection must be specified");
	return 0;
	}

	/* Each HTTP/3 request is represented by a stream. */
	if ((s_req = h3_conn_create_stream(conn, OSSL_DEMO_H3_STREAM_TYPE_REQ)) == NULL)
	goto err;

	s_req->user_data = user_data;

	ec = nghttp3_conn_submit_request(conn->h3conn, s_req->id, nva, nvlen,
	dr, s_req);
	if (ec < 0) {
	ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
	"cannot submit HTTP/3 request: %s (%d)",
	nghttp3_strerror(ec), ec);
	goto err;
	}

	return 1;

	err:
	h3_conn_remove_stream(conn, s_req);
	return 0;
	}