Kernel TLS with Openssl and Nginx

2022-01-20

A simple demo of Kernel TLS using OpenSSL and Nginx. Really, this sample here does nothing new other than show how to set this up end-to-end in a VM. It draws directly from the following articles.

As the name suggests, kernelTLS offloads parts of the TLS operation used by openssl from the userspace to the kernel. The advantage of using it in the kernel is notable efficiency of curtailing the number of syscalls used between user and kernel space. You can read much more about the advantages in the following articles here:

What this article will do is just set it up for you to test with …or just for amusement (which is why i did this!)

images/ktls.png

_*image taken from the intel presentation cited above

Create Ubuntu VM

Lets get started…for me i just used 20.04.3 LTS (Focal Fossa) as that is currently one of the VM types supporting openssl and kTLS. You can find the list of os, kernel and openssl versions supported for nginx in the document above as well.

Note, i’m using qemu-system-x86_64 below…you are certainly free to use any other virtualization system you want.

# create a disk
qemu-img create -f qcow2 boot.img 10G

# get the iso
wget https://releases.ubuntu.com/20.04/ubuntu-20.04.6-live-server-amd64.iso

# create a VM (remember to install openssl_server; i created a user called 'sal' with any password
qemu-system-x86_64 -hda boot.img -net nic -net user,hostfwd=tcp::10022-:22,hostfwd=tcp::8443-:8443 -cpu host -smp `nproc` \
    -cdrom  ubuntu-20.04.6-live-server-amd64.iso --enable-kvm -m 2048 -boot menu=on   -machine q35,smm=on  --vga vmware

# if you reboot, remove the cd
qemu-system-x86_64 -hda boot.img -net nic -net user,hostfwd=tcp::10022-:22,hostfwd=tcp::8443-:8443 \
   -cpu host -smp `nproc`   --enable-kvm -m 2048 -boot menu=on   -machine q35,smm=on  --vga vmware

Now ssh to the VM and verify

ssh -o UserKnownHostsFile=/dev/null     -o CheckHostIP=no -o StrictHostKeyChecking=no sal@127.0.0.1 -p 10022

$ cat /etc/os-release
    NAME="Ubuntu"
    VERSION="20.04.3 LTS (Focal Fossa)"
    ID=ubuntu
    ID_LIKE=debian
    PRETTY_NAME="Ubuntu 20.04.3 LTS"
    VERSION_ID="20.04"
    HOME_URL="https://www.ubuntu.com/"
    SUPPORT_URL="https://help.ubuntu.com/"
    BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/"
    PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"
    VERSION_CODENAME=focal
    UBUNTU_CODENAME=focal

$ uname -r
    5.4.0-96-generic

Install openssl 3.0 with kTLS

apt-get update && apt-get install gcc build-essential git \
   wget curl vim libpcre3 libpcre3-dev zlib1g zlib1g-dev kmod \
   libunwind-dev zip -y

git clone https://github.com/openssl/openssl.git

# this is just the version i used i know that works when i did this demo
cd openssl 
git checkout 5288303da96084b41b062d99eb37177fb4cf471e

./config enable-ssl-trace enable-ktls && make -j`nproc` && make install

verify that the module is installed

export LD_LIBRARY_PATH=/usr/local/lib/:/usr/local/lib64/

modprobe tls
lsmod | grep tls

Install nginx

We’re ready to install nginx.

The configuration below is taken directly from here (with the exception that we’re using the now system default openssl3.0 installed previously)

cd
wget https://nginx.org/download/nginx-1.21.4.tar.gz && tar xzf nginx-1.21.4.tar.gz && \
   cd nginx-1.21.4 && \
   ./configure \
    --with-debug \
    --prefix=/usr/local \
    --conf-path=/usr/local/etc/nginx/nginx.conf \
    --error-log-path=/var/log/nginx/error.log \
    --http-log-path=/var/log/nginx/access.log \
    --pid-path=/var/run/nginx.pid \
    --lock-path=/var/run/nginx.lock \
    --http-client-body-temp-path=/var/cache/nginx/client_temp \
    --http-proxy-temp-path=/var/cache/nginx/proxy_temp \
    --http-fastcgi-temp-path=/var/cache/nginx/fastcgi_temp \
    --http-uwsgi-temp-path=/var/cache/nginx/uwsgi_temp \
    --http-scgi-temp-path=/var/cache/nginx/scgi_temp \
    --user=nginx \
    --group=nginx \
    --with-compat \
    --with-file-aio \
    --with-threads \
    --with-http_addition_module \
    --with-http_auth_request_module \
    --with-http_dav_module \
    --with-http_flv_module \
    --with-http_gunzip_module \
    --with-http_gzip_static_module \
    --with-http_mp4_module \
    --with-http_random_index_module \
    --with-http_realip_module \
    --with-http_secure_link_module \
    --with-http_slice_module \
    --with-http_ssl_module \
    --with-http_stub_status_module \
    --with-http_sub_module \
    --with-http_v2_module \
    --with-mail \
    --with-mail_ssl_module \
    --with-stream \
    --with-stream_realip_module \
    --with-stream_ssl_module \
    --with-stream_ssl_preread_module \
    --with-cc-opt='-g -O2 -fstack-protector-strong -Wformat -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -fPIC'  \
    --with-ld-opt='-Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-z,now -Wl,--as-needed -pie' && \
    make -j`nproc` && make install


mkdir -p /var/cache/nginx/
mkdir -p  /usr/share/nginx/html
echo foo >  /usr/share/nginx/html/index.html
useradd --no-create-home nginx

We need some sample TLS certs for nginx, so we’ll just use these

cd /usr/local/etc/nginx/
mkdir ssl

wget -O ssl/server_crt.pem https://raw.githubusercontent.com/salrashid123/squid_proxy/master/server_crt.pem
wget -O ssl/server_key.pem https://raw.githubusercontent.com/salrashid123/squid_proxy/master/server_key.pem
wget -O ssl/tls-ca.crt https://raw.githubusercontent.com/salrashid123/squid_proxy/master/tls-ca.crt

Now edit the nginx config to specify the kTLS and certificates

rm /usr/local/etc/nginx/nginx.conf
vi /usr/local/etc/nginx/nginx.conf

set nginx.conf to

worker_processes auto;
error_log /var/log/nginx/error.log debug;
 
daemon off;
events {}

http {
    sendfile on;

    server {
        listen 8443 ssl;
        ssl_certificate ssl/server_crt.pem;
        ssl_certificate_key ssl/server_key.pem;
        ssl_conf_command Options KTLS;
        ssl_protocols TLSv1.2;
        ssl_ciphers ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256;
        ssl_prefer_server_ciphers   on;

        location / {
            root /usr/share/nginx/html;
    	}
    }
}

start nginx

export LD_LIBRARY_PATH=/usr/local/lib/:/usr/local/lib64/
/usr/local/sbin/nginx -c /usr/local/etc/nginx/nginx.conf

In a new window, send some traffic in

curl -vvvk https://localhost:8443/index.html

Now check the nginx logs:

$ grep BIO /var/log/nginx/error.log
    2022/01/20 22:43:16 [debug] 39753#39753: *1 BIO_get_ktls_send(): 1

$ grep SSL_sendfile /var/log/nginx/error.log
    2022/01/20 22:43:16 [debug] 39753#39753: *1 SSL_sendfile: 4

The commands shows tls delegated to the kernel!

Openssl

Now do the same thing with openssl (which we’ve already enabled for ktls)

Start s_server listener though openssl

$ openssl version
OpenSSL 3.1.0-dev  (Library: OpenSSL 3.1.0-dev )

cd
echo foo > index.html
openssl s_server   \
      -ktls -sendfile -cert /usr/local/etc/nginx/ssl/server_crt.pem \
      -key /usr/local/etc/nginx/ssl/server_key.pem \
      -port 8443 \
      -CAfile /usr/local/etc/nginx/ssl/tls-ca.crt \
      -tlsextdebug \
      -tls1_2  \
      -trace \
      -WWW

Notice the -ktls and -sendfile switches which enables kTLS for s_server.

Or you can compile and run a test TLS Server shown below

export LD_LIBRARY_PATH=/usr/local/lib/:/usr/local/lib64/
 gcc  -o tls_server -std=gnu99 tls_server.c -L/usr/local/lib64 -lssl -lcrypto

# ldd tls_server
	linux-vdso.so.1 (0x00007ffd15881000)
	libssl.so.3 => /usr/local/lib64/libssl.so.3 (0x00007fb1ba20c000)
	libcrypto.so.3 => /usr/local/lib64/libcrypto.so.3 (0x00007fb1b9d87000)
	libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007fb1b9b8d000)
	libpthread.so.0 => /lib/x86_64-linux-gnu/libpthread.so.0 (0x00007fb1b9b6a000)
	libdl.so.2 => /lib/x86_64-linux-gnu/libdl.so.2 (0x00007fb1b9b64000)
	/lib64/ld-linux-x86-64.so.2 (0x00007fb1ba2c1000)


# get certs
wget -O server_crt.pem https://raw.githubusercontent.com/salrashid123/squid_proxy/master/server_crt.pem
wget -O server_key.pem https://raw.githubusercontent.com/salrashid123/squid_proxy/master/server_key.pem

# run tls_server
./tls_server

Now send client traffic:

openssl  s_client  -cipher 'ECDHE-RSA-AES256-GCM-SHA384' -tls1_2  --connect localhost:8443 -debug

Observe strace

If you enable strace on either openssl or the tls_server, you may see something like the following

{sa_family=AF_INET, sin_port=htons(46200), sin_addr=inet_addr("127.0.0.1")}, [16]) = 4

***************
setsockopt(4, SOL_TCP, TCP_ULP, [7564404], 4) = 0
***************

read(4, "\26\3\1\0\211", 5)             = 5
read(4, "\1\0\0\205\3\3\3033\236\365\200\345\244\306\317uY\5\270#\10\10\5lH\264Gc*o\236\240"..., 137) = 137
getpid()                                = 38491
getpid()                                = 38491
getpid()                                = 38491
getpid()                                = 38491
write(4, "\26\3\3\0A\2\0\0=\3\3!.\262\3'\241\206\356\177Bj\307\262\f\4\372\333SOW\307"..., 1487) = 1487
read(4, "\26\3\3\0%", 5)                = 5
read(4, "\20\0\0! \f\201#\357\253B\200\326\366Q?\366\367\363\340c\2507\275\363\323\177v\26\266\262\300"..., 37) = 37
read(4, "\24\3\3\0\1", 5)               = 5
read(4, "\1", 1)                        = 1
brk(0x560ec0722000)                     = 0x560ec0722000

***************
setsockopt(4, SOL_TLS, TLS_RX, "\3\0034\0\0\0\0\0\0\0\0\0\355\362\255\204\231&\343\373\\K\266\331U\270m\236\5X\2433"..., 56) = 0
***************

recvmsg(4, {msg_name=NULL, msg_namelen=0, msg_iov=[{iov_base="\24\0\0\f\353\245S\326\316\211']\307\17\223\r", iov_len=16688}], msg_iovlen=1, msg_control=[{cmsg_len=17, cmsg_level=SOL_TLS, cmsg_type=0x2}], msg_controllen=24, msg_flags=MSG_EOR}, 0) = 16
getpid()                                = 38491
write(4, "\26\3\3\0\252\4\0\0\246\0\0\34 \0\240\34i\"1\24\f\222;U\\\22\20C\217\tD\323"..., 181) = 181
getpid()                                = 38491

***************
setsockopt(4, SOL_TLS, TLS_TX, "\3\0034\0\356\321\6\204\310\255\2\302\236\17\370\1:\220\34\2\2753\251\262\250\205\334\210\36\r\252\301"..., 56) = 0
***************

sendmsg(4, {msg_name=NULL, msg_namelen=0, msg_iov=[{iov_base="\24\0\0\f\215`:\3762u\34\264.~\344\214", iov_len=16}], msg_iovlen=1, msg_control=[{cmsg_len=17, cmsg_level=SOL_TLS, cmsg_type=0x1}], msg_controllen=17, msg_flags=0}, 0) = 16
write(4, "test\n", 5)                   = 5
sendmsg(4, {msg_name=NULL, msg_namelen=0, msg_iov=[{iov_base="\1\0", iov_len=2}], msg_iovlen=1, msg_control=[{cmsg_len=17, cmsg_level=SOL_TLS, cmsg_type=0x1}], msg_controllen=17, msg_flags=0}, 0) = 2
brk(0x560ec0721000)                     = 0x560ec0721000
close(4)                                = 0
accept(3,                              = 0

Compare that with the same tls_server and s_client on a system that does not have ktls

{sa_family=AF_INET, sin_port=htons(50972), sin_addr=inet_addr("127.0.0.1")}, [16]) = 4
read(4, "\26\3\1\0\211", 5)             = 5
read(4, "\1\0\0\205\3\3\314!\330~C\331\261(\17\254\257(Y\317\35425Wxp\276gy\235\235b"..., 137) = 137
getpid()                                = 296309
getpid()                                = 296309
getpid()                                = 296309
getpid()                                = 296309
getpid()                                = 296309
getpid()                                = 296309
write(4, "\26\3\3\0A\2\0\0=\3\3\335?\326k\236\253\316!&\267:\256\237&\2502\347\375<\251\7"..., 1487) = 1487
read(4, "\26\3\3\0%", 5)                = 5
read(4, "\20\0\0! XB\373\30\7\342mt\323z\334x}\252\235T\233\224\377\335\31\316\4\245\242g9"..., 37) = 37
read(4, "\24\3\3\0\1", 5)               = 5
read(4, "\1", 1)                        = 1
read(4, "\26\3\3\0(", 5)                = 5
read(4, "\216\23b\20\240\223\336\306\302\374~X\354C^\336k\n\330-\350\365\t\317\365y\212v\277\263\224\263"..., 40) = 40
getpid()                                = 296309
getpid()                                = 296309
getpid()                                = 296309
getpid()                                = 296309
write(4, "\26\3\3\0\252\4\0\0\246\0\0\34 \0\240\225\32}\264\2439R\251a\36\30\356\3553\"B^"..., 226) = 226
write(4, "\27\3\3\0\35\366\300\276?; \244Rp\376\0\322f\26}\370?\342\24\310\5\265\337~?n\276"..., 34) = 34
write(4, "\25\3\3\0\32\366\300\276?; \244S#\267\0B\224.\215\305\304]1\273BDs\2174\211", 31) = 31
close(4)                                = 0

well, thats it..like i said, its just a rehash and setup, nothing more to see.

  • tls_server.c
#include <stdio.h>
#include <unistd.h>
#include <string.h>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <openssl/ssl.h>
#include <openssl/err.h>

int create_socket(int port)
{
    int s;
    struct sockaddr_in addr;

    addr.sin_family = AF_INET;
    addr.sin_port = htons(port);
    addr.sin_addr.s_addr = htonl(INADDR_ANY);

    s = socket(AF_INET, SOCK_STREAM, 0);
    if (s < 0) {
        perror("Unable to create socket");
        exit(EXIT_FAILURE);
    }

    if (bind(s, (struct sockaddr*)&addr, sizeof(addr)) < 0) {
        perror("Unable to bind");
        exit(EXIT_FAILURE);
    }

    if (listen(s, 1) < 0) {
        perror("Unable to listen");
        exit(EXIT_FAILURE);
    }

    return s;
}

SSL_CTX *create_context()
{
    const SSL_METHOD *method;
    SSL_CTX *ctx;

    method = TLS_server_method();

    ctx = SSL_CTX_new(method);

    SSL_CTX_set_options(ctx, SSL_OP_ENABLE_KTLS);
    SSL_CTX_set_min_proto_version(ctx, TLS1_2_VERSION);
    SSL_CTX_set_max_proto_version(ctx, TLS1_2_VERSION);

    if (!ctx) {
        perror("Unable to create SSL context");
        ERR_print_errors_fp(stderr);
        exit(EXIT_FAILURE);
    }

    return ctx;
}

void configure_context(SSL_CTX *ctx)
{
    /* Set the key and cert */
    if (SSL_CTX_use_certificate_file(ctx, "server_crt.pem", SSL_FILETYPE_PEM) <= 0) {
        ERR_print_errors_fp(stderr);
        exit(EXIT_FAILURE);
    }

    if (SSL_CTX_use_PrivateKey_file(ctx, "server_key.pem", SSL_FILETYPE_PEM) <= 0 ) {
        ERR_print_errors_fp(stderr);
        exit(EXIT_FAILURE);
    }
}

int main(int argc, char **argv)
{
    int sock;
    SSL_CTX *ctx;

    ctx = create_context();

    configure_context(ctx);

    sock = create_socket(8443);

    /* Handle connections */
    while(1) {
        struct sockaddr_in addr;
        unsigned int len = sizeof(addr);
        SSL *ssl;
        const char reply[] = "test\n";

        int client = accept(sock, (struct sockaddr*)&addr, &len);
        if (client < 0) {
            perror("Unable to accept");
            exit(EXIT_FAILURE);
        }

        ssl = SSL_new(ctx);
        SSL_set_fd(ssl, client);

        if (SSL_accept(ssl) <= 0) {
            ERR_print_errors_fp(stderr);
        } else {
            SSL_write(ssl, reply, strlen(reply));
        }

        SSL_shutdown(ssl);
        SSL_free(ssl);
        close(client);
    }

    close(sock);
    SSL_CTX_free(ctx);
}

This site supports webmentions. Send me a mention via this form.