From 4ff42f92373aa4b6b0563c760e2d77d7b6a9b778 Mon Sep 17 00:00:00 2001 From: jesperpedersen Date: Mon, 13 Dec 2021 03:47:25 -0500 Subject: [PATCH] [#70] TLS support: pgagroal - PostgreSQL Pooling support by saving the SSL session and reusing it --- src/include/pgagroal.h | 10 ++- src/include/security.h | 18 ++++ src/libpgagroal/management.c | 13 ++- src/libpgagroal/message.c | 28 +++++-- src/libpgagroal/pool.c | 45 +++++++++- src/libpgagroal/security.c | 158 +++++++++++++++++++++++++++++++---- 6 files changed, 242 insertions(+), 30 deletions(-) diff --git a/src/include/pgagroal.h b/src/include/pgagroal.h index ac6f5de2..628c66e1 100644 --- a/src/include/pgagroal.h +++ b/src/include/pgagroal.h @@ -48,9 +48,10 @@ extern "C" { #define MAIN_UDS ".s.pgagroal" -#define MAX_BUFFER_SIZE 65535 -#define DEFAULT_BUFFER_SIZE 65535 -#define SECURITY_BUFFER_SIZE 512 +#define MAX_BUFFER_SIZE 65535 +#define DEFAULT_BUFFER_SIZE 65535 +#define SECURITY_BUFFER_SIZE 512 +#define SSL_SESSION_BUFFER_SIZE 2048 #define MAX_USERNAME_LENGTH 128 #define MAX_DATABASE_LENGTH 256 @@ -176,6 +177,9 @@ struct connection ssize_t security_lengths[NUMBER_OF_SECURITY_MESSAGES]; /**< The lengths of the security messages */ char security_messages[NUMBER_OF_SECURITY_MESSAGES][SECURITY_BUFFER_SIZE]; /**< The security messages */ + long ssl_session_length; /**< The length of the SSL session */ + unsigned char ssl_session[SSL_SESSION_BUFFER_SIZE]; /**< The SSL session (ASN.1) */ + signed char limit_rule; /**< The limit rule used */ time_t timestamp; /**< The last used timestamp */ pid_t pid; /**< The associated process id */ diff --git a/src/include/security.h b/src/include/security.h index 7125c4bb..96e8b73f 100644 --- a/src/include/security.h +++ b/src/include/security.h @@ -138,6 +138,24 @@ pgagroal_user_known(char* user); int pgagroal_tls_valid(void); +/** + * Load a SSL connection from a slot + * @param slot The slot + * @param ssl The resulting SSL connection (can be NULL) + * @return 0 upon success, otherwise 1 + */ +int +pgagroal_load_tls_connection(int slot, SSL** ssl); + +/** + * Save a TLS connection to a slot + * @param ssl The SSL connection + * @param slot The slot + * @return 0 upon success, otherwise 1 + */ +int +pgagroal_save_tls_connection(SSL* ssl, int slot); + #ifdef __cplusplus } #endif diff --git a/src/libpgagroal/management.c b/src/libpgagroal/management.c index 80767db2..03bdc023 100644 --- a/src/libpgagroal/management.c +++ b/src/libpgagroal/management.c @@ -1376,6 +1376,7 @@ write_ssl(SSL* ssl, void* buf, size_t size) } else { + long derr; int err = SSL_get_error(ssl, numbytes); switch (err) @@ -1397,12 +1398,20 @@ write_ssl(SSL* ssl, void* buf, size_t size) keep_write = true; break; case SSL_ERROR_SYSCALL: - pgagroal_log_error("SSL_ERROR_SYSCALL: %s (%d)", strerror(errno), SSL_get_fd(ssl)); + derr = ERR_get_error(); + pgagroal_log_error("SSL_ERROR_SYSCALL: FD %d", SSL_get_fd(ssl)); + pgagroal_log_error("%s", ERR_error_string(derr, NULL)); + pgagroal_log_error("%s", ERR_lib_error_string(derr)); + pgagroal_log_error("%s", ERR_reason_error_string(derr)); errno = 0; keep_write = false; break; case SSL_ERROR_SSL: - pgagroal_log_error("SSL_ERROR_SSL: %s (%d)", strerror(errno), SSL_get_fd(ssl)); + derr = ERR_get_error(); + pgagroal_log_error("SSL_ERROR_SSL: FD %d", SSL_get_fd(ssl)); + pgagroal_log_error("%s", ERR_error_string(derr, NULL)); + pgagroal_log_error("%s", ERR_lib_error_string(derr)); + pgagroal_log_error("%s", ERR_reason_error_string(derr)); errno = 0; keep_write = false; break; diff --git a/src/libpgagroal/message.c b/src/libpgagroal/message.c index 29374d6b..325d9721 100644 --- a/src/libpgagroal/message.c +++ b/src/libpgagroal/message.c @@ -1234,6 +1234,7 @@ ssl_read_message(SSL* ssl, int timeout, struct message** msg) } else { + long derr; int err; err = SSL_get_error(ssl, numbytes); @@ -1269,11 +1270,19 @@ ssl_read_message(SSL* ssl, int timeout, struct message** msg) keep_read = true; break; case SSL_ERROR_SYSCALL: - pgagroal_log_error("SSL_ERROR_SYSCALL: %s (%d)", strerror(errno), SSL_get_fd(ssl)); + derr = ERR_get_error(); + pgagroal_log_error("SSL_ERROR_SYSCALL: FD %d", SSL_get_fd(ssl)); + pgagroal_log_error("%s", ERR_error_string(derr, NULL)); + pgagroal_log_error("%s", ERR_lib_error_string(derr)); + pgagroal_log_error("%s", ERR_reason_error_string(derr)); errno = 0; break; case SSL_ERROR_SSL: - pgagroal_log_error("SSL_ERROR_SSL: %s (%d)", strerror(errno), SSL_get_fd(ssl)); + derr = ERR_get_error(); + pgagroal_log_error("SSL_ERROR_SSL: FD %d", SSL_get_fd(ssl)); + pgagroal_log_error("%s", ERR_error_string(derr, NULL)); + pgagroal_log_error("%s", ERR_lib_error_string(derr)); + pgagroal_log_error("%s", ERR_reason_error_string(derr)); break; } ERR_clear_error(); @@ -1328,6 +1337,7 @@ ssl_write_message(SSL* ssl, struct message* msg) } else { + long derr; int err = SSL_get_error(ssl, numbytes); switch (err) @@ -1349,17 +1359,19 @@ ssl_write_message(SSL* ssl, struct message* msg) keep_write = true; break; case SSL_ERROR_SYSCALL: + derr = ERR_get_error(); pgagroal_log_error("SSL_ERROR_SYSCALL: FD %d", SSL_get_fd(ssl)); - pgagroal_log_error("%s", ERR_error_string(err, NULL)); - pgagroal_log_error("%s", ERR_lib_error_string(err)); - pgagroal_log_error("%s", ERR_reason_error_string(err)); + pgagroal_log_error("%s", ERR_error_string(derr, NULL)); + pgagroal_log_error("%s", ERR_lib_error_string(derr)); + pgagroal_log_error("%s", ERR_reason_error_string(derr)); errno = 0; break; case SSL_ERROR_SSL: + derr = ERR_get_error(); pgagroal_log_error("SSL_ERROR_SSL: FD %d", SSL_get_fd(ssl)); - pgagroal_log_error("%s", ERR_error_string(err, NULL)); - pgagroal_log_error("%s", ERR_lib_error_string(err)); - pgagroal_log_error("%s", ERR_reason_error_string(err)); + pgagroal_log_error("%s", ERR_error_string(derr, NULL)); + pgagroal_log_error("%s", ERR_lib_error_string(derr)); + pgagroal_log_error("%s", ERR_reason_error_string(derr)); errno = 0; break; } diff --git a/src/libpgagroal/pool.c b/src/libpgagroal/pool.c index 5716b9cf..af7318d3 100644 --- a/src/libpgagroal/pool.c +++ b/src/libpgagroal/pool.c @@ -223,8 +223,14 @@ pgagroal_get_connection(char* username, char* database, bool reuse, bool transac } else { + SSL* s = NULL; bool kill = false; + if (pgagroal_load_tls_connection(*slot, &s)) + { + kill = true; + } + /* Verify the socket for the slot */ if (!pgagroal_socket_isvalid(config->connections[*slot].fd)) { @@ -250,7 +256,8 @@ pgagroal_get_connection(char* username, char* database, bool reuse, bool transac pgagroal_log_debug("pgagroal_get_connection: Slot %d FD %d - Error", *slot, config->connections[*slot].fd); pgagroal_tracking_event_slot(TRACKER_BAD_CONNECTION, *slot); - status = pgagroal_kill_connection(*slot, *ssl); + status = pgagroal_kill_connection(*slot, s); + s = NULL; if (config->number_of_users > 0 && config->number_of_limits > 0) { @@ -269,6 +276,8 @@ pgagroal_get_connection(char* username, char* database, bool reuse, bool transac goto timeout; } } + + *ssl = s; } config->connections[*slot].timestamp = time(NULL); @@ -388,8 +397,7 @@ pgagroal_return_connection(int slot, SSL* ssl, bool transaction_mode) if (config->connections[slot].has_security != SECURITY_INVALID && (config->connections[slot].has_security != SECURITY_SCRAM256 || (config->connections[slot].has_security == SECURITY_SCRAM256 && - (config->authquery || pgagroal_user_known(config->connections[slot].username)))) && - ssl == NULL) + (config->authquery || pgagroal_user_known(config->connections[slot].username))))) { state = atomic_load(&config->states[slot]); @@ -406,6 +414,20 @@ pgagroal_return_connection(int slot, SSL* ssl, bool transaction_mode) } } + if (pgagroal_save_tls_connection(ssl, slot)) + { + goto kill_connection; + } + + if (ssl != NULL) + { + SSL_CTX* ctx; + + ctx = SSL_get_SSL_CTX(ssl); + SSL_free(ssl); + SSL_CTX_free(ctx); + } + pgagroal_tracking_event_slot(TRACKER_RETURN_CONNECTION_SUCCESS, slot); config->connections[slot].timestamp = time(NULL); @@ -515,6 +537,9 @@ pgagroal_kill_connection(int slot, SSL* ssl) memset(&config->connections[slot].security_messages[i], 0, SECURITY_BUFFER_SIZE); } + config->connections[slot].ssl_session_length = 0; + memset(&config->connections[slot].ssl_session, 0, sizeof(config->connections[slot].ssl_session)); + config->connections[slot].limit_rule = -1; config->connections[slot].timestamp = -1; config->connections[slot].fd = -1; @@ -1117,6 +1142,8 @@ connection_details(int slot) pgagroal_log_trace(" Size: %zd", connection.security_lengths[i]); pgagroal_log_mem(&connection.security_messages[i], connection.security_lengths[i]); } + pgagroal_log_trace(" Session length: %d", connection.ssl_session_length); + pgagroal_log_mem(&connection.ssl_session, connection.ssl_session_length); break; case STATE_IN_USE: pgagroal_log_debug("pgagroal_pool_status: State: IN_USE"); @@ -1135,6 +1162,8 @@ connection_details(int slot) pgagroal_log_trace(" Size: %zd", connection.security_lengths[i]); pgagroal_log_mem(&connection.security_messages[i], connection.security_lengths[i]); } + pgagroal_log_trace(" Session length: %d", connection.ssl_session_length); + pgagroal_log_mem(&connection.ssl_session, connection.ssl_session_length); break; case STATE_GRACEFULLY: pgagroal_log_debug("pgagroal_pool_status: State: GRACEFULLY"); @@ -1153,6 +1182,8 @@ connection_details(int slot) pgagroal_log_trace(" Size: %zd", connection.security_lengths[i]); pgagroal_log_mem(&connection.security_messages[i], connection.security_lengths[i]); } + pgagroal_log_trace(" Session length: %d", connection.ssl_session_length); + pgagroal_log_mem(&connection.ssl_session, connection.ssl_session_length); break; case STATE_FLUSH: pgagroal_log_debug("pgagroal_pool_status: State: FLUSH"); @@ -1171,6 +1202,8 @@ connection_details(int slot) pgagroal_log_trace(" Size: %zd", connection.security_lengths[i]); pgagroal_log_mem(&connection.security_messages[i], connection.security_lengths[i]); } + pgagroal_log_trace(" Session length: %d", connection.ssl_session_length); + pgagroal_log_mem(&connection.ssl_session, connection.ssl_session_length); break; case STATE_IDLE_CHECK: pgagroal_log_debug("pgagroal_pool_status: State: IDLE CHECK"); @@ -1189,6 +1222,8 @@ connection_details(int slot) pgagroal_log_trace(" Size: %zd", connection.security_lengths[i]); pgagroal_log_mem(&connection.security_messages[i], connection.security_lengths[i]); } + pgagroal_log_trace(" Session length: %d", connection.ssl_session_length); + pgagroal_log_mem(&connection.ssl_session, connection.ssl_session_length); break; case STATE_VALIDATION: pgagroal_log_debug("pgagroal_pool_status: State: VALIDATION"); @@ -1207,6 +1242,8 @@ connection_details(int slot) pgagroal_log_trace(" Size: %zd", connection.security_lengths[i]); pgagroal_log_mem(&connection.security_messages[i], connection.security_lengths[i]); } + pgagroal_log_trace(" Session length: %d", connection.ssl_session_length); + pgagroal_log_mem(&connection.ssl_session, connection.ssl_session_length); break; case STATE_REMOVE: pgagroal_log_debug("pgagroal_pool_status: State: REMOVE"); @@ -1225,6 +1262,8 @@ connection_details(int slot) pgagroal_log_trace(" Size: %zd", connection.security_lengths[i]); pgagroal_log_mem(&connection.security_messages[i], connection.security_lengths[i]); } + pgagroal_log_trace(" Session length: %d", connection.ssl_session_length); + pgagroal_log_mem(&connection.ssl_session, connection.ssl_session_length); break; default: pgagroal_log_debug("pgagroal_pool_status: State %d Slot %d FD %d", state, slot, connection.fd); diff --git a/src/libpgagroal/security.c b/src/libpgagroal/security.c index 3dd6759f..7ab55e0f 100644 --- a/src/libpgagroal/security.c +++ b/src/libpgagroal/security.c @@ -125,7 +125,7 @@ static int create_ssl_ctx(bool client, SSL_CTX** ctx); static int create_ssl_client(SSL_CTX* ctx, char* key, char* cert, char* root, int socket, SSL** ssl); static int create_ssl_server(SSL_CTX* ctx, int socket, SSL** ssl); static int establish_client_tls_connection(int server, int fd, SSL** ssl); -static int create_client_tls_connection(int fd, SSL** ssl); +static int create_client_tls_connection(int slot, int fd, SSL** ssl); static int auth_query(SSL* c_ssl, int client_fd, int slot, char* username, char* database, int hba_method); static int auth_query_get_connection(char* username, char* password, char* database, int* server_fd, SSL** server_ssl); @@ -495,6 +495,9 @@ pgagroal_prefill_auth(char* username, char* password, char* database, int* slot, } server_fd = config->connections[*slot].fd; + /* TLS support */ + establish_client_tls_connection(config->connections[*slot].server, server_fd, server_ssl); + status = pgagroal_create_startup_message(username, database, &startup_msg); if (status != MESSAGE_STATUS_OK) { @@ -887,6 +890,7 @@ pgagroal_remote_management_scram_sha256(char* username, char* password, int serv if (status != 1) { + long derr; int err = SSL_get_error(ssl, status); switch (err) { @@ -905,15 +909,20 @@ pgagroal_remote_management_scram_sha256(char* username, char* password, int serv #endif break; case SSL_ERROR_SYSCALL: - pgagroal_log_error("SSL_ERROR_SYSCALL: %s (%d)", strerror(errno), server_fd); + derr = ERR_get_error(); + pgagroal_log_error("SSL_ERROR_SYSCALL: FD %d", server_fd); + pgagroal_log_error("%s", ERR_error_string(derr, NULL)); + pgagroal_log_error("%s", ERR_lib_error_string(derr)); + pgagroal_log_error("%s", ERR_reason_error_string(derr)); errno = 0; goto error; break; case SSL_ERROR_SSL: - pgagroal_log_error("SSL_ERROR_SSL: %s (%d)", strerror(errno), server_fd); - pgagroal_log_error("%s", ERR_error_string(err, NULL)); - pgagroal_log_error("%s", ERR_lib_error_string(err)); - pgagroal_log_error("%s", ERR_reason_error_string(err)); + derr = ERR_get_error(); + pgagroal_log_error("SSL_ERROR_SSL: FD %d", server_fd); + pgagroal_log_error("%s", ERR_error_string(derr, NULL)); + pgagroal_log_error("%s", ERR_lib_error_string(derr)); + pgagroal_log_error("%s", ERR_reason_error_string(derr)); errno = 0; goto error; break; @@ -3338,6 +3347,58 @@ pgagroal_tls_valid(void) return 1; } +int +pgagroal_load_tls_connection(int slot, SSL** ssl) +{ + int result = 0; + struct configuration* config; + + config = (struct configuration*)shmem; + + if (config->connections[slot].ssl_session_length > 0) + { + result = create_client_tls_connection(slot, config->connections[slot].fd, ssl); + } + + return result; +} + +int +pgagroal_save_tls_connection(SSL* ssl, int slot) +{ + int length; + SSL_SESSION* session = NULL; + unsigned char* p = NULL; + struct configuration* config; + + config = (struct configuration*)shmem; + + config->connections[slot].ssl_session_length = 0; + memset(&config->connections[slot].ssl_session, 0, SSL_SESSION_BUFFER_SIZE); + + if (ssl != NULL) + { + p = (unsigned char*)config->connections[slot].ssl_session; + + session = SSL_get_session(ssl); + + length = i2d_SSL_SESSION(session, NULL); + if (length > SSL_SESSION_BUFFER_SIZE) + { + pgagroal_log_error("Could not save TLS session: %d (%d)", length, SSL_SESSION_BUFFER_SIZE); + goto error; + } + + config->connections[slot].ssl_session_length = i2d_SSL_SESSION(session, &p); + } + + return 0; + +error: + + return 1; +} + static int derive_key_iv(char *password, unsigned char *key, unsigned char *iv) { @@ -5666,7 +5727,7 @@ establish_client_tls_connection(int server, int fd, SSL** ssl) if (msg->kind == 'S') { - create_client_tls_connection(fd, ssl); + create_client_tls_connection(-1, fd, ssl); } } @@ -5684,11 +5745,16 @@ establish_client_tls_connection(int server, int fd, SSL** ssl) } static int -create_client_tls_connection(int fd, SSL** ssl) +create_client_tls_connection(int slot, int fd, SSL** ssl) { SSL_CTX* ctx = NULL; SSL* s = NULL; + SSL_SESSION* session = NULL; + unsigned char* p = NULL; int status = -1; + struct configuration* config = NULL; + + config = (struct configuration*)shmem; /* We are acting as a client against the server */ if (create_ssl_ctx(true, &ctx)) @@ -5704,12 +5770,74 @@ create_client_tls_connection(int fd, SSL** ssl) goto error; } + /* If we have an existing session then load it */ + if (slot >= 0 && config->connections[slot].ssl_session_length > 0) + { + p = (unsigned char*)config->connections[slot].ssl_session; + + session = d2i_SSL_SESSION(NULL, (const unsigned char**)&p, config->connections[slot].ssl_session_length); + + pgagroal_log_error("Loading session: %p", session); + + if (session == NULL) + { + goto error; + } + + if (SSL_set_session(s, session) != 1) + { + goto error; + } + + if (SSL_set_fd(s, fd) != 1) + { + goto error; + } + } + do { + if (SSL_pending(s) > 0) + { + char pending[SSL_pending(s)]; + struct message msg; + + memset(&msg, 0, sizeof(struct message)); + memset(&pending, 0, sizeof(pending)); + + SSL_peek(s, &pending, SSL_pending(s)); + + msg.kind = 0; + msg.length = SSL_pending(s); + msg.data = &pending; + + pgagroal_log_error("BEFORE CONNECT: %d", SSL_pending(s)); + pgagroal_log_message(&msg); + } + status = SSL_connect(s); + if (SSL_pending(s) > 0) + { + char pending[SSL_pending(s)]; + struct message msg; + + memset(&msg, 0, sizeof(struct message)); + memset(&pending, 0, sizeof(pending)); + + SSL_peek(s, &pending, SSL_pending(s)); + + msg.kind = 0; + msg.length = SSL_pending(s); + msg.data = &pending; + + pgagroal_log_error("AFTER CONNECT: %d", SSL_pending(s)); + pgagroal_log_message(&msg); + } + if (status != 1) { + long derr; int err = SSL_get_error(s, status); switch (err) { @@ -5724,18 +5852,20 @@ create_client_tls_connection(int fd, SSL** ssl) case SSL_ERROR_WANT_CLIENT_HELLO_CB: break; case SSL_ERROR_SYSCALL: + derr = ERR_get_error(); pgagroal_log_error("SSL_ERROR_SYSCALL: FD %d", fd); - pgagroal_log_error("%s", ERR_error_string(err, NULL)); - pgagroal_log_error("%s", ERR_lib_error_string(err)); - pgagroal_log_error("%s", ERR_reason_error_string(err)); + pgagroal_log_error("%s", ERR_error_string(derr, NULL)); + pgagroal_log_error("%s", ERR_lib_error_string(derr)); + pgagroal_log_error("%s", ERR_reason_error_string(derr)); errno = 0; goto error; break; case SSL_ERROR_SSL: + derr = ERR_get_error(); pgagroal_log_error("SSL_ERROR_SSL: FD %d", fd); - pgagroal_log_error("%s", ERR_error_string(err, NULL)); - pgagroal_log_error("%s", ERR_lib_error_string(err)); - pgagroal_log_error("%s", ERR_reason_error_string(err)); + pgagroal_log_error("%s", ERR_error_string(derr, NULL)); + pgagroal_log_error("%s", ERR_lib_error_string(derr)); + pgagroal_log_error("%s", ERR_reason_error_string(derr)); errno = 0; goto error; break;