diff --git a/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.Crypto.cs b/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.Crypto.cs index 27b047b8308ed3..ff72852caa75c6 100644 --- a/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.Crypto.cs +++ b/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.Crypto.cs @@ -3,10 +3,13 @@ using System; using System.Diagnostics; +using System.Globalization; using System.Runtime.InteropServices; using System.Security.Cryptography.X509Certificates; using Microsoft.Win32.SafeHandles; +using TrackedAllocationDelegate = System.Action; + internal static partial class Interop { internal static partial class Crypto @@ -164,5 +167,52 @@ internal static byte[] GetDynamicBuffer(NegativeSizeReadMethod return bytes; } + + [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_GetMemoryUse")] + private static partial void GetMemoryUse(ref long memoryUse, ref long allocationCount); + + internal static long GetOpenSslAllocatedMemory() + { + long used = 0; + long count = 0; + GetMemoryUse(ref used, ref count); + return used; + } + + internal static long GetOpenSslAllocationCount() + { + long used = 0; + long count = 0; + GetMemoryUse(ref used, ref count); + return count; + } + + [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_EnableMemoryTracking")] + internal static unsafe partial void EnableMemoryTracking(int enable); + + [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_ForEachTrackedAllocation")] + private static unsafe partial void ForEachTrackedAllocation(delegate* unmanaged callback, IntPtr ctx); + + internal static unsafe void ForEachTrackedAllocation(TrackedAllocationDelegate callback) + { + ForEachTrackedAllocation(&MemoryTrackingCallback, (IntPtr)(&callback)); + } + + [UnmanagedCallersOnly] + private static unsafe void MemoryTrackingCallback(IntPtr ptr, ulong size, char* file, int line, IntPtr ctx) + { + TrackedAllocationDelegate callback = *(TrackedAllocationDelegate*)ctx; + callback(ptr, size, (IntPtr)file, line); + } + + internal static unsafe void EnableMemoryTracking() + { + EnableMemoryTracking(1); + } + + internal static unsafe void DisableMemoryTracking() + { + EnableMemoryTracking(0); + } } } diff --git a/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/README.md b/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/README.md new file mode 100644 index 00000000000000..8d081002ad1a9d --- /dev/null +++ b/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/README.md @@ -0,0 +1,77 @@ +# System.Security.Cryptography.Native + +This folder contains C# bindings for native shim (libSystem.Security.Cryptography.Native.OpenSsl.so), shimming functionality provided by the OpenSSL library. + +## Memory allocation hooks + +One extra feature exposed by the native shim is tracking of memory used by +OpenSSL by hooking the memory allocation routines via +`CRYPTO_set_mem_functions`. + +The functionality is enabled by setting +`DOTNET_OPENSSL_MEMORY_DEBUG` to 1. This environment +variable must be set before launching the program (calling +`Environment.SetEnvironmentVariable` at the start of the program is not +sufficient). The diagnostic API is not officially exposed and needs to be +accessed via private reflection on the `Interop.Crypto` type located in the +`System.Security.Cryptography` assembly. On this type, you can use following static +methods: + +- `int GetOpenSslAllocatedMemory()` + - Gets the total amount of memory allocated by OpenSSL +- `int GetOpenSslAllocationCount()` + - Gets the number of allocations made by OpenSSL +- `void EnableMemoryTracking()`/`void DisableMemoryTracking()` + - toggles tracking of individual live allocations via internal data + structures. I.e. will keep track of live memory allocated since the start of + tracking. +- `void ForEachTrackedAllocation(Action callback)` + - Accepts an callback and calls it for each allocation performed since the + last `EnableMemoryTracking` call. The order of reported information does not + correspond to the order of allocation. This method holds an internal lock + which prevents other threads from allocating any memory from OpenSSL. + - Callback parameters are + - IntPtr - The pointer to the allocated object + - ulong - size of the allocation in bytes + - IntPtr - Pointer to a null-terminated string (`const char*`) containing the name of the file from which the allocation was made. + - int - line number within the file specified by the previous parameter where the allocation was called from. + +The debug functionality brings some overhead (header for each allocation, +locks/synchronization during each allocation) and may cause performance penalty. + +### Example usage + +```cs +// all above mentioned APIs are accessible via "private reflection" +BindingFlags flags = BindingFlags.InvokeMethod | BindingFlags.NonPublic | BindingFlags.Static; +var cryptoInterop = typeof(RandomNumberGenerator).Assembly.GetTypes().First(t => t.Name == "Crypto"); + +// enable tracking, this clears up any previously tracked allocations +cryptoInterop.InvokeMember("EnableMemoryTracking", flags, null, null, null); + +// do some work that includes OpenSSL +HttpClient client = new HttpClient(); +await client.GetAsync("https://www.microsoft.com"); + +// stop tracking (this step is optional) +cryptoInterop.InvokeMember("DisableMemoryTracking", flags, null, null, null); + +using var process = Process.GetCurrentProcess(); +Console.WriteLine($"Bytes known to GC [{GC.GetTotalMemory(false)}], process working set [{process.WorkingSet64}]"); +Console.WriteLine("OpenSSL - currently allocated memory: {0} B", cryptoInterop.InvokeMember("GetOpenSslAllocatedMemory", flags, null, null, null)); +Console.WriteLine("OpenSSL - total allocations since start: {0}", cryptoInterop.InvokeMember("GetOpenSslAllocationCount", flags, null, null, null)); + +Dictionary<(IntPtr file, int line), ulong> allAllocations = new(); +Action callback = (ptr, size, namePtr, line) => +{ + CollectionsMarshal.GetValueRefOrAddDefault(allAllocations, (namePtr, line), out _) += size; +}; +cryptoInterop.InvokeMember("ForEachTrackedAllocation", flags, null, null, [callback]); + +Console.WriteLine("Total allocated OpenSSL memory by location"); +foreach (var ((filenameptr, line), total) in allAllocations.OrderByDescending(kvp => kvp.Value).Take(10)) +{ + string filename = Marshal.PtrToStringUTF8(filenameptr); + Console.WriteLine($"{total:N0} B from {filename}:{line}"); +} +``` \ No newline at end of file diff --git a/src/libraries/System.Security.Cryptography/src/ILLink/ILLink.Descriptors.LibraryBuild.xml b/src/libraries/System.Security.Cryptography/src/ILLink/ILLink.Descriptors.LibraryBuild.xml new file mode 100644 index 00000000000000..6a151eaf89bdcb --- /dev/null +++ b/src/libraries/System.Security.Cryptography/src/ILLink/ILLink.Descriptors.LibraryBuild.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/src/native/libs/System.Security.Cryptography.Native/CMakeLists.txt b/src/native/libs/System.Security.Cryptography.Native/CMakeLists.txt index ca536c1697a601..4a1d2e9e5e8e61 100644 --- a/src/native/libs/System.Security.Cryptography.Native/CMakeLists.txt +++ b/src/native/libs/System.Security.Cryptography.Native/CMakeLists.txt @@ -21,6 +21,7 @@ include_directories(${OPENSSL_INCLUDE_DIR}) set(NATIVECRYPTO_SOURCES apibridge.c apibridge_30.c + memory_debug.c openssl.c pal_asn1.c pal_bignum.c diff --git a/src/native/libs/System.Security.Cryptography.Native/apibridge.h b/src/native/libs/System.Security.Cryptography.Native/apibridge.h index 06bfa4c2fe8fb9..ae305b977629dd 100644 --- a/src/native/libs/System.Security.Cryptography.Native/apibridge.h +++ b/src/native/libs/System.Security.Cryptography.Native/apibridge.h @@ -65,3 +65,9 @@ int32_t local_X509_get_version(const X509* x509); int32_t local_X509_up_ref(X509* x509); typedef void (*SSL_CTX_keylog_cb_func)(const SSL *ssl, const char *line); void local_SSL_CTX_set_keylog_callback(SSL_CTX *ctx, SSL_CTX_keylog_cb_func cb); + +typedef void *(*CRYPTO_malloc_fn)(size_t num, const char *file, int line); +typedef void *(*CRYPTO_realloc_fn)(void *addr, size_t num, const char *file, int line); +typedef void (*CRYPTO_free_fn)(void *addr, const char *file, int line); + +int CRYPTO_set_mem_functions(CRYPTO_malloc_fn malloc_fn, CRYPTO_realloc_fn realloc_fn, CRYPTO_free_fn free_fn); diff --git a/src/native/libs/System.Security.Cryptography.Native/entrypoints.c b/src/native/libs/System.Security.Cryptography.Native/entrypoints.c index 329d0932ecf72e..7d59afc0d33d1b 100644 --- a/src/native/libs/System.Security.Cryptography.Native/entrypoints.c +++ b/src/native/libs/System.Security.Cryptography.Native/entrypoints.c @@ -5,6 +5,7 @@ // Include System.Security.Cryptography.Native headers #include "openssl.h" +#include "memory_debug.h" #include "pal_asn1.h" #include "pal_bignum.h" #include "pal_bio.h" @@ -188,6 +189,9 @@ static const Entry s_cryptoNative[] = DllImportEntry(CryptoNative_GetECKeyParameters) DllImportEntry(CryptoNative_GetMaxMdSize) DllImportEntry(CryptoNative_GetMemoryBioSize) + DllImportEntry(CryptoNative_GetMemoryUse) + DllImportEntry(CryptoNative_EnableMemoryTracking) + DllImportEntry(CryptoNative_ForEachTrackedAllocation) DllImportEntry(CryptoNative_GetObjectDefinitionByName) DllImportEntry(CryptoNative_GetOcspRequestDerSize) DllImportEntry(CryptoNative_GetPkcs7Certificates) diff --git a/src/native/libs/System.Security.Cryptography.Native/memory_debug.c b/src/native/libs/System.Security.Cryptography.Native/memory_debug.c new file mode 100644 index 00000000000000..ed24da7086028a --- /dev/null +++ b/src/native/libs/System.Security.Cryptography.Native/memory_debug.c @@ -0,0 +1,308 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#include "opensslshim.h" +#include "memory_debug.h" + +#include +#include +#include +#include + +// +// OpenSSL memory tracking/debugging facilities. +// +// We can use CRYPTO_set_mem_functions to replace allocation routines in +// OpenSSL. This allows us to prepend each allocated memory with a header that +// contains the size and source of the allocation, which allows us to track how +// much memory is OpenSSL consuming (not including malloc overhead). +// +// Additionally, if requested, we can track all allocations over a period of +// time and present them to managed code for analysis (via reflection APIs). +// +// Given that there is an overhead associated with tracking, the feature is +// gated behind the DOTNET_OPENSSL_MEMORY_DEBUG environment variable and should +// not be enabled by default in production. +// +// To track all allocated objects in a given period, the allocated objects are +// stringed in a circular, doubly-linked list. This allows us to do O(1) +// insertion and deletion. All operations are done under lock to prevent data +// corruption. To prevent lock contention over a single list, we maintain +// multiple lists, each with its own lock and allocate them round-robin. +// + +// helper macro for getting the pointer to containing type from a pointer to a member +#define container_of(ptr, type, member) ((type*)((char*)(ptr) - offsetof(type, member))) + +typedef struct link_t +{ + struct link_t* next; + struct link_t* prev; +} link_t; + +typedef struct list_t +{ + link_t head; + pthread_mutex_t lock; +} list_t; + +// Whether outstanding allocations should be tracked +static int32_t g_trackingEnabled = 0; +// Static lists of tracked allocations. +static list_t* g_trackedMemory = NULL; +// Number of partitions (distinct lists in g_trackedMemory) to track outstanding +// allocations in. +static uint32_t kPartitionCount = 32; + +// header for each tracked allocation +typedef struct header_t +{ + // link for the circular list + struct link_t link; + + // size of the allocation (of the data field) + uint64_t size; + + // filename from where the allocation was made + const char* file; + + // line number in the file where the allocation was made + int32_t line; + + // index of the list where this entry is stored + uint32_t index; + + // the start of actual allocated memory + __attribute__((aligned(8))) uint8_t data[]; +} header_t; + +static uint64_t g_allocatedMemory; +static uint64_t g_allocationCount; + +static void list_link_init(link_t* link) +{ + link->next = link; + link->prev = link; +} + +static void list_init(list_t* list) +{ + list_link_init(&list->head); + + int res = pthread_mutex_init(&list->lock, NULL); + assert(res == 0); +} + +static void list_insert_link(link_t* item, link_t* prev, link_t* next) +{ + next->prev = item; + item->next = next; + item->prev = prev; + prev->next = item; +} + +static void list_unlink_item(link_t* item) +{ + link_t* prev = item->prev; + link_t* next = item->next; + + prev->next = next; + next->prev = prev; + + list_link_init(item); +} + +static void init_memory_entry(header_t* entry, size_t size, const char* file, int32_t line) +{ + uint64_t newCount = __atomic_fetch_add(&g_allocationCount, 1, __ATOMIC_SEQ_CST); + + entry->size = size; + entry->line = line; + entry->file = file; + list_link_init(&entry->link); + entry->index = (uint32_t)(newCount % kPartitionCount); +} + +static list_t* get_item_bucket(header_t* entry) +{ + uint32_t index = entry->index % kPartitionCount; + return &g_trackedMemory[index]; +} + +static void do_track_entry(header_t* entry, int32_t add) +{ + __atomic_fetch_add(&g_allocatedMemory, (add != 0 ? entry->size : -entry->size), __ATOMIC_SEQ_CST); + + if (add != 0 && !g_trackingEnabled) + { + // don't track this (new) allocation individually + return; + } + + if (add == 0 && entry->link.next == &entry->link) + { + // freeing allocation, which is not in any list, skip taking the lock + return; + } + + list_t* list = get_item_bucket(entry); + int res = pthread_mutex_lock(&list->lock); + assert (res == 0); + + if (add != 0) + { + list_insert_link(&entry->link, &list->head, list->head.next); + } + else + { + list_unlink_item(&entry->link); + } + + res = pthread_mutex_unlock(&list->lock); + assert (res == 0); +} + +static void* mallocFunction(size_t size, const char *file, int line) +{ + header_t* entry = malloc(size + sizeof(header_t)); + if (entry != NULL) + { + init_memory_entry(entry, size, file, line); + do_track_entry(entry, 1); + } + + return (void*)(&entry->data); +} + +static void* reallocFunction (void *ptr, size_t size, const char *file, int line) +{ + struct header_t* entry = NULL; + + if (ptr != NULL) + { + entry = container_of(ptr, header_t, data); + + // untrack the item as realloc will free the memory and copy the contents elsewhere + do_track_entry(entry, 0); + } + + void* toReturn = NULL; + header_t* newEntry = (header_t*) realloc((void*)entry, size + sizeof(header_t)); + if (newEntry != NULL) + { + entry = newEntry; + + init_memory_entry(entry, size, file, line); + toReturn = (void*)(&entry->data); + } + + // either track the new memory, or add back the original one if realloc failed + if (entry) + { + do_track_entry(entry, 1); + } + + return toReturn; +} + +static void freeFunction(void *ptr, const char *file, int line) +{ + (void)file; + (void)line; + + if (ptr != NULL) + { + header_t* entry = container_of(ptr, header_t, data); + do_track_entry(entry, 0); + free(entry); + } +} + +void CryptoNative_GetMemoryUse(uint64_t* totalUsed, uint64_t* allocationCount) +{ + assert(totalUsed != NULL); + assert(allocationCount != NULL); + + *totalUsed = g_allocatedMemory; + *allocationCount = g_allocationCount; +} + +void CryptoNative_EnableMemoryTracking(int32_t enable) +{ + if (g_trackedMemory == NULL) + { + return; + } + + if (enable) + { + // Clear the lists by unlinking the list heads, any existing items in + // the list will become orphaned in a "floating" circular list. + // we will keep removing items from the list as they are freed + for (uint32_t i = 0; i < kPartitionCount; i++) + { + list_t* list = &g_trackedMemory[i]; + + pthread_mutex_lock(&list->lock); + + list_unlink_item(&list->head); + + pthread_mutex_unlock(&list->lock); + } + } + + g_trackingEnabled = enable; +} + +void CryptoNative_ForEachTrackedAllocation(void (*callback)(void* ptr, uint64_t size, const char* file, int32_t line, void* ctx), void* ctx) +{ + assert(callback != NULL); + + if (g_trackedMemory == NULL) + { + return; + } + + for (uint32_t i = 0; i < kPartitionCount; i++) + { + list_t* list = &g_trackedMemory[i]; + + pthread_mutex_lock(&list->lock); + for (link_t* node = list->head.next; node != &list->head; node = node->next) + { + header_t* entry = container_of(node, header_t, link); + callback(entry->data, entry->size, entry->file, entry->line, ctx); + } + pthread_mutex_unlock(&list->lock); + } +} + +static void init_tracking_lists(void) +{ + g_trackedMemory = malloc(kPartitionCount * sizeof(list_t)); + for (uint32_t i = 0; i < kPartitionCount; i++) + { + list_init(&g_trackedMemory[i]); + } +} + +void InitializeMemoryDebug(void) +{ + const char* debug = getenv("DOTNET_OPENSSL_MEMORY_DEBUG"); + if (debug != NULL && strcmp(debug, "1") == 0) + { +#ifdef FEATURE_DISTRO_AGNOSTIC_SSL + if (API_EXISTS(CRYPTO_set_mem_functions)) + { + // This should cover 1.1.1+ + CRYPTO_set_mem_functions(mallocFunction, reallocFunction, freeFunction); + init_tracking_lists(); + } +#elif OPENSSL_VERSION_NUMBER >= OPENSSL_VERSION_1_1_1_RTM + // OpenSSL 1.0 has different prototypes and it is out of support so we enable this only + // on 1.1.1+ + CRYPTO_set_mem_functions(mallocFunction, reallocFunction, freeFunction); + init_tracking_lists(); +#endif + } +} diff --git a/src/native/libs/System.Security.Cryptography.Native/memory_debug.h b/src/native/libs/System.Security.Cryptography.Native/memory_debug.h new file mode 100644 index 00000000000000..38572ea227651e --- /dev/null +++ b/src/native/libs/System.Security.Cryptography.Native/memory_debug.h @@ -0,0 +1,12 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#include "pal_types.h" + +void InitializeMemoryDebug(void); + +PALEXPORT void CryptoNative_EnableMemoryTracking(int32_t enable); + +PALEXPORT void CryptoNative_ForEachTrackedAllocation(void (*callback)(void* ptr, uint64_t size, const char* file, int32_t line, void* ctx), void* ctx); + +PALEXPORT void CryptoNative_GetMemoryUse(uint64_t* totalUsed, uint64_t* allocationCount); diff --git a/src/native/libs/System.Security.Cryptography.Native/openssl.c b/src/native/libs/System.Security.Cryptography.Native/openssl.c index d63481421a1323..f752da891b287c 100644 --- a/src/native/libs/System.Security.Cryptography.Native/openssl.c +++ b/src/native/libs/System.Security.Cryptography.Native/openssl.c @@ -7,6 +7,7 @@ #include "pal_safecrt.h" #include "pal_x509.h" #include "pal_ssl.h" +#include "memory_debug.h" #include "openssl.h" #ifdef FEATURE_DISTRO_AGNOSTIC_SSL @@ -1508,7 +1509,12 @@ static int32_t EnsureOpenSslInitializedCore(void) // Otherwise call the 1.1 one. #ifdef FEATURE_DISTRO_AGNOSTIC_SSL InitializeOpenSSLShim(); +#endif + // This needs to be done before any allocation is done e.g. EnsureOpenSsl* is called. + // And it also needs to be after the pointers are loaded for DISTRO_AGNOSTIC_SSL + InitializeMemoryDebug(); +#ifdef FEATURE_DISTRO_AGNOSTIC_SSL if (API_EXISTS(SSL_state)) { ret = EnsureOpenSsl10Initialized(); diff --git a/src/native/libs/System.Security.Cryptography.Native/opensslshim.h b/src/native/libs/System.Security.Cryptography.Native/opensslshim.h index 9f7228d06e13ca..400ecf9275b6d3 100644 --- a/src/native/libs/System.Security.Cryptography.Native/opensslshim.h +++ b/src/native/libs/System.Security.Cryptography.Native/opensslshim.h @@ -313,6 +313,7 @@ extern bool g_libSslUses32BitTime; REQUIRED_FUNCTION(CRYPTO_malloc) \ LEGACY_FUNCTION(CRYPTO_num_locks) \ LEGACY_FUNCTION(CRYPTO_set_locking_callback) \ + REQUIRED_FUNCTION(CRYPTO_set_mem_functions) \ REQUIRED_FUNCTION(d2i_OCSP_RESPONSE) \ REQUIRED_FUNCTION(d2i_PKCS12_fp) \ REQUIRED_FUNCTION(d2i_PKCS7) \ @@ -861,6 +862,7 @@ extern TYPEOF(OPENSSL_gmtime)* OPENSSL_gmtime_ptr; #define CRYPTO_malloc CRYPTO_malloc_ptr #define CRYPTO_num_locks CRYPTO_num_locks_ptr #define CRYPTO_set_locking_callback CRYPTO_set_locking_callback_ptr +#define CRYPTO_set_mem_functions CRYPTO_set_mem_functions_ptr #define d2i_OCSP_RESPONSE d2i_OCSP_RESPONSE_ptr #define d2i_PKCS12_fp d2i_PKCS12_fp_ptr #define d2i_PKCS7 d2i_PKCS7_ptr