| /* |
| * Copyright (C) 2019 The Android Open Source Project |
| * All rights reserved. |
| * |
| * Redistribution and use in source and binary forms, with or without |
| * modification, are permitted provided that the following conditions |
| * are met: |
| * * Redistributions of source code must retain the above copyright |
| * notice, this list of conditions and the following disclaimer. |
| * * Redistributions in binary form must reproduce the above copyright |
| * notice, this list of conditions and the following disclaimer in |
| * the documentation and/or other materials provided with the |
| * distribution. |
| * |
| * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS |
| * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE |
| * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, |
| * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, |
| * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS |
| * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED |
| * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, |
| * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT |
| * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
| * SUCH DAMAGE. |
| */ |
| |
| #if defined(LIBC_STATIC) |
| #errorThis file should not be compiledforstatic targets. |
| #endif |
| |
| #include<dlfcn.h> |
| #include<fcntl.h> |
| #include<signal.h> |
| #include<stdio.h> |
| #include<stdlib.h> |
| #include<unistd.h> |
| |
| #include<platform/bionic/malloc.h> |
| #include<private/bionic_config.h> |
| #include<private/bionic_malloc_dispatch.h> |
| #include<sys/system_properties.h> |
| |
| #include"gwp_asan_wrappers.h" |
| #include"malloc_common.h" |
| #include"malloc_common_dynamic.h" |
| #include"malloc_heapprofd.h" |
| #include"malloc_limit.h" |
| |
| // Installing heapprofd hooks is a multi step process, as outlined below. |
| // |
| // The incremental hooking and a dedicated task thread are used since we cannot |
| // do heavy work within a signal handler, or when blocking a malloc invocation. |
| // |
| // +--->+-------------+------------------+ |
| // | +->+kInitialState+----------------+ | malloc functions are not intercepted in any way. |
| // | | +-------+-----+ | | |
| // | | | HandleHeapprofd | | |
| // | | v Signal() | | |
| // | | +-------+----------------+ | | currently installing the ephemeral hooks. |
| // | | |kInstallingEphemeralHook|<--+ | | |
| // | | +-------+----------------+ | | | |
| // | | | | | | |
| // | | v | | | |
| // | | +-------+---------------+ | | | ephemeral hooks are installed. on the first call to |
| // | | |kEphemeralHookInstalled| | | | malloc these hooks spawn a thread that installs the |
| // | | +-------+---------------+ A B C heapprofd hooks. |
| // | | | MallocInit | | | |
| // | | v HeapprofdHook () | | | |
| // | | +-------+--------------+ | | | first call to malloc happened. the hooks are reset to |
| // | +--|kRemovingEphemeralHook| | | | kInitialState. |
| // | +----------------------+ | | | |
| // | | | | |
| // | | | | |
| // | +---------------+ | | | currently installing the heapprofd hook |
| // | |kInstallingHook|<-----------|-+ | |
| // | +-------+-------+ | | |
| // | | | | |
| // | v | | |
| // | +-------+------+ | | heapprofd hooks are installed. these forward calls to |
| // | |kHookInstalled|-------------+ | malloc / free / etc. to heapprofd_client.so. |
| // | +-------+------+ | |
| // | | DispatchReset() | |
| // | v | |
| // | +-------+---------+ | currently resetting the hooks to default. |
| // |----+kUninstallingHook| | |
| // +-----------------+ | |
| // | |
| // | |
| // +------------------+ | malloc debug / malloc hooks are active. these take |
| // |kIncompatibleHooks+<------------+ precendence over heapprofd, so heapprofd will not get |
| // +------------------+ enabled. this is a terminal state. |
| // |
| // |
| // A) HandleHeapprofdSignal() |
| // B) HeapprofdInstallHooksAtInit() / InitHeapprofd() |
| // C) HeapprofdRememberHookConflict() |
| enumMallocHeapprofdState:uint8_t{ |
| kInitialState, |
| kInstallingEphemeralHook, |
| kEphemeralHookInstalled, |
| kRemovingEphemeralHook, |
| kInstallingHook, |
| kHookInstalled, |
| kUninstallingHook, |
| kIncompatibleHooks |
| }; |
| |
| enumModifyGlobalsMode{ |
| kWithLock,// all calls to MaybeModifyGlobals with kWithLock will serialise. they can fail |
| // due to a concurrent call with kWithoutLock. |
| kWithoutLock// calls to MaybeModifyGlobals with kWithoutLock do not serialise. they can fail |
| // due to concurrent calls with kWithoutLock or kWithLock. |
| }; |
| |
| // Provide mutual exclusion so no two threads try to modify the globals at the same time. |
| template<typenameFn> |
| boolMaybeModifyGlobals(ModifyGlobalsMode mode,Fn f){ |
| bool success=false; |
| if(mode== kWithLock){ |
| pthread_mutex_lock(&gGlobalsMutateLock); |
| } |
| // As we have grabbed the mutex, the following condition should always hold, except |
| // if we are currently running HandleHeapprofdSignal. |
| if(!atomic_exchange(&gGlobalsMutating,true)){ |
| f(); |
| success=true; |
| atomic_store(&gGlobalsMutating,false); |
| }else{ |
| error_log("%s: heapprofd client: concurrent modification.", getprogname()); |
| } |
| if(mode== kWithLock){ |
| pthread_mutex_unlock(&gGlobalsMutateLock); |
| } |
| return success; |
| } |
| |
| extern"C"void*MallocInitHeapprofdHook(size_t); |
| |
| staticconstexprchar kHeapprofdSharedLib[]="heapprofd_client.so"; |
| staticconstexprchar kHeapprofdPrefix[]="heapprofd"; |
| staticconstexprchar kHeapprofdPropertyEnable[]="heapprofd.enable"; |
| |
| constexprchar kHeapprofdProgramPropertyPrefix[]="heapprofd.enable."; |
| constexprsize_t kHeapprofdProgramPropertyPrefixSize=sizeof(kHeapprofdProgramPropertyPrefix)-1; |
| constexprsize_t kMaxCmdlineSize=512; |
| |
| // The handle returned by dlopen when previously loading the heapprofd |
| // hooks. nullptr if shared library has not been already been loaded. |
| static_Atomic(void*) gHeapprofdHandle=nullptr; |
| static_AtomicMallocHeapprofdState gHeapprofdState= kInitialState; |
| |
| staticboolGetHeapprofdProgramProperty(char* data,size_t size){ |
| if(size< kHeapprofdProgramPropertyPrefixSize){ |
| error_log("%s: Overflow constructing heapprofd property", getprogname()); |
| returnfalse; |
| } |
| memcpy(data, kHeapprofdProgramPropertyPrefix, kHeapprofdProgramPropertyPrefixSize); |
| |
| int fd= open("/proc/self/cmdline", O_RDONLY| O_CLOEXEC); |
| if(fd==-1){ |
| error_log("%s: Failed to open /proc/self/cmdline", getprogname()); |
| returnfalse; |
| } |
| char cmdline[kMaxCmdlineSize]; |
| ssize_t rd= read(fd, cmdline,sizeof(cmdline)-1); |
| close(fd); |
| if(rd==-1){ |
| error_log("%s: Failed to read /proc/self/cmdline", getprogname()); |
| returnfalse; |
| } |
| cmdline[rd]='\0'; |
| char* first_arg=static_cast<char*>(memchr(cmdline,'\0', rd)); |
| if(first_arg==nullptr){ |
| error_log("%s: Overflow reading cmdline", getprogname()); |
| returnfalse; |
| } |
| // For consistency with what we do with Java app cmdlines, trim everything |
| // after the @ sign of the first arg. |
| char* first_at=static_cast<char*>(memchr(cmdline,'@', rd)); |
| if(first_at!=nullptr&& first_at< first_arg){ |
| *first_at='\0'; |
| first_arg= first_at; |
| } |
| |
| char* start=static_cast<char*>(memrchr(cmdline,'/', first_arg- cmdline)); |
| if(start== first_arg){ |
| // The first argument ended in a slash. |
| error_log("%s: cmdline ends in /", getprogname()); |
| returnfalse; |
| }elseif(start==nullptr){ |
| start= cmdline; |
| }else{ |
| // Skip the /. |
| start++; |
| } |
| |
| size_t name_size=static_cast<size_t>(first_arg- start); |
| if(name_size>= size- kHeapprofdProgramPropertyPrefixSize){ |
| error_log("%s: overflow constructing heapprofd property.", getprogname()); |
| returnfalse; |
| } |
| // + 1 to also copy the trailing null byte. |
| memcpy(data+ kHeapprofdProgramPropertyPrefixSize, start, name_size+1); |
| returntrue; |
| } |
| |
| // Runtime triggering entry-point. Two possible call sites: |
| // * when receiving a profiling signal with a si_value indicating heapprofd. |
| // * when a Zygote child is marking itself as profileable, and there's a |
| // matching profiling request for this process (in which case heapprofd client |
| // is loaded synchronously). |
| // In both cases, the caller is responsible for verifying that the process is |
| // considered profileable. |
| |
| // Previously installed default dispatch table, if it exists. This is used to |
| // load heapprofd properly when GWP-ASan was already installed. If GWP-ASan was |
| // already installed, heapprofd will take over the dispatch table, but will use |
| // GWP-ASan as the backing dispatch. Writes to this variable is atomically |
| // protected by MaybeModifyGlobals. |
| // Reads are not protected, so this is atomic. We cannot fail the call in |
| // MallocInitHeapprofdHook. |
| static_Atomic(constMallocDispatch*) gPreviousDefaultDispatchTable=nullptr; |
| staticMallocDispatch gEphemeralDispatch; |
| |
| voidHandleHeapprofdSignal(){ |
| if(atomic_load(&gHeapprofdState)== kIncompatibleHooks){ |
| error_log("%s: not enabling heapprofd, malloc_debug/malloc_hooks are enabled.", getprogname()); |
| return; |
| } |
| |
| // We cannot grab the mutex here, as this is used in a signal handler. |
| MaybeModifyGlobals(kWithoutLock,[]{ |
| MallocHeapprofdState expected= kInitialState; |
| // If hooks are already installed, we still want to install ephemeral hooks to retrigger |
| // heapprofd client initialization. |
| MallocHeapprofdState expected2= kHookInstalled; |
| if(atomic_compare_exchange_strong(&gHeapprofdState,&expected, |
| kInstallingEphemeralHook)){ |
| constMallocDispatch* default_dispatch=GetDefaultDispatchTable(); |
| |
| // Below, we initialize heapprofd lazily by redirecting libc's malloc() to |
| // call MallocInitHeapprofdHook, which spawns off a thread and initializes |
| // heapprofd. During the short period between now and when heapprofd is |
| // initialized, allocations may need to be serviced. There are three |
| // possible configurations: |
| |
| if(DispatchIsGwpAsan(default_dispatch)){ |
| // 1. GWP-ASan was installed. We should use GWP-ASan for everything but |
| // malloc() in the interim period before heapprofd is properly |
| // installed. After heapprofd is finished installing, we will use |
| // GWP-ASan as heapprofd's backing allocator to allow heapprofd and |
| // GWP-ASan to coexist. |
| atomic_store(&gPreviousDefaultDispatchTable, default_dispatch); |
| gEphemeralDispatch=*default_dispatch; |
| }else{ |
| // Either, |
| // 2. No malloc hooking has been done (heapprofd, GWP-ASan, etc.). In |
| // this case, everything but malloc() should come from the system |
| // allocator. |
| // |
| // or, |
| // |
| // 3. It may be possible at this point in time that heapprofd is |
| // *already* the default dispatch, and when it was initialized there |
| // was no default dispatch installed. As such we don't want to use |
| // heapprofd as the backing store for itself (otherwise infinite |
| // recursion occurs). We will use the system allocator functions. Note: |
| // We've checked that no other malloc interceptors are being used by |
| // validating `gHeapprofdIncompatibleHooks` above, so we don't need to |
| // worry about that case here. |
| atomic_store(&gPreviousDefaultDispatchTable,nullptr); |
| gEphemeralDispatch=*NativeAllocatorDispatch(); |
| } |
| }elseif(expected== kEphemeralHookInstalled){ |
| // Nothing to do here. The ephemeral hook was installed, but |
| // MallocInitHeapprofdHook() was never called. Since the ephemeral hook |
| // is already there, no need to reinstall it. |
| return; |
| }elseif(atomic_compare_exchange_strong(&gHeapprofdState,&expected2, |
| kInstallingEphemeralHook)){ |
| // if we still have hook installed, we can reuse the previous |
| // decision. THIS IS REQUIRED FOR CORRECTNESS, because otherwise the |
| // following can happen |
| // 1. Assume DispatchIsGwpAsan(default_dispatch) |
| // 2. This function is ran, sets gPreviousDefaultDispatchTable to |
| // GWP ASan. |
| // 3. The sessions ends, DispatchReset FAILS due to a race. Now |
| // heapprofd hooks are default dispatch. |
| // 4. We re-enter this function later. If we did NOT look at the |
| // previously recorded gPreviousDefaultDispatchTable, we would |
| // incorrectly reach case 3. below. |
| // 5. The session ends, DispatchReset now resets the hooks to the |
| // system allocator. This is incorrect. |
| constMallocDispatch* prev_dispatch= |
| atomic_load(&gPreviousDefaultDispatchTable); |
| gEphemeralDispatch= prev_dispatch?*prev_dispatch:*NativeAllocatorDispatch(); |
| }else{ |
| error_log("%s: heapprofd: failed to transition kInitialState -> kInstallingEphemeralHook. " |
| "current state (possible race): %d", getprogname(), expected2); |
| return; |
| } |
| // Now, replace the malloc function so that the next call to malloc() will |
| // initialize heapprofd. |
| gEphemeralDispatch.malloc=MallocInitHeapprofdHook; |
| |
| // And finally, install these new malloc-family interceptors. |
| __libc_globals.mutate([](libc_globals* globals){ |
| atomic_store(&globals->default_dispatch_table,&gEphemeralDispatch); |
| if(!MallocLimitInstalled()){ |
| atomic_store(&globals->current_dispatch_table,&gEphemeralDispatch); |
| } |
| }); |
| atomic_store(&gHeapprofdState, kEphemeralHookInstalled); |
| }); |
| // Otherwise, we're racing against malloc_limit's enable logic (at most once |
| // per process, and a niche feature). This is highly unlikely, so simply give |
| // up if it does happen. |
| } |
| |
| boolHeapprofdShouldLoad(){ |
| // First check for heapprofd.enable. If it is set to "all", enable |
| // heapprofd for all processes. Otherwise, check heapprofd.enable.${prog}, |
| // if it is set and not 0, enable heap profiling for this process. |
| char property_value[PROP_VALUE_MAX]; |
| if(__system_property_get(kHeapprofdPropertyEnable, property_value)==0){ |
| returnfalse; |
| } |
| if(strcmp(property_value,"all")==0){ |
| returntrue; |
| } |
| |
| char program_property[kHeapprofdProgramPropertyPrefixSize+ kMaxCmdlineSize]; |
| if(!GetHeapprofdProgramProperty(program_property, |
| sizeof(program_property))){ |
| returnfalse; |
| } |
| if(__system_property_get(program_property, property_value)==0){ |
| returnfalse; |
| } |
| return property_value[0]!='\0'; |
| } |
| |
| voidHeapprofdRememberHookConflict(){ |
| atomic_store(&gHeapprofdState, kIncompatibleHooks); |
| } |
| |
| staticvoidCommonInstallHooks(libc_globals* globals){ |
| void* impl_handle= atomic_load(&gHeapprofdHandle); |
| if(impl_handle==nullptr){ |
| impl_handle=LoadSharedLibrary(kHeapprofdSharedLib, kHeapprofdPrefix,&globals->malloc_dispatch_table); |
| if(impl_handle==nullptr){ |
| return; |
| } |
| atomic_store(&gHeapprofdHandle, impl_handle); |
| }elseif(!InitSharedLibrary(impl_handle, kHeapprofdSharedLib, kHeapprofdPrefix,&globals->malloc_dispatch_table)){ |
| return; |
| } |
| |
| FinishInstallHooks(globals,nullptr, kHeapprofdPrefix); |
| } |
| |
| voidHeapprofdInstallHooksAtInit(libc_globals*globals){ |
| // Before we set the new default_dispatch_table in FinishInstallHooks, save |
| // the previous dispatch table. If DispatchReset() gets called later, we want |
| // to be able to restore the dispatch. We're still under |
| // MaybeModifyGlobals locks at this point. |
| atomic_store(&gPreviousDefaultDispatchTable,GetDefaultDispatchTable()); |
| MaybeModifyGlobals(kWithoutLock,[globals]{ |
| MallocHeapprofdState expected= kInitialState; |
| if(atomic_compare_exchange_strong(&gHeapprofdState,&expected, kInstallingHook)){ |
| CommonInstallHooks(globals); |
| atomic_store(&gHeapprofdState, kHookInstalled); |
| }else{ |
| error_log("%s: heapprofd: failed to transition kInitialState -> kInstallingHook. " |
| "current state (possible race): %d", getprogname(), expected); |
| } |
| }); |
| } |
| |
| staticvoid*InitHeapprofd(void*){ |
| MaybeModifyGlobals(kWithLock,[]{ |
| MallocHeapprofdState expected= kInitialState; |
| if(atomic_compare_exchange_strong(&gHeapprofdState,&expected, kInstallingHook)){ |
| __libc_globals.mutate([](libc_globals* globals){ |
| CommonInstallHooks(globals); |
| }); |
| atomic_store(&gHeapprofdState, kHookInstalled); |
| }else{ |
| error_log("%s: heapprofd: failed to transition kInitialState -> kInstallingHook. " |
| "current state (possible race): %d", getprogname(), expected); |
| } |
| }); |
| returnnullptr; |
| } |
| |
| extern"C"void*MallocInitHeapprofdHook(size_t bytes){ |
| MaybeModifyGlobals(kWithLock,[]{ |
| MallocHeapprofdState expected= kEphemeralHookInstalled; |
| if(atomic_compare_exchange_strong(&gHeapprofdState,&expected, kRemovingEphemeralHook)){ |
| __libc_globals.mutate([](libc_globals* globals){ |
| constMallocDispatch* previous_dispatch= atomic_load(&gPreviousDefaultDispatchTable); |
| atomic_store(&globals->default_dispatch_table, previous_dispatch); |
| if(!MallocLimitInstalled()){ |
| atomic_store(&globals->current_dispatch_table, previous_dispatch); |
| } |
| }); |
| atomic_store(&gHeapprofdState, kInitialState); |
| |
| pthread_t thread_id; |
| if(pthread_create(&thread_id,nullptr,InitHeapprofd,nullptr)!=0){ |
| error_log("%s: heapprofd: failed to pthread_create.", getprogname()); |
| }elseif(pthread_setname_np(thread_id,"heapprofdinit")!=0){ |
| error_log("%s: heapprod: failed to pthread_setname_np", getprogname()); |
| }elseif(pthread_detach(thread_id)!=0){ |
| error_log("%s: heapprofd: failed to pthread_detach", getprogname()); |
| } |
| }else{ |
| warning_log("%s: heapprofd: could not transition kEphemeralHookInstalled -> " |
| "kRemovingEphemeralHook. current state (possible race): %d. this can be benign " |
| "if two threads try this transition at the same time", getprogname(), |
| expected); |
| } |
| }); |
| // If we had a previous dispatch table, use that to service the allocation, |
| // otherwise fall back to the native allocator. |
| // This could be modified by a concurrent HandleHeapprofdSignal, but that is |
| // benign as we will dispatch to the ephemeral handler, which will then dispatch |
| // to the underlying one. |
| constMallocDispatch* previous_dispatch= atomic_load(&gPreviousDefaultDispatchTable); |
| if(previous_dispatch){ |
| return previous_dispatch->malloc(bytes); |
| } |
| returnNativeAllocatorDispatch()->malloc(bytes); |
| } |
| |
| boolHeapprofdInitZygoteChildProfiling(){ |
| // Conditionally start "from startup" profiling. |
| if(HeapprofdShouldLoad()){ |
| // Directly call the signal handler codepath (properly protects against |
| // concurrent invocations). |
| HandleHeapprofdSignal(); |
| } |
| returntrue; |
| } |
| |
| staticboolDispatchReset(){ |
| if(atomic_load(&gHeapprofdState)== kInitialState){ |
| returntrue; |
| } |
| |
| bool success=false; |
| MaybeModifyGlobals(kWithLock,[&success]{ |
| MallocHeapprofdState expected= kHookInstalled; |
| |
| if(atomic_compare_exchange_strong(&gHeapprofdState,&expected, kUninstallingHook)){ |
| __libc_globals.mutate([](libc_globals* globals){ |
| constMallocDispatch* previous_dispatch= atomic_load(&gPreviousDefaultDispatchTable); |
| atomic_store(&globals->default_dispatch_table, previous_dispatch); |
| if(!MallocLimitInstalled()){ |
| atomic_store(&globals->current_dispatch_table, previous_dispatch); |
| } |
| }); |
| atomic_store(&gHeapprofdState, kInitialState); |
| success=true; |
| }else{ |
| error_log("%s: heapprofd: failed to transition kHookInstalled -> kUninstallingHook. " |
| "current state (possible race): %d", getprogname(), |
| expected); |
| } |
| }); |
| if(!success){ |
| errno= EAGAIN; |
| } |
| return success; |
| } |
| |
| boolHeapprofdMallopt(int opcode,void* arg,size_t arg_size){ |
| if(opcode== M_RESET_HOOKS){ |
| if(arg!=nullptr|| arg_size!=0){ |
| errno= EINVAL; |
| returnfalse; |
| } |
| returnDispatchReset(); |
| } |
| errno= ENOTSUP; |
| returnfalse; |
| } |
