/* Copyright (c) 2013 The Chromium Authors. All rights reserved. * Use of this source code is governed by a BSD-style license that can be * found in the LICENSE file. */ /* XRay -- a simple profiler for Native Client */ #include #include #include #include #include #include #include #include #include #include "xray/xray_priv.h" #if defined(XRAY) /* GTSC - Get Time Stamp Counter */ #if defined(__amd64__) && !defined(XRAY_NO_RDTSC) XRAY_INLINE uint64_t RDTSC64(); uint64_t RDTSC64() { uint64_t a, d; __asm__ __volatile__("rdtsc" : "=a" (a), "=d" (d)); return ((uint64_t)a) | (((uint64_t)d) << 32); } #define GTSC(_x) _x = RDTSC64() #elif defined(__i386__) && !defined(XRAY_NO_RDTSC) #define GTSC(_x) __asm__ __volatile__ ("rdtsc" : "=A" (_x)); #else XRAY_INLINE uint64_t GTOD(); uint64_t GTOD() { struct timeval tv; gettimeofday(&tv, NULL); return (uint64_t)tv.tv_sec * 1000000 + (uint64_t)tv.tv_usec; } #define GTSC(_x) _x = GTOD(); #endif /* Use a TLS variable for cheap thread uid. */ __thread struct XRayTraceCapture* g_xray_capture = NULL; __thread int g_xray_thread_id_placeholder = 0; struct XRayTraceStackEntry { uint32_t depth_addr; uint64_t tsc; uint32_t dest; uint32_t annotation_index; }; struct XRayTraceFrameEntry { /* Indices into global tracebuffer */ int start; int end; uint64_t start_tsc; uint64_t end_tsc; uint64_t total_ticks; int annotation_count; bool valid; #ifndef XRAY_DISABLE_BROWSER_INTEGRATION struct XRayTimestampPair start_time; struct XRayTimestampPair end_time; #endif }; struct XRayTraceFrame { struct XRayTraceFrameEntry* entry; int head; int tail; int count; }; struct XRayTraceCapture { /* Common variables share cache line */ bool recording; uint32_t stack_depth; uint32_t max_stack_depth; int buffer_index; int buffer_size; int disabled; int annotation_count; struct XRaySymbolTable* symbols; bool initialized; uint32_t annotation_filter; uint32_t guard0; struct XRayTraceStackEntry stack[XRAY_TRACE_STACK_SIZE] XRAY_ALIGN64; uint32_t guard1; uint32_t guard2; char annotation[XRAY_ANNOTATION_STACK_SIZE] XRAY_ALIGN64; uint32_t guard3; struct XRayTraceBufferEntry* buffer; struct XRayTraceFrame frame; #ifndef XRAY_DISABLE_BROWSER_INTEGRATION int32_t thread_id; #endif } XRAY_ALIGN64; #ifdef __cplusplus extern "C" { #endif #if defined(__pnacl__) XRAY_NO_INSTRUMENT void __pnacl_profile_func_enter(const char* fname); XRAY_NO_INSTRUMENT void __pnacl_profile_func_exit(const char* fname); #else XRAY_NO_INSTRUMENT void __cyg_profile_func_enter(void* this_fn, void* call_site); XRAY_NO_INSTRUMENT void __cyg_profile_func_exit(void* this_fn, void* call_site); #endif XRAY_INLINE int XRayTraceDecrementIndexInline( struct XRayTraceCapture* capture, int index); XRAY_INLINE int XRayTraceIncrementIndexInline( struct XRayTraceCapture* capture, int index); XRAY_NO_INSTRUMENT void __xray_profile_append_annotation( struct XRayTraceCapture* capture, struct XRayTraceStackEntry* se, struct XRayTraceBufferEntry* be); #ifdef __cplusplus } #endif /* Asserts that the guard values haven't changed. */ void XRayCheckGuards(struct XRayTraceCapture* capture) { assert(capture->guard0 == XRAY_GUARD_VALUE_0x12345678); assert(capture->guard1 == XRAY_GUARD_VALUE_0x12345678); assert(capture->guard2 == XRAY_GUARD_VALUE_0x87654321); assert(capture->guard3 == XRAY_GUARD_VALUE_0x12345678); } /* Decrements the trace index, wrapping around if needed. */ int XRayTraceDecrementIndexInline( struct XRayTraceCapture* capture, int index) { --index; if (index < 0) index = capture->buffer_size - 1; return index; } /* Increments the trace index, wrapping around if needed. */ int XRayTraceIncrementIndexInline( struct XRayTraceCapture* capture, int index) { ++index; if (index >= capture->buffer_size) index = 0; return index; } /* Returns true if the trace entry is an annotation string. */ bool XRayTraceIsAnnotation( struct XRayTraceCapture* capture, int index) { struct XRayTraceBufferEntry* be = &capture->buffer[index]; char* dst = (char*)be; return 0 == *dst; } int XRayTraceIncrementIndex(struct XRayTraceCapture* capture, int index) { return XRayTraceIncrementIndexInline(capture, index); } int XRayTraceDecrementIndex(struct XRayTraceCapture* capture, int index) { return XRayTraceDecrementIndexInline(capture, index); } /* The entry in the tracebuffer at index is an annotation string. */ /* Calculate the next index value representing the next trace entry. */ int XRayTraceSkipAnnotation(struct XRayTraceCapture* capture, int index) { /* Annotations are strings embedded in the trace buffer. */ /* An annotation string can span multiple trace entries. */ /* Skip over the string by looking for zero termination. */ assert(capture); assert(XRayTraceIsAnnotation(capture, index)); bool done = false; int start_index = 1; int i; while (!done) { char* str = (char*) &capture->buffer[index]; const int num = sizeof(capture->buffer[index]); for (i = start_index; i < num; ++i) { if (0 == str[i]) { done = true; break; } } index = XRayTraceIncrementIndexInline(capture, index); start_index = 0; } return index; } struct XRayTraceBufferEntry* XRayTraceGetEntry( struct XRayTraceCapture* capture, int index) { return &capture->buffer[index]; } /* Starting at index, return the index into the trace buffer */ /* for the next trace entry. Index can wrap (ringbuffer) */ int XRayTraceNextEntry(struct XRayTraceCapture* capture, int index) { if (XRayTraceIsAnnotation(capture, index)) index = XRayTraceSkipAnnotation(capture, index); else index = XRayTraceIncrementIndexInline(capture, index); return index; } int XRayFrameGetTraceStartIndex(struct XRayTraceCapture* capture, int frame) { assert(capture); assert(capture->initialized); assert(!capture->recording); return capture->frame.entry[frame].start; } int XRayFrameGetTraceEndIndex(struct XRayTraceCapture* capture, int frame) { assert(capture); assert(capture->initialized); assert(!capture->recording); return capture->frame.entry[frame].end; } /* Not very accurate, annotation strings will also be counted as "entries" */ int XRayFrameGetTraceCount( struct XRayTraceCapture* capture, int frame) { assert(true == capture->initialized); assert(frame >= 0); assert(frame < capture->frame.count); assert(!capture->recording); int start = capture->frame.entry[frame].start; int end = capture->frame.entry[frame].end; int num; if (start < end) num = end - start; else num = capture->buffer_size - (start - end); return num; } /* Append a string to trace buffer. */ void XRayTraceAppendString(struct XRayTraceCapture* capture, char* src) { int index = capture->buffer_index; bool done = false; int start_index = 1; int s = 0; int i; char* dst = (char*)&capture->buffer[index]; const int num = sizeof(capture->buffer[index]); dst[0] = 0; while (!done) { for (i = start_index; i < num; ++i) { dst[i] = src[s]; if (0 == src[s]) { dst[i] = 0; done = true; break; } ++s; } index = XRayTraceIncrementIndexInline(capture, index); dst = (char*)&capture->buffer[index]; start_index = 0; } capture->buffer_index = index; } /* Copies annotation from trace buffer to output string. */ int XRayTraceCopyToString( struct XRayTraceCapture* capture, int index, char* dst) { assert(XRayTraceIsAnnotation(capture, index)); bool done = false; int i; int d = 0; int start_index = 1; while (!done) { char* src = (char*) &capture->buffer[index]; const int num = sizeof(capture->buffer[index]); for (i = start_index; i < num; ++i) { dst[d] = src[i]; if (0 == src[i]) { done = true; break; } ++d; } index = XRayTraceIncrementIndexInline(capture, index); start_index = 0; } return index; } /* Generic memory malloc for XRay */ /* validates pointer returned by malloc */ /* memsets memory block to zero */ void* XRayMalloc(size_t t) { void* data; data = calloc(1, t); if (NULL == data) { printf("XRay: malloc(%d) failed, panic shutdown!\n", t); exit(-1); } return data; } /* Generic memory free for XRay */ void XRayFree(void* data) { assert(NULL != data); free(data); } /* Main profile capture function that is called at the start */ /* of every instrumented function. This function is implicitly */ /* called when code is compilied with the -finstrument-functions option */ #if defined(__pnacl__) void __pnacl_profile_func_enter(const char* this_fn) { #else void __cyg_profile_func_enter(void* this_fn, void* call_site) { #endif struct XRayTraceCapture* capture = g_xray_capture; if (capture && capture->recording) { uint32_t depth = capture->stack_depth; if (depth < capture->max_stack_depth) { struct XRayTraceStackEntry* se = &capture->stack[depth]; uint32_t addr = (uint32_t)(uintptr_t)this_fn; se->depth_addr = XRAY_PACK_DEPTH_ADDR(depth, addr); se->dest = capture->buffer_index; se->annotation_index = 0; GTSC(se->tsc); capture->buffer_index = XRayTraceIncrementIndexInline(capture, capture->buffer_index); } ++capture->stack_depth; } } /* Main profile capture function that is called at the exit of */ /* every instrumented function. This function is implicity called */ /* when the code is compiled with the -finstrument-functions option */ #if defined(__pnacl__) void __pnacl_profile_func_exit(const char* this_fn) { #else void __cyg_profile_func_exit(void* this_fn, void* call_site) { #endif struct XRayTraceCapture* capture = g_xray_capture; if (capture && capture->recording) { --capture->stack_depth; if (capture->stack_depth < capture->max_stack_depth) { uint32_t depth = capture->stack_depth; struct XRayTraceStackEntry* se = &capture->stack[depth]; uint32_t buffer_index = se->dest; uint64_t tsc; struct XRayTraceBufferEntry* be = &capture->buffer[buffer_index]; GTSC(tsc); be->depth_addr = se->depth_addr; be->start_tick = se->tsc; be->end_tick = tsc; be->annotation_index = 0; if (0 != se->annotation_index) __xray_profile_append_annotation(capture, se, be); } } } #ifndef XRAY_DISABLE_BROWSER_INTEGRATION void XRayGetTSC(uint64_t* tsc) { GTSC(*tsc); } int32_t XRayGetSavedThreadID(struct XRayTraceCapture* capture) { return capture->thread_id; } struct XRayTimestampPair XRayFrameGetStartTimestampPair( struct XRayTraceCapture* capture, int frame) { return capture->frame.entry[frame].start_time; } struct XRayTimestampPair XRayFrameGetEndTimestampPair( struct XRayTraceCapture* capture, int frame) { return capture->frame.entry[frame].end_time; } #endif /* Special case appending annotation string to trace buffer */ /* this function should only ever be called from __cyg_profile_func_exit() */ void __xray_profile_append_annotation(struct XRayTraceCapture* capture, struct XRayTraceStackEntry* se, struct XRayTraceBufferEntry* be) { struct XRayTraceStackEntry* parent = se - 1; int start = parent->annotation_index; be->annotation_index = capture->buffer_index; char* str = &capture->annotation[start]; XRayTraceAppendString(capture, str); *str = 0; ++capture->annotation_count; } /* Annotates the trace buffer. no filtering. */ void __XRayAnnotate(const char* fmt, ...) { va_list args; struct XRayTraceCapture* capture = g_xray_capture; /* Only annotate functions recorded in the trace buffer. */ if (capture && capture->initialized) { if (0 == capture->disabled) { if (capture->recording) { char buffer[1024]; int r; va_start(args, fmt); r = vsnprintf(buffer, sizeof(buffer), fmt, args); va_end(args); { /* Get current string ptr */ int depth = capture->stack_depth - 1; struct XRayTraceStackEntry* se = &capture->stack[depth]; if (0 == se->annotation_index) { struct XRayTraceStackEntry* parent = se - 1; se->annotation_index = parent->annotation_index; } char* dst = &capture->annotation[se->annotation_index]; strcpy(dst, buffer); int len = strlen(dst); se->annotation_index += len; } } } } } /* Annotates the trace buffer with user strings. Can be filtered. */ void __XRayAnnotateFiltered(const uint32_t filter, const char* fmt, ...) { va_list args; struct XRayTraceCapture* capture = g_xray_capture; if (capture && capture->initialized) { if (0 != (filter & capture->annotation_filter)) { if (0 == capture->disabled) { if (capture->recording) { char buffer[XRAY_TRACE_ANNOTATION_LENGTH]; int r; va_start(args, fmt); r = vsnprintf(buffer, sizeof(buffer), fmt, args); va_end(args); { /* get current string ptr */ int depth = capture->stack_depth - 1; struct XRayTraceStackEntry* se = &capture->stack[depth]; if (0 == se->annotation_index) { struct XRayTraceStackEntry* parent = se - 1; se->annotation_index = parent->annotation_index; } char* dst = &capture->annotation[se->annotation_index]; strcpy(dst, buffer); int len = strlen(dst); se->annotation_index += len; } } } } } } /* Allows user to specify annotation filter value, a 32 bit mask. */ void XRaySetAnnotationFilter(struct XRayTraceCapture* capture, uint32_t filter) { capture->annotation_filter = filter; } /* Reset xray profiler. */ void XRayReset(struct XRayTraceCapture* capture) { assert(capture); assert(capture->initialized); assert(!capture->recording); capture->buffer_index = 0; capture->stack_depth = 0; capture->disabled = 0; capture->frame.head = 0; capture->frame.tail = 0; memset(capture->frame.entry, 0, sizeof(capture->frame.entry[0]) * capture->frame.count); memset(&capture->stack, 0, sizeof(capture->stack[0]) * XRAY_TRACE_STACK_SIZE); XRayCheckGuards(capture); } /* Change the maximum stack depth captures are made. */ void XRaySetMaxStackDepth(struct XRayTraceCapture* capture, int stack_depth) { assert(capture); assert(capture->initialized); assert(!capture->recording); if (stack_depth < 1) stack_depth = 1; if (stack_depth >= XRAY_TRACE_STACK_SIZE) stack_depth = (XRAY_TRACE_STACK_SIZE - 1); capture->max_stack_depth = stack_depth; } int XRayFrameGetCount(struct XRayTraceCapture* capture) { return capture->frame.count; } int XRayFrameGetTail(struct XRayTraceCapture* capture) { return capture->frame.tail; } int XRayFrameGetHead(struct XRayTraceCapture* capture) { return capture->frame.head; } int XRayFrameGetPrev(struct XRayTraceCapture* capture, int i) { i = i - 1; if (i < 0) i = capture->frame.count - 1; return i; } int XRayFrameGetNext(struct XRayTraceCapture* capture, int i) { i = i + 1; if (i >= capture->frame.count) i = 0; return i; } bool XRayFrameIsValid(struct XRayTraceCapture* capture, int i) { return capture->frame.entry[i].valid; } uint64_t XRayFrameGetTotalTicks(struct XRayTraceCapture* capture, int i) { return capture->frame.entry[i].total_ticks; } int XRayFrameGetAnnotationCount(struct XRayTraceCapture* capture, int i) { return capture->frame.entry[i].annotation_count; } void XRayFrameMakeLabel(struct XRayTraceCapture* capture, int counter, char* label) { snprintf(label, XRAY_MAX_LABEL, "@@@frame%d@@@", counter); } /* Scans the ring buffer going backwards to find last valid complete frame. */ /* Will mark whether frames are valid or invalid during the traversal. */ int XRayFrameFindTail(struct XRayTraceCapture* capture) { int head = capture->frame.head; int index = XRayFrameGetPrev(capture, head); int total_capture = 0; int last_valid_frame = index; /* Check for no captures */ if (capture->frame.head == capture->frame.tail) return capture->frame.head; /* Go back and invalidate all captures that have been stomped. */ while (index != head) { bool valid = capture->frame.entry[index].valid; if (valid) { total_capture += XRayFrameGetTraceCount(capture, index) + 1; if (total_capture < capture->buffer_size) { last_valid_frame = index; capture->frame.entry[index].valid = true; } else { capture->frame.entry[index].valid = false; } } index = XRayFrameGetPrev(capture, index); } return last_valid_frame; } /* Starts a new frame and enables capturing, and must be paired with */ /* XRayEndFrame() Trace capturing only occurs on the thread which called */ /* XRayBeginFrame() and each instance of capture can only trace one thread */ /* at a time. */ void XRayStartFrame(struct XRayTraceCapture* capture) { int i; assert(NULL == g_xray_capture); assert(capture->initialized); assert(!capture->recording); i = capture->frame.head; XRayCheckGuards(capture); /* Add a trace entry marker so we can detect wrap around stomping */ struct XRayTraceBufferEntry* be = &capture->buffer[capture->buffer_index]; be->depth_addr = XRAY_FRAME_MARKER; capture->buffer_index = XRayTraceIncrementIndex(capture, capture->buffer_index); /* Set start of the frame we're about to trace */ capture->frame.entry[i].start = capture->buffer_index; capture->disabled = 0; capture->stack_depth = 1; /* The trace stack[0] is reserved */ memset(&capture->stack[0], 0, sizeof(capture->stack[0])); /* Annotation index 0 is reserved to indicate no annotation */ capture->stack[0].annotation_index = 1; capture->annotation[0] = 0; capture->annotation[1] = 0; capture->annotation_count = 0; capture->recording = true; GTSC(capture->frame.entry[i].start_tsc); g_xray_capture = capture; #ifndef XRAY_DISABLE_BROWSER_INTEGRATION capture->frame.entry[i].start_time = XRayGenerateTimestampsNow(); #endif } /* Ends a frame and disables capturing. Advances to the next frame. */ /* Must be paired with XRayStartFrame(), and called from the same thread. */ void XRayEndFrame(struct XRayTraceCapture* capture) { int i; assert(capture); assert(capture->initialized); assert(capture->recording); assert(g_xray_capture == capture); assert(0 == capture->disabled); assert(1 == capture->stack_depth); i = capture->frame.head; GTSC(capture->frame.entry[i].end_tsc); capture->frame.entry[i].total_ticks = capture->frame.entry[i].end_tsc - capture->frame.entry[i].start_tsc; capture->recording = NULL; capture->frame.entry[i].end = capture->buffer_index; capture->frame.entry[i].valid = true; capture->frame.entry[i].annotation_count = capture->annotation_count; capture->frame.head = XRayFrameGetNext(capture, capture->frame.head); /* If the table is filled, bump the tail. */ if (capture->frame.head == capture->frame.tail) capture->frame.tail = XRayFrameGetNext(capture, capture->frame.tail); capture->frame.tail = XRayFrameFindTail(capture); /* Check that we didn't stomp over trace entry marker. */ int marker = XRayTraceDecrementIndex(capture, capture->frame.entry[i].start); struct XRayTraceBufferEntry* be = &capture->buffer[marker]; if (be->depth_addr != XRAY_FRAME_MARKER) { fprintf(stderr, "XRay: XRayStopFrame() detects insufficient trace buffer size!\n"); XRayReset(capture); } else { /* Replace marker with an empty annotation string. */ be->depth_addr = XRAY_NULL_ANNOTATION; XRayCheckGuards(capture); } g_xray_capture = NULL; #ifndef XRAY_DISABLE_BROWSER_INTEGRATION capture->frame.entry[i].end_time = XRayGenerateTimestampsNow(); #endif } /* Get the last frame captured. Do not call while capturing. */ /* (ie call outside of XRayStartFrame() / XRayStopFrame() pair) */ int XRayGetLastFrame(struct XRayTraceCapture* capture) { assert(capture); assert(capture->initialized); assert(!capture->recording); assert(0 == capture->disabled); assert(1 == capture->stack_depth); int last_frame = XRayFrameGetPrev(capture, capture->frame.head); return last_frame; } /* Disables capturing until a paired XRayEnableCapture() is called */ /* This call can be nested, but must be paired with an enable */ /* (If you need to just exclude a specific function and not its */ /* children, the XRAY_NO_INSTRUMENT modifier might be better) */ /* Must be called from same thread as XRayBeginFrame() / XRayEndFrame() */ void XRayDisableCapture(struct XRayTraceCapture* capture) { assert(capture); assert(capture == g_xray_capture); assert(capture->initialized); ++capture->disabled; capture->recording = false; } /* Re-enables capture. Must be paired with XRayDisableCapture() */ void XRayEnableCapture(struct XRayTraceCapture* capture) { assert(capture); assert(capture == g_xray_capture); assert(capture->initialized); assert(0 < capture->disabled); --capture->disabled; if (0 == capture->disabled) { capture->recording = true; } } struct XRaySymbolTable* XRayGetSymbolTable(struct XRayTraceCapture* capture) { return capture->symbols; } /* Initialize XRay */ struct XRayTraceCapture* XRayInit(int stack_depth, int buffer_size, int frame_count, const char* mapfilename) { struct XRayTraceCapture* capture; capture = (struct XRayTraceCapture*)XRayMalloc( sizeof(struct XRayTraceCapture)); int adj_frame_count = frame_count + 1; size_t buffer_size_in_bytes = sizeof(capture->buffer[0]) * buffer_size; size_t frame_size_in_bytes = sizeof(capture->frame.entry[0]) * adj_frame_count; capture->buffer = (struct XRayTraceBufferEntry *)XRayMalloc(buffer_size_in_bytes); capture->frame.entry = (struct XRayTraceFrameEntry *)XRayMalloc(frame_size_in_bytes); capture->buffer_size = buffer_size; capture->frame.count = adj_frame_count; capture->frame.head = 0; capture->frame.tail = 0; capture->disabled = 0; capture->annotation_filter = 0xFFFFFFFF; capture->guard0 = XRAY_GUARD_VALUE_0x12345678; capture->guard1 = XRAY_GUARD_VALUE_0x12345678; capture->guard2 = XRAY_GUARD_VALUE_0x87654321; capture->guard3 = XRAY_GUARD_VALUE_0x12345678; capture->initialized = true; capture->recording = false; XRaySetMaxStackDepth(capture, stack_depth); XRayReset(capture); /* Mapfile is optional; we don't need it for captures, only for reports. */ capture->symbols = XRaySymbolTableCreate(XRAY_DEFAULT_SYMBOL_TABLE_SIZE); if (NULL != mapfilename) XRaySymbolTableParseMapfile(capture->symbols, mapfilename); #ifndef XRAY_DISABLE_BROWSER_INTEGRATION /* Use the address of a thread local variable as a fake thread id. */ capture->thread_id = (int32_t)(&g_xray_thread_id_placeholder); #endif return capture; } /* Shut down and free memory used by XRay. */ void XRayShutdown(struct XRayTraceCapture* capture) { assert(capture); assert(capture->initialized); assert(!capture->recording); XRayCheckGuards(capture); if (NULL != capture->symbols) { XRaySymbolTableFree(capture->symbols); } XRayFree(capture->frame.entry); XRayFree(capture->buffer); capture->initialized = false; XRayFree(capture); } #endif /* XRAY */