diff options
| -rw-r--r-- | numpy/core/src/multiarray/multiarraymodule.c | 33 | ||||
| -rw-r--r-- | tools/allocation_tracking/alloc_hook.pyx | 42 | ||||
| -rw-r--r-- | tools/allocation_tracking/setup.py | 9 | ||||
| -rw-r--r-- | tools/allocation_tracking/sorttable.js | 493 | ||||
| -rw-r--r-- | tools/allocation_tracking/track_allocations.py | 131 |
5 files changed, 701 insertions, 7 deletions
diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 3917c1611..54e26c08f 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -3672,22 +3672,29 @@ NPY_NO_EXPORT void *_PyDataMem_eventhook_user_data; * Returns a pointer to the previous hook or NULL. If old_data is * non-NULL, the previous user_data pointer will be copied to it. * - * * If not NULL, hook will be called at the end of each PyDataMem_NEW/FREE/RENEW: * result = PyDataMem_NEW(size) -> (*hook)(NULL, result, size, user_data) * PyDataMem_FREE(ptr) -> (*hook)(ptr, NULL, 0, user_data) * result = PyDataMem_RENEW(ptr, size) -> (*hook)(ptr, result, size, user_data) + * + * When the hook is called, the GIL will be held by the calling + * thread. The hook should be written to be reentrant, if it performs + * operations that might cause new allocation events (such as the + * creation/descruction numpy objects, or creating/destroying Python + * objects which might cause a gc) */ NPY_NO_EXPORT PyDataMem_EventHookFunc * PyDataMem_SetEventHook(PyDataMem_EventHookFunc *newhook, void *user_data, void **old_data) { + PyGILState_STATE gilstate = PyGILState_Ensure(); PyDataMem_EventHookFunc *temp = _PyDataMem_eventhook; _PyDataMem_eventhook = newhook; if (old_data != NULL) { *old_data = _PyDataMem_eventhook_user_data; } _PyDataMem_eventhook_user_data = user_data; + PyGILState_Release(gilstate); return temp; } @@ -3701,8 +3708,12 @@ PyDataMem_NEW(size_t size) result = malloc(size); if (_PyDataMem_eventhook != NULL) { - (*_PyDataMem_eventhook)(NULL, result, size, - _PyDataMem_eventhook_user_data); + PyGILState_STATE gilstate = PyGILState_Ensure(); + if (_PyDataMem_eventhook != NULL) { + (*_PyDataMem_eventhook)(NULL, result, size, + _PyDataMem_eventhook_user_data); + } + PyGILState_Release(gilstate); } return (char *)result; } @@ -3715,8 +3726,12 @@ PyDataMem_FREE(void *ptr) { free(ptr); if (_PyDataMem_eventhook != NULL) { - (*_PyDataMem_eventhook)(ptr, NULL, 0, - _PyDataMem_eventhook_user_data); + PyGILState_STATE gilstate = PyGILState_Ensure(); + if (_PyDataMem_eventhook != NULL) { + (*_PyDataMem_eventhook)(ptr, NULL, 0, + _PyDataMem_eventhook_user_data); + } + PyGILState_Release(gilstate); } } @@ -3730,8 +3745,12 @@ PyDataMem_RENEW(void *ptr, size_t size) result = realloc(ptr, size); if (_PyDataMem_eventhook != NULL) { - (*_PyDataMem_eventhook)(ptr, result, size, - _PyDataMem_eventhook_user_data); + PyGILState_STATE gilstate = PyGILState_Ensure(); + if (_PyDataMem_eventhook != NULL) { + (*_PyDataMem_eventhook)(ptr, result, size, + _PyDataMem_eventhook_user_data); + } + PyGILState_Release(gilstate); } return (char *)result; } diff --git a/tools/allocation_tracking/alloc_hook.pyx b/tools/allocation_tracking/alloc_hook.pyx new file mode 100644 index 000000000..d1e656f90 --- /dev/null +++ b/tools/allocation_tracking/alloc_hook.pyx @@ -0,0 +1,42 @@ +# A cython wrapper for using python functions as callbacks for +# PyDataMem_SetEventHook. + +cimport numpy as np + +cdef extern from "Python.h": + object PyLong_FromVoidPtr(void *) + void *PyLong_AsVoidPtr(object) + +ctypedef void PyDataMem_EventHookFunc(void *inp, void *outp, size_t size, + void *user_data) +cdef extern from "numpy/arrayobject.h": + PyDataMem_EventHookFunc * \ + PyDataMem_SetEventHook(PyDataMem_EventHookFunc *newhook, + void *user_data, void **old_data) + +np.import_array() + +cdef void pyhook(void *old, void *new, size_t size, void *user_data): + cdef object pyfunc = <object> user_data + pyfunc(PyLong_FromVoidPtr(old), + PyLong_FromVoidPtr(new), + size) + +class NumpyAllocHook(object): + def __init__(self, callback): + self.callback = callback + + def __enter__(self): + cdef void *old_hook, *old_data + old_hook = <void *> \ + PyDataMem_SetEventHook(<PyDataMem_EventHookFunc *> pyhook, + <void *> self.callback, + <void **> &old_data) + self.old_hook = PyLong_FromVoidPtr(old_hook) + self.old_data = PyLong_FromVoidPtr(old_data) + + def __exit__(self): + PyDataMem_SetEventHook(<PyDataMem_EventHookFunc *> \ + PyLong_AsVoidPtr(self.old_hook), + <void *> PyLong_AsVoidPtr(self.old_data), + <void **> 0) diff --git a/tools/allocation_tracking/setup.py b/tools/allocation_tracking/setup.py new file mode 100644 index 000000000..4462f9f4e --- /dev/null +++ b/tools/allocation_tracking/setup.py @@ -0,0 +1,9 @@ +from distutils.core import setup +from distutils.extension import Extension +from Cython.Distutils import build_ext +import numpy + +setup( + cmdclass = {'build_ext': build_ext}, + ext_modules = [Extension("alloc_hook", ["alloc_hook.pyx"], + include_dirs=[numpy.get_include()])]) diff --git a/tools/allocation_tracking/sorttable.js b/tools/allocation_tracking/sorttable.js new file mode 100644 index 000000000..25bccb2b6 --- /dev/null +++ b/tools/allocation_tracking/sorttable.js @@ -0,0 +1,493 @@ +/* + SortTable + version 2 + 7th April 2007 + Stuart Langridge, http://www.kryogenix.org/code/browser/sorttable/ + + Instructions: + Download this file + Add <script src="sorttable.js"></script> to your HTML + Add class="sortable" to any table you'd like to make sortable + Click on the headers to sort + + Thanks to many, many people for contributions and suggestions. + Licenced as X11: http://www.kryogenix.org/code/browser/licence.html + This basically means: do what you want with it. +*/ + + +var stIsIE = /*@cc_on!@*/false; + +sorttable = { + init: function() { + // quit if this function has already been called + if (arguments.callee.done) return; + // flag this function so we don't do the same thing twice + arguments.callee.done = true; + // kill the timer + if (_timer) clearInterval(_timer); + + if (!document.createElement || !document.getElementsByTagName) return; + + sorttable.DATE_RE = /^(\d\d?)[\/\.-](\d\d?)[\/\.-]((\d\d)?\d\d)$/; + + forEach(document.getElementsByTagName('table'), function(table) { + if (table.className.search(/\bsortable\b/) != -1) { + sorttable.makeSortable(table); + } + }); + + }, + + makeSortable: function(table) { + if (table.getElementsByTagName('thead').length == 0) { + // table doesn't have a tHead. Since it should have, create one and + // put the first table row in it. + the = document.createElement('thead'); + the.appendChild(table.rows[0]); + table.insertBefore(the,table.firstChild); + } + // Safari doesn't support table.tHead, sigh + if (table.tHead == null) table.tHead = table.getElementsByTagName('thead')[0]; + + if (table.tHead.rows.length != 1) return; // can't cope with two header rows + + // Sorttable v1 put rows with a class of "sortbottom" at the bottom (as + // "total" rows, for example). This is B&R, since what you're supposed + // to do is put them in a tfoot. So, if there are sortbottom rows, + // for backwards compatibility, move them to tfoot (creating it if needed). + sortbottomrows = []; + for (var i=0; i<table.rows.length; i++) { + if (table.rows[i].className.search(/\bsortbottom\b/) != -1) { + sortbottomrows[sortbottomrows.length] = table.rows[i]; + } + } + if (sortbottomrows) { + if (table.tFoot == null) { + // table doesn't have a tfoot. Create one. + tfo = document.createElement('tfoot'); + table.appendChild(tfo); + } + for (var i=0; i<sortbottomrows.length; i++) { + tfo.appendChild(sortbottomrows[i]); + } + delete sortbottomrows; + } + + // work through each column and calculate its type + headrow = table.tHead.rows[0].cells; + for (var i=0; i<headrow.length; i++) { + // manually override the type with a sorttable_type attribute + if (!headrow[i].className.match(/\bsorttable_nosort\b/)) { // skip this col + mtch = headrow[i].className.match(/\bsorttable_([a-z0-9]+)\b/); + if (mtch) { override = mtch[1]; } + if (mtch && typeof sorttable["sort_"+override] == 'function') { + headrow[i].sorttable_sortfunction = sorttable["sort_"+override]; + } else { + headrow[i].sorttable_sortfunction = sorttable.guessType(table,i); + } + // make it clickable to sort + headrow[i].sorttable_columnindex = i; + headrow[i].sorttable_tbody = table.tBodies[0]; + dean_addEvent(headrow[i],"click", function(e) { + + if (this.className.search(/\bsorttable_sorted\b/) != -1) { + // if we're already sorted by this column, just + // reverse the table, which is quicker + sorttable.reverse(this.sorttable_tbody); + this.className = this.className.replace('sorttable_sorted', + 'sorttable_sorted_reverse'); + this.removeChild(document.getElementById('sorttable_sortfwdind')); + sortrevind = document.createElement('span'); + sortrevind.id = "sorttable_sortrevind"; + sortrevind.innerHTML = stIsIE ? ' <font face="webdings">5</font>' : ' ▴'; + this.appendChild(sortrevind); + return; + } + if (this.className.search(/\bsorttable_sorted_reverse\b/) != -1) { + // if we're already sorted by this column in reverse, just + // re-reverse the table, which is quicker + sorttable.reverse(this.sorttable_tbody); + this.className = this.className.replace('sorttable_sorted_reverse', + 'sorttable_sorted'); + this.removeChild(document.getElementById('sorttable_sortrevind')); + sortfwdind = document.createElement('span'); + sortfwdind.id = "sorttable_sortfwdind"; + sortfwdind.innerHTML = stIsIE ? ' <font face="webdings">6</font>' : ' ▾'; + this.appendChild(sortfwdind); + return; + } + + // remove sorttable_sorted classes + theadrow = this.parentNode; + forEach(theadrow.childNodes, function(cell) { + if (cell.nodeType == 1) { // an element + cell.className = cell.className.replace('sorttable_sorted_reverse',''); + cell.className = cell.className.replace('sorttable_sorted',''); + } + }); + sortfwdind = document.getElementById('sorttable_sortfwdind'); + if (sortfwdind) { sortfwdind.parentNode.removeChild(sortfwdind); } + sortrevind = document.getElementById('sorttable_sortrevind'); + if (sortrevind) { sortrevind.parentNode.removeChild(sortrevind); } + + this.className += ' sorttable_sorted'; + sortfwdind = document.createElement('span'); + sortfwdind.id = "sorttable_sortfwdind"; + sortfwdind.innerHTML = stIsIE ? ' <font face="webdings">6</font>' : ' ▾'; + this.appendChild(sortfwdind); + + // build an array to sort. This is a Schwartzian transform thing, + // i.e., we "decorate" each row with the actual sort key, + // sort based on the sort keys, and then put the rows back in order + // which is a lot faster because you only do getInnerText once per row + row_array = []; + col = this.sorttable_columnindex; + rows = this.sorttable_tbody.rows; + for (var j=0; j<rows.length; j++) { + row_array[row_array.length] = [sorttable.getInnerText(rows[j].cells[col]), rows[j]]; + } + /* If you want a stable sort, uncomment the following line */ + //sorttable.shaker_sort(row_array, this.sorttable_sortfunction); + /* and comment out this one */ + row_array.sort(this.sorttable_sortfunction); + + tb = this.sorttable_tbody; + for (var j=0; j<row_array.length; j++) { + tb.appendChild(row_array[j][1]); + } + + delete row_array; + }); + } + } + }, + + guessType: function(table, column) { + // guess the type of a column based on its first non-blank row + sortfn = sorttable.sort_alpha; + for (var i=0; i<table.tBodies[0].rows.length; i++) { + text = sorttable.getInnerText(table.tBodies[0].rows[i].cells[column]); + if (text != '') { + if (text.match(/^-?[£$¤]?[\d,.]+%?$/)) { + return sorttable.sort_numeric; + } + // check for a date: dd/mm/yyyy or dd/mm/yy + // can have / or . or - as separator + // can be mm/dd as well + possdate = text.match(sorttable.DATE_RE) + if (possdate) { + // looks like a date + first = parseInt(possdate[1]); + second = parseInt(possdate[2]); + if (first > 12) { + // definitely dd/mm + return sorttable.sort_ddmm; + } else if (second > 12) { + return sorttable.sort_mmdd; + } else { + // looks like a date, but we can't tell which, so assume + // that it's dd/mm (English imperialism!) and keep looking + sortfn = sorttable.sort_ddmm; + } + } + } + } + return sortfn; + }, + + getInnerText: function(node) { + // gets the text we want to use for sorting for a cell. + // strips leading and trailing whitespace. + // this is *not* a generic getInnerText function; it's special to sorttable. + // for example, you can override the cell text with a customkey attribute. + // it also gets .value for <input> fields. + + hasInputs = (typeof node.getElementsByTagName == 'function') && + node.getElementsByTagName('input').length; + + if (node.getAttribute("sorttable_customkey") != null) { + return node.getAttribute("sorttable_customkey"); + } + else if (typeof node.textContent != 'undefined' && !hasInputs) { + return node.textContent.replace(/^\s+|\s+$/g, ''); + } + else if (typeof node.innerText != 'undefined' && !hasInputs) { + return node.innerText.replace(/^\s+|\s+$/g, ''); + } + else if (typeof node.text != 'undefined' && !hasInputs) { + return node.text.replace(/^\s+|\s+$/g, ''); + } + else { + switch (node.nodeType) { + case 3: + if (node.nodeName.toLowerCase() == 'input') { + return node.value.replace(/^\s+|\s+$/g, ''); + } + case 4: + return node.nodeValue.replace(/^\s+|\s+$/g, ''); + break; + case 1: + case 11: + var innerText = ''; + for (var i = 0; i < node.childNodes.length; i++) { + innerText += sorttable.getInnerText(node.childNodes[i]); + } + return innerText.replace(/^\s+|\s+$/g, ''); + break; + default: + return ''; + } + } + }, + + reverse: function(tbody) { + // reverse the rows in a tbody + newrows = []; + for (var i=0; i<tbody.rows.length; i++) { + newrows[newrows.length] = tbody.rows[i]; + } + for (var i=newrows.length-1; i>=0; i--) { + tbody.appendChild(newrows[i]); + } + delete newrows; + }, + + /* sort functions + each sort function takes two parameters, a and b + you are comparing a[0] and b[0] */ + sort_numeric: function(a,b) { + aa = parseFloat(a[0].replace(/[^0-9.-]/g,'')); + if (isNaN(aa)) aa = 0; + bb = parseFloat(b[0].replace(/[^0-9.-]/g,'')); + if (isNaN(bb)) bb = 0; + return aa-bb; + }, + sort_alpha: function(a,b) { + if (a[0]==b[0]) return 0; + if (a[0]<b[0]) return -1; + return 1; + }, + sort_ddmm: function(a,b) { + mtch = a[0].match(sorttable.DATE_RE); + y = mtch[3]; m = mtch[2]; d = mtch[1]; + if (m.length == 1) m = '0'+m; + if (d.length == 1) d = '0'+d; + dt1 = y+m+d; + mtch = b[0].match(sorttable.DATE_RE); + y = mtch[3]; m = mtch[2]; d = mtch[1]; + if (m.length == 1) m = '0'+m; + if (d.length == 1) d = '0'+d; + dt2 = y+m+d; + if (dt1==dt2) return 0; + if (dt1<dt2) return -1; + return 1; + }, + sort_mmdd: function(a,b) { + mtch = a[0].match(sorttable.DATE_RE); + y = mtch[3]; d = mtch[2]; m = mtch[1]; + if (m.length == 1) m = '0'+m; + if (d.length == 1) d = '0'+d; + dt1 = y+m+d; + mtch = b[0].match(sorttable.DATE_RE); + y = mtch[3]; d = mtch[2]; m = mtch[1]; + if (m.length == 1) m = '0'+m; + if (d.length == 1) d = '0'+d; + dt2 = y+m+d; + if (dt1==dt2) return 0; + if (dt1<dt2) return -1; + return 1; + }, + + shaker_sort: function(list, comp_func) { + // A stable sort function to allow multi-level sorting of data + // see: http://en.wikipedia.org/wiki/Cocktail_sort + // thanks to Joseph Nahmias + var b = 0; + var t = list.length - 1; + var swap = true; + + while(swap) { + swap = false; + for(var i = b; i < t; ++i) { + if ( comp_func(list[i], list[i+1]) > 0 ) { + var q = list[i]; list[i] = list[i+1]; list[i+1] = q; + swap = true; + } + } // for + t--; + + if (!swap) break; + + for(var i = t; i > b; --i) { + if ( comp_func(list[i], list[i-1]) < 0 ) { + var q = list[i]; list[i] = list[i-1]; list[i-1] = q; + swap = true; + } + } // for + b++; + + } // while(swap) + } +} + +/* ****************************************************************** + Supporting functions: bundled here to avoid depending on a library + ****************************************************************** */ + +// Dean Edwards/Matthias Miller/John Resig + +/* for Mozilla/Opera9 */ +if (document.addEventListener) { + document.addEventListener("DOMContentLoaded", sorttable.init, false); +} + +/* for Internet Explorer */ +/*@cc_on @*/ +/*@if (@_win32) + document.write("<script id=__ie_onload defer src=javascript:void(0)><\/script>"); + var script = document.getElementById("__ie_onload"); + script.onreadystatechange = function() { + if (this.readyState == "complete") { + sorttable.init(); // call the onload handler + } + }; +/*@end @*/ + +/* for Safari */ +if (/WebKit/i.test(navigator.userAgent)) { // sniff + var _timer = setInterval(function() { + if (/loaded|complete/.test(document.readyState)) { + sorttable.init(); // call the onload handler + } + }, 10); +} + +/* for other browsers */ +window.onload = sorttable.init; + +// written by Dean Edwards, 2005 +// with input from Tino Zijdel, Matthias Miller, Diego Perini + +// http://dean.edwards.name/weblog/2005/10/add-event/ + +function dean_addEvent(element, type, handler) { + if (element.addEventListener) { + element.addEventListener(type, handler, false); + } else { + // assign each event handler a unique ID + if (!handler.$$guid) handler.$$guid = dean_addEvent.guid++; + // create a hash table of event types for the element + if (!element.events) element.events = {}; + // create a hash table of event handlers for each element/event pair + var handlers = element.events[type]; + if (!handlers) { + handlers = element.events[type] = {}; + // store the existing event handler (if there is one) + if (element["on" + type]) { + handlers[0] = element["on" + type]; + } + } + // store the event handler in the hash table + handlers[handler.$$guid] = handler; + // assign a global event handler to do all the work + element["on" + type] = handleEvent; + } +}; +// a counter used to create unique IDs +dean_addEvent.guid = 1; + +function removeEvent(element, type, handler) { + if (element.removeEventListener) { + element.removeEventListener(type, handler, false); + } else { + // delete the event handler from the hash table + if (element.events && element.events[type]) { + delete element.events[type][handler.$$guid]; + } + } +}; + +function handleEvent(event) { + var returnValue = true; + // grab the event object (IE uses a global event object) + event = event || fixEvent(((this.ownerDocument || this.document || this).parentWindow || window).event); + // get a reference to the hash table of event handlers + var handlers = this.events[event.type]; + // execute each event handler + for (var i in handlers) { + this.$$handleEvent = handlers[i]; + if (this.$$handleEvent(event) === false) { + returnValue = false; + } + } + return returnValue; +}; + +function fixEvent(event) { + // add W3C standard event methods + event.preventDefault = fixEvent.preventDefault; + event.stopPropagation = fixEvent.stopPropagation; + return event; +}; +fixEvent.preventDefault = function() { + this.returnValue = false; +}; +fixEvent.stopPropagation = function() { + this.cancelBubble = true; +} + +// Dean's forEach: http://dean.edwards.name/base/forEach.js +/* + forEach, version 1.0 + Copyright 2006, Dean Edwards + License: http://www.opensource.org/licenses/mit-license.php +*/ + +// array-like enumeration +if (!Array.forEach) { // mozilla already supports this + Array.forEach = function(array, block, context) { + for (var i = 0; i < array.length; i++) { + block.call(context, array[i], i, array); + } + }; +} + +// generic enumeration +Function.prototype.forEach = function(object, block, context) { + for (var key in object) { + if (typeof this.prototype[key] == "undefined") { + block.call(context, object[key], key, object); + } + } +}; + +// character enumeration +String.forEach = function(string, block, context) { + Array.forEach(string.split(""), function(chr, index) { + block.call(context, chr, index, string); + }); +}; + +// globally resolve forEach enumeration +var forEach = function(object, block, context) { + if (object) { + var resolve = Object; // default + if (object instanceof Function) { + // functions have a "length" property + resolve = Function; + } else if (object.forEach instanceof Function) { + // the object implements a custom forEach method so use that + object.forEach(block, context); + return; + } else if (typeof object == "string") { + // the object is a string + resolve = String; + } else if (typeof object.length == "number") { + // the object is array-like + resolve = Array; + } + resolve.forEach(object, block, context); + } +}; + diff --git a/tools/allocation_tracking/track_allocations.py b/tools/allocation_tracking/track_allocations.py new file mode 100644 index 000000000..e9d937817 --- /dev/null +++ b/tools/allocation_tracking/track_allocations.py @@ -0,0 +1,131 @@ +import numpy as np +import inspect +from alloc_hook import NumpyAllocHook + +class AllocationTracker(object): + def __init__(self, threshold=0): + '''track numpy allocations of size threshold bytes or more.''' + + self.threshold = threshold + + # The total number of bytes currently allocated with size above + # threshold + self.total_bytes = 0 + + # We buffer requests line by line and move them into the allocation + # trace when a new line occurs + self.current_line = None + self.pending_allocations = [] + + self.blocksizes = {} + + # list of (lineinfo, bytes allocated, bytes freed, # allocations, # + # frees, maximum memory usage, long-lived bytes allocated) + self.allocation_trace = [] + + self.numpy_hook = NumpyAllocHook(self.hook) + + def __enter__(self): + self.numpy_hook.__enter__() + + def __exit__(self, type, value, traceback): + self.check_line_changed() # forces pending events to be handled + self.numpy_hook.__exit__() + + def hook(self, inptr, outptr, size): + if outptr == 0: # it's a free + self.free_cb(inptr) + elif inptr != 0: # realloc + self.realloc_cb(inptr, outptr, size) + else: # malloc + self.alloc_cb(outptr, size) + + def alloc_cb(self, ptr, size): + if size >= self.threshold: + self.check_line_changed() + self.blocksizes[ptr] = size + self.pending_allocations.append(size) + + def free_cb(self, ptr): + size = self.blocksizes.pop(ptr, 0) + if size: + self.check_line_changed() + self.pending_allocations.append(-size) + + def realloc_cb(self, newptr, oldptr, size): + if (size >= self.threshold) or (oldptr in self.blocksizes): + self.check_line_changed() + oldsize = self.blocksizes.pop(oldptr, 0) + self.pending_allocations.append(size - oldsize) + self.blocksizes[newptr] = size + + def get_code_line(self): + # first frame is this line, then check_line_changed(), then 2 callbacks, + # then actual code. + try: + return inspect.stack()[4][1:] + except: + return inspect.stack()[0][1:] + + def check_line_changed(self): + line = self.get_code_line() + if line != self.current_line and (self.current_line is not None): + # move pending events into the allocation_trace + max_size = self.total_bytes + bytes_allocated = 0 + bytes_freed = 0 + num_allocations = 0 + num_frees = 0 + before_size = self.total_bytes + for allocation in self.pending_allocations: + self.total_bytes += allocation + if allocation > 0: + bytes_allocated += allocation + num_allocations += 1 + else: + bytes_freed += -allocation + num_frees += 1 + max_size = max(max_size, self.total_bytes) + long_lived = max(self.total_bytes - before_size, 0) + self.allocation_trace.append((self.current_line, bytes_allocated, + bytes_freed, num_allocations, + num_frees, max_size, long_lived)) + # clear pending allocations + self.pending_allocations = [] + # move to the new line + self.current_line = line + + def write_html(self, filename): + f = open(filename, "w") + f.write('<HTML><HEAD><script src="sorttable.js"></script></HEAD><BODY>\n') + f.write('<TABLE class="sortable" width=100%>\n') + f.write("<TR>\n") + cols = "event#,lineinfo,bytes allocated,bytes freed,#allocations,#frees,max memory usage,long lived bytes".split(',') + for header in cols: + f.write(" <TH>{0}</TH>".format(header)) + f.write("\n</TR>\n") + for idx, event in enumerate(self.allocation_trace): + f.write("<TR>\n") + event = [idx] + list(event) + for col, val in zip(cols, event): + if col == 'lineinfo': + # special handling + try: + filename, line, module, code, index = val + val = "{0}({1}): {2}".format(filename, line, code[index]) + except: + # sometimes this info is not available (from eval()?) + val = str(val) + f.write(" <TD>{0}</TD>".format(val)) + f.write("\n</TR>\n") + f.write("</TABLE></BODY></HTML>\n") + f.close() + + +if __name__ == '__main__': + tracker = AllocationTracker(1000) + with tracker: + for i in range(100): + np.zeros(i * 100) + np.zeros(i * 200) + tracker.write_html("allocations.html") |