diff options
author | Zhizhou Yang <zhizhouy@google.com> | 2020-02-11 16:28:03 -0800 |
---|---|---|
committer | Zhizhou Yang <zhizhouy@google.com> | 2020-02-12 01:35:23 +0000 |
commit | 6ba01883794a0e69cd05bb516162a60e582b9ebc (patch) | |
tree | d1618831fdc4dd2c65a2dcd59d78109f7c21ba7b /deprecated/mem_tests | |
parent | 7bda3eb62f8125f641cc9e3defcc5278cde2ba7b (diff) | |
download | toolchain-utils-6ba01883794a0e69cd05bb516162a60e582b9ebc.tar.gz |
toolchain-utils: deprecate cwp and mem_tests scripts
These two projects are very old and seems no one is touching them.
BUG=chromium:1051236
TEST=None
Change-Id: I81fb0fb69d0f505c931c153926eeb0d7a7d59b07
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2051387
Tested-by: Zhizhou Yang <zhizhouy@google.com>
Reviewed-by: Manoj Gupta <manojgupta@chromium.org>
Reviewed-by: Gabriel Marin <gmx@chromium.org>
Commit-Queue: Zhizhou Yang <zhizhouy@google.com>
Auto-Submit: Zhizhou Yang <zhizhouy@google.com>
Diffstat (limited to 'deprecated/mem_tests')
-rw-r--r-- | deprecated/mem_tests/README.md | 52 | ||||
-rwxr-xr-x | deprecated/mem_tests/clean_data.py | 29 | ||||
-rwxr-xr-x | deprecated/mem_tests/mem_groups.py | 55 | ||||
-rwxr-xr-x | deprecated/mem_tests/total_mem_actual.py | 37 | ||||
-rwxr-xr-x | deprecated/mem_tests/total_mem_sampled.py | 31 | ||||
-rw-r--r-- | deprecated/mem_tests/utils.py | 22 |
6 files changed, 226 insertions, 0 deletions
diff --git a/deprecated/mem_tests/README.md b/deprecated/mem_tests/README.md new file mode 100644 index 00000000..44bf16c9 --- /dev/null +++ b/deprecated/mem_tests/README.md @@ -0,0 +1,52 @@ +# Memory tests + +## Usage + +These scripts are made to parse TCMalloc output in order to extract certain +info from them. + +In particular, these scripts rely on the error logging system for ChromeOS in +order to extract information. In order to use a script (e.g. `total_mem.py`), +you just have the command: + +``` +./total_mem.py FILENAME +``` + +where `FILENAME` is the name of the log file to be parsed. + +## Codebase Changes + +There are two ideas that motivate these changes: + +1. Turn on TCMalloc sampling. +2. Use perf to collect the sample information. + + +The following files have to be changed: + +in `chrome/browser/metrics/perf_provider_chrome_os`, add: + +``` +#include "third_party/tcmalloc/chromium/src/gperftools/malloc_extension.h" +``` + +Change the perf profiling interval to something small (60*1000 milliseconds). + +Inside DoPeriodicCollection, insert the following code: + +``` +std::string output; +char* chr_arr = new char[9999]; +MallocExtension::instance() ->GetHeapSample(&output); +MallocExtension::instance() ->GetStats(chr_arr, 9999); +LOG(ERROR) << "Output Heap Data: "; +LOG(ERROR) << output; +LOG(ERROR) << "Output Heap Stats: "; +output = ""; +for (unsigned int i = 0; i < strlen(chr_arr); i++) { + output += chr_arr[i]; +} +LOG(ERROR) << output; +delete[] chr_arr; +``` diff --git a/deprecated/mem_tests/clean_data.py b/deprecated/mem_tests/clean_data.py new file mode 100755 index 00000000..1433ba41 --- /dev/null +++ b/deprecated/mem_tests/clean_data.py @@ -0,0 +1,29 @@ +#!/usr/bin/python2 +"""Cleans output from other scripts to eliminate duplicates. + +When frequently sampling data, we see that records occasionally will contain +the same timestamp (due to perf recording twice in the same second). + +This removes all of the duplicate timestamps for every record. Order with +respect to timestamps is not preserved. Also, the assumption is that the log +file is a csv with the first value in each row being the time in seconds from a +standard time. + +""" + +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument('filename') +args = parser.parse_args() + +my_file = open(args.filename) +output_file = open('clean2.csv', 'a') +dictionary = dict() + +for line in my_file: + new_time = int(line.split(',')[0]) + dictionary[new_time] = line + +for key in dictionary.keys(): + output_file.write(dictionary[key]) diff --git a/deprecated/mem_tests/mem_groups.py b/deprecated/mem_tests/mem_groups.py new file mode 100755 index 00000000..5d593872 --- /dev/null +++ b/deprecated/mem_tests/mem_groups.py @@ -0,0 +1,55 @@ +#!/usr/bin/python2 +"""Groups memory by allocation sizes. + +Takes a log entry and sorts sorts everything into groups based on what size +chunks the memory has been allocated in. groups is an array that contains the +divisions (in bytes). + +The output format is: + +timestamp, percent of memory in chunks < groups[0], percent between groups[0] +and groups[1], etc. + +""" + +import argparse +from cros_utils import compute_total_diff +from datetime import datetime + +pretty_print = True + +parser = argparse.ArgumentParser() +parser.add_argument('filename') +args = parser.parse_args() + +my_file = open(args.filename) +output_file = open('groups.csv', 'a') + +# The cutoffs for each group in the output (in bytes) +groups = [1024, 8192, 65536, 524288, 4194304] + +base_time = datetime(2014, 6, 11, 0, 0) +prev_line = '' +half_entry = (None, None) + +for line in my_file: + if 'heap profile:' in line: + if half_entry[0] is not None: + group_totals = half_entry[1] + total = sum(group_totals) * 1.0 + to_join = [half_entry[0]] + [value / total for value in group_totals] + to_output = ','.join([str(elem) for elem in to_join]) + output_file.write(to_output) + total_diff = compute_total_diff(line, base_time) + half_entry = (total_diff, [0] * (len(groups) + 1)) + if '] @ ' in line and 'heap profile:' not in line: + mem_samples = line.strip().split('[')[0] + num_samples, total_mem = map(int, mem_samples.strip().split(':')) + mem_per_sample = total_mem // num_samples + group_totals = half_entry[1] + for cutoff_index in range(len(groups)): + if mem_per_sample <= groups[cutoff_index]: + group_totals[cutoff_index] += total_mem + break + if mem_per_sample > groups[-1]: + group_totals[-1] += total_mem diff --git a/deprecated/mem_tests/total_mem_actual.py b/deprecated/mem_tests/total_mem_actual.py new file mode 100755 index 00000000..d2a0cedf --- /dev/null +++ b/deprecated/mem_tests/total_mem_actual.py @@ -0,0 +1,37 @@ +#!/usr/bin/python2 +"""Parses the actual memory usage from TCMalloc. + +This goes through logs that have the actual allocated memory (not sampled) in +the logs. The output is of the form of: + +time (in seconds from some base time), amount of memory allocated by the +application + +""" + +import argparse +from cros_utils import compute_total_diff +from datetime import datetime + +pretty_print = True + +parser = argparse.ArgumentParser() +parser.add_argument('filename') +args = parser.parse_args() + +my_file = open(args.filename) +output_file = open('raw_memory_data.csv', 'a') + +base_time = datetime(2014, 6, 11, 0, 0) +prev_line = '' +half_entry = (None, None) + +for line in my_file: + if 'Output Heap Stats:' in line: + total_diff = compute_total_diff(line, base_time) + half_entry = (total_diff, None) + if 'Bytes in use by application' in line: + total_diff = half_entry[0] + memory_used = int(line.strip().split()[1]) + half_entry = (None, None) + output_file.write('{0},{1}\n'.format(total_diff, memory_used)) diff --git a/deprecated/mem_tests/total_mem_sampled.py b/deprecated/mem_tests/total_mem_sampled.py new file mode 100755 index 00000000..32aa527c --- /dev/null +++ b/deprecated/mem_tests/total_mem_sampled.py @@ -0,0 +1,31 @@ +#!/usr/bin/python2 +"""Parses the total amount of sampled memory from log files. + +This file outputs the total amount of memory that has been sampled by tcmalloc. +The output is of the format: + +time in seconds from a base time, amount of memory that has been sampled + +""" + +import argparse +from cros_utils import compute_total_diff +from datetime import datetime + +parser = argparse.ArgumentParser() +parser.add_argument('filename') +args = parser.parse_args() + +my_file = open(args.filename) +output_file = open('memory_data.csv', 'a') + +base_time = datetime(2014, 6, 11, 0, 0) +prev_line = '' +half_entry = (None, None) + +for line in my_file: + if 'heap profile: ' not in line: + continue + memory_used = line.strip().split(':')[-1].strip().split(']')[0].strip() + total_diff = compute_total_diff(line, base_time) + output_file.write('{0},{1}\n'.format(int(total_diff), memory_used)) diff --git a/deprecated/mem_tests/utils.py b/deprecated/mem_tests/utils.py new file mode 100644 index 00000000..38bd89ca --- /dev/null +++ b/deprecated/mem_tests/utils.py @@ -0,0 +1,22 @@ +"""Utility functions for the memory tests. +""" + +from datetime import datetime + + +def compute_total_diff(line, base_time): + """ + Computes the difference in time the line was recorded from the base time. + + An example of a line is: + [4688:4688:0701/010151:ERROR:perf_provider_chromeos.cc(228)]... + + Here, the month is 07, the day is 01 and the time is 01:01:51. + + line- the line that contains the time the record was taken + base_time- the base time to measure our timestamp from + """ + date = line.strip().split(':')[2].split('/') + timestamp = datetime(2014, int(date[0][0:2]), int(date[0][2:4]), + int(date[1][0:2]), int(date[1][2:4]), int(date[1][4:6])) + return (timestamp - base_time).total_seconds() |