diff options
author | Tiancong Wang <tcwang@google.com> | 2020-02-13 21:08:49 +0000 |
---|---|---|
committer | Tiancong Wang <tcwang@google.com> | 2020-02-13 21:08:49 +0000 |
commit | b75f321fc8978b92ce3db6886ccb966768f0c7a8 (patch) | |
tree | 35fa0fbaeaaddd9cc2a126a05eee3527b51e83a8 /deprecated/cwp/demo_pipeline.sh | |
parent | cddd960b0ba2eb62c372c0d3176c75f0bd05d5e8 (diff) | |
parent | e617e3393dd24003aa976ece5050bb291070041c (diff) | |
download | toolchain-utils-b75f321fc8978b92ce3db6886ccb966768f0c7a8.tar.gz |
Merging 18 commit(s) from Chromium's toolchain-utils am: 0ae38c8498 am: 2a19d36a82 am: e617e3393dr_aml_301500702android-mainline-12.0.0_r55android-mainline-11.0.0_r9android-mainline-11.0.0_r8android-mainline-11.0.0_r7android-mainline-11.0.0_r6android-mainline-11.0.0_r5android-mainline-11.0.0_r45android-mainline-11.0.0_r44android-mainline-11.0.0_r43android-mainline-11.0.0_r42android-mainline-11.0.0_r41android-mainline-11.0.0_r40android-mainline-11.0.0_r4android-mainline-11.0.0_r39android-mainline-11.0.0_r38android-mainline-11.0.0_r37android-mainline-11.0.0_r36android-mainline-11.0.0_r35android-mainline-11.0.0_r34android-mainline-11.0.0_r33android-mainline-11.0.0_r32android-mainline-11.0.0_r31android-mainline-11.0.0_r30android-mainline-11.0.0_r3android-mainline-11.0.0_r29android-mainline-11.0.0_r28android-mainline-11.0.0_r27android-mainline-11.0.0_r26android-mainline-11.0.0_r25android-mainline-11.0.0_r24android-mainline-11.0.0_r23android-mainline-11.0.0_r22android-mainline-11.0.0_r21android-mainline-11.0.0_r20android-mainline-11.0.0_r2android-mainline-11.0.0_r19android-mainline-11.0.0_r18android-mainline-11.0.0_r17android-mainline-11.0.0_r16android-mainline-11.0.0_r15android-mainline-11.0.0_r14android-mainline-11.0.0_r13android-mainline-11.0.0_r12android-mainline-11.0.0_r10android-mainline-11.0.0_r1android-11.0.0_r48android-11.0.0_r47android-11.0.0_r46android-11.0.0_r45android-11.0.0_r44android-11.0.0_r43android-11.0.0_r42android-11.0.0_r41android-11.0.0_r40android-11.0.0_r39android-11.0.0_r38android-11.0.0_r37android-11.0.0_r36android-11.0.0_r35android-11.0.0_r34android-11.0.0_r33android-11.0.0_r32android-11.0.0_r31android-11.0.0_r30android-11.0.0_r29android-11.0.0_r28android-11.0.0_r27android-11.0.0_r26android-11.0.0_r24android-11.0.0_r23android-11.0.0_r22android-11.0.0_r21android-11.0.0_r20android-11.0.0_r19android-11.0.0_r18android-11.0.0_r16android11-qpr3-s1-releaseandroid11-qpr3-releaseandroid11-qpr2-releaseandroid11-qpr1-s2-releaseandroid11-qpr1-s1-releaseandroid11-qpr1-releaseandroid11-qpr1-d-s1-releaseandroid11-qpr1-d-releaseandroid11-qpr1-c-releaseandroid11-mainline-tethering-releaseandroid11-mainline-sparse-2021-jan-releaseandroid11-mainline-sparse-2020-dec-releaseandroid11-mainline-releaseandroid11-mainline-permission-releaseandroid11-mainline-os-statsd-releaseandroid11-mainline-networkstack-releaseandroid11-mainline-media-swcodec-releaseandroid11-mainline-media-releaseandroid11-mainline-extservices-releaseandroid11-mainline-documentsui-releaseandroid11-mainline-conscrypt-releaseandroid11-mainline-cellbroadcast-releaseandroid11-mainline-captiveportallogin-releaseandroid11-devandroid11-d2-releaseandroid11-d1-b-release
Change-Id: I3f25c7ee034b2e20e37ed941b8eae24eec7043eb
Diffstat (limited to 'deprecated/cwp/demo_pipeline.sh')
-rw-r--r-- | deprecated/cwp/demo_pipeline.sh | 55 |
1 files changed, 55 insertions, 0 deletions
diff --git a/deprecated/cwp/demo_pipeline.sh b/deprecated/cwp/demo_pipeline.sh new file mode 100644 index 00000000..d45c5c44 --- /dev/null +++ b/deprecated/cwp/demo_pipeline.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +# These should be on the local filesystem. We'll be hitting it hard. +DATA_DIR=/usr/local/google/home/${USER}/ +SYMBOL_CACHE=${DATA_DIR}cache/ +REPORT_DIR=${DATA_DIR}reports/ +SAMPLE_DIR=${DATA_DIR}samples/ +RECORD_FILE=/tmp/profiles.rio +COLUMN_FILE=/tmp/profiles.cio +mkdir -p ${SYMBOL_CACHE} +mkdir -p ${REPORT_DIR} +mkdir -p ${SAMPLE_DIR} + +# Directory that has the scripts app_engine_pull.py and symbolizer.py +INTERPRETER_DIR=/google/src/files/p2/head/depot2/gcctools/chromeos/v14/cwp/interpreter/ +V14_DIR=$(dirname $(dirname ${INTERPRETER_DIR})) + +PYTHONPATH=$PYTHONPATH:$V14_DIR + +# Profile util binary +PROFILE_UTIL_BINARY=/home/mrdmnd/${USER}-profiledb/google3/blaze-bin/perftools/gwp/chromeos/profile_util + +# mr-convert binary +MR_CONVERT_BINARY=/home/build/static/projects/dremel/mr-convert + +CNS_LOC=/cns/ag-d/home/${USER}/profiledb/ + +# Protofile location +PROTO_LOC=${CNS_LOC}cwp_profile_db_entry.proto + +echo "0. Cleaning up old data." +rm /tmp/profiles.* +rm ${REPORT_DIR}* +rm ${SAMPLE_DIR}* + + +echo "Starting CWP Pipeline..." +echo "1. Pulling samples to local filesystem from server." +echo " For demo purposes, UN=${USER}@google.com, PW=xjpbmshkzefutlrm" +python ${INTERPRETER_DIR}app_engine_pull.py --output_dir=${SAMPLE_DIR} +echo "2. Symbolizing samples to perf reports. Hold on..." + +python ${INTERPRETER_DIR}symbolizer.py --in=${SAMPLE_DIR} --out=${REPORT_DIR} --cache=${SYMBOL_CACHE} +echo "3. Loading reports into RecordIO format..." +# Will need to make append_dir more clever / incremental +${PROFILE_UTIL_BINARY} --record=${RECORD_FILE} --append_dir=${REPORT_DIR} +echo "Done." +echo "4. Converting records to columnio." +${MR_CONVERT_BINARY} --mapreduce_input_map=recordio:${RECORD_FILE} --mapreduce_output_map=${COLUMN_FILE}@1 --columnio_mroutput_message_type=CwpProfileDbEntry --columnio_mroutput_protofiles=${PROTO_LOC} +echo "5. Uploading columnio to colossus." +fileutil cp -f /tmp/profiles.cio-* ${CNS_LOC} +echo "6. Let's try some dremel queries..." +echo " dremel> define table t /cns/ag-d/home/${USER}/profiledb/profiles.cio-*" +echo " Like, say, dremel> select sum(frames.count) as count, left(frames.function_name, 80) as name from t group by name order by count desc limit 25;" + |