diff options
-rw-r--r-- | .gitignore (renamed from v14/.gitignore) | 0 | ||||
-rw-r--r-- | automation/PRESUBMIT.py (renamed from v14/automation/PRESUBMIT.py) | 0 | ||||
-rw-r--r-- | automation/__init__.py (renamed from v14/automation/__init__.py) | 0 | ||||
-rw-r--r-- | automation/all_tests.py (renamed from v14/automation/all_tests.py) | 0 | ||||
-rw-r--r-- | automation/clients/__init__.py (renamed from v14/automation/clients/__init__.py) | 0 | ||||
-rwxr-xr-x | automation/clients/android.py (renamed from v14/automation/clients/android.py) | 0 | ||||
-rwxr-xr-x | automation/clients/chromeos.py (renamed from v14/automation/clients/chromeos.py) | 0 | ||||
-rwxr-xr-x | automation/clients/crosstool.py (renamed from v14/automation/clients/crosstool.py) | 0 | ||||
-rwxr-xr-x | automation/clients/dejagnu_compiler.py (renamed from v14/automation/clients/dejagnu_compiler.py) | 0 | ||||
-rw-r--r-- | automation/clients/helper/__init__.py (renamed from v14/automation/clients/helper/__init__.py) | 0 | ||||
-rw-r--r-- | automation/clients/helper/android.py (renamed from v14/automation/clients/helper/android.py) | 0 | ||||
-rw-r--r-- | automation/clients/helper/chromeos.py (renamed from v14/automation/clients/helper/chromeos.py) | 0 | ||||
-rw-r--r-- | automation/clients/helper/crosstool.py (renamed from v14/automation/clients/helper/crosstool.py) | 0 | ||||
-rw-r--r-- | automation/clients/helper/jobs.py (renamed from v14/automation/clients/helper/jobs.py) | 0 | ||||
-rw-r--r-- | automation/clients/helper/perforce.py (renamed from v14/automation/clients/helper/perforce.py) | 0 | ||||
-rwxr-xr-x | automation/clients/nightly.py (renamed from v14/automation/clients/nightly.py) | 0 | ||||
-rwxr-xr-x | automation/clients/output_test.py (renamed from v14/automation/clients/output_test.py) | 0 | ||||
-rwxr-xr-x | automation/clients/pwd_test.py (renamed from v14/automation/clients/pwd_test.py) | 0 | ||||
-rwxr-xr-x | automation/clients/report/dejagnu.sh (renamed from v14/automation/clients/report/dejagnu.sh) | 0 | ||||
-rw-r--r-- | automation/clients/report/dejagnu/__init__.py (renamed from v14/automation/clients/report/dejagnu/__init__.py) | 0 | ||||
-rw-r--r-- | automation/clients/report/dejagnu/main.py (renamed from v14/automation/clients/report/dejagnu/main.py) | 0 | ||||
-rw-r--r-- | automation/clients/report/dejagnu/manifest.py (renamed from v14/automation/clients/report/dejagnu/manifest.py) | 0 | ||||
-rw-r--r-- | automation/clients/report/dejagnu/report.html (renamed from v14/automation/clients/report/dejagnu/report.html) | 0 | ||||
-rw-r--r-- | automation/clients/report/dejagnu/report.py (renamed from v14/automation/clients/report/dejagnu/report.py) | 0 | ||||
-rw-r--r-- | automation/clients/report/dejagnu/summary.py (renamed from v14/automation/clients/report/dejagnu/summary.py) | 0 | ||||
-rwxr-xr-x | automation/clients/report/validate_failures.py (renamed from v14/automation/clients/report/validate_failures.py) | 0 | ||||
-rw-r--r-- | automation/common/__init__.py (renamed from v14/automation/common/__init__.py) | 0 | ||||
-rw-r--r-- | automation/common/command.py (renamed from v14/automation/common/command.py) | 0 | ||||
-rw-r--r-- | automation/common/command_executer.py (renamed from v14/automation/common/command_executer.py) | 0 | ||||
-rwxr-xr-x | automation/common/command_executer_test.py (renamed from v14/automation/common/command_executer_test.py) | 0 | ||||
-rw-r--r-- | automation/common/events.py (renamed from v14/automation/common/events.py) | 0 | ||||
-rw-r--r-- | automation/common/job.py (renamed from v14/automation/common/job.py) | 0 | ||||
-rw-r--r-- | automation/common/job_group.py (renamed from v14/automation/common/job_group.py) | 0 | ||||
-rw-r--r-- | automation/common/logger.py (renamed from v14/automation/common/logger.py) | 0 | ||||
-rw-r--r-- | automation/common/machine.py (renamed from v14/automation/common/machine.py) | 0 | ||||
-rwxr-xr-x | automation/common/machine_test.py (renamed from v14/automation/common/machine_test.py) | 0 | ||||
-rw-r--r-- | automation/common/state_machine.py (renamed from v14/automation/common/state_machine.py) | 0 | ||||
-rw-r--r-- | automation/server/__init__.py (renamed from v14/automation/server/__init__.py) | 0 | ||||
-rw-r--r-- | automation/server/job_executer.py (renamed from v14/automation/server/job_executer.py) | 0 | ||||
-rw-r--r-- | automation/server/job_group_manager.py (renamed from v14/automation/server/job_group_manager.py) | 0 | ||||
-rw-r--r-- | automation/server/job_manager.py (renamed from v14/automation/server/job_manager.py) | 0 | ||||
-rw-r--r-- | automation/server/machine_manager.py (renamed from v14/automation/server/machine_manager.py) | 0 | ||||
-rwxr-xr-x | automation/server/machine_manager_test.py (renamed from v14/automation/server/machine_manager_test.py) | 0 | ||||
-rw-r--r-- | automation/server/monitor/__init__.py (renamed from v14/automation/server/monitor/__init__.py) | 0 | ||||
-rw-r--r-- | automation/server/monitor/dashboard.py (renamed from v14/automation/server/monitor/dashboard.py) | 0 | ||||
-rwxr-xr-x | automation/server/monitor/manage.py (renamed from v14/automation/server/monitor/manage.py) | 0 | ||||
-rw-r--r-- | automation/server/monitor/settings.py (renamed from v14/automation/server/monitor/settings.py) | 0 | ||||
-rwxr-xr-x | automation/server/monitor/start.sh (renamed from v14/automation/server/monitor/start.sh) | 0 | ||||
-rw-r--r-- | automation/server/monitor/static/style.css (renamed from v14/automation/server/monitor/static/style.css) | 0 | ||||
-rw-r--r-- | automation/server/monitor/templates/base.html (renamed from v14/automation/server/monitor/templates/base.html) | 0 | ||||
-rw-r--r-- | automation/server/monitor/templates/job.html (renamed from v14/automation/server/monitor/templates/job.html) | 0 | ||||
-rw-r--r-- | automation/server/monitor/templates/job_group.html (renamed from v14/automation/server/monitor/templates/job_group.html) | 0 | ||||
-rw-r--r-- | automation/server/monitor/templates/job_group_list.html (renamed from v14/automation/server/monitor/templates/job_group_list.html) | 0 | ||||
-rw-r--r-- | automation/server/monitor/templates/job_log.html (renamed from v14/automation/server/monitor/templates/job_log.html) | 0 | ||||
-rw-r--r-- | automation/server/monitor/templates/machine_list.html (renamed from v14/automation/server/monitor/templates/machine_list.html) | 0 | ||||
-rw-r--r-- | automation/server/monitor/templates/snippet_attribute_table.html (renamed from v14/automation/server/monitor/templates/snippet_attribute_table.html) | 0 | ||||
-rw-r--r-- | automation/server/monitor/templates/snippet_code.html (renamed from v14/automation/server/monitor/templates/snippet_code.html) | 0 | ||||
-rw-r--r-- | automation/server/monitor/templates/snippet_links.html (renamed from v14/automation/server/monitor/templates/snippet_links.html) | 0 | ||||
-rw-r--r-- | automation/server/monitor/urls.py (renamed from v14/automation/server/monitor/urls.py) | 0 | ||||
-rwxr-xr-x | automation/server/server.py (renamed from v14/automation/server/server.py) | 0 | ||||
-rwxr-xr-x | automation/server/server_test.py (renamed from v14/automation/server/server_test.py) | 0 | ||||
-rw-r--r-- | automation/server/test_pool.csv (renamed from v14/automation/server/test_pool.csv) | 0 | ||||
-rw-r--r-- | build-binutils/opts.sh (renamed from v14/build-binutils/opts.sh) | 0 | ||||
-rw-r--r-- | build-gcc/opts.sh (renamed from v14/build-gcc/opts.sh) | 0 | ||||
-rwxr-xr-x | build_benchmarks.py (renamed from v14/build_benchmarks.py) | 0 | ||||
-rwxr-xr-x | build_chrome_browser.py (renamed from v14/build_chrome_browser.py) | 0 | ||||
-rwxr-xr-x | build_chromeos.py (renamed from v14/build_chromeos.py) | 0 | ||||
-rwxr-xr-x | build_tc.py (renamed from v14/build_tc.py) | 0 | ||||
-rwxr-xr-x | command_executer_timeout_test.py (renamed from v14/command_executer_timeout_test.py) | 0 | ||||
-rwxr-xr-x | compare_benchmarks.py (renamed from v14/compare_benchmarks.py) | 0 | ||||
-rw-r--r-- | crb/autotest_gatherer.py (renamed from v14/crb/autotest_gatherer.py) | 0 | ||||
-rw-r--r-- | crb/autotest_run.py (renamed from v14/crb/autotest_run.py) | 0 | ||||
-rwxr-xr-x | crb/crb_driver.py (renamed from v14/crb/crb_driver.py) | 0 | ||||
-rw-r--r-- | crb/machine_manager_singleton.py (renamed from v14/crb/machine_manager_singleton.py) | 0 | ||||
-rw-r--r-- | crb/table_formatter.py (renamed from v14/crb/table_formatter.py) | 0 | ||||
-rwxr-xr-x | cros_login.py (renamed from v14/cros_login.py) | 0 | ||||
-rw-r--r-- | crosperf/benchmark.py | 4 | ||||
-rw-r--r-- | crosperf/benchmark_run.py | 7 | ||||
-rwxr-xr-x | crosperf/benchmark_run_unittest.py | 12 | ||||
-rw-r--r-- | crosperf/config.py (renamed from v14/crosperf/config.py) | 0 | ||||
-rwxr-xr-x | crosperf/crosperf.py | 13 | ||||
-rw-r--r-- | crosperf/experiment.py | 23 | ||||
-rw-r--r-- | crosperf/experiment_factory.py | 27 | ||||
-rwxr-xr-x | crosperf/experiment_factory_unittest.py | 4 | ||||
-rw-r--r-- | crosperf/experiment_runner.py | 8 | ||||
-rw-r--r-- | crosperf/image_checksummer.py | 38 | ||||
-rw-r--r-- | crosperf/label.py | 8 | ||||
-rw-r--r-- | crosperf/machine_manager.py | 47 | ||||
-rwxr-xr-x | crosperf/machine_manager_unittest.py | 8 | ||||
-rw-r--r-- | crosperf/perf_table.py | 39 | ||||
-rw-r--r-- | crosperf/results_cache.py | 84 | ||||
-rw-r--r-- | crosperf/results_organizer.py | 13 | ||||
-rw-r--r-- | crosperf/results_report.py | 15 | ||||
-rw-r--r-- | crosperf/settings_factory.py | 29 | ||||
-rw-r--r-- | cwp/bartlett/app.yaml (renamed from v14/cwp/bartlett/app.yaml) | 0 | ||||
-rwxr-xr-x | cwp/bartlett/server.py (renamed from v14/cwp/bartlett/server.py) | 0 | ||||
-rw-r--r-- | cwp/bartlett/static/favicon.ico (renamed from v14/cwp/bartlett/static/favicon.ico) | bin | 198 -> 198 bytes | |||
-rw-r--r-- | cwp/bartlett/test/server_tester.py (renamed from v14/cwp/bartlett/test/server_tester.py) | 0 | ||||
-rwxr-xr-x | cwp/bartlett/update_appengine_server (renamed from v14/cwp/bartlett/update_appengine_server) | 0 | ||||
-rw-r--r-- | cwp/demo_pipeline.sh (renamed from v14/cwp/demo_pipeline.sh) | 0 | ||||
-rw-r--r-- | cwp/interpreter/app_engine_pull.py (renamed from v14/cwp/interpreter/app_engine_pull.py) | 0 | ||||
-rw-r--r-- | cwp/interpreter/symbolizer.py (renamed from v14/cwp/interpreter/symbolizer.py) | 0 | ||||
-rw-r--r-- | cwp/performance/experiment_gen.py (renamed from v14/cwp/performance/experiment_gen.py) | 0 | ||||
-rw-r--r-- | dejagnu/__init__.py (renamed from v14/dejagnu/__init__.py) | 0 | ||||
-rw-r--r-- | dejagnu/boards/chromeos-machine.exp (renamed from v14/dejagnu/boards/chromeos-machine.exp) | 0 | ||||
-rw-r--r-- | dejagnu/chromeos.exp.in (renamed from v14/dejagnu/chromeos.exp.in) | 0 | ||||
-rwxr-xr-x | dejagnu/run_dejagnu.py (renamed from v14/dejagnu/run_dejagnu.py) | 0 | ||||
-rw-r--r-- | dejagnu/site.exp (renamed from v14/dejagnu/site.exp) | 0 | ||||
-rwxr-xr-x | fdo_scripts/divide_and_merge_profiles.py (renamed from v14/fdo_scripts/divide_and_merge_profiles.py) | 0 | ||||
-rwxr-xr-x | fdo_scripts/divide_and_merge_profiles_test.py (renamed from v14/fdo_scripts/divide_and_merge_profiles_test.py) | 0 | ||||
-rwxr-xr-x | fdo_scripts/profile_cycler.py (renamed from v14/fdo_scripts/profile_cycler.py) | 0 | ||||
-rw-r--r-- | fdo_scripts/summarize_hot_blocks.py (renamed from v14/fdo_scripts/summarize_hot_blocks.py) | 0 | ||||
-rw-r--r-- | fdo_scripts/vanilla_vs_fdo.py (renamed from v14/fdo_scripts/vanilla_vs_fdo.py) | 0 | ||||
-rwxr-xr-x | get_common_image_version.py (renamed from v14/get_common_image_version.py) | 0 | ||||
-rwxr-xr-x | image_chromeos.py | 18 | ||||
-rwxr-xr-x | lock_machine.py (renamed from v14/lock_machine.py) | 0 | ||||
-rw-r--r-- | lock_machine_test.py (renamed from v14/lock_machine_test.py) | 0 | ||||
-rwxr-xr-x | produce_output.py (renamed from v14/produce_output.py) | 0 | ||||
-rwxr-xr-x | pyrun (renamed from v14/pyrun) | 0 | ||||
-rwxr-xr-x | remote_gcc_build.py (renamed from v14/remote_gcc_build.py) | 0 | ||||
-rwxr-xr-x | remote_kill_test.py (renamed from v14/remote_kill_test.py) | 0 | ||||
-rwxr-xr-x | remote_test.py (renamed from v14/remote_test.py) | 0 | ||||
-rwxr-xr-x | repo_to_repo.py (renamed from v14/repo_to_repo.py) | 0 | ||||
-rw-r--r-- | repo_to_repo_files/binutils-master.rtr (renamed from v14/repo_to_repo_files/binutils-master.rtr) | 0 | ||||
-rw-r--r-- | repo_to_repo_files/binutils-mobile_toolchain_v16.rtr (renamed from v14/repo_to_repo_files/binutils-mobile_toolchain_v16.rtr) | 0 | ||||
-rw-r--r-- | repo_to_repo_files/crosperf.rtr (renamed from v14/repo_to_repo_files/crosperf.rtr) | 0 | ||||
-rw-r--r-- | repo_to_repo_files/gcc-branches_google_4_7.rtr (renamed from v14/repo_to_repo_files/gcc-branches_google_4_7.rtr) | 0 | ||||
-rw-r--r-- | repo_to_repo_files/gcc-branches_google_main.rtr (renamed from v14/repo_to_repo_files/gcc-branches_google_main.rtr) | 0 | ||||
-rw-r--r-- | repo_to_repo_files/gcc-master.rtr (renamed from v14/repo_to_repo_files/gcc-master.rtr) | 0 | ||||
-rw-r--r-- | repo_to_repo_files/gdb-master.rtr (renamed from v14/repo_to_repo_files/gdb-master.rtr) | 0 | ||||
-rw-r--r-- | repo_to_repo_files/toolchain-utils.rtr (renamed from v14/repo_to_repo_files/toolchain-utils.rtr) | 0 | ||||
-rwxr-xr-x | report_generator.py (renamed from v14/report_generator.py) | 0 | ||||
-rwxr-xr-x | run_benchmarks.py (renamed from v14/run_benchmarks.py) | 0 | ||||
-rwxr-xr-x | run_tests.py (renamed from v14/run_tests.py) | 0 | ||||
-rwxr-xr-x | setup_chromeos.py (renamed from v14/setup_chromeos.py) | 0 | ||||
-rwxr-xr-x | sheriff_rotation.py (renamed from v14/sheriff_rotation.py) | 0 | ||||
-rwxr-xr-x | summarize_results.py (renamed from v14/summarize_results.py) | 0 | ||||
-rwxr-xr-x | tc_enter_chroot.py (renamed from v14/tc_enter_chroot.py) | 0 | ||||
-rwxr-xr-x | test_gcc_dejagnu.py (renamed from v14/test_gcc_dejagnu.py) | 0 | ||||
-rw-r--r-- | test_toolchains.py (renamed from v14/test_toolchains.py) | 0 | ||||
-rwxr-xr-x | utils/buildbot_json.py (renamed from v14/utils/buildbot_json.py) | 0 | ||||
-rw-r--r-- | utils/command_executer.py | 15 | ||||
-rwxr-xr-x | utils/command_executer_unittest.py (renamed from v14/utils/command_executer_unittest.py) | 0 | ||||
-rw-r--r-- | utils/logger.py | 10 | ||||
-rw-r--r-- | utils/misc.py | 25 | ||||
-rwxr-xr-x | utils/perf_diff.py | 12 | ||||
-rw-r--r-- | utils/tabulator.py | 169 | ||||
-rw-r--r-- | utils/tabulator_test.py | 4 | ||||
-rw-r--r-- | v14/README | 10 | ||||
-rw-r--r-- | v14/crosperf/autotest_runner.py | 40 | ||||
-rw-r--r-- | v14/crosperf/benchmark.py | 26 | ||||
-rw-r--r-- | v14/crosperf/benchmark_run.py | 198 | ||||
-rwxr-xr-x | v14/crosperf/benchmark_run_unittest.py | 50 | ||||
-rw-r--r-- | v14/crosperf/column_chart.py | 57 | ||||
-rwxr-xr-x | v14/crosperf/crosperf | 2 | ||||
-rwxr-xr-x | v14/crosperf/crosperf.py | 108 | ||||
-rwxr-xr-x | v14/crosperf/crosperf_test.py | 40 | ||||
-rw-r--r-- | v14/crosperf/default_remotes | 5 | ||||
-rw-r--r-- | v14/crosperf/experiment.py | 127 | ||||
-rw-r--r-- | v14/crosperf/experiment_factory.py | 135 | ||||
-rwxr-xr-x | v14/crosperf/experiment_factory_unittest.py | 55 | ||||
-rw-r--r-- | v14/crosperf/experiment_file.py | 185 | ||||
-rwxr-xr-x | v14/crosperf/experiment_file_unittest.py | 110 | ||||
-rw-r--r-- | v14/crosperf/experiment_files/README | 26 | ||||
-rw-r--r-- | v14/crosperf/experiment_files/aes_perf | 7 | ||||
-rw-r--r-- | v14/crosperf/experiment_files/bloat_perf | 6 | ||||
-rw-r--r-- | v14/crosperf/experiment_files/morejs_perf | 6 | ||||
-rw-r--r-- | v14/crosperf/experiment_files/page_cycler | 6 | ||||
-rw-r--r-- | v14/crosperf/experiment_files/page_cycler_perf | 43 | ||||
-rw-r--r-- | v14/crosperf/experiment_files/toolchain | 16 | ||||
-rw-r--r-- | v14/crosperf/experiment_runner.py | 132 | ||||
-rw-r--r-- | v14/crosperf/experiment_status.py | 88 | ||||
-rw-r--r-- | v14/crosperf/field.py | 108 | ||||
-rw-r--r-- | v14/crosperf/help.py | 106 | ||||
-rw-r--r-- | v14/crosperf/image_checksummer.py | 63 | ||||
-rw-r--r-- | v14/crosperf/label.py | 54 | ||||
-rw-r--r-- | v14/crosperf/machine_manager.py | 419 | ||||
-rwxr-xr-x | v14/crosperf/machine_manager_unittest.py | 67 | ||||
-rw-r--r-- | v14/crosperf/perf_table.py | 85 | ||||
-rw-r--r-- | v14/crosperf/results_cache.py | 417 | ||||
-rw-r--r-- | v14/crosperf/results_organizer.py | 108 | ||||
-rw-r--r-- | v14/crosperf/results_report.py | 502 | ||||
-rw-r--r-- | v14/crosperf/results_sorter.py | 49 | ||||
-rwxr-xr-x | v14/crosperf/run_tests.sh | 13 | ||||
-rw-r--r-- | v14/crosperf/settings.py | 62 | ||||
-rw-r--r-- | v14/crosperf/settings_factory.py | 144 | ||||
-rw-r--r-- | v14/crosperf/test_flag.py | 16 | ||||
-rwxr-xr-x | v14/image_chromeos.py | 320 | ||||
-rw-r--r-- | v14/utils/__init__.py | 0 | ||||
-rw-r--r-- | v14/utils/colortrans.py | 376 | ||||
-rw-r--r-- | v14/utils/command_executer.py | 338 | ||||
-rw-r--r-- | v14/utils/constants.py | 10 | ||||
-rw-r--r-- | v14/utils/email_sender.py | 62 | ||||
-rw-r--r-- | v14/utils/file_utils.py | 89 | ||||
-rw-r--r-- | v14/utils/html_tools.py | 93 | ||||
-rw-r--r-- | v14/utils/logger.py | 195 | ||||
-rw-r--r-- | v14/utils/misc.py | 327 | ||||
-rw-r--r-- | v14/utils/misc_test.py | 50 | ||||
-rwxr-xr-x | v14/utils/perf_diff.py | 324 | ||||
-rw-r--r-- | v14/utils/pstat.py | 1068 | ||||
-rw-r--r-- | v14/utils/stats.py | 4528 | ||||
-rw-r--r-- | v14/utils/tabulator.py | 1125 | ||||
-rw-r--r-- | v14/utils/tabulator_test.py | 135 | ||||
-rw-r--r-- | v14/utils/timeline.py | 51 | ||||
-rw-r--r-- | v14/utils/timeline_test.py | 54 |
205 files changed, 465 insertions, 13003 deletions
diff --git a/v14/.gitignore b/.gitignore index d0b39e30..d0b39e30 100644 --- a/v14/.gitignore +++ b/.gitignore diff --git a/v14/automation/PRESUBMIT.py b/automation/PRESUBMIT.py index 56a43e6c..56a43e6c 100644 --- a/v14/automation/PRESUBMIT.py +++ b/automation/PRESUBMIT.py diff --git a/v14/automation/__init__.py b/automation/__init__.py index e69de29b..e69de29b 100644 --- a/v14/automation/__init__.py +++ b/automation/__init__.py diff --git a/v14/automation/all_tests.py b/automation/all_tests.py index 3ca4f52a..3ca4f52a 100644 --- a/v14/automation/all_tests.py +++ b/automation/all_tests.py diff --git a/v14/automation/clients/__init__.py b/automation/clients/__init__.py index e69de29b..e69de29b 100644 --- a/v14/automation/clients/__init__.py +++ b/automation/clients/__init__.py diff --git a/v14/automation/clients/android.py b/automation/clients/android.py index 069fcafb..069fcafb 100755 --- a/v14/automation/clients/android.py +++ b/automation/clients/android.py diff --git a/v14/automation/clients/chromeos.py b/automation/clients/chromeos.py index ec80a0eb..ec80a0eb 100755 --- a/v14/automation/clients/chromeos.py +++ b/automation/clients/chromeos.py diff --git a/v14/automation/clients/crosstool.py b/automation/clients/crosstool.py index 9844174b..9844174b 100755 --- a/v14/automation/clients/crosstool.py +++ b/automation/clients/crosstool.py diff --git a/v14/automation/clients/dejagnu_compiler.py b/automation/clients/dejagnu_compiler.py index 4bc881e0..4bc881e0 100755 --- a/v14/automation/clients/dejagnu_compiler.py +++ b/automation/clients/dejagnu_compiler.py diff --git a/v14/automation/clients/helper/__init__.py b/automation/clients/helper/__init__.py index 8b137891..8b137891 100644 --- a/v14/automation/clients/helper/__init__.py +++ b/automation/clients/helper/__init__.py diff --git a/v14/automation/clients/helper/android.py b/automation/clients/helper/android.py index f49ccb86..f49ccb86 100644 --- a/v14/automation/clients/helper/android.py +++ b/automation/clients/helper/android.py diff --git a/v14/automation/clients/helper/chromeos.py b/automation/clients/helper/chromeos.py index 4005390e..4005390e 100644 --- a/v14/automation/clients/helper/chromeos.py +++ b/automation/clients/helper/chromeos.py diff --git a/v14/automation/clients/helper/crosstool.py b/automation/clients/helper/crosstool.py index b78e2d86..b78e2d86 100644 --- a/v14/automation/clients/helper/crosstool.py +++ b/automation/clients/helper/crosstool.py diff --git a/v14/automation/clients/helper/jobs.py b/automation/clients/helper/jobs.py index e1827015..e1827015 100644 --- a/v14/automation/clients/helper/jobs.py +++ b/automation/clients/helper/jobs.py diff --git a/v14/automation/clients/helper/perforce.py b/automation/clients/helper/perforce.py index 5e9c261c..5e9c261c 100644 --- a/v14/automation/clients/helper/perforce.py +++ b/automation/clients/helper/perforce.py diff --git a/v14/automation/clients/nightly.py b/automation/clients/nightly.py index bf1ef5a7..bf1ef5a7 100755 --- a/v14/automation/clients/nightly.py +++ b/automation/clients/nightly.py diff --git a/v14/automation/clients/output_test.py b/automation/clients/output_test.py index 7cff8eda..7cff8eda 100755 --- a/v14/automation/clients/output_test.py +++ b/automation/clients/output_test.py diff --git a/v14/automation/clients/pwd_test.py b/automation/clients/pwd_test.py index 2c2118bb..2c2118bb 100755 --- a/v14/automation/clients/pwd_test.py +++ b/automation/clients/pwd_test.py diff --git a/v14/automation/clients/report/dejagnu.sh b/automation/clients/report/dejagnu.sh index fadd8a0c..fadd8a0c 100755 --- a/v14/automation/clients/report/dejagnu.sh +++ b/automation/clients/report/dejagnu.sh diff --git a/v14/automation/clients/report/dejagnu/__init__.py b/automation/clients/report/dejagnu/__init__.py index e69de29b..e69de29b 100644 --- a/v14/automation/clients/report/dejagnu/__init__.py +++ b/automation/clients/report/dejagnu/__init__.py diff --git a/v14/automation/clients/report/dejagnu/main.py b/automation/clients/report/dejagnu/main.py index c05e3e48..c05e3e48 100644 --- a/v14/automation/clients/report/dejagnu/main.py +++ b/automation/clients/report/dejagnu/main.py diff --git a/v14/automation/clients/report/dejagnu/manifest.py b/automation/clients/report/dejagnu/manifest.py index 75d907cd..75d907cd 100644 --- a/v14/automation/clients/report/dejagnu/manifest.py +++ b/automation/clients/report/dejagnu/manifest.py diff --git a/v14/automation/clients/report/dejagnu/report.html b/automation/clients/report/dejagnu/report.html index 39b39e09..39b39e09 100644 --- a/v14/automation/clients/report/dejagnu/report.html +++ b/automation/clients/report/dejagnu/report.html diff --git a/v14/automation/clients/report/dejagnu/report.py b/automation/clients/report/dejagnu/report.py index 7f2375de..7f2375de 100644 --- a/v14/automation/clients/report/dejagnu/report.py +++ b/automation/clients/report/dejagnu/report.py diff --git a/v14/automation/clients/report/dejagnu/summary.py b/automation/clients/report/dejagnu/summary.py index c45fed44..c45fed44 100644 --- a/v14/automation/clients/report/dejagnu/summary.py +++ b/automation/clients/report/dejagnu/summary.py diff --git a/v14/automation/clients/report/validate_failures.py b/automation/clients/report/validate_failures.py index e99c9054..e99c9054 100755 --- a/v14/automation/clients/report/validate_failures.py +++ b/automation/clients/report/validate_failures.py diff --git a/v14/automation/common/__init__.py b/automation/common/__init__.py index e69de29b..e69de29b 100644 --- a/v14/automation/common/__init__.py +++ b/automation/common/__init__.py diff --git a/v14/automation/common/command.py b/automation/common/command.py index 205467cb..205467cb 100644 --- a/v14/automation/common/command.py +++ b/automation/common/command.py diff --git a/v14/automation/common/command_executer.py b/automation/common/command_executer.py index 63e5af71..63e5af71 100644 --- a/v14/automation/common/command_executer.py +++ b/automation/common/command_executer.py diff --git a/v14/automation/common/command_executer_test.py b/automation/common/command_executer_test.py index d3cef898..d3cef898 100755 --- a/v14/automation/common/command_executer_test.py +++ b/automation/common/command_executer_test.py diff --git a/v14/automation/common/events.py b/automation/common/events.py index 79a69422..79a69422 100644 --- a/v14/automation/common/events.py +++ b/automation/common/events.py diff --git a/v14/automation/common/job.py b/automation/common/job.py index 94cd6e2b..94cd6e2b 100644 --- a/v14/automation/common/job.py +++ b/automation/common/job.py diff --git a/v14/automation/common/job_group.py b/automation/common/job_group.py index 6f9466b1..6f9466b1 100644 --- a/v14/automation/common/job_group.py +++ b/automation/common/job_group.py diff --git a/v14/automation/common/logger.py b/automation/common/logger.py index 6bdd0c82..6bdd0c82 100644 --- a/v14/automation/common/logger.py +++ b/automation/common/logger.py diff --git a/v14/automation/common/machine.py b/automation/common/machine.py index aa576c13..aa576c13 100644 --- a/v14/automation/common/machine.py +++ b/automation/common/machine.py diff --git a/v14/automation/common/machine_test.py b/automation/common/machine_test.py index 0d5f32c7..0d5f32c7 100755 --- a/v14/automation/common/machine_test.py +++ b/automation/common/machine_test.py diff --git a/v14/automation/common/state_machine.py b/automation/common/state_machine.py index d97f0fde..d97f0fde 100644 --- a/v14/automation/common/state_machine.py +++ b/automation/common/state_machine.py diff --git a/v14/automation/server/__init__.py b/automation/server/__init__.py index e69de29b..e69de29b 100644 --- a/v14/automation/server/__init__.py +++ b/automation/server/__init__.py diff --git a/v14/automation/server/job_executer.py b/automation/server/job_executer.py index 33080e84..33080e84 100644 --- a/v14/automation/server/job_executer.py +++ b/automation/server/job_executer.py diff --git a/v14/automation/server/job_group_manager.py b/automation/server/job_group_manager.py index 56af1985..56af1985 100644 --- a/v14/automation/server/job_group_manager.py +++ b/automation/server/job_group_manager.py diff --git a/v14/automation/server/job_manager.py b/automation/server/job_manager.py index a858b0de..a858b0de 100644 --- a/v14/automation/server/job_manager.py +++ b/automation/server/job_manager.py diff --git a/v14/automation/server/machine_manager.py b/automation/server/machine_manager.py index 615b22b4..615b22b4 100644 --- a/v14/automation/server/machine_manager.py +++ b/automation/server/machine_manager.py diff --git a/v14/automation/server/machine_manager_test.py b/automation/server/machine_manager_test.py index d84607b1..d84607b1 100755 --- a/v14/automation/server/machine_manager_test.py +++ b/automation/server/machine_manager_test.py diff --git a/v14/automation/server/monitor/__init__.py b/automation/server/monitor/__init__.py index e69de29b..e69de29b 100644 --- a/v14/automation/server/monitor/__init__.py +++ b/automation/server/monitor/__init__.py diff --git a/v14/automation/server/monitor/dashboard.py b/automation/server/monitor/dashboard.py index 236b9cbc..236b9cbc 100644 --- a/v14/automation/server/monitor/dashboard.py +++ b/automation/server/monitor/dashboard.py diff --git a/v14/automation/server/monitor/manage.py b/automation/server/monitor/manage.py index 047c90ee..047c90ee 100755 --- a/v14/automation/server/monitor/manage.py +++ b/automation/server/monitor/manage.py diff --git a/v14/automation/server/monitor/settings.py b/automation/server/monitor/settings.py index e6b7e633..e6b7e633 100644 --- a/v14/automation/server/monitor/settings.py +++ b/automation/server/monitor/settings.py diff --git a/v14/automation/server/monitor/start.sh b/automation/server/monitor/start.sh index 4fc53bef..4fc53bef 100755 --- a/v14/automation/server/monitor/start.sh +++ b/automation/server/monitor/start.sh diff --git a/v14/automation/server/monitor/static/style.css b/automation/server/monitor/static/style.css index b571b059..b571b059 100644 --- a/v14/automation/server/monitor/static/style.css +++ b/automation/server/monitor/static/style.css diff --git a/v14/automation/server/monitor/templates/base.html b/automation/server/monitor/templates/base.html index 95ffc222..95ffc222 100644 --- a/v14/automation/server/monitor/templates/base.html +++ b/automation/server/monitor/templates/base.html diff --git a/v14/automation/server/monitor/templates/job.html b/automation/server/monitor/templates/job.html index 90acd969..90acd969 100644 --- a/v14/automation/server/monitor/templates/job.html +++ b/automation/server/monitor/templates/job.html diff --git a/v14/automation/server/monitor/templates/job_group.html b/automation/server/monitor/templates/job_group.html index b6ed8ea8..b6ed8ea8 100644 --- a/v14/automation/server/monitor/templates/job_group.html +++ b/automation/server/monitor/templates/job_group.html diff --git a/v14/automation/server/monitor/templates/job_group_list.html b/automation/server/monitor/templates/job_group_list.html index b82fa730..b82fa730 100644 --- a/v14/automation/server/monitor/templates/job_group_list.html +++ b/automation/server/monitor/templates/job_group_list.html diff --git a/v14/automation/server/monitor/templates/job_log.html b/automation/server/monitor/templates/job_log.html index 937b21b0..937b21b0 100644 --- a/v14/automation/server/monitor/templates/job_log.html +++ b/automation/server/monitor/templates/job_log.html diff --git a/v14/automation/server/monitor/templates/machine_list.html b/automation/server/monitor/templates/machine_list.html index f81422d3..f81422d3 100644 --- a/v14/automation/server/monitor/templates/machine_list.html +++ b/automation/server/monitor/templates/machine_list.html diff --git a/v14/automation/server/monitor/templates/snippet_attribute_table.html b/automation/server/monitor/templates/snippet_attribute_table.html index 24bacc17..24bacc17 100644 --- a/v14/automation/server/monitor/templates/snippet_attribute_table.html +++ b/automation/server/monitor/templates/snippet_attribute_table.html diff --git a/v14/automation/server/monitor/templates/snippet_code.html b/automation/server/monitor/templates/snippet_code.html index 281754d6..281754d6 100644 --- a/v14/automation/server/monitor/templates/snippet_code.html +++ b/automation/server/monitor/templates/snippet_code.html diff --git a/v14/automation/server/monitor/templates/snippet_links.html b/automation/server/monitor/templates/snippet_links.html index f19fa6e5..f19fa6e5 100644 --- a/v14/automation/server/monitor/templates/snippet_links.html +++ b/automation/server/monitor/templates/snippet_links.html diff --git a/v14/automation/server/monitor/urls.py b/automation/server/monitor/urls.py index 80aa7d2a..80aa7d2a 100644 --- a/v14/automation/server/monitor/urls.py +++ b/automation/server/monitor/urls.py diff --git a/v14/automation/server/server.py b/automation/server/server.py index ae761893..ae761893 100755 --- a/v14/automation/server/server.py +++ b/automation/server/server.py diff --git a/v14/automation/server/server_test.py b/automation/server/server_test.py index 40cf491f..40cf491f 100755 --- a/v14/automation/server/server_test.py +++ b/automation/server/server_test.py diff --git a/v14/automation/server/test_pool.csv b/automation/server/test_pool.csv index b0700c9b..b0700c9b 100644 --- a/v14/automation/server/test_pool.csv +++ b/automation/server/test_pool.csv diff --git a/v14/build-binutils/opts.sh b/build-binutils/opts.sh index e69de29b..e69de29b 100644 --- a/v14/build-binutils/opts.sh +++ b/build-binutils/opts.sh diff --git a/v14/build-gcc/opts.sh b/build-gcc/opts.sh index d1e6fadb..d1e6fadb 100644 --- a/v14/build-gcc/opts.sh +++ b/build-gcc/opts.sh diff --git a/v14/build_benchmarks.py b/build_benchmarks.py index 41079f80..41079f80 100755 --- a/v14/build_benchmarks.py +++ b/build_benchmarks.py diff --git a/v14/build_chrome_browser.py b/build_chrome_browser.py index f162c1d4..f162c1d4 100755 --- a/v14/build_chrome_browser.py +++ b/build_chrome_browser.py diff --git a/v14/build_chromeos.py b/build_chromeos.py index 483e44a8..483e44a8 100755 --- a/v14/build_chromeos.py +++ b/build_chromeos.py diff --git a/v14/build_tc.py b/build_tc.py index 0c009b3a..0c009b3a 100755 --- a/v14/build_tc.py +++ b/build_tc.py diff --git a/v14/command_executer_timeout_test.py b/command_executer_timeout_test.py index 446651d3..446651d3 100755 --- a/v14/command_executer_timeout_test.py +++ b/command_executer_timeout_test.py diff --git a/v14/compare_benchmarks.py b/compare_benchmarks.py index e63dfea7..e63dfea7 100755 --- a/v14/compare_benchmarks.py +++ b/compare_benchmarks.py diff --git a/v14/crb/autotest_gatherer.py b/crb/autotest_gatherer.py index da39040d..da39040d 100644 --- a/v14/crb/autotest_gatherer.py +++ b/crb/autotest_gatherer.py diff --git a/v14/crb/autotest_run.py b/crb/autotest_run.py index d6dc70f5..d6dc70f5 100644 --- a/v14/crb/autotest_run.py +++ b/crb/autotest_run.py diff --git a/v14/crb/crb_driver.py b/crb/crb_driver.py index 6cf7af35..6cf7af35 100755 --- a/v14/crb/crb_driver.py +++ b/crb/crb_driver.py diff --git a/v14/crb/machine_manager_singleton.py b/crb/machine_manager_singleton.py index 93629c4c..93629c4c 100644 --- a/v14/crb/machine_manager_singleton.py +++ b/crb/machine_manager_singleton.py diff --git a/v14/crb/table_formatter.py b/crb/table_formatter.py index b3b82f09..b3b82f09 100644 --- a/v14/crb/table_formatter.py +++ b/crb/table_formatter.py diff --git a/v14/cros_login.py b/cros_login.py index 44b22643..44b22643 100755 --- a/v14/cros_login.py +++ b/cros_login.py diff --git a/crosperf/benchmark.py b/crosperf/benchmark.py index bc7f1fa8..8fe8a492 100644 --- a/crosperf/benchmark.py +++ b/crosperf/benchmark.py @@ -14,11 +14,13 @@ class Benchmark(object): """ def __init__(self, name, autotest_name, autotest_args, iterations, - outlier_range, perf_args): + outlier_range, key_results_only, rm_chroot_tmp, perf_args): self.name = name self.autotest_name = autotest_name self.autotest_args = autotest_args self.iterations = iterations self.outlier_range = outlier_range self.perf_args = perf_args + self.key_results_only = key_results_only + self.rm_chroot_tmp = rm_chroot_tmp self.iteration_adjusted = False diff --git a/crosperf/benchmark_run.py b/crosperf/benchmark_run.py index dc837937..80c95c4d 100644 --- a/crosperf/benchmark_run.py +++ b/crosperf/benchmark_run.py @@ -29,7 +29,8 @@ class BenchmarkRun(threading.Thread): iteration, cache_conditions, machine_manager, - logger_to_use): + logger_to_use, + share_users): threading.Thread.__init__(self) self.name = name self._logger = logger_to_use @@ -53,6 +54,7 @@ class BenchmarkRun(threading.Thread): self._ce = command_executer.GetCommandExecuter(self._logger) self.timeline = timeline.Timeline() self.timeline.Record(STATUS_PENDING) + self.share_users = share_users def run(self): try: @@ -67,7 +69,8 @@ class BenchmarkRun(threading.Thread): self.label.board, self.cache_conditions, self._logger, - self.label + self.label, + self.share_users ) self.result = self.cache.ReadResult() diff --git a/crosperf/benchmark_run_unittest.py b/crosperf/benchmark_run_unittest.py index c4670c9c..47e027f4 100755 --- a/crosperf/benchmark_run_unittest.py +++ b/crosperf/benchmark_run_unittest.py @@ -19,14 +19,19 @@ from results_cache import MockResultsCache class BenchmarkRunTest(unittest.TestCase): def testDryRun(self): my_label = MockLabel("test1", "image1", "/tmp/test_benchmark_run", - "x86-alex", "chromeos-alex1", "") - m = MockMachineManager("/tmp/chromeos_root") + "x86-alex", "chromeos-alex1", + image_args="", + image_md5sum="", + cache_dir="") + m = MockMachineManager("/tmp/chromeos_root", 0) m.AddMachine("chromeos-alex1") bench = Benchmark("PageCyler", "Pyautoperf", "", 1, 0.2, + False, + False, "") b = MockBenchmarkRun("test run", bench, @@ -34,7 +39,8 @@ class BenchmarkRunTest(unittest.TestCase): 1, [], m, - logger.GetLogger()) + logger.GetLogger(), + "") b.cache = MockResultsCache() b.autotest_runner = MockAutotestRunner() b.start() diff --git a/v14/crosperf/config.py b/crosperf/config.py index 45a3d000..45a3d000 100644 --- a/v14/crosperf/config.py +++ b/crosperf/config.py diff --git a/crosperf/crosperf.py b/crosperf/crosperf.py index cfb48d7c..cb7911fd 100755 --- a/crosperf/crosperf.py +++ b/crosperf/crosperf.py @@ -18,8 +18,6 @@ from utils import logger import test_flag -l = logger.GetLogger() - class MyIndentedHelpFormatter(optparse.IndentedHelpFormatter): def format_description(self, description): @@ -65,12 +63,20 @@ def Main(argv): description=Help().GetHelp(), formatter=MyIndentedHelpFormatter(), version="%prog 0.1") + + parser.add_option("-l", "--log_dir", + dest="log_dir", + default="", + help="The log_dir, default is under <crosperf_logs>/logs") + SetupParserOptions(parser) options, args = parser.parse_args(argv) # Convert the relevant options that are passed in into a settings # object which will override settings in the experiment file. option_settings = ConvertOptionsToSettings(options) + log_dir = os.path.abspath(os.path.expanduser(options.log_dir)) + logger.GetLogger(log_dir) if len(args) == 2: experiment_filename = args[1] @@ -87,7 +93,8 @@ def Main(argv): experiment_name = os.path.basename(experiment_filename) experiment_file.GetGlobalSettings().SetField("name", experiment_name) experiment = ExperimentFactory().GetExperiment(experiment_file, - working_directory) + working_directory, + log_dir) atexit.register(Cleanup, experiment) diff --git a/crosperf/experiment.py b/crosperf/experiment.py index e9dc3d07..2a4590c4 100644 --- a/crosperf/experiment.py +++ b/crosperf/experiment.py @@ -9,23 +9,20 @@ import time from utils import logger -from autotest_runner import AutotestRunner from benchmark_run import BenchmarkRun from machine_manager import MachineManager from machine_manager import MockMachineManager -from results_cache import ResultsCache -from results_report import HTMLResultsReport import test_flag class Experiment(object): """Class representing an Experiment to be run.""" - def __init__(self, name, remote, rerun_if_failed, working_directory, + def __init__(self, name, remote, working_directory, chromeos_root, cache_conditions, labels, benchmarks, - experiment_file, email_to): + experiment_file, email_to, acquire_timeout, log_dir, + share_users): self.name = name - self.rerun_if_failed = rerun_if_failed self.working_directory = working_directory self.remote = remote self.chromeos_root = chromeos_root @@ -34,11 +31,12 @@ class Experiment(object): self.email_to = email_to self.results_directory = os.path.join(self.working_directory, self.name + "_results") - + self.log_dir = log_dir self.labels = labels self.benchmarks = benchmarks self.num_complete = 0 self.num_run_complete = 0 + self.share_users = share_users # We need one chromeos_root to run the benchmarks in, but it doesn't # matter where it is, unless the ABIs are different. @@ -51,10 +49,10 @@ class Experiment(object): "the image path.") if test_flag.GetTestMode(): - self.machine_manager = MockMachineManager(chromeos_root) + self.machine_manager = MockMachineManager(chromeos_root, acquire_timeout) else: - self.machine_manager = MachineManager(chromeos_root) - self.l = logger.GetLogger() + self.machine_manager = MachineManager(chromeos_root, acquire_timeout) + self.l = logger.GetLogger(log_dir) for machine in remote: self.machine_manager.AddMachine(machine) @@ -75,7 +73,7 @@ class Experiment(object): benchmark_run_name = "%s: %s (%s)" % (label.name, benchmark.name, iteration) full_name = "%s_%s_%s" % (label.name, benchmark.name, iteration) - logger_to_use = logger.Logger(os.path.dirname(__file__), + logger_to_use = logger.Logger(self.log_dir, "run.%s" % (full_name), True) benchmark_run = BenchmarkRun(benchmark_run_name, @@ -84,7 +82,8 @@ class Experiment(object): iteration, self.cache_conditions, self.machine_manager, - logger_to_use) + logger_to_use, + self.share_users) benchmark_runs.append(benchmark_run) return benchmark_runs diff --git a/crosperf/experiment_factory.py b/crosperf/experiment_factory.py index bd3076dd..3c92ee3e 100644 --- a/crosperf/experiment_factory.py +++ b/crosperf/experiment_factory.py @@ -7,6 +7,7 @@ import os import socket from benchmark import Benchmark +import config from experiment import Experiment from label import Label from label import MockLabel @@ -22,13 +23,18 @@ class ExperimentFactory(object): of experiments could be produced. """ - def GetExperiment(self, experiment_file, working_directory): + def GetExperiment(self, experiment_file, working_directory, log_dir): """Construct an experiment from an experiment file.""" global_settings = experiment_file.GetGlobalSettings() experiment_name = global_settings.GetField("name") remote = global_settings.GetField("remote") - rerun_if_failed = global_settings.GetField("rerun_if_failed") chromeos_root = global_settings.GetField("chromeos_root") + rm_chroot_tmp = global_settings.GetField("rm_chroot_tmp") + key_results_only = global_settings.GetField("key_results_only") + acquire_timeout= global_settings.GetField("acquire_timeout") + cache_dir = global_settings.GetField("cache_dir") + config.AddConfig("no_email", global_settings.GetField("no_email")) + share_users = global_settings.GetField("share_users") # Default cache hit conditions. The image checksum in the cache and the # computed checksum of the image must match. Also a cache file must exist. @@ -55,8 +61,13 @@ class ExperimentFactory(object): iterations = benchmark_settings.GetField("iterations") outlier_range = benchmark_settings.GetField("outlier_range") perf_args = benchmark_settings.GetField("perf_args") + rm_chroot_tmp = benchmark_settings.GetField("rm_chroot_tmp") + key_results_only = benchmark_settings.GetField("key_results_only") + benchmark = Benchmark(benchmark_name, autotest_name, autotest_args, - iterations, outlier_range, perf_args) + iterations, outlier_range, + key_results_only, rm_chroot_tmp, + perf_args) benchmarks.append(benchmark) # Construct labels. @@ -69,6 +80,8 @@ class ExperimentFactory(object): chromeos_root = label_settings.GetField("chromeos_root") board = label_settings.GetField("board") my_remote = label_settings.GetField("remote") + image_md5sum = label_settings.GetField("md5sum") + cache_dir = label_settings.GetField("cache_dir") # TODO(yunlian): We should consolidate code in machine_manager.py # to derermine whether we are running from within google or not if ("corp.google.com" in socket.gethostname() and @@ -83,19 +96,19 @@ class ExperimentFactory(object): image_args = label_settings.GetField("image_args") if test_flag.GetTestMode(): label = MockLabel(label_name, image, chromeos_root, board, my_remote, - image_args) + image_args, image_md5sum, cache_dir) else: label = Label(label_name, image, chromeos_root, board, my_remote, - image_args) + image_args, image_md5sum, cache_dir) labels.append(label) email = global_settings.GetField("email") all_remote = list(set(all_remote)) - experiment = Experiment(experiment_name, all_remote, rerun_if_failed, + experiment = Experiment(experiment_name, all_remote, working_directory, chromeos_root, cache_conditions, labels, benchmarks, experiment_file.Canonicalize(), - email) + email, acquire_timeout, log_dir, share_users) return experiment diff --git a/crosperf/experiment_factory_unittest.py b/crosperf/experiment_factory_unittest.py index fa943519..6cee6b74 100755 --- a/crosperf/experiment_factory_unittest.py +++ b/crosperf/experiment_factory_unittest.py @@ -32,7 +32,9 @@ EXPERIMENT_FILE_1 = """ class ExperimentFactoryTest(unittest.TestCase): def testLoadExperimentFile1(self): experiment_file = ExperimentFile(StringIO.StringIO(EXPERIMENT_FILE_1)) - experiment = ExperimentFactory().GetExperiment(experiment_file, "") + experiment = ExperimentFactory().GetExperiment(experiment_file, + working_directory="", + log_dir="") self.assertEqual(experiment.remote, ["chromeos-alex3"]) self.assertEqual(len(experiment.benchmarks), 1) diff --git a/crosperf/experiment_runner.py b/crosperf/experiment_runner.py index b905bbdc..9212ba56 100644 --- a/crosperf/experiment_runner.py +++ b/crosperf/experiment_runner.py @@ -12,6 +12,7 @@ from utils import logger from utils.email_sender import EmailSender from utils.file_utils import FileUtils +import config from experiment_status import ExperimentStatus from results_report import HTMLResultsReport from results_report import TextResultsReport @@ -25,7 +26,7 @@ class ExperimentRunner(object): def __init__(self, experiment): self._experiment = experiment - self.l = logger.GetLogger() + self.l = logger.GetLogger(experiment.log_dir) self._ce = command_executer.GetCommandExecuter(self.l) self._terminated = False @@ -58,7 +59,8 @@ class ExperimentRunner(object): if not benchmark_run.cache_hit: send_mail = True break - if not send_mail and not experiment.email_to: + if (not send_mail and not experiment.email_to + or config.GetConfig("no_email")): return label_names = [] @@ -100,7 +102,7 @@ class ExperimentRunner(object): benchmark_run_path = os.path.join(results_directory, benchmark_run_name) benchmark_run.result.CopyResultsTo(benchmark_run_path) - benchmark_run.result.CleanUp() + benchmark_run.result.CleanUp(benchmark_run.benchmark.rm_chroot_tmp) def Run(self): self._Run(self._experiment) diff --git a/crosperf/image_checksummer.py b/crosperf/image_checksummer.py index f75dc944..dcc1cb02 100644 --- a/crosperf/image_checksummer.py +++ b/crosperf/image_checksummer.py @@ -2,24 +2,38 @@ # Copyright 2011 Google Inc. All Rights Reserved. +import os import threading + from utils import logger from utils.file_utils import FileUtils class ImageChecksummer(object): class PerImageChecksummer(object): - def __init__(self, filename): + def __init__(self, label): self._lock = threading.Lock() - self.filename = filename + self.label = label self._checksum = None def Checksum(self): with self._lock: if not self._checksum: - logger.GetLogger().LogOutput("Computing checksum for '%s'." % - self.filename) - self._checksum = FileUtils().Md5File(self.filename) + logger.GetLogger().LogOutput("Acquiring checksum for '%s'." % + self.label.name) + self._checksum = None + if self.label.chromeos_image: + if os.path.exists(self.label.chromeos_image): + self._checksum = FileUtils().Md5File(self.label.chromeos_image) + logger.GetLogger().LogOutput("Computed checksum is " + ": %s" % self._checksum) + if not self._checksum: + if self.label.image_md5sum: + self._checksum = self.label.image_md5sum + logger.GetLogger().LogOutput("Checksum in experiment file is " + ": %s" % self._checksum) + else: + raise Exception("Checksum computing error.") logger.GetLogger().LogOutput("Checksum is: %s" % self._checksum) return self._checksum @@ -34,16 +48,16 @@ class ImageChecksummer(object): *args, **kwargs) return cls._instance - def Checksum(self, filename): + def Checksum(self, label): with self._lock: - if filename not in self._per_image_checksummers: - self._per_image_checksummers[filename] = (ImageChecksummer. - PerImageChecksummer(filename)) - checksummer = self._per_image_checksummers[filename] + if label.name not in self._per_image_checksummers: + self._per_image_checksummers[label.name] = (ImageChecksummer. + PerImageChecksummer(label)) + checksummer = self._per_image_checksummers[label.name] try: return checksummer.Checksum() except Exception, e: - logger.GetLogger().LogError("Could not compute checksum of file '%s'." - % filename) + logger.GetLogger().LogError("Could not compute checksum of image in label" + " '%s'."% label.name) raise e diff --git a/crosperf/label.py b/crosperf/label.py index 64ce352f..be7a868e 100644 --- a/crosperf/label.py +++ b/crosperf/label.py @@ -10,7 +10,7 @@ from utils.file_utils import FileUtils class Label(object): def __init__(self, name, chromeos_image, chromeos_root, board, remote, - image_args): + image_args, image_md5sum, cache_dir): # Expand ~ chromeos_root = os.path.expanduser(chromeos_root) chromeos_image = os.path.expanduser(chromeos_image) @@ -20,6 +20,8 @@ class Label(object): self.board = board self.remote = remote self.image_args = image_args + self.image_md5sum = image_md5sum + self.cache_dir = cache_dir if not chromeos_root: chromeos_root = FileUtils().ChromeOSRootFromImage(chromeos_image) @@ -38,13 +40,15 @@ class Label(object): class MockLabel(object): def __init__(self, name, chromeos_image, chromeos_root, board, remote, - image_args): + image_args, image_md5sum, cache_dir): self.name = name self.chromeos_image = chromeos_image self.board = board self.remote = remote + self.cache_dir = cache_dir if not chromeos_root: self.chromeos_root = "/tmp/chromeos_root" else: self.chromeos_root = chromeos_root self.image_args = image_args + self.image_md5sum = image_md5sum diff --git a/crosperf/machine_manager.py b/crosperf/machine_manager.py index 9eb9bcdf..29a4df7a 100644 --- a/crosperf/machine_manager.py +++ b/crosperf/machine_manager.py @@ -111,12 +111,12 @@ class CrosMachine(object): def _GetMachineID(self): ce = command_executer.GetCommandExecuter() - command = "ifconfig" + command = "dump_vpd_log --full --stdout" ret, if_out, _ = ce.CrosRunCommand( command, return_output=True, machine=self.name, chromeos_root=self.chromeos_root) b = if_out.splitlines() - a = [l for l in b if "lan" in l] + a = [l for l in b if "Product" in l] self.machine_id = a[0] assert ret == 0, "Could not get machine_id from machine: %s" % self.name @@ -131,7 +131,7 @@ class CrosMachine(object): class MachineManager(object): - def __init__(self, chromeos_root): + def __init__(self, chromeos_root, acquire_timeout): self._lock = threading.RLock() self._all_machines = [] self._machines = [] @@ -140,6 +140,7 @@ class MachineManager(object): self.chromeos_root = None self.machine_checksum = {} self.machine_checksum_string = {} + self.acquire_timeout = acquire_timeout if os.path.isdir(lock_machine.Machine.LOCKS_DIR): self.no_lock = False @@ -149,7 +150,7 @@ class MachineManager(object): self.chromeos_root = chromeos_root def ImageMachine(self, machine, label): - checksum = ImageChecksummer().Checksum(label.chromeos_image) + checksum = ImageChecksummer().Checksum(label) if machine.checksum == checksum: return chromeos_root = label.chromeos_root @@ -193,8 +194,8 @@ class MachineManager(object): with self._lock: assert cros_machine, "Machine can't be None" for m in self._machines: - assert m.name != cros_machine.name, ( - "Tried to double-lock %s" % cros_machine.name) + if m.name == cros_machine.name: + return if self.no_lock: locked = True else: @@ -226,25 +227,33 @@ class MachineManager(object): return len(set(checksums)) == 1 def AcquireMachine(self, chromeos_image, label): - image_checksum = ImageChecksummer().Checksum(chromeos_image) + image_checksum = ImageChecksummer().Checksum(label) machines = self.GetMachines(label) + check_interval_time = 120 with self._lock: # Lazily external lock machines - - for m in machines: - if m not in self._initialized_machines: - self._initialized_machines.append(m) + while self.acquire_timeout >= 0: + for m in machines: + new_machine = m not in self._all_machines self._TryToLockMachine(m) - m.released_time = time.time() - if not self.AreAllMachineSame(label): - logger.GetLogger().LogFatal("-- not all the machine are identical") - if not self.GetAvailableMachines(label): + if new_machine: + m.released_time = time.time() + if not self.AreAllMachineSame(label): + logger.GetLogger().LogFatal("-- not all the machine are identical") + if self.GetAvailableMachines(label): + break + else: + sleep_time = max(1, min(self.acquire_timeout, check_interval_time)) + time.sleep(sleep_time) + self.acquire_timeout -= sleep_time + + if self.acquire_timeout < 0: machine_names = [] for machine in machines: machine_names.append(machine.name) logger.GetLogger().LogFatal("Could not acquire any of the " - "following machines: '%s'" - % ", ".join(machine_names)) + "following machines: '%s'" + % ", ".join(machine_names)) ### for m in self._machines: ### if (m.locked and time.time() - m.released_time < 10 and @@ -374,8 +383,8 @@ class MockCrosMachine(CrosMachine): class MockMachineManager(MachineManager): - def __init__(self, chromeos_root): - super(MockMachineManager, self).__init__(chromeos_root) + def __init__(self, chromeos_root, acquire_timeout): + super(MockMachineManager, self).__init__(chromeos_root, acquire_timeout) def _TryToLockMachine(self, cros_machine): self._machines.append(cros_machine) diff --git a/crosperf/machine_manager_unittest.py b/crosperf/machine_manager_unittest.py index 98baf456..84266d5e 100755 --- a/crosperf/machine_manager_unittest.py +++ b/crosperf/machine_manager_unittest.py @@ -12,7 +12,7 @@ import machine_manager class MyMachineManager(machine_manager.MachineManager): def __init__(self, chromeos_root): - super(MyMachineManager, self).__init__(chromeos_root) + super(MyMachineManager, self).__init__(chromeos_root, 0) def _TryToLockMachine(self, cros_machine): self._machines.append(cros_machine) @@ -30,9 +30,11 @@ class MyMachineManager(machine_manager.MachineManager): CHROMEOS_ROOT = "/tmp/chromeos-root" MACHINE_NAMES = ["lumpy1", "lumpy2", "lumpy3", "daisy1", "daisy2"] LABEL_LUMPY = label.MockLabel("lumpy", "image", CHROMEOS_ROOT, "lumpy", - ["lumpy1", "lumpy2", "lumpy3", "lumpy4"], "") + ["lumpy1", "lumpy2", "lumpy3", "lumpy4"], + "", "", "") LABEL_MIX = label.MockLabel("mix", "image", CHROMEOS_ROOT, "mix", - ["daisy1", "daisy2", "lumpy3", "lumpy4"], "") + ["daisy1", "daisy2", "lumpy3", "lumpy4"], + "", "", "") class MachineManagerTest(unittest.TestCase): diff --git a/crosperf/perf_table.py b/crosperf/perf_table.py index b3387ea8..3c8b88b8 100644 --- a/crosperf/perf_table.py +++ b/crosperf/perf_table.py @@ -7,6 +7,7 @@ import os from utils import perf_diff + def ParsePerfReport(perf_file): """It should return a dict.""" @@ -22,9 +23,16 @@ class PerfTable(object): self._label_names = label_names self.perf_data = {} self.GenerateData() - # {benchmark:{perf_event1:[[{func1:number, func2:number}, - # {func1: number, func2: number}]], ...}, + + # {benchmark:{perf_event1:[[{func1:number, func2:number, + # rows_to_show: number} + # {func1: number, func2: number + # rows_to_show: number}]], ...}, # benchmark2:...} + # The rows_to_show is temp data recording how many + # rows have over 1% running time. + self.row_info = {} + self.GetRowsToShow() def GenerateData(self): for label in self._label_names: @@ -39,9 +47,10 @@ class PerfTable(object): def ReadPerfReport(self, perf_file, label, benchmark_name, iteration): """Add the data from one run to the dict.""" - if not os.path.isfile(perf_file): - return - perf_of_run = perf_diff.GetPerfDictFromReport(perf_file) + if os.path.isfile(perf_file): + perf_of_run = perf_diff.GetPerfDictFromReport(perf_file) + else: + perf_of_run = {} if benchmark_name not in self.perf_data: self.perf_data[benchmark_name] = {} for event in perf_of_run: @@ -55,4 +64,22 @@ class PerfTable(object): data_for_label = ben_data[event][label_index] while len(data_for_label) <= iteration: data_for_label.append({}) - data_for_label[iteration] = perf_of_run[event] + if perf_of_run: + data_for_label[iteration] = perf_of_run[event] + else: + data_for_label[iteration] = {} + + def GetRowsToShow(self): + for benchmark in self.perf_data: + if benchmark not in self.row_info: + self.row_info[benchmark] = {} + for event in self.perf_data[benchmark]: + rows = 0 + for run in self.perf_data[benchmark][event]: + for iteration in run: + if perf_diff.ROWS_TO_SHOW in iteration: + rows = max(iteration[perf_diff.ROWS_TO_SHOW], rows) + # delete the temp data which stores how many rows of + # the perf data have over 1% running time. + del iteration[perf_diff.ROWS_TO_SHOW] + self.row_info[benchmark][event] = rows diff --git a/crosperf/results_cache.py b/crosperf/results_cache.py index c0600962..0357275d 100644 --- a/crosperf/results_cache.py +++ b/crosperf/results_cache.py @@ -15,7 +15,8 @@ from utils import misc from image_checksummer import ImageChecksummer -SCRATCH_DIR = "/home/%s/cros_scratch" % getpass.getuser() +SCRATCH_BASE = "/home/%s/cros_scratch" +SCRATCH_DIR = SCRATCH_BASE % getpass.getuser() RESULTS_FILE = "results.txt" MACHINE_FILE = "machine.txt" AUTOTEST_TARBALL = "autotest.tbz2" @@ -55,16 +56,18 @@ class Result(object): self._CopyFilesTo(dest_dir, self.perf_report_files) def _GetKeyvals(self): - generate_test_report = os.path.join(self._chromeos_root, - "src", - "platform", - "crostestutils", - "utils_py", - "generate_test_report.py") - command = ("python %s --no-color --csv %s" % - (generate_test_report, - self.results_dir)) - [_, out, _] = self._ce.RunCommand(command, return_output=True) + results_in_chroot = os.path.join(self._chromeos_root, + "chroot", "tmp") + if not self._temp_dir: + self._temp_dir = tempfile.mkdtemp(dir=results_in_chroot) + command = "cp -r {0}/* {1}".format(self.results_dir, self._temp_dir) + self._ce.RunCommand(command) + + command = ("python generate_test_report --no-color --csv %s" % + (os.path.join("/tmp", os.path.basename(self._temp_dir)))) + [_, out, _] = self._ce.ChrootRunCommand(self._chromeos_root, + command, + return_output=True) keyvals_dict = {} for line in out.splitlines(): tokens = re.split("=|,", line) @@ -177,7 +180,9 @@ class Result(object): self.retval = pickle.load(f) # Untar the tarball to a temporary directory - self._temp_dir = tempfile.mkdtemp() + self._temp_dir = tempfile.mkdtemp(dir=os.path.join(self._chromeos_root, + "chroot", "tmp")) + command = ("cd %s && tar xf %s" % (self._temp_dir, os.path.join(cache_dir, AUTOTEST_TARBALL))) @@ -189,24 +194,25 @@ class Result(object): self.perf_report_files = self._GetPerfReportFiles() self._ProcessResults() - def CleanUp(self): + def CleanUp(self, rm_chroot_tmp): + if rm_chroot_tmp: + command = "rm -rf %s" % self.results_dir + self._ce.RunCommand(command) if self._temp_dir: command = "rm -rf %s" % self._temp_dir self._ce.RunCommand(command) def StoreToCacheDir(self, cache_dir, machine_manager): # Create the dir if it doesn't exist. - command = "mkdir -p %s" % cache_dir - ret = self._ce.RunCommand(command) - if ret: - raise Exception("Could not create cache dir: %s" % cache_dir) - # Store to the cache directory. - with open(os.path.join(cache_dir, RESULTS_FILE), "w") as f: + temp_dir = tempfile.mkdtemp() + + # Store to the temp directory. + with open(os.path.join(temp_dir, RESULTS_FILE), "w") as f: pickle.dump(self.out, f) pickle.dump(self.err, f) pickle.dump(self.retval, f) - tarball = os.path.join(cache_dir, AUTOTEST_TARBALL) + tarball = os.path.join(temp_dir, AUTOTEST_TARBALL) command = ("cd %s && " "tar " "--exclude=var/spool " @@ -218,9 +224,22 @@ class Result(object): # Store machine info. # TODO(asharif): Make machine_manager a singleton, and don't pass it into # this function. - with open(os.path.join(cache_dir, MACHINE_FILE), "w") as f: + with open(os.path.join(temp_dir, MACHINE_FILE), "w") as f: f.write(machine_manager.machine_checksum_string[self.label_name]) + if os.path.exists(cache_dir): + command = "rm -rf {0}".format(cache_dir) + self._ce.RunCommand(command) + + command = "mkdir -p {0} && ".format(os.path.dirname(cache_dir)) + command += "mv {0} {1}".format(temp_dir, cache_dir) + ret = self._ce.RunCommand(command) + if ret: + command = "rm -rf {0}".format(temp_dir) + self._ce.RunCommand(command) + raise Exception("Could not move dir %s to dir %s" % + (temp_dir, cache_dir)) + @classmethod def CreateFromRun(cls, logger, chromeos_root, board, label_name, out, err, retval): @@ -273,7 +292,7 @@ class ResultsCache(object): def Init(self, chromeos_image, chromeos_root, autotest_name, iteration, autotest_args, machine_manager, board, cache_conditions, - logger_to_use, label): + logger_to_use, label, share_users): self.chromeos_image = chromeos_image self.chromeos_root = chromeos_root self.autotest_name = autotest_name @@ -285,10 +304,12 @@ class ResultsCache(object): self._logger = logger_to_use self._ce = command_executer.GetCommandExecuter(self._logger) self.label = label + self.share_users = share_users def _GetCacheDirForRead(self): - glob_path = self._FormCacheDir(self._GetCacheKeyList(True)) - matching_dirs = glob.glob(glob_path) + matching_dirs = [] + for glob_path in self._FormCacheDir(self._GetCacheKeyList(True)): + matching_dirs += glob.glob(glob_path) if matching_dirs: # Cache file found. @@ -297,12 +318,21 @@ class ResultsCache(object): return None def _GetCacheDirForWrite(self): - return self._FormCacheDir(self._GetCacheKeyList(False)) + return self._FormCacheDir(self._GetCacheKeyList(False))[0] def _FormCacheDir(self, list_of_strings): cache_key = " ".join(list_of_strings) cache_dir = misc.GetFilenameFromString(cache_key) - cache_path = os.path.join(SCRATCH_DIR, cache_dir) + if self.label.cache_dir: + cache_home = os.path.abspath(os.path.expanduser(self.label.cache_dir)) + cache_path = [os.path.join(cache_home, cache_dir)] + else: + cache_path = [os.path.join(SCRATCH_DIR, cache_dir)] + + for i in [x.strip() for x in self.share_users.split(",")]: + path = SCRATCH_BASE % i + cache_path.append(os.path.join(path, cache_dir)) + return cache_path def _GetCacheKeyList(self, read): @@ -313,7 +343,7 @@ class ResultsCache(object): if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions: checksum = "*" else: - checksum = ImageChecksummer().Checksum(self.chromeos_image) + checksum = ImageChecksummer().Checksum(self.label) if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions: image_path_checksum = "*" diff --git a/crosperf/results_organizer.py b/crosperf/results_organizer.py index 810186b2..2e5c9296 100644 --- a/crosperf/results_organizer.py +++ b/crosperf/results_organizer.py @@ -19,6 +19,11 @@ class ResultOrganizer(object): [ ]}. """ + key_filter = ["milliseconds_", + "retval", + "iterations", + "ms_", + "score_"] def __init__(self, benchmark_runs, labels, benchmarks=None): self.result = {} @@ -43,7 +48,15 @@ class ResultOrganizer(object): cur_dict = cur_table[index] if not benchmark_run.result: continue + benchmark = benchmark_run.benchmark + key_filter_on = (benchmark.key_results_only and + "PyAutoPerfTest" in benchmark.name + benchmark.autotest_name and + "perf." not in benchmark.autotest_args) for autotest_key in benchmark_run.result.keyvals: + if (key_filter_on and + not any([key for key in self.key_filter if key in autotest_key]) + ): + continue result_value = benchmark_run.result.keyvals[autotest_key] cur_dict[autotest_key] = result_value self._DuplicatePass() diff --git a/crosperf/results_report.py b/crosperf/results_report.py index f7434132..61c67d5b 100644 --- a/crosperf/results_report.py +++ b/crosperf/results_report.py @@ -11,6 +11,7 @@ from perf_table import PerfTable class ResultsReport(object): MAX_COLOR_CODE = 255 + PERF_ROWS = 5 def __init__(self, experiment): self.experiment = experiment @@ -40,7 +41,13 @@ class ResultsReport(object): Column(AmeanResult(), Format()), Column(StdResult(), - Format()) + Format(), "StdDev"), + Column(CoeffVarResult(), + CoeffVarFormat(), "StdDev/Mean"), + Column(GmeanRatioResult(), + RatioFormat(), "GmeanSpeedup"), + Column(PValueResult(), + PValueFormat(), "p-value") ] if not perf: return self._GetTables(self.labels, self.benchmark_runs, columns) @@ -138,10 +145,12 @@ class ResultsReport(object): ben_table = self._GetTableHeader(ben) tables.append(ben_table) benchmark_data = p_table.perf_data[benchmark] + row_info = p_table.row_info[benchmark] table = [] for event in benchmark_data: - tg = TableGenerator(benchmark_data[event], label_names) - table = tg.GetTable() + tg = TableGenerator(benchmark_data[event], label_names, + sort=TableGenerator.SORT_BY_VALUES_DESC) + table = tg.GetTable(max(self.PERF_ROWS, row_info[event])) parsed_columns = self._ParseColumn(columns, ben.iterations) tf = TableFormatter(table, parsed_columns) tf.GenerateCellTable() diff --git a/crosperf/settings_factory.py b/crosperf/settings_factory.py index 11fa4b4b..924bc114 100644 --- a/crosperf/settings_factory.py +++ b/crosperf/settings_factory.py @@ -26,6 +26,12 @@ class BenchmarkSettings(Settings): self.AddField(FloatField("outlier_range", default=0.2, description="The percentage of highest/lowest " "values to omit when computing the average.")) + self.AddField(BooleanField("rm_chroot_tmp", default=False, + description="Whether remove the run_remote_test" + "result in the chroot")) + self.AddField(BooleanField("key_results_only", default=True, + description="Whether only show the key results" + "of pyautoperf")) self.AddField(TextField("perf_args", default="", description="The optional profile command. It " "enables perf commands to record perforamance " @@ -44,6 +50,8 @@ class LabelSettings(Settings): "contains a src/scripts directory. Defaults to " "the chromeos checkout which contains the " "chromeos_image.")) + self.AddField(TextField("md5sum", default="", + description="The md5sum of this image")) self.AddField(TextField("board", required=True, description="The target " "board for running experiments on, e.g. x86-alex.")) self.AddField(ListField("remote", description= @@ -53,6 +61,8 @@ class LabelSettings(Settings): default="", description="Extra arguments to pass to " "image_chromeos.py.")) + self.AddField(TextField("cache_dir", default="", + description="The cache dir for this image.")) class GlobalSettings(Settings): @@ -69,6 +79,9 @@ class GlobalSettings(Settings): self.AddField(BooleanField("rerun_if_failed", description="Whether to " "re-run failed autotest runs or not.", default=False)) + self.AddField(BooleanField("rm_chroot_tmp", default=False, + description="Whether remove the run_remote_test" + "result in the chroot")) self.AddField(ListField("email", description="Space-seperated" "list of email addresses to send email to.")) self.AddField(BooleanField("rerun", description="Whether to ignore the " @@ -89,11 +102,27 @@ class GlobalSettings(Settings): "contains a src/scripts directory. Defaults to " "the chromeos checkout which contains the " "chromeos_image.")) + self.AddField(BooleanField("key_results_only", default=True, + description="Whether only show the key results" + "of pyautoperf")) + self.AddField(IntegerField("acquire_timeout", default=0, + description="Number of seconds to wait for " + "machine before exit if all the machines in " + "the experiment file are busy. Default is 0")) self.AddField(TextField("perf_args", default="", description="The optional profile command. It " "enables perf commands to record perforamance " "related counters. It must start with perf " "command record or stat followed by arguments.")) + self.AddField(TextField("cache_dir", default="", + description="The abs path of cache dir. " + "Default is /home/$(whoami)/cros_scratch.")) + self.AddField(BooleanField("no_email", default=False, + description="Whether to disable the email to " + "user after crosperf finishes.")) + self.AddField(TextField("share_users", default="", + description="Who's cache data you want to " + "use. It accepts multiple users seperated by \",\"")) class SettingsFactory(object): diff --git a/v14/cwp/bartlett/app.yaml b/cwp/bartlett/app.yaml index 60010f70..60010f70 100644 --- a/v14/cwp/bartlett/app.yaml +++ b/cwp/bartlett/app.yaml diff --git a/v14/cwp/bartlett/server.py b/cwp/bartlett/server.py index 3e4f1be7..3e4f1be7 100755 --- a/v14/cwp/bartlett/server.py +++ b/cwp/bartlett/server.py diff --git a/v14/cwp/bartlett/static/favicon.ico b/cwp/bartlett/static/favicon.ico Binary files differindex 19b58c2e..19b58c2e 100644 --- a/v14/cwp/bartlett/static/favicon.ico +++ b/cwp/bartlett/static/favicon.ico diff --git a/v14/cwp/bartlett/test/server_tester.py b/cwp/bartlett/test/server_tester.py index e5a2341d..e5a2341d 100644 --- a/v14/cwp/bartlett/test/server_tester.py +++ b/cwp/bartlett/test/server_tester.py diff --git a/v14/cwp/bartlett/update_appengine_server b/cwp/bartlett/update_appengine_server index f3812057..f3812057 100755 --- a/v14/cwp/bartlett/update_appengine_server +++ b/cwp/bartlett/update_appengine_server diff --git a/v14/cwp/demo_pipeline.sh b/cwp/demo_pipeline.sh index d45c5c44..d45c5c44 100644 --- a/v14/cwp/demo_pipeline.sh +++ b/cwp/demo_pipeline.sh diff --git a/v14/cwp/interpreter/app_engine_pull.py b/cwp/interpreter/app_engine_pull.py index 65f67940..65f67940 100644 --- a/v14/cwp/interpreter/app_engine_pull.py +++ b/cwp/interpreter/app_engine_pull.py diff --git a/v14/cwp/interpreter/symbolizer.py b/cwp/interpreter/symbolizer.py index 3e589538..3e589538 100644 --- a/v14/cwp/interpreter/symbolizer.py +++ b/cwp/interpreter/symbolizer.py diff --git a/v14/cwp/performance/experiment_gen.py b/cwp/performance/experiment_gen.py index 7752c11e..7752c11e 100644 --- a/v14/cwp/performance/experiment_gen.py +++ b/cwp/performance/experiment_gen.py diff --git a/v14/dejagnu/__init__.py b/dejagnu/__init__.py index e69de29b..e69de29b 100644 --- a/v14/dejagnu/__init__.py +++ b/dejagnu/__init__.py diff --git a/v14/dejagnu/boards/chromeos-machine.exp b/dejagnu/boards/chromeos-machine.exp index b168b677..b168b677 100644 --- a/v14/dejagnu/boards/chromeos-machine.exp +++ b/dejagnu/boards/chromeos-machine.exp diff --git a/v14/dejagnu/chromeos.exp.in b/dejagnu/chromeos.exp.in index 889653aa..889653aa 100644 --- a/v14/dejagnu/chromeos.exp.in +++ b/dejagnu/chromeos.exp.in diff --git a/v14/dejagnu/run_dejagnu.py b/dejagnu/run_dejagnu.py index 8943bb20..8943bb20 100755 --- a/v14/dejagnu/run_dejagnu.py +++ b/dejagnu/run_dejagnu.py diff --git a/v14/dejagnu/site.exp b/dejagnu/site.exp index beaa958d..beaa958d 100644 --- a/v14/dejagnu/site.exp +++ b/dejagnu/site.exp diff --git a/v14/fdo_scripts/divide_and_merge_profiles.py b/fdo_scripts/divide_and_merge_profiles.py index c75e353a..c75e353a 100755 --- a/v14/fdo_scripts/divide_and_merge_profiles.py +++ b/fdo_scripts/divide_and_merge_profiles.py diff --git a/v14/fdo_scripts/divide_and_merge_profiles_test.py b/fdo_scripts/divide_and_merge_profiles_test.py index f42db4e9..f42db4e9 100755 --- a/v14/fdo_scripts/divide_and_merge_profiles_test.py +++ b/fdo_scripts/divide_and_merge_profiles_test.py diff --git a/v14/fdo_scripts/profile_cycler.py b/fdo_scripts/profile_cycler.py index efdafffa..efdafffa 100755 --- a/v14/fdo_scripts/profile_cycler.py +++ b/fdo_scripts/profile_cycler.py diff --git a/v14/fdo_scripts/summarize_hot_blocks.py b/fdo_scripts/summarize_hot_blocks.py index c19f5477..c19f5477 100644 --- a/v14/fdo_scripts/summarize_hot_blocks.py +++ b/fdo_scripts/summarize_hot_blocks.py diff --git a/v14/fdo_scripts/vanilla_vs_fdo.py b/fdo_scripts/vanilla_vs_fdo.py index 3cf5f4b6..3cf5f4b6 100644 --- a/v14/fdo_scripts/vanilla_vs_fdo.py +++ b/fdo_scripts/vanilla_vs_fdo.py diff --git a/v14/get_common_image_version.py b/get_common_image_version.py index cdac8006..cdac8006 100755 --- a/v14/get_common_image_version.py +++ b/get_common_image_version.py diff --git a/image_chromeos.py b/image_chromeos.py index 30b29a45..d5d5f7fe 100755 --- a/image_chromeos.py +++ b/image_chromeos.py @@ -171,6 +171,7 @@ def DoImage(argv): options.remote) logger.GetLogger().LogFatalIf(not successfully_imaged, "Image verification failed!") + TryRemountPartitionAsRW(options.chromeos_root, options.remote) else: l.LogOutput("Checksums match. Skipping reimage") return retval @@ -268,6 +269,23 @@ def VerifyChromeChecksum(chromeos_root, image, remote): else: return False +# Remount partition as writable. +# TODO: auto-detect if an image is built using --noenable_rootfs_verification. +def TryRemountPartitionAsRW(chromeos_root, remote): + l = logger.GetLogger() + cmd_executer = command_executer.GetCommandExecuter() + command = "sudo mount -o remount,rw /" + retval = cmd_executer.CrosRunCommand(\ + command, chromeos_root=chromeos_root, machine=remote, terminated_timeout=10) + if retval: + ## Safely ignore. + l.LogWarning("Failed to remount partition as rw, " + "probably the image was not built with " + "\"--noenable_rootfs_verification\", " + "you can safely ignore this.") + else: + l.LogOutput("Re-mounted partition as writable.") + def EnsureMachineUp(chromeos_root, remote): l = logger.GetLogger() diff --git a/v14/lock_machine.py b/lock_machine.py index 0f948c3d..0f948c3d 100755 --- a/v14/lock_machine.py +++ b/lock_machine.py diff --git a/v14/lock_machine_test.py b/lock_machine_test.py index d61878b8..d61878b8 100644 --- a/v14/lock_machine_test.py +++ b/lock_machine_test.py diff --git a/v14/produce_output.py b/produce_output.py index 0e982bf7..0e982bf7 100755 --- a/v14/produce_output.py +++ b/produce_output.py diff --git a/v14/remote_gcc_build.py b/remote_gcc_build.py index b2dfee38..b2dfee38 100755 --- a/v14/remote_gcc_build.py +++ b/remote_gcc_build.py diff --git a/v14/remote_kill_test.py b/remote_kill_test.py index e56ef209..e56ef209 100755 --- a/v14/remote_kill_test.py +++ b/remote_kill_test.py diff --git a/v14/remote_test.py b/remote_test.py index 2ef27f9d..2ef27f9d 100755 --- a/v14/remote_test.py +++ b/remote_test.py diff --git a/v14/repo_to_repo.py b/repo_to_repo.py index 88433dfe..88433dfe 100755 --- a/v14/repo_to_repo.py +++ b/repo_to_repo.py diff --git a/v14/repo_to_repo_files/binutils-master.rtr b/repo_to_repo_files/binutils-master.rtr index 7cb31941..7cb31941 100644 --- a/v14/repo_to_repo_files/binutils-master.rtr +++ b/repo_to_repo_files/binutils-master.rtr diff --git a/v14/repo_to_repo_files/binutils-mobile_toolchain_v16.rtr b/repo_to_repo_files/binutils-mobile_toolchain_v16.rtr index 51f73cb2..51f73cb2 100644 --- a/v14/repo_to_repo_files/binutils-mobile_toolchain_v16.rtr +++ b/repo_to_repo_files/binutils-mobile_toolchain_v16.rtr diff --git a/v14/repo_to_repo_files/crosperf.rtr b/repo_to_repo_files/crosperf.rtr index 0efb1cde..0efb1cde 100644 --- a/v14/repo_to_repo_files/crosperf.rtr +++ b/repo_to_repo_files/crosperf.rtr diff --git a/v14/repo_to_repo_files/gcc-branches_google_4_7.rtr b/repo_to_repo_files/gcc-branches_google_4_7.rtr index ec7fcf34..ec7fcf34 100644 --- a/v14/repo_to_repo_files/gcc-branches_google_4_7.rtr +++ b/repo_to_repo_files/gcc-branches_google_4_7.rtr diff --git a/v14/repo_to_repo_files/gcc-branches_google_main.rtr b/repo_to_repo_files/gcc-branches_google_main.rtr index 289f531b..289f531b 100644 --- a/v14/repo_to_repo_files/gcc-branches_google_main.rtr +++ b/repo_to_repo_files/gcc-branches_google_main.rtr diff --git a/v14/repo_to_repo_files/gcc-master.rtr b/repo_to_repo_files/gcc-master.rtr index 82a91c01..82a91c01 100644 --- a/v14/repo_to_repo_files/gcc-master.rtr +++ b/repo_to_repo_files/gcc-master.rtr diff --git a/v14/repo_to_repo_files/gdb-master.rtr b/repo_to_repo_files/gdb-master.rtr index c29e36e0..c29e36e0 100644 --- a/v14/repo_to_repo_files/gdb-master.rtr +++ b/repo_to_repo_files/gdb-master.rtr diff --git a/v14/repo_to_repo_files/toolchain-utils.rtr b/repo_to_repo_files/toolchain-utils.rtr index 09a74730..09a74730 100644 --- a/v14/repo_to_repo_files/toolchain-utils.rtr +++ b/repo_to_repo_files/toolchain-utils.rtr diff --git a/v14/report_generator.py b/report_generator.py index a5f11ca6..a5f11ca6 100755 --- a/v14/report_generator.py +++ b/report_generator.py diff --git a/v14/run_benchmarks.py b/run_benchmarks.py index 9dc49004..9dc49004 100755 --- a/v14/run_benchmarks.py +++ b/run_benchmarks.py diff --git a/v14/run_tests.py b/run_tests.py index 075f29b1..075f29b1 100755 --- a/v14/run_tests.py +++ b/run_tests.py diff --git a/v14/setup_chromeos.py b/setup_chromeos.py index 7d3c22da..7d3c22da 100755 --- a/v14/setup_chromeos.py +++ b/setup_chromeos.py diff --git a/v14/sheriff_rotation.py b/sheriff_rotation.py index 4ccf896d..4ccf896d 100755 --- a/v14/sheriff_rotation.py +++ b/sheriff_rotation.py diff --git a/v14/summarize_results.py b/summarize_results.py index 3e561f3b..3e561f3b 100755 --- a/v14/summarize_results.py +++ b/summarize_results.py diff --git a/v14/tc_enter_chroot.py b/tc_enter_chroot.py index 50eb312c..50eb312c 100755 --- a/v14/tc_enter_chroot.py +++ b/tc_enter_chroot.py diff --git a/v14/test_gcc_dejagnu.py b/test_gcc_dejagnu.py index 0edf2cb9..0edf2cb9 100755 --- a/v14/test_gcc_dejagnu.py +++ b/test_gcc_dejagnu.py diff --git a/v14/test_toolchains.py b/test_toolchains.py index 18448ab2..18448ab2 100644 --- a/v14/test_toolchains.py +++ b/test_toolchains.py diff --git a/v14/utils/buildbot_json.py b/utils/buildbot_json.py index 6ea7f8fb..6ea7f8fb 100755 --- a/v14/utils/buildbot_json.py +++ b/utils/buildbot_json.py diff --git a/utils/command_executer.py b/utils/command_executer.py index 3edb6262..0aedc47c 100644 --- a/utils/command_executer.py +++ b/utils/command_executer.py @@ -74,11 +74,14 @@ class CommandExecuter: out = err = None pipes = [p.stdout, p.stderr] + my_poll = select.poll() + my_poll.register(p.stdout, select.POLLIN) + my_poll.register(p.stderr, select.POLLIN) + terminated_time = None started_time = time.time() while len(pipes): - fds = select.select(pipes, [], [], 0.1) if command_terminator and command_terminator.IsTerminated(): self.RunCommand("sudo kill -9 " + str(p.pid), print_to_console=print_to_console) @@ -88,21 +91,25 @@ class CommandExecuter: return (p.wait, full_stdout, full_stderr) else: return wait - for fd in fds[0]: - if fd == p.stdout: + + l=my_poll.poll(100) + for (fd, evt) in l: + if fd == p.stdout.fileno(): out = os.read(p.stdout.fileno(), 16384) if return_output: full_stdout += out self.logger.LogCommandOutput(out, print_to_console) if out == "": pipes.remove(p.stdout) - if fd == p.stderr: + my_poll.unregister(p.stdout) + if fd == p.stderr.fileno(): err = os.read(p.stderr.fileno(), 16384) if return_output: full_stderr += err self.logger.LogCommandError(err, print_to_console) if err == "": pipes.remove(p.stderr) + my_poll.unregister(p.stderr) if p.poll() is not None: if terminated_time is None: diff --git a/v14/utils/command_executer_unittest.py b/utils/command_executer_unittest.py index ea0e918c..ea0e918c 100755 --- a/v14/utils/command_executer_unittest.py +++ b/utils/command_executer_unittest.py diff --git a/utils/logger.py b/utils/logger.py index faad9666..99c679ab 100644 --- a/utils/logger.py +++ b/utils/logger.py @@ -167,17 +167,19 @@ class Logger(object): main_logger = None -def InitLogger(script_name, print_console=True): +def InitLogger(script_name, log_dir, print_console=True): """Initialize a global logger. To be called only once.""" global main_logger assert not main_logger, "The logger has already been initialized" rootdir, basefilename = GetRoot(script_name) - main_logger = Logger(rootdir, basefilename, print_console) + if not log_dir: + log_dir = rootdir + main_logger = Logger(log_dir, basefilename, print_console) -def GetLogger(): +def GetLogger(log_dir=""): if not main_logger: - InitLogger(sys.argv[0]) + InitLogger(sys.argv[0], log_dir) return main_logger diff --git a/utils/misc.py b/utils/misc.py index 40622512..175a6e07 100644 --- a/utils/misc.py +++ b/utils/misc.py @@ -145,19 +145,25 @@ def DoesLabelExist(chromeos_root, board, label): return os.path.exists(image_label) -def GetBuildPackagesCommand(board, usepkg=False): +def GetBuildPackagesCommand(board, usepkg=False, debug=False): if usepkg: usepkg_flag = "--usepkg" else: usepkg_flag = "--nousepkg" + if debug: + withdebug_flag = '--withdebug' + else: + withdebug_flag = '--nowithdebug' return ("./build_packages %s --withdev --withtest --withautotest " - "--skip_toolchain_update --nowithdebug --board=%s" % - (usepkg_flag, board)) - + "--skip_toolchain_update %s --board=%s" % + (usepkg_flag, withdebug_flag, board)) -def GetBuildImageCommand(board): - return "./build_image --board=%s test" % board +def GetBuildImageCommand(board, dev=False): + dev_args = "" + if dev: + dev_args = "--noenable_rootfs_verification --disk_layout=2gb-rootfs" + return "./build_image --board=%s %s test" % (board, dev_args) def GetSetupBoardCommand(board, gcc_version=None, binutils_version=None, usepkg=None, force=None): @@ -252,14 +258,17 @@ def AcquireLock(lock_file, timeout=1200): abs_path = os.path.abspath(lock_file) dir_path = os.path.dirname(abs_path) sleep_time = min(10, timeout/10.0) + reason = "pid: {0}, commad: {1}".format(os.getpid(), + sys.argv[0]) if not os.path.exists(dir_path): try: - os.makedirs(dir_path) + with lock_machine.FileCreationMask(0002): + os.makedirs(dir_path) except OSError: print "Cannot create dir {0}, exiting...".format(dir_path) exit(0) while True: - locked = (lock_machine.Lock(lock_file).NonBlockingLock(True, sys.argv[0])) + locked = (lock_machine.Lock(lock_file).NonBlockingLock(True, reason)) if locked: break time.sleep(sleep_time) diff --git a/utils/perf_diff.py b/utils/perf_diff.py index cfe90a9b..d56eba16 100755 --- a/utils/perf_diff.py +++ b/utils/perf_diff.py @@ -15,16 +15,20 @@ import sys import misc import tabulator +ROWS_TO_SHOW = "Rows_to_show_in_the_perf_table" -def GetPerfDictFromReport(report_file, num_functions=5): +def GetPerfDictFromReport(report_file): output = {} perf_report = PerfReport(report_file) for k, v in perf_report.sections.items(): if k not in output: output[k] = {} - for function in v.functions[:num_functions]: + output[k][ROWS_TO_SHOW] = 0 + for function in v.functions: out_key = "%s" % (function.name) output[k][out_key] = function.count + if function.percent > 1: + output[k][ROWS_TO_SHOW] += 1 return output @@ -85,6 +89,7 @@ class Function(object): def __init__(self): self.count = 0 self.name = "" + self.percent = 0 class Section(object): @@ -110,6 +115,7 @@ class Section(object): if not line.startswith("#"): fields = [f for f in line.split(" ") if f] function = Function() + function.percent = float(fields[0].strip("%")) function.count = int(fields[1]) function.name = " ".join(fields[2:]) self.functions.append(function) @@ -164,7 +170,7 @@ class PerfReport(object): def _SplitSections(self): self._section_contents = [] - indices = [m.start() for m in re.finditer("Events:", self._perf_contents)] + indices = [m.start() for m in re.finditer("# Events:", self._perf_contents)] indices.append(len(self._perf_contents)) for i in range(len(indices) - 1): section_content = self._perf_contents[indices[i]:indices[i+1]] diff --git a/utils/tabulator.py b/utils/tabulator.py index ed83a04b..f75419ab 100644 --- a/utils/tabulator.py +++ b/utils/tabulator.py @@ -2,8 +2,69 @@ # Copyright 2011 Google Inc. All Rights Reserved. +"""Table generating, analyzing and printing functions. + +This defines several classes that are used to generate, analyze and print +tables. + +Example usage: + + from utils import tabulator + + data = [["benchmark1", "33", "44"],["benchmark2", "44", "33"]] + tabulator.GetSimpleTable(data) + +You could also use it to generate more complex tables with analysis such as +p-values, custom colors, etc. Tables are generated by TableGenerator and +analyzed/formatted by TableFormatter. TableFormatter can take in a list of +columns with custom result computation and coloring, and will compare values in +each row according to taht scheme. Here is a complex example on printing a +table: + + from utils import tabulator + + runs = [[{"k1": "10", "k2": "12", "k5": "40", "k6": "40", + "ms_1": "20", "k7": "FAIL", "k8": "PASS", "k9": "PASS", + "k10": "0"}, + {"k1": "13", "k2": "14", "k3": "15", "ms_1": "10", "k8": "PASS", + "k9": "FAIL", "k10": "0"}], + [{"k1": "50", "k2": "51", "k3": "52", "k4": "53", "k5": "35", "k6": + "45", "ms_1": "200", "ms_2": "20", "k7": "FAIL", "k8": "PASS", "k9": + "PASS"}]] + labels = ["vanilla", "modified"] + tg = TableGenerator(runs, labels, TableGenerator.SORT_BY_VALUES_DESC) + table = tg.GetTable() + columns = [Column(LiteralResult(), + Format(), + "Literal"), + Column(AmeanResult(), + Format()), + Column(StdResult(), + Format()), + Column(CoeffVarResult(), + CoeffVarFormat()), + Column(NonEmptyCountResult(), + Format()), + Column(AmeanRatioResult(), + PercentFormat()), + Column(AmeanRatioResult(), + RatioFormat()), + Column(GmeanRatioResult(), + RatioFormat()), + Column(PValueResult(), + PValueFormat()), + ] + tf = TableFormatter(table, columns) + cell_table = tf.GetCellTable() + tp = TablePrinter(cell_table, out_to) + print tp.Print() + +""" + + import getpass import math +import sys import numpy import colortrans @@ -58,8 +119,10 @@ class TableGenerator(object): for run in run_list: if key in run: values.append(run[key]) - ret = max(_StripNone(values)) - return ret + values = _StripNone(values) + if _AllFloat(values): + values = _GetFloats(values) + return max(values) def _GetLowestValue(self, key): values = [] @@ -67,15 +130,19 @@ class TableGenerator(object): for run in run_list: if key in run: values.append(run[key]) - ret = min(_StripNone(values)) - return ret + values = _StripNone(values) + if _AllFloat(values): + values = _GetFloats(values) + return min(values) def _SortKeys(self, keys): if self._sort == self.SORT_BY_KEYS: return sorted(keys) elif self._sort == self.SORT_BY_VALUES: + # pylint: disable=unnecessary-lambda return sorted(keys, key=lambda x: self._GetLowestValue(x)) elif self._sort == self.SORT_BY_VALUES_DESC: + # pylint: disable=unnecessary-lambda return sorted(keys, key=lambda x: self._GetHighestValue(x), reverse=True) else: assert 0, "Unimplemented sort %s" % self._sort @@ -84,7 +151,7 @@ class TableGenerator(object): keys = self._AggregateKeys() return self._SortKeys(keys) - def GetTable(self): + def GetTable(self, number_of_rows=sys.maxint): """Returns a table from a list of list of dicts. The list of list of dicts is passed into the constructor of TableGenerator. @@ -92,7 +159,7 @@ class TableGenerator(object): table of values. Args: - None + number_of_rows: Maximum number of rows to return from the table. Returns: A list of lists which is the table. @@ -113,6 +180,7 @@ class TableGenerator(object): keys = self._GetKeys() header = [self._key_name] + self._labels table = [header] + rows = 0 for k in keys: row = [k] for run_list in self._runs: @@ -124,6 +192,9 @@ class TableGenerator(object): v.append(None) row.append(v) table.append(row) + rows += 1 + if rows == number_of_rows: + break return table @@ -144,6 +215,7 @@ class Result(object): def NeedsBaseline(self): return False + # pylint: disable=unused-argument def _Literal(self, cell, values, baseline_values): cell.value = " ".join([str(v) for v in values]) @@ -178,7 +250,7 @@ class Result(object): """ all_floats = True values = _StripNone(values) - if not len(values): + if not values: cell.value = "" return if _AllFloat(values): @@ -211,16 +283,25 @@ class LiteralResult(Result): def Compute(self, cell, values, baseline_values): try: - if values[self.iteration]: - cell.value = values[self.iteration] - else: - cell.value = "-" + cell.value = values[self.iteration] except IndexError: cell.value = "-" class NonEmptyCountResult(Result): + """A class that counts the number of non-empty results. + + The number of non-empty values will be stored in the cell. + """ + def Compute(self, cell, values, baseline_values): + """Put the number of non-empty values in the cell result. + + Args: + cell: Put the result in cell.value. + values: A list of values for the row. + baseline_values: A list of baseline values for the row. + """ cell.value = len(_StripNone(values)) if not baseline_values: return @@ -231,8 +312,8 @@ class NonEmptyCountResult(Result): len_values = len(values) len_baseline_values = len(baseline_values) tmp_cell = Cell() - tmp_cell.value= 1.0 + (float(cell.value - base_value) / - (max(len_values, len_baseline_values))) + tmp_cell.value = 1.0 + (float(cell.value - base_value) / + (max(len_values, len_baseline_values))) f.Compute(tmp_cell) cell.bgcolor = tmp_cell.bgcolor @@ -244,6 +325,7 @@ class StringMeanResult(Result): else: cell.value = "?" + class AmeanResult(StringMeanResult): def _ComputeFloat(self, cell, values, baseline_values): cell.value = numpy.mean(values) @@ -316,7 +398,7 @@ class PValueResult(ComparisonResult): if len(values) < 2 or len(baseline_values) < 2: cell.value = float("nan") return - import stats + import stats # pylint: disable=g-import-not-at-top _, cell.value = stats.lttest_ind(values, baseline_values) def _ComputeString(self, cell, values, baseline_values): @@ -462,12 +544,11 @@ class PValueFormat(Format): cell.string_value = "%0.2f" % float(cell.value) if float(cell.value) < 0.05: cell.bgcolor = self._GetColor(cell.value, - Color(255, 255, 0, 0), - Color(255, 255, 255, 0), - Color(255, 255, 255, 0), - mid_value=0.05, - power=1) - cell.bgcolor_row = True + Color(255, 255, 0, 0), + Color(255, 255, 255, 0), + Color(255, 255, 255, 0), + mid_value=0.05, + power=1) class StorageFormat(Format): @@ -802,6 +883,7 @@ class TablePrinter(object): if self._output_type == self.CONSOLE: rgb = color.GetRGB() prefix, _ = colortrans.rgb2short(rgb) + # pylint: disable=anomalous-backslash-in-string prefix = "\033[48;5;%sm" % prefix suffix = "\033[0m" elif self._output_type in [self.EMAIL, self.HTML]: @@ -818,6 +900,7 @@ class TablePrinter(object): if self._output_type == self.CONSOLE: rgb = color.GetRGB() prefix, _ = colortrans.rgb2short(rgb) + # pylint: disable=anomalous-backslash-in-string prefix = "\033[38;5;%sm" % prefix suffix = "\033[0m" elif self._output_type in [self.EMAIL, self.HTML]: @@ -965,7 +1048,21 @@ def GetSimpleTable(table, out_to=TablePrinter.CONSOLE): return tp.Print() +# pylint: disable=redefined-outer-name def GetComplexTable(runs, labels, out_to=TablePrinter.CONSOLE): + """Prints a complex table. + + This can be used to generate a table with arithmetic mean, standard deviation, + coefficient of variation, p-values, etc. + + Args: + runs: A list of lists with data to tabulate. + labels: A list of labels that correspond to the runs. + out_to: specifies the format of the table (example CONSOLE or HTML). + + Returns: + A string table that can be printed to the console or put in an HTML file. + """ tg = TableGenerator(runs, labels, TableGenerator.SORT_BY_VALUES_DESC) table = tg.GetTable() columns = [Column(LiteralResult(), @@ -995,33 +1092,21 @@ def GetComplexTable(runs, labels, out_to=TablePrinter.CONSOLE): if __name__ == "__main__": # Run a few small tests here. - runs = [ - [ - {"k1": "10", "k2": "12", "k5": "40", "k6": "40", - "ms_1": "20", "k7": "FAIL", "k8": "PASS", "k9": "PASS", - "k10": "0"}, - {"k1": "13", "k2": "14", "k3": "15", "ms_1": "10", "k8": "PASS", - "k9": "FAIL", "k10": "0"} - ], - [ - {"k1": "50", "k2": "51", "k3": "52", "k4": "53", "k5": "35", "k6": - "45", "ms_1": "200", "ms_2": "20", "k7": "FAIL", "k8": "PASS", "k9": - "PASS"}, - ], - ] + runs = [[{"k1": "10", "k2": "12", "k5": "40", "k6": "40", + "ms_1": "20", "k7": "FAIL", "k8": "PASS", "k9": "PASS", + "k10": "0"}, + {"k1": "13", "k2": "14", "k3": "15", "ms_1": "10", "k8": "PASS", + "k9": "FAIL", "k10": "0"}], + [{"k1": "50", "k2": "51", "k3": "52", "k4": "53", "k5": "35", "k6": + "45", "ms_1": "200", "ms_2": "20", "k7": "FAIL", "k8": "PASS", "k9": + "PASS"}]] labels = ["vanilla", "modified"] t = GetComplexTable(runs, labels, TablePrinter.CONSOLE) print t email = GetComplexTable(runs, labels, TablePrinter.EMAIL) - runs = [ - [ - {"k1": "1",}, {"k1": "1.1"}, {"k1": "1.2"}, - ], - [ - {"k1": "5",}, {"k1": "5.1"}, {"k1": "5.2"}, - ], - ] + runs = [[{"k1": "1"}, {"k1": "1.1"}, {"k1": "1.2"}], + [{"k1": "5"}, {"k1": "5.1"}, {"k1": "5.2"}]] t = GetComplexTable(runs, labels, TablePrinter.CONSOLE) print t diff --git a/utils/tabulator_test.py b/utils/tabulator_test.py index 54b455d3..16406107 100644 --- a/utils/tabulator_test.py +++ b/utils/tabulator_test.py @@ -1,6 +1,6 @@ # Copyright 2012 Google Inc. All Rights Reserved. -"""Tests for misc.""" +"""Tests for the tabulator module.""" __author__ = "asharif@google.com (Ahmad Sharif)" @@ -69,7 +69,7 @@ class TabulatorTest(unittest.TestCase): def testTableGenerator(self): runs = [[{"k1": "10", "k2": "12"}, {"k1": "13", "k2": "14", "k3": "15"}], - [{"k1": "50", "k2": "51", "k3": "52", "k4": "53"},]] + [{"k1": "50", "k2": "51", "k3": "52", "k4": "53"}]] labels = ["vanilla", "modified"] tg = tabulator.TableGenerator(runs, labels) table = tg.GetTable() diff --git a/v14/README b/v14/README deleted file mode 100644 index 3ab3d7b7..00000000 --- a/v14/README +++ /dev/null @@ -1,10 +0,0 @@ -To run scripts in this directory, first run: - -export PYTHONPATH=$(readlink -f .):$PYTHONPATH - -from this directory. - -Then you can run any script. - -To get help on any script, type in python <script> --help, or refer to the -header of the script for more information. diff --git a/v14/crosperf/autotest_runner.py b/v14/crosperf/autotest_runner.py deleted file mode 100644 index 5611b655..00000000 --- a/v14/crosperf/autotest_runner.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/python -# -# Copyright 2011 Google Inc. All Rights Reserved. - -from utils import command_executer - - -class AutotestRunner(object): - """ This defines the interface from crosperf to ./run_remote_tests.sh. - """ - def __init__(self, logger_to_use=None): - self._logger = logger_to_use - self._ce = command_executer.GetCommandExecuter(self._logger) - self._ct = command_executer.CommandTerminator() - - def Run(self, machine_name, chromeos_root, board, autotest_name, - autotest_args): - """Run the run_remote_test.""" - options = "" - if board: - options += " --board=%s" % board - if autotest_args: - options += " %s" % autotest_args - command = "rm -rf /usr/local/autotest/results/*" - self._ce.CrosRunCommand(command, machine=machine_name, username="root", - chromeos_root=chromeos_root) - command = ("./run_remote_tests.sh --remote=%s %s %s" % - (machine_name, options, autotest_name)) - return self._ce.ChrootRunCommand(chromeos_root, command, True, self._ct) - - def Terminate(self): - self._ct.Terminate() - - -class MockAutotestRunner(object): - def __init__(self): - pass - - def Run(self, *args): - return ["", "", 0] diff --git a/v14/crosperf/benchmark.py b/v14/crosperf/benchmark.py deleted file mode 100644 index 8fe8a492..00000000 --- a/v14/crosperf/benchmark.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/python - -# Copyright 2011 Google Inc. All Rights Reserved. - - -class Benchmark(object): - """Class representing a benchmark to be run. - - Contains details of the autotest, arguments to pass to the autotest, - iterations to run the autotest and so on. Note that the benchmark name - can be different to the autotest name. For example, you may want to have - two different benchmarks which run the same autotest with different - arguments. - """ - - def __init__(self, name, autotest_name, autotest_args, iterations, - outlier_range, key_results_only, rm_chroot_tmp, perf_args): - self.name = name - self.autotest_name = autotest_name - self.autotest_args = autotest_args - self.iterations = iterations - self.outlier_range = outlier_range - self.perf_args = perf_args - self.key_results_only = key_results_only - self.rm_chroot_tmp = rm_chroot_tmp - self.iteration_adjusted = False diff --git a/v14/crosperf/benchmark_run.py b/v14/crosperf/benchmark_run.py deleted file mode 100644 index 80c95c4d..00000000 --- a/v14/crosperf/benchmark_run.py +++ /dev/null @@ -1,198 +0,0 @@ -#!/usr/bin/python - -# Copyright 2011 Google Inc. All Rights Reserved. - -import datetime -import os -import threading -import time -import traceback - -from utils import command_executer -from utils import timeline - -from autotest_runner import AutotestRunner -from results_cache import Result -from results_cache import ResultsCache - -STATUS_FAILED = "FAILED" -STATUS_SUCCEEDED = "SUCCEEDED" -STATUS_IMAGING = "IMAGING" -STATUS_RUNNING = "RUNNING" -STATUS_WAITING = "WAITING" -STATUS_PENDING = "PENDING" - - -class BenchmarkRun(threading.Thread): - def __init__(self, name, benchmark, - label, - iteration, - cache_conditions, - machine_manager, - logger_to_use, - share_users): - threading.Thread.__init__(self) - self.name = name - self._logger = logger_to_use - self.benchmark = benchmark - self.iteration = iteration - self.label = label - self.result = None - self.terminated = False - self.retval = None - self.run_completed = False - self.machine_manager = machine_manager - self.cache = ResultsCache() - self.autotest_runner = AutotestRunner(self._logger) - self.machine = None - self.cache_conditions = cache_conditions - self.runs_complete = 0 - self.cache_hit = False - self.failure_reason = "" - self.autotest_args = "%s %s" % (benchmark.autotest_args, - self._GetExtraAutotestArgs()) - self._ce = command_executer.GetCommandExecuter(self._logger) - self.timeline = timeline.Timeline() - self.timeline.Record(STATUS_PENDING) - self.share_users = share_users - - def run(self): - try: - # Just use the first machine for running the cached version, - # without locking it. - self.cache.Init(self.label.chromeos_image, - self.label.chromeos_root, - self.benchmark.autotest_name, - self.iteration, - self.autotest_args, - self.machine_manager, - self.label.board, - self.cache_conditions, - self._logger, - self.label, - self.share_users - ) - - self.result = self.cache.ReadResult() - self.cache_hit = (self.result is not None) - - if self.result: - self._logger.LogOutput("%s: Cache hit." % self.name) - self._logger.LogOutput(self.result.out, print_to_console=False) - self._logger.LogError(self.result.err, print_to_console=False) - - else: - self._logger.LogOutput("%s: No cache hit." % self.name) - self.timeline.Record(STATUS_WAITING) - # Try to acquire a machine now. - self.machine = self.AcquireMachine() - self.cache.remote = self.machine.name - self.result = self.RunTest(self.machine) - self.cache.StoreResult(self.result) - - if self.terminated: - return - - if not self.result.retval: - self.timeline.Record(STATUS_SUCCEEDED) - else: - if self.timeline.GetLastEvent() != STATUS_FAILED: - self.failure_reason = "Return value of autotest was non-zero." - self.timeline.Record(STATUS_FAILED) - - except Exception, e: - self._logger.LogError("Benchmark run: '%s' failed: %s" % (self.name, e)) - traceback.print_exc() - if self.timeline.GetLastEvent() != STATUS_FAILED: - self.timeline.Record(STATUS_FAILED) - self.failure_reason = str(e) - finally: - if self.machine: - self._logger.LogOutput("Releasing machine: %s" % self.machine.name) - self.machine_manager.ReleaseMachine(self.machine) - self._logger.LogOutput("Released machine: %s" % self.machine.name) - - def Terminate(self): - self.terminated = True - self.autotest_runner.Terminate() - if self.timeline.GetLastEvent() != STATUS_FAILED: - self.timeline.Record(STATUS_FAILED) - self.failure_reason = "Thread terminated." - - def AcquireMachine(self): - while True: - if self.terminated: - raise Exception("Thread terminated while trying to acquire machine.") - machine = self.machine_manager.AcquireMachine(self.label.chromeos_image, - self.label) - - if machine: - self._logger.LogOutput("%s: Machine %s acquired at %s" % - (self.name, - machine.name, - datetime.datetime.now())) - break - else: - sleep_duration = 10 - time.sleep(sleep_duration) - return machine - - def _GetExtraAutotestArgs(self): - if self.benchmark.perf_args: - perf_args_list = self.benchmark.perf_args.split(" ") - perf_args_list = [perf_args_list[0]] + ["-a"] + perf_args_list[1:] - perf_args = " ".join(perf_args_list) - if not perf_args_list[0] in ["record", "stat"]: - raise Exception("perf_args must start with either record or stat") - extra_autotest_args = ["--profiler=custom_perf", - ("--profiler_args='perf_options=\"%s\"'" % - perf_args)] - return " ".join(extra_autotest_args) - else: - return "" - - def RunTest(self, machine): - self.timeline.Record(STATUS_IMAGING) - self.machine_manager.ImageMachine(machine, - self.label) - self.timeline.Record(STATUS_RUNNING) - [retval, out, err] = self.autotest_runner.Run(machine.name, - self.label.chromeos_root, - self.label.board, - self.benchmark.autotest_name, - self.autotest_args) - self.run_completed = True - - return Result.CreateFromRun(self._logger, - self.label.chromeos_root, - self.label.board, - self.label.name, - out, - err, - retval) - - def SetCacheConditions(self, cache_conditions): - self.cache_conditions = cache_conditions - - -class MockBenchmarkRun(BenchmarkRun): - """Inherited from BenchmarkRun, just overide RunTest for testing.""" - - def RunTest(self, machine): - """Remove Result.CreateFromRun for testing.""" - self.timeline.Record(STATUS_IMAGING) - self.machine_manager.ImageMachine(machine, - self.label) - self.timeline.Record(STATUS_RUNNING) - [retval, out, err] = self.autotest_runner.Run(machine.name, - self.label.chromeos_root, - self.label.board, - self.benchmark.autotest_name, - self.autotest_args) - self.run_completed = True - rr = Result("Results placed in /tmp/test", "", 0) - rr.out = out - rr.err = err - rr.retval = retval - return rr - diff --git a/v14/crosperf/benchmark_run_unittest.py b/v14/crosperf/benchmark_run_unittest.py deleted file mode 100755 index 47e027f4..00000000 --- a/v14/crosperf/benchmark_run_unittest.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/python - -# Copyright 2011 Google Inc. All Rights Reserved. - -"""Testing of benchmark_run.""" - -import unittest - -from utils import logger - -from autotest_runner import MockAutotestRunner -from benchmark_run import MockBenchmarkRun -from label import MockLabel -from benchmark import Benchmark -from machine_manager import MockMachineManager -from results_cache import MockResultsCache - - -class BenchmarkRunTest(unittest.TestCase): - def testDryRun(self): - my_label = MockLabel("test1", "image1", "/tmp/test_benchmark_run", - "x86-alex", "chromeos-alex1", - image_args="", - image_md5sum="", - cache_dir="") - m = MockMachineManager("/tmp/chromeos_root", 0) - m.AddMachine("chromeos-alex1") - bench = Benchmark("PageCyler", - "Pyautoperf", - "", - 1, - 0.2, - False, - False, - "") - b = MockBenchmarkRun("test run", - bench, - my_label, - 1, - [], - m, - logger.GetLogger(), - "") - b.cache = MockResultsCache() - b.autotest_runner = MockAutotestRunner() - b.start() - - -if __name__ == "__main__": - unittest.main() diff --git a/v14/crosperf/column_chart.py b/v14/crosperf/column_chart.py deleted file mode 100644 index 22a45c5b..00000000 --- a/v14/crosperf/column_chart.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/python - -# Copyright 2011 Google Inc. All Rights Reserved. - - -class ColumnChart(object): - def __init__(self, title, width, height): - self.title = title - self.chart_div = filter(str.isalnum, title) - self.width = width - self.height = height - self.columns = [] - self.rows = [] - self.series = [] - - def AddSeries(self, column_name, series_type, color): - for i in range(len(self.columns)): - if column_name == self.columns[i][1]: - self.series.append((i - 1, series_type, color)) - break - - def AddColumn(self, name, column_type): - self.columns.append((column_type, name)) - - def AddRow(self, row): - self.rows.append(row) - - def GetJavascript(self): - res = "var data = new google.visualization.DataTable();\n" - for column in self.columns: - res += "data.addColumn('%s', '%s');\n" % column - res += "data.addRows(%s);\n" % len(self.rows) - for row in range(len(self.rows)): - for column in range(len(self.columns)): - val = self.rows[row][column] - if isinstance(val, str): - val = "'%s'" % val - res += "data.setValue(%s, %s, %s);\n" % (row, column, val) - - series_javascript = "" - for series in self.series: - series_javascript += "%s: {type: '%s', color: '%s'}, " % series - - chart_add_javascript = """ -var chart_%s = new google.visualization.ComboChart( - document.getElementById('%s')); -chart_%s.draw(data, {width: %s, height: %s, title: '%s', legend: 'none', - seriesType: "bars", lineWidth: 0, pointSize: 5, series: {%s}, - vAxis: {minValue: 0}}) -""" - res += chart_add_javascript % (self.chart_div, self.chart_div, - self.chart_div, self.width, - self.height, self.title, series_javascript) - return res - - def GetDiv(self): - return "<div id='%s' class='chart'></div>" % self.chart_div diff --git a/v14/crosperf/crosperf b/v14/crosperf/crosperf deleted file mode 100755 index 904a172a..00000000 --- a/v14/crosperf/crosperf +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -PYTHONPATH=$(dirname $0)/..:$PYTHONPATH python $(dirname $0)/crosperf.py "$@" diff --git a/v14/crosperf/crosperf.py b/v14/crosperf/crosperf.py deleted file mode 100755 index cb7911fd..00000000 --- a/v14/crosperf/crosperf.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/python - -# Copyright 2011 Google Inc. All Rights Reserved. - -"""The driver script for running performance benchmarks on ChromeOS.""" - -import atexit -import optparse -import os -import sys -from experiment_runner import ExperimentRunner -from experiment_runner import MockExperimentRunner -from experiment_factory import ExperimentFactory -from experiment_file import ExperimentFile -from help import Help -from settings_factory import GlobalSettings -from utils import logger - -import test_flag - - -class MyIndentedHelpFormatter(optparse.IndentedHelpFormatter): - def format_description(self, description): - return description - - -def SetupParserOptions(parser): - """Add all options to the parser.""" - parser.add_option("--dry_run", - dest="dry_run", - help=("Parse the experiment file and " - "show what will be done"), - action="store_true", - default=False) - # Allow each of the global fields to be overridden by passing in - # options. Add each global field as an option. - option_settings = GlobalSettings("") - for field_name in option_settings.fields: - field = option_settings.fields[field_name] - parser.add_option("--%s" % field.name, - dest=field.name, - help=field.description, - action="store") - - -def ConvertOptionsToSettings(options): - """Convert options passed in into global settings.""" - option_settings = GlobalSettings("option_settings") - for option_name in options.__dict__: - if (options.__dict__[option_name] is not None and - option_name in option_settings.fields): - option_settings.SetField(option_name, options.__dict__[option_name]) - return option_settings - - -def Cleanup(experiment): - """Handler function which is registered to the atexit handler.""" - experiment.Cleanup() - - -def Main(argv): - parser = optparse.OptionParser(usage=Help().GetUsage(), - description=Help().GetHelp(), - formatter=MyIndentedHelpFormatter(), - version="%prog 0.1") - - parser.add_option("-l", "--log_dir", - dest="log_dir", - default="", - help="The log_dir, default is under <crosperf_logs>/logs") - - SetupParserOptions(parser) - options, args = parser.parse_args(argv) - - # Convert the relevant options that are passed in into a settings - # object which will override settings in the experiment file. - option_settings = ConvertOptionsToSettings(options) - log_dir = os.path.abspath(os.path.expanduser(options.log_dir)) - logger.GetLogger(log_dir) - - if len(args) == 2: - experiment_filename = args[1] - else: - parser.error("Invalid number arguments.") - - working_directory = os.getcwd() - if options.dry_run: - test_flag.SetTestMode(True) - - experiment_file = ExperimentFile(open(experiment_filename, "rb"), - option_settings) - if not experiment_file.GetGlobalSettings().GetField("name"): - experiment_name = os.path.basename(experiment_filename) - experiment_file.GetGlobalSettings().SetField("name", experiment_name) - experiment = ExperimentFactory().GetExperiment(experiment_file, - working_directory, - log_dir) - - atexit.register(Cleanup, experiment) - - if options.dry_run: - runner = MockExperimentRunner(experiment) - else: - runner = ExperimentRunner(experiment) - runner.Run() - -if __name__ == "__main__": - Main(sys.argv) diff --git a/v14/crosperf/crosperf_test.py b/v14/crosperf/crosperf_test.py deleted file mode 100755 index 0c50e7b5..00000000 --- a/v14/crosperf/crosperf_test.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/python - -# Copyright 2011 Google Inc. All Rights Reserved. - -import os -import tempfile -import unittest -import crosperf -from utils.file_utils import FileUtils - - -EXPERIMENT_FILE_1 = """ - board: x86-alex - remote: chromeos-alex3 - - benchmark: PageCycler { - iterations: 3 - } - - image1 { - chromeos_image: /usr/local/google/cros_image1.bin - } - - image2 { - chromeos_image: /usr/local/google/cros_image2.bin - } - """ - - -class CrosPerfTest(unittest.TestCase): - def testDryRun(self): - filehandle, filename = tempfile.mkstemp() - os.write(filehandle, EXPERIMENT_FILE_1) - crosperf.Main(["", filename, "--dry_run"]) - os.remove(filename) - - -if __name__ == "__main__": - FileUtils.Configure(True) - unittest.main() diff --git a/v14/crosperf/default_remotes b/v14/crosperf/default_remotes deleted file mode 100644 index 5efaebcd..00000000 --- a/v14/crosperf/default_remotes +++ /dev/null @@ -1,5 +0,0 @@ -x86-zgb : chromeos1-rack3-host4.cros chromeos1-rack3-host5.cros chromeos1-rack3-host6.cros -x86-alex : chromeos2-row1-rack4-host7.cros chromeos2-row1-rack4-host8.cros chromeos2-row1-rack4-host9.cros -lumpy : chromeos2-row2-rack4-host10.cros chromeos2-row2-rack4-host11.cros chromeos2-row2-rack4-host12.cros -stumpy : chromeos2-row3-rack7-host1.cros chromeos2-row3-rack7-host2.cros chromeos2-row3-rack7-host3.cros - diff --git a/v14/crosperf/experiment.py b/v14/crosperf/experiment.py deleted file mode 100644 index 2a4590c4..00000000 --- a/v14/crosperf/experiment.py +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/python - -# Copyright 2011 Google Inc. All Rights Reserved. - -"""The experiment setting module.""" - -import os -import time - -from utils import logger - -from benchmark_run import BenchmarkRun -from machine_manager import MachineManager -from machine_manager import MockMachineManager -import test_flag - - -class Experiment(object): - """Class representing an Experiment to be run.""" - - def __init__(self, name, remote, working_directory, - chromeos_root, cache_conditions, labels, benchmarks, - experiment_file, email_to, acquire_timeout, log_dir, - share_users): - self.name = name - self.working_directory = working_directory - self.remote = remote - self.chromeos_root = chromeos_root - self.cache_conditions = cache_conditions - self.experiment_file = experiment_file - self.email_to = email_to - self.results_directory = os.path.join(self.working_directory, - self.name + "_results") - self.log_dir = log_dir - self.labels = labels - self.benchmarks = benchmarks - self.num_complete = 0 - self.num_run_complete = 0 - self.share_users = share_users - - # We need one chromeos_root to run the benchmarks in, but it doesn't - # matter where it is, unless the ABIs are different. - if not chromeos_root: - for label in self.labels: - if label.chromeos_root: - chromeos_root = label.chromeos_root - if not chromeos_root: - raise Exception("No chromeos_root given and could not determine one from " - "the image path.") - - if test_flag.GetTestMode(): - self.machine_manager = MockMachineManager(chromeos_root, acquire_timeout) - else: - self.machine_manager = MachineManager(chromeos_root, acquire_timeout) - self.l = logger.GetLogger(log_dir) - - for machine in remote: - self.machine_manager.AddMachine(machine) - for label in labels: - self.machine_manager.ComputeCommonCheckSum(label) - self.machine_manager.ComputeCommonCheckSumString(label) - - self.start_time = None - self.benchmark_runs = self._GenerateBenchmarkRuns() - - def _GenerateBenchmarkRuns(self): - """Generate benchmark runs from labels and benchmark defintions.""" - benchmark_runs = [] - for label in self.labels: - for benchmark in self.benchmarks: - for iteration in range(1, benchmark.iterations + 1): - - benchmark_run_name = "%s: %s (%s)" % (label.name, benchmark.name, - iteration) - full_name = "%s_%s_%s" % (label.name, benchmark.name, iteration) - logger_to_use = logger.Logger(self.log_dir, - "run.%s" % (full_name), - True) - benchmark_run = BenchmarkRun(benchmark_run_name, - benchmark, - label, - iteration, - self.cache_conditions, - self.machine_manager, - logger_to_use, - self.share_users) - - benchmark_runs.append(benchmark_run) - return benchmark_runs - - def Build(self): - pass - - def Terminate(self): - for t in self.benchmark_runs: - if t.isAlive(): - self.l.LogError("Terminating run: '%s'." % t.name) - t.Terminate() - - def IsComplete(self): - if self.active_threads: - for t in self.active_threads: - if t.isAlive(): - t.join(0) - if not t.isAlive(): - self.num_complete += 1 - if not t.cache_hit: - self.num_run_complete += 1 - self.active_threads.remove(t) - return False - return True - - def Run(self): - self.start_time = time.time() - self.active_threads = [] - for benchmark_run in self.benchmark_runs: - # Set threads to daemon so program exits when ctrl-c is pressed. - benchmark_run.daemon = True - benchmark_run.start() - self.active_threads.append(benchmark_run) - - def SetCacheConditions(self, cache_conditions): - for benchmark_run in self.benchmark_runs: - benchmark_run.SetCacheConditions(cache_conditions) - - def Cleanup(self): - self.machine_manager.Cleanup() diff --git a/v14/crosperf/experiment_factory.py b/v14/crosperf/experiment_factory.py deleted file mode 100644 index 3c92ee3e..00000000 --- a/v14/crosperf/experiment_factory.py +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/python - -# Copyright 2011 Google Inc. All Rights Reserved. -"""A module to generate experments.""" - -import os -import socket - -from benchmark import Benchmark -import config -from experiment import Experiment -from label import Label -from label import MockLabel -from results_cache import CacheConditions -import test_flag - - -class ExperimentFactory(object): - """Factory class for building an Experiment, given an ExperimentFile as input. - - This factory is currently hardcoded to produce an experiment for running - ChromeOS benchmarks, but the idea is that in the future, other types - of experiments could be produced. - """ - - def GetExperiment(self, experiment_file, working_directory, log_dir): - """Construct an experiment from an experiment file.""" - global_settings = experiment_file.GetGlobalSettings() - experiment_name = global_settings.GetField("name") - remote = global_settings.GetField("remote") - chromeos_root = global_settings.GetField("chromeos_root") - rm_chroot_tmp = global_settings.GetField("rm_chroot_tmp") - key_results_only = global_settings.GetField("key_results_only") - acquire_timeout= global_settings.GetField("acquire_timeout") - cache_dir = global_settings.GetField("cache_dir") - config.AddConfig("no_email", global_settings.GetField("no_email")) - share_users = global_settings.GetField("share_users") - - # Default cache hit conditions. The image checksum in the cache and the - # computed checksum of the image must match. Also a cache file must exist. - cache_conditions = [CacheConditions.CACHE_FILE_EXISTS, - CacheConditions.CHECKSUMS_MATCH] - if global_settings.GetField("rerun_if_failed"): - cache_conditions.append(CacheConditions.RUN_SUCCEEDED) - if global_settings.GetField("rerun"): - cache_conditions.append(CacheConditions.FALSE) - if global_settings.GetField("same_machine"): - cache_conditions.append(CacheConditions.SAME_MACHINE_MATCH) - if global_settings.GetField("same_specs"): - cache_conditions.append(CacheConditions.MACHINES_MATCH) - - # Construct benchmarks. - benchmarks = [] - all_benchmark_settings = experiment_file.GetSettings("benchmark") - for benchmark_settings in all_benchmark_settings: - benchmark_name = benchmark_settings.name - autotest_name = benchmark_settings.GetField("autotest_name") - if not autotest_name: - autotest_name = benchmark_name - autotest_args = benchmark_settings.GetField("autotest_args") - iterations = benchmark_settings.GetField("iterations") - outlier_range = benchmark_settings.GetField("outlier_range") - perf_args = benchmark_settings.GetField("perf_args") - rm_chroot_tmp = benchmark_settings.GetField("rm_chroot_tmp") - key_results_only = benchmark_settings.GetField("key_results_only") - - benchmark = Benchmark(benchmark_name, autotest_name, autotest_args, - iterations, outlier_range, - key_results_only, rm_chroot_tmp, - perf_args) - benchmarks.append(benchmark) - - # Construct labels. - labels = [] - all_label_settings = experiment_file.GetSettings("label") - all_remote = list(remote) - for label_settings in all_label_settings: - label_name = label_settings.name - image = label_settings.GetField("chromeos_image") - chromeos_root = label_settings.GetField("chromeos_root") - board = label_settings.GetField("board") - my_remote = label_settings.GetField("remote") - image_md5sum = label_settings.GetField("md5sum") - cache_dir = label_settings.GetField("cache_dir") - # TODO(yunlian): We should consolidate code in machine_manager.py - # to derermine whether we are running from within google or not - if ("corp.google.com" in socket.gethostname() and - (not my_remote - or my_remote == remote - and global_settings.GetField("board") != board)): - my_remote = self.GetDefaultRemotes(board) - if global_settings.GetField("same_machine") and len(my_remote) > 1: - raise Exception("Only one remote is allowed when same_machine " - "is turned on") - all_remote += my_remote - image_args = label_settings.GetField("image_args") - if test_flag.GetTestMode(): - label = MockLabel(label_name, image, chromeos_root, board, my_remote, - image_args, image_md5sum, cache_dir) - else: - label = Label(label_name, image, chromeos_root, board, my_remote, - image_args, image_md5sum, cache_dir) - labels.append(label) - - email = global_settings.GetField("email") - all_remote = list(set(all_remote)) - experiment = Experiment(experiment_name, all_remote, - working_directory, chromeos_root, - cache_conditions, labels, benchmarks, - experiment_file.Canonicalize(), - email, acquire_timeout, log_dir, share_users) - - return experiment - - def GetDefaultRemotes(self, board): - default_remotes_file = os.path.join(os.path.dirname(__file__), - "default_remotes") - try: - with open(default_remotes_file) as f: - for line in f: - key, v = line.split(":") - if key.strip() == board: - remotes = v.strip().split(" ") - if remotes: - return remotes - else: - raise Exception("There is not remote for {0}".format(board)) - except IOError: - raise Exception("IOError while reading file {0}" - .format(default_remotes_file)) - else: - raise Exception("There is not remote for {0}".format(board)) - - - diff --git a/v14/crosperf/experiment_factory_unittest.py b/v14/crosperf/experiment_factory_unittest.py deleted file mode 100755 index 6cee6b74..00000000 --- a/v14/crosperf/experiment_factory_unittest.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/python - -# Copyright 2011 Google Inc. All Rights Reserved. - -import StringIO -import unittest - -from utils.file_utils import FileUtils - -from experiment_factory import ExperimentFactory -from experiment_file import ExperimentFile -import test_flag - -EXPERIMENT_FILE_1 = """ - board: x86-alex - remote: chromeos-alex3 - - benchmark: PageCycler { - iterations: 3 - } - - image1 { - chromeos_image: /usr/local/google/cros_image1.bin - } - - image2 { - chromeos_image: /usr/local/google/cros_image2.bin - } - """ - - -class ExperimentFactoryTest(unittest.TestCase): - def testLoadExperimentFile1(self): - experiment_file = ExperimentFile(StringIO.StringIO(EXPERIMENT_FILE_1)) - experiment = ExperimentFactory().GetExperiment(experiment_file, - working_directory="", - log_dir="") - self.assertEqual(experiment.remote, ["chromeos-alex3"]) - - self.assertEqual(len(experiment.benchmarks), 1) - self.assertEqual(experiment.benchmarks[0].name, "PageCycler") - self.assertEqual(experiment.benchmarks[0].autotest_name, "PageCycler") - self.assertEqual(experiment.benchmarks[0].iterations, 3) - - self.assertEqual(len(experiment.labels), 2) - self.assertEqual(experiment.labels[0].chromeos_image, - "/usr/local/google/cros_image1.bin") - self.assertEqual(experiment.labels[0].board, - "x86-alex") - - -if __name__ == "__main__": - FileUtils.Configure(True) - test_flag.SetTestMode(True) - unittest.main() diff --git a/v14/crosperf/experiment_file.py b/v14/crosperf/experiment_file.py deleted file mode 100644 index fc0f16ab..00000000 --- a/v14/crosperf/experiment_file.py +++ /dev/null @@ -1,185 +0,0 @@ -#!/usr/bin/python - -# Copyright (c) 2011 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import os.path -import re -from settings import Settings -from settings_factory import SettingsFactory - - -class ExperimentFile(object): - """Class for parsing the experiment file format. - - The grammar for this format is: - - experiment = { _FIELD_VALUE_RE | settings } - settings = _OPEN_SETTINGS_RE - { _FIELD_VALUE_RE } - _CLOSE_SETTINGS_RE - - Where the regexes are terminals defined below. This results in an format - which looks something like: - - field_name: value - settings_type: settings_name { - field_name: value - field_name: value - } - """ - - # Field regex, e.g. "iterations: 3" - _FIELD_VALUE_RE = re.compile("(\+)?\s*(\w+?)(?:\.(\S+))?\s*:\s*(.*)") - # Open settings regex, e.g. "label {" - _OPEN_SETTINGS_RE = re.compile("(?:(\w+):)?\s*(\w+)\s*{") - # Close settings regex. - _CLOSE_SETTINGS_RE = re.compile("}") - - def __init__(self, experiment_file, overrides=None): - """Construct object from file-like experiment_file. - - Args: - experiment_file: file-like object with text description of experiment. - overrides: A settings object that will override fields in other settings. - - Raises: - Exception: if invalid build type or description is invalid. - """ - self.all_settings = [] - self.global_settings = SettingsFactory().GetSettings("global", "global") - self.all_settings.append(self.global_settings) - - self._Parse(experiment_file) - - for settings in self.all_settings: - settings.Inherit() - settings.Validate() - if overrides: - settings.Override(overrides) - - def GetSettings(self, settings_type): - """Return nested fields from the experiment file.""" - res = [] - for settings in self.all_settings: - if settings.settings_type == settings_type: - res.append(settings) - return res - - def GetGlobalSettings(self): - """Return the global fields from the experiment file.""" - return self.global_settings - - def _ParseField(self, reader): - """Parse a key/value field.""" - line = reader.CurrentLine().strip() - match = ExperimentFile._FIELD_VALUE_RE.match(line) - append, name, _, text_value = match.groups() - return (name, text_value, append) - - def _ParseSettings(self, reader): - """Parse a settings block.""" - line = reader.CurrentLine().strip() - match = ExperimentFile._OPEN_SETTINGS_RE.match(line) - settings_type = match.group(1) - if settings_type is None: - settings_type = "" - settings_name = match.group(2) - settings = SettingsFactory().GetSettings(settings_name, settings_type) - settings.SetParentSettings(self.global_settings) - - while reader.NextLine(): - line = reader.CurrentLine().strip() - - if not line: - continue - elif ExperimentFile._FIELD_VALUE_RE.match(line): - field = self._ParseField(reader) - settings.SetField(field[0], field[1], field[2]) - elif ExperimentFile._CLOSE_SETTINGS_RE.match(line): - return settings - - raise Exception("Unexpected EOF while parsing settings block.") - - def _Parse(self, experiment_file): - """Parse experiment file and create settings.""" - reader = ExperimentFileReader(experiment_file) - settings_names = {} - try: - while reader.NextLine(): - line = reader.CurrentLine().strip() - - if not line: - continue - elif ExperimentFile._OPEN_SETTINGS_RE.match(line): - new_settings = self._ParseSettings(reader) - if new_settings.name in settings_names: - raise Exception("Duplicate settings name: '%s'." % - new_settings.name) - settings_names[new_settings.name] = True - self.all_settings.append(new_settings) - elif ExperimentFile._FIELD_VALUE_RE.match(line): - field = self._ParseField(reader) - self.global_settings.SetField(field[0], field[1], field[2]) - else: - raise Exception("Unexpected line.") - except Exception, err: - raise Exception("Line %d: %s\n==> %s" % (reader.LineNo(), str(err), - reader.CurrentLine(False))) - - def Canonicalize(self): - """Convert parsed experiment file back into an experiment file.""" - res = "" - for field_name in self.global_settings.fields: - field = self.global_settings.fields[field_name] - if field.assigned: - res += "%s: %s\n" % (field.name, field.GetString()) - res += "\n" - - for settings in self.all_settings: - if settings.settings_type != "global": - res += "%s: %s {\n" % (settings.settings_type, settings.name) - for field_name in settings.fields: - field = settings.fields[field_name] - if field.assigned: - res += "\t%s: %s\n" % (field.name, field.GetString()) - if field.name == "chromeos_image": - real_file = (os.path.realpath - (os.path.expanduser(field.GetString()))) - if real_file != field.GetString(): - res += "\t#actual_image: %s\n" % real_file - res += "}\n\n" - - return res - - -class ExperimentFileReader(object): - """Handle reading lines from an experiment file.""" - - def __init__(self, file_object): - self.file_object = file_object - self.current_line = None - self.current_line_no = 0 - - def CurrentLine(self, strip_comment=True): - """Return the next line from the file, without advancing the iterator.""" - if strip_comment: - return self._StripComment(self.current_line) - return self.current_line - - def NextLine(self, strip_comment=True): - """Advance the iterator and return the next line of the file.""" - self.current_line_no += 1 - self.current_line = self.file_object.readline() - return self.CurrentLine(strip_comment) - - def _StripComment(self, line): - """Strip comments starting with # from a line.""" - if "#" in line: - line = line[:line.find("#")] + line[-1] - return line - - def LineNo(self): - """Return the current line number.""" - return self.current_line_no diff --git a/v14/crosperf/experiment_file_unittest.py b/v14/crosperf/experiment_file_unittest.py deleted file mode 100755 index d08c7eb5..00000000 --- a/v14/crosperf/experiment_file_unittest.py +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/python - -# Copyright (c) 2011 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import StringIO -import unittest -from experiment_file import ExperimentFile - -EXPERIMENT_FILE_1 = """ - board: x86-alex - remote: chromeos-alex3 - perf_args: record -a -e cycles - benchmark: PageCycler { - iterations: 3 - } - - image1 { - chromeos_image: /usr/local/google/cros_image1.bin - } - - image2 { - board: lumpy - remote: chromeos-lumpy1 - chromeos_image: /usr/local/google/cros_image2.bin - } - """ - -EXPERIMENT_FILE_2 = """ - board: x86-alex - remote: chromeos-alex3 - iterations: 3 - - benchmark: PageCycler { - } - - benchmark: AndroidBench { - iterations: 2 - } - - image1 { - chromeos_image:/usr/local/google/cros_image1.bin - } - - image2 { - chromeos_image: /usr/local/google/cros_image2.bin - } - """ - -EXPERIMENT_FILE_3 = """ - board: x86-alex - remote: chromeos-alex3 - iterations: 3 - - benchmark: PageCycler { - } - - image1 { - chromeos_image:/usr/local/google/cros_image1.bin - } - - image1 { - chromeos_image: /usr/local/google/cros_image2.bin - } - """ - - -class ExperimentFileTest(unittest.TestCase): - def testLoadExperimentFile1(self): - input_file = StringIO.StringIO(EXPERIMENT_FILE_1) - experiment_file = ExperimentFile(input_file) - global_settings = experiment_file.GetGlobalSettings() - self.assertEqual(global_settings.GetField("remote"), ["chromeos-alex3"]) - self.assertEqual(global_settings.GetField("perf_args"), - "record -a -e cycles") - benchmark_settings = experiment_file.GetSettings("benchmark") - self.assertEqual(len(benchmark_settings), 1) - self.assertEqual(benchmark_settings[0].name, "PageCycler") - self.assertEqual(benchmark_settings[0].GetField("iterations"), 3) - - label_settings = experiment_file.GetSettings("label") - self.assertEqual(len(label_settings), 2) - self.assertEqual(label_settings[0].name, "image1") - self.assertEqual(label_settings[0].GetField("board"), "x86-alex") - self.assertEqual(label_settings[0].GetField("chromeos_image"), - "/usr/local/google/cros_image1.bin") - self.assertEqual(label_settings[1].GetField("remote"), ["chromeos-lumpy1"]) - self.assertEqual(label_settings[0].GetField("remote"), ["chromeos-alex3"]) - - def testOverrideSetting(self): - input_file = StringIO.StringIO(EXPERIMENT_FILE_2) - experiment_file = ExperimentFile(input_file) - global_settings = experiment_file.GetGlobalSettings() - self.assertEqual(global_settings.GetField("remote"), ["chromeos-alex3"]) - - benchmark_settings = experiment_file.GetSettings("benchmark") - self.assertEqual(len(benchmark_settings), 2) - self.assertEqual(benchmark_settings[0].name, "PageCycler") - self.assertEqual(benchmark_settings[0].GetField("iterations"), 3) - self.assertEqual(benchmark_settings[1].name, "AndroidBench") - self.assertEqual(benchmark_settings[1].GetField("iterations"), 2) - - def testDuplicateLabel(self): - input_file = StringIO.StringIO(EXPERIMENT_FILE_3) - self.assertRaises(Exception, ExperimentFile, input_file) - - -if __name__ == "__main__": - unittest.main() diff --git a/v14/crosperf/experiment_files/README b/v14/crosperf/experiment_files/README deleted file mode 100644 index 5c0e3d41..00000000 --- a/v14/crosperf/experiment_files/README +++ /dev/null @@ -1,26 +0,0 @@ -To use these experiment files, add board, remote and images and run crosperf -on them. - -Further information about crosperf: -https://sites.google.com/a/google.com/compiler-chromeos-workflows/crosperf - -The final experiment file should look something like the following: - -board: <board> -remote: <ip address or machine name> - -# Add images you want to test: -label: myimage { - chromeos_image: <path to image> -} - -# Paste experiment benchmarks here. Example, I pasted aes_perf here. - -# This experiment just runs a short autotest which measures the performance of -# aes encryption. In addition, it profiles - -profile_type: record -profile_counters: instructions cycles - -benchmark: platform_AesThroughput { -} diff --git a/v14/crosperf/experiment_files/aes_perf b/v14/crosperf/experiment_files/aes_perf deleted file mode 100644 index 0c54ccbd..00000000 --- a/v14/crosperf/experiment_files/aes_perf +++ /dev/null @@ -1,7 +0,0 @@ -# This experiment just runs a short autotest which measures the performance of -# aes encryption. In addition, it profiles - -profile_args: record -e cycles -e instructions - -benchmark: platform_AesThroughput { -} diff --git a/v14/crosperf/experiment_files/bloat_perf b/v14/crosperf/experiment_files/bloat_perf deleted file mode 100644 index f8258ee1..00000000 --- a/v14/crosperf/experiment_files/bloat_perf +++ /dev/null @@ -1,6 +0,0 @@ -perf_args: record -e cycles - -benchmark: bloat { - autotest_name: desktopui_PyAutoPerfTests - autotest_args: --args='--iterations=1 perf.PageCyclerTest.testBloatFile' -} diff --git a/v14/crosperf/experiment_files/morejs_perf b/v14/crosperf/experiment_files/morejs_perf deleted file mode 100644 index a02f15f5..00000000 --- a/v14/crosperf/experiment_files/morejs_perf +++ /dev/null @@ -1,6 +0,0 @@ -perf_args: record -e cycles - -benchmark: morejs { - autotest_name: desktopui_PyAutoPerfTests - autotest_args: --args='--iterations=1 perf.PageCyclerTest.testMoreJSFile' -} diff --git a/v14/crosperf/experiment_files/page_cycler b/v14/crosperf/experiment_files/page_cycler deleted file mode 100644 index ada9ed67..00000000 --- a/v14/crosperf/experiment_files/page_cycler +++ /dev/null @@ -1,6 +0,0 @@ -# This experiment runs page cycler tests. - -benchmark: AllPageCyclers { - autotest_name: desktopui_PyAutoPerfTests - autotest_args: --args='perf.PageCyclerTest' -} diff --git a/v14/crosperf/experiment_files/page_cycler_perf b/v14/crosperf/experiment_files/page_cycler_perf deleted file mode 100644 index 7f5e7118..00000000 --- a/v14/crosperf/experiment_files/page_cycler_perf +++ /dev/null @@ -1,43 +0,0 @@ -# This experiment profiles all page cyclers. - -perf_args: record -e cycles - -benchmark: morejs { - autotest_name: desktopui_PyAutoPerfTests - autotest_args: --args='--iterations=10 perf.PageCyclerTest.testMoreJSFile' -} - -benchmark: alexa { - autotest_name: desktopui_PyAutoPerfTests - autotest_args: --args='--iterations=10 perf.PageCyclerTest.testAlexaFile' -} - -benchmark: bloat { - autotest_name: desktopui_PyAutoPerfTests - autotest_args: --args='--iterations=10 perf.PageCyclerTest.testBloatFile' -} - -benchmark: dhtml { - autotest_name: desktopui_PyAutoPerfTests - autotest_args: --args='--iterations=10 perf.PageCyclerTest.testDHTMLFile' -} - -benchmark: intl1 { - autotest_name: desktopui_PyAutoPerfTests - autotest_args: --args='--iterations=10 perf.PageCyclerTest.testIntl1File' -} - -benchmark: intl2 { - autotest_name: desktopui_PyAutoPerfTests - autotest_args: --args='--iterations=10 perf.PageCyclerTest.testIntl2File' -} - -benchmark: moz { - autotest_name: desktopui_PyAutoPerfTests - autotest_args: --args='--iterations=10 perf.PageCyclerTest.testMozFile' -} - -benchmark: moz2 { - autotest_name: desktopui_PyAutoPerfTests - autotest_args: --args='--iterations=10 perf.PageCyclerTest.testMoz2File' -} diff --git a/v14/crosperf/experiment_files/toolchain b/v14/crosperf/experiment_files/toolchain deleted file mode 100644 index 9156998b..00000000 --- a/v14/crosperf/experiment_files/toolchain +++ /dev/null @@ -1,16 +0,0 @@ -# Use this experiment whenever the toolchain is upgraded. - -benchmark: bvt { - autotest_name: suite:bvt -} - -benchmark: suite_Smoke { - autotest_name: suite:smoke -} - -benchmark: PyAutoPerfTests { -} - -benchmark: BootPerfServer { - autotest_name: ^server/site_tests/platform_BootPerfServer/control$ -} diff --git a/v14/crosperf/experiment_runner.py b/v14/crosperf/experiment_runner.py deleted file mode 100644 index 9212ba56..00000000 --- a/v14/crosperf/experiment_runner.py +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/python - -# Copyright 2011 Google Inc. All Rights Reserved. - -"""The experiment runner module.""" -import getpass -import os -import time - -from utils import command_executer -from utils import logger -from utils.email_sender import EmailSender -from utils.file_utils import FileUtils - -import config -from experiment_status import ExperimentStatus -from results_report import HTMLResultsReport -from results_report import TextResultsReport - - -class ExperimentRunner(object): - """ExperimentRunner Class.""" - - STATUS_TIME_DELAY = 30 - THREAD_MONITOR_DELAY = 2 - - def __init__(self, experiment): - self._experiment = experiment - self.l = logger.GetLogger(experiment.log_dir) - self._ce = command_executer.GetCommandExecuter(self.l) - self._terminated = False - - def _Run(self, experiment): - status = ExperimentStatus(experiment) - experiment.Run() - last_status_time = 0 - try: - while not experiment.IsComplete(): - if last_status_time + self.STATUS_TIME_DELAY < time.time(): - last_status_time = time.time() - border = "==============================" - self.l.LogOutput(border) - self.l.LogOutput(status.GetProgressString()) - self.l.LogOutput(status.GetStatusString()) - logger.GetLogger().LogOutput(border) - time.sleep(self.THREAD_MONITOR_DELAY) - except KeyboardInterrupt: - self._terminated = True - self.l.LogError("Ctrl-c pressed. Cleaning up...") - experiment.Terminate() - - def _PrintTable(self, experiment): - self.l.LogOutput(TextResultsReport(experiment).GetReport()) - - def _Email(self, experiment): - # Only email by default if a new run was completed. - send_mail = False - for benchmark_run in experiment.benchmark_runs: - if not benchmark_run.cache_hit: - send_mail = True - break - if (not send_mail and not experiment.email_to - or config.GetConfig("no_email")): - return - - label_names = [] - for label in experiment.labels: - label_names.append(label.name) - subject = "%s: %s" % (experiment.name, " vs. ".join(label_names)) - - text_report = TextResultsReport(experiment, True).GetReport() - text_report = "<pre style='font-size: 13px'>%s</pre>" % text_report - html_report = HTMLResultsReport(experiment).GetReport() - attachment = EmailSender.Attachment("report.html", html_report) - email_to = [getpass.getuser()] + experiment.email_to - EmailSender().SendEmail(email_to, - subject, - text_report, - attachments=[attachment], - msg_type="html") - - def _StoreResults (self, experiment): - if self._terminated: - return - results_directory = experiment.results_directory - FileUtils().RmDir(results_directory) - FileUtils().MkDirP(results_directory) - self.l.LogOutput("Storing experiment file.") - experiment_file_path = os.path.join(results_directory, - "experiment.exp") - FileUtils().WriteFile(experiment_file_path, experiment.experiment_file) - - self.l.LogOutput("Storing results report.") - results_table_path = os.path.join(results_directory, "results.html") - report = HTMLResultsReport(experiment).GetReport() - FileUtils().WriteFile(results_table_path, report) - - self.l.LogOutput("Storing results of each benchmark run.") - for benchmark_run in experiment.benchmark_runs: - if benchmark_run.result: - benchmark_run_name = filter(str.isalnum, benchmark_run.name) - benchmark_run_path = os.path.join(results_directory, - benchmark_run_name) - benchmark_run.result.CopyResultsTo(benchmark_run_path) - benchmark_run.result.CleanUp(benchmark_run.benchmark.rm_chroot_tmp) - - def Run(self): - self._Run(self._experiment) - self._PrintTable(self._experiment) - if not self._terminated: - self._StoreResults(self._experiment) - self._Email(self._experiment) - - -class MockExperimentRunner(ExperimentRunner): - """Mocked ExperimentRunner for testing.""" - - def __init__(self, experiment): - super(MockExperimentRunner, self).__init__(experiment) - - def _Run(self, experiment): - self.l.LogOutput("Would run the following experiment: '%s'." % - experiment.name) - - def _PrintTable(self, experiment): - self.l.LogOutput("Would print the experiment table.") - - def _Email(self, experiment): - self.l.LogOutput("Would send result email.") - - def _StoreResults(self, experiment): - self.l.LogOutput("Would store the results.") diff --git a/v14/crosperf/experiment_status.py b/v14/crosperf/experiment_status.py deleted file mode 100644 index 3a270663..00000000 --- a/v14/crosperf/experiment_status.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/python - -# Copyright 2011 Google Inc. All Rights Reserved. - -"""The class to show the banner.""" - -import datetime -import time - - -class ExperimentStatus(object): - """The status class.""" - - def __init__(self, experiment): - self.experiment = experiment - self.num_total = len(self.experiment.benchmark_runs) - self.completed = 0 - self.new_job_start_time = time.time() - - def _GetProgressBar(self, num_complete, num_total): - ret = "Done: %s%%" % int(100.0 * num_complete / num_total) - bar_length = 50 - done_char = ">" - undone_char = " " - num_complete_chars = bar_length * num_complete / num_total - num_undone_chars = bar_length - num_complete_chars - ret += " [%s%s]" % (num_complete_chars * done_char, num_undone_chars * - undone_char) - return ret - - def GetProgressString(self): - """Get the elapsed_time, ETA.""" - current_time = time.time() - if self.experiment.start_time: - elapsed_time = current_time - self.experiment.start_time - else: - elapsed_time = 0 - try: - if self.completed != self.experiment.num_complete: - self.completed = self.experiment.num_complete - self.new_job_start_time = current_time - time_completed_jobs = (elapsed_time - - (current_time - self.new_job_start_time)) - eta_seconds = (float(self.num_total - self.experiment.num_complete -1) * - time_completed_jobs / self.experiment.num_run_complete - + (time_completed_jobs / self.experiment.num_run_complete - - (current_time - self.new_job_start_time))) - - eta_seconds = int(eta_seconds) - eta = datetime.timedelta(seconds=eta_seconds) - except ZeroDivisionError: - eta = "Unknown" - strings = [] - strings.append("Current time: %s Elapsed: %s ETA: %s" % - (datetime.datetime.now(), - datetime.timedelta(seconds=int(elapsed_time)), - eta)) - strings.append(self._GetProgressBar(self.experiment.num_complete, - self.num_total)) - return "\n".join(strings) - - def GetStatusString(self): - """Get the status string of all the benchmark_runs.""" - status_bins = {} - for benchmark_run in self.experiment.benchmark_runs: - if benchmark_run.timeline.GetLastEvent() not in status_bins: - status_bins[benchmark_run.timeline.GetLastEvent()] = [] - status_bins[benchmark_run.timeline.GetLastEvent()].append(benchmark_run) - - status_strings = [] - for key, val in status_bins.items(): - status_strings.append("%s: %s" % - (key, self._GetNamesAndIterations(val))) - result = "Thread Status:\n%s" % "\n".join(status_strings) - - # Add the machine manager status. - result += "\n" + self.experiment.machine_manager.AsString() + "\n" - - return result - - def _GetNamesAndIterations(self, benchmark_runs): - strings = [] - t = time.time() - for benchmark_run in benchmark_runs: - t_last = benchmark_run.timeline.GetLastEventTime() - elapsed = str(datetime.timedelta(seconds=int(t-t_last))) - strings.append("'{0}' {1}".format(benchmark_run.name, elapsed)) - return " %s (%s)" % (len(strings), ", ".join(strings)) diff --git a/v14/crosperf/field.py b/v14/crosperf/field.py deleted file mode 100644 index b3cdaa23..00000000 --- a/v14/crosperf/field.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/python - -# Copyright 2011 Google Inc. All Rights Reserved. - - -class Field(object): - """Class representing a Field in an experiment file.""" - - def __init__(self, name, required, default, inheritable, description): - self.name = name - self.required = required - self.assigned = False - self.default = default - self._value = default - self.inheritable = inheritable - self.description = description - - def Set(self, value, parse=True): - if parse: - self._value = self._Parse(value) - else: - self._value = value - self.assigned = True - - def Append(self, value): - self._value += self._Parse(value) - self.assigned = True - - def _Parse(self, value): - return value - - def Get(self): - return self._value - - def GetString(self): - return str(self._value) - - -class TextField(Field): - def __init__(self, name, required=False, default="", inheritable=False, - description=""): - super(TextField, self).__init__(name, required, default, inheritable, - description) - - def _Parse(self, value): - return str(value) - - -class BooleanField(Field): - def __init__(self, name, required=False, default=False, inheritable=False, - description=""): - super(BooleanField, self).__init__(name, required, default, inheritable, - description) - - def _Parse(self, value): - if value.lower() == "true": - return True - elif value.lower() == "false": - return False - raise Exception("Invalid value for '%s'. Must be true or false." % - self.name) - - -class IntegerField(Field): - def __init__(self, name, required=False, default=0, inheritable=False, - description=""): - super(IntegerField, self).__init__(name, required, default, inheritable, - description) - - def _Parse(self, value): - return int(value) - - -class FloatField(Field): - def __init__(self, name, required=False, default=0, inheritable=False, - description=""): - super(FloatField, self).__init__(name, required, default, inheritable, - description) - - def _Parse(self, value): - return float(value) - - -class ListField(Field): - def __init__(self, name, required=False, default=[], inheritable=False, - description=""): - super(ListField, self).__init__(name, required, default, inheritable, - description) - - def _Parse(self, value): - return value.split() - - def GetString(self): - return " ".join(self._value) - - -class EnumField(Field): - def __init__(self, name, options, required=False, default="", - inheritable=False, description=""): - super(EnumField, self).__init__(name, required, default, inheritable, - description) - self.options = options - - def _Parse(self, value): - if value not in self.options: - raise Exception("Invalid enum value for field '%s'. Must be one of (%s)" - % (self.name, ", ".join(self.options))) - return str(value) diff --git a/v14/crosperf/help.py b/v14/crosperf/help.py deleted file mode 100644 index cf74d93e..00000000 --- a/v14/crosperf/help.py +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/python - -# Copyright 2011 Google Inc. All Rights Reserved. - -import sys -import textwrap -from settings_factory import BenchmarkSettings -from settings_factory import GlobalSettings -from settings_factory import LabelSettings - - -class Help(object): - def GetUsage(self): - return """%s [OPTIONS] [ACTION] EXPERIMENT_FILE""" % (sys.argv[0]) - - def _WrapLine(self, line): - return "\n".join(textwrap.wrap(line, 80)) - - def _GetFieldDescriptions(self, fields): - res = "" - for field_name in fields: - field = fields[field_name] - res += "Field:\t\t%s\n" % field.name - res += self._WrapLine("Description:\t%s" % field.description) + "\n" - res += "Type:\t\t%s\n" % type(field).__name__.replace("Field", "") - res += "Required:\t%s\n" % field.required - if field.default: - res += "Default:\t%s\n" % field.default - res += "\n" - return res - - def GetHelp(self): - global_fields = self._GetFieldDescriptions(GlobalSettings("").fields) - benchmark_fields = self._GetFieldDescriptions(BenchmarkSettings("").fields) - label_fields = self._GetFieldDescriptions(LabelSettings("").fields) - - return """%s is a script for running performance experiments on ChromeOS. It -allows one to run ChromeOS Autotest benchmarks over several images and compare -the results to determine whether there is a performance difference. - -Comparing several images using %s is referred to as running an -"experiment". An "experiment file" is a configuration file which holds all the -information that describes the experiment and how it should be run. An example -of a simple experiment file is below: - ---------------------------------- test.exp --------------------------------- -name: my_experiment -board: x86-alex -remote: chromeos-alex5 172.18.122.132 - -benchmark: PageCycler { - iterations: 3 -} - -my_first_image { - chromeos_image: /usr/local/chromeos-1/chromiumos_image.bin -} - -my_second_image { - chromeos_image: /usr/local/chromeos-2/chromiumos_image.bin -} ----------------------------------------------------------------------------- - -This experiment file names the experiment "my_experiment". It will be run -on the board x86-alex. Benchmarks will be run using two remote devices, -one is a device specified by a hostname and the other is a device specified -by it's IP address. Benchmarks will be run in parallel across these devices. -There is currently no way to specify which benchmark will run on each device. - -We define one "benchmark" that will be run, PageCycler. This benchmark has one -"field" which specifies how many iterations it will run for. - -We specify 2 "labels" or images which will be compared. The PageCycler benchmark -will be run on each of these images 3 times and a result table will be output -which compares the two. - -The full list of fields that can be specified are as follows: -================= -Global Fields -================= -%s -================= -Benchmark Fields -================= -%s -================= -Label Fields -================= -%s - -Note that global fields are overidden by label or benchmark fields, if they can -be specified in both places. Fields that are specified as arguments override -fields specified in experiment files. - -%s is invoked by passing it a path to an experiment file, as well as an action -to execute on that experiment file. The possible actions to use are: - -run\t\tRun the experiment and cache the results. - -table\t\tDisplay cached results of an experiment, without running anything. - -email\t\tEmail a summary of the results to the user. - -do\t\tThe default action. Executes the following actions: run, table, email. -""" % (sys.argv[0], sys.argv[0], global_fields, - benchmark_fields, label_fields, sys.argv[0]) diff --git a/v14/crosperf/image_checksummer.py b/v14/crosperf/image_checksummer.py deleted file mode 100644 index dcc1cb02..00000000 --- a/v14/crosperf/image_checksummer.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/python - -# Copyright 2011 Google Inc. All Rights Reserved. - -import os -import threading - -from utils import logger -from utils.file_utils import FileUtils - - -class ImageChecksummer(object): - class PerImageChecksummer(object): - def __init__(self, label): - self._lock = threading.Lock() - self.label = label - self._checksum = None - - def Checksum(self): - with self._lock: - if not self._checksum: - logger.GetLogger().LogOutput("Acquiring checksum for '%s'." % - self.label.name) - self._checksum = None - if self.label.chromeos_image: - if os.path.exists(self.label.chromeos_image): - self._checksum = FileUtils().Md5File(self.label.chromeos_image) - logger.GetLogger().LogOutput("Computed checksum is " - ": %s" % self._checksum) - if not self._checksum: - if self.label.image_md5sum: - self._checksum = self.label.image_md5sum - logger.GetLogger().LogOutput("Checksum in experiment file is " - ": %s" % self._checksum) - else: - raise Exception("Checksum computing error.") - logger.GetLogger().LogOutput("Checksum is: %s" % self._checksum) - return self._checksum - - _instance = None - _lock = threading.Lock() - _per_image_checksummers = {} - - def __new__(cls, *args, **kwargs): - with cls._lock: - if not cls._instance: - cls._instance = super(ImageChecksummer, cls).__new__(cls, - *args, **kwargs) - return cls._instance - - def Checksum(self, label): - with self._lock: - if label.name not in self._per_image_checksummers: - self._per_image_checksummers[label.name] = (ImageChecksummer. - PerImageChecksummer(label)) - checksummer = self._per_image_checksummers[label.name] - - try: - return checksummer.Checksum() - except Exception, e: - logger.GetLogger().LogError("Could not compute checksum of image in label" - " '%s'."% label.name) - raise e diff --git a/v14/crosperf/label.py b/v14/crosperf/label.py deleted file mode 100644 index be7a868e..00000000 --- a/v14/crosperf/label.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/python - -# Copyright 2011 Google Inc. All Rights Reserved. - -"""The label of benchamrks.""" - -import os -from utils.file_utils import FileUtils - - -class Label(object): - def __init__(self, name, chromeos_image, chromeos_root, board, remote, - image_args, image_md5sum, cache_dir): - # Expand ~ - chromeos_root = os.path.expanduser(chromeos_root) - chromeos_image = os.path.expanduser(chromeos_image) - - self.name = name - self.chromeos_image = chromeos_image - self.board = board - self.remote = remote - self.image_args = image_args - self.image_md5sum = image_md5sum - self.cache_dir = cache_dir - - if not chromeos_root: - chromeos_root = FileUtils().ChromeOSRootFromImage(chromeos_image) - if not chromeos_root: - raise Exception("No ChromeOS root given for label '%s' and could not " - "determine one from image path: '%s'." % - (name, chromeos_image)) - else: - chromeos_root = FileUtils().CanonicalizeChromeOSRoot(chromeos_root) - if not chromeos_root: - raise Exception("Invalid ChromeOS root given for label '%s': '%s'." - % (name, chromeos_root)) - - self.chromeos_root = chromeos_root - - -class MockLabel(object): - def __init__(self, name, chromeos_image, chromeos_root, board, remote, - image_args, image_md5sum, cache_dir): - self.name = name - self.chromeos_image = chromeos_image - self.board = board - self.remote = remote - self.cache_dir = cache_dir - if not chromeos_root: - self.chromeos_root = "/tmp/chromeos_root" - else: - self.chromeos_root = chromeos_root - self.image_args = image_args - self.image_md5sum = image_md5sum diff --git a/v14/crosperf/machine_manager.py b/v14/crosperf/machine_manager.py deleted file mode 100644 index 29a4df7a..00000000 --- a/v14/crosperf/machine_manager.py +++ /dev/null @@ -1,419 +0,0 @@ -#!/usr/bin/python -# -# Copyright 2012 Google Inc. All Rights Reserved. - -import hashlib -import image_chromeos -import lock_machine -import math -import os.path -import re -import sys -import threading -import time - -from utils import command_executer -from utils import logger -from utils.file_utils import FileUtils - -from image_checksummer import ImageChecksummer - -CHECKSUM_FILE = "/usr/local/osimage_checksum_file" - - -class CrosMachine(object): - def __init__(self, name, chromeos_root): - self.name = name - self.image = None - self.checksum = None - self.locked = False - self.released_time = time.time() - self.autotest_run = None - self.chromeos_root = chromeos_root - self._GetMemoryInfo() - self._GetCPUInfo() - self._ComputeMachineChecksumString() - self._GetMachineID() - self.machine_checksum = self._GetMD5Checksum(self.checksum_string) - self.machine_id_checksum = self._GetMD5Checksum(self.machine_id) - - def _ParseMemoryInfo(self): - line = self.meminfo.splitlines()[0] - usable_kbytes = int(line.split()[1]) - # This code is from src/third_party/autotest/files/client/bin/base_utils.py - # usable_kbytes is system's usable DRAM in kbytes, - # as reported by memtotal() from device /proc/meminfo memtotal - # after Linux deducts 1.5% to 9.5% for system table overhead - # Undo the unknown actual deduction by rounding up - # to next small multiple of a big power-of-two - # eg 12GB - 5.1% gets rounded back up to 12GB - mindeduct = 0.005 # 0.5 percent - maxdeduct = 0.095 # 9.5 percent - # deduction range 1.5% .. 9.5% supports physical mem sizes - # 6GB .. 12GB in steps of .5GB - # 12GB .. 24GB in steps of 1 GB - # 24GB .. 48GB in steps of 2 GB ... - # Finer granularity in physical mem sizes would require - # tighter spread between min and max possible deductions - - # increase mem size by at least min deduction, without rounding - min_kbytes = int(usable_kbytes / (1.0 - mindeduct)) - # increase mem size further by 2**n rounding, by 0..roundKb or more - round_kbytes = int(usable_kbytes / (1.0 - maxdeduct)) - min_kbytes - # find least binary roundup 2**n that covers worst-cast roundKb - mod2n = 1 << int(math.ceil(math.log(round_kbytes, 2))) - # have round_kbytes <= mod2n < round_kbytes*2 - # round min_kbytes up to next multiple of mod2n - phys_kbytes = min_kbytes + mod2n - 1 - phys_kbytes -= phys_kbytes % mod2n # clear low bits - self.phys_kbytes = phys_kbytes - - def _GetMemoryInfo(self): - #TODO yunlian: when the machine in rebooting, it will not return - #meminfo, the assert does not catch it either - ce = command_executer.GetCommandExecuter() - command = "cat /proc/meminfo" - ret, self.meminfo, _ = ce.CrosRunCommand( - command, return_output=True, - machine=self.name, username="root", chromeos_root=self.chromeos_root) - assert ret == 0, "Could not get meminfo from machine: %s" % self.name - if ret == 0: - self._ParseMemoryInfo() - - #cpuinfo format is different across architecture - #need to find a better way to parse it. - def _ParseCPUInfo(self,cpuinfo): - return 0 - - def _GetCPUInfo(self): - ce = command_executer.GetCommandExecuter() - command = "cat /proc/cpuinfo" - ret, self.cpuinfo, _ = ce.CrosRunCommand( - command, return_output=True, - machine=self.name, username="root", chromeos_root=self.chromeos_root) - assert ret == 0, "Could not get cpuinfo from machine: %s" % self.name - if ret == 0: - self._ParseCPUInfo(self.cpuinfo) - - def _ComputeMachineChecksumString(self): - self.checksum_string = "" - exclude_lines_list = ["MHz", "BogoMIPS", "bogomips"] - for line in self.cpuinfo.splitlines(): - if not any([e in line for e in exclude_lines_list]): - self.checksum_string += line - self.checksum_string += " " + str(self.phys_kbytes) - - def _GetMD5Checksum(self, ss): - if ss: - return hashlib.md5(ss).hexdigest() - else: - return "" - - def _GetMachineID(self): - ce = command_executer.GetCommandExecuter() - command = "dump_vpd_log --full --stdout" - ret, if_out, _ = ce.CrosRunCommand( - command, return_output=True, - machine=self.name, chromeos_root=self.chromeos_root) - b = if_out.splitlines() - a = [l for l in b if "Product" in l] - self.machine_id = a[0] - assert ret == 0, "Could not get machine_id from machine: %s" % self.name - - def __str__(self): - l = [] - l.append(self.name) - l.append(str(self.image)) - l.append(str(self.checksum)) - l.append(str(self.locked)) - l.append(str(self.released_time)) - return ", ".join(l) - - -class MachineManager(object): - def __init__(self, chromeos_root, acquire_timeout): - self._lock = threading.RLock() - self._all_machines = [] - self._machines = [] - self.image_lock = threading.Lock() - self.num_reimages = 0 - self.chromeos_root = None - self.machine_checksum = {} - self.machine_checksum_string = {} - self.acquire_timeout = acquire_timeout - - if os.path.isdir(lock_machine.Machine.LOCKS_DIR): - self.no_lock = False - else: - self.no_lock = True - self._initialized_machines = [] - self.chromeos_root = chromeos_root - - def ImageMachine(self, machine, label): - checksum = ImageChecksummer().Checksum(label) - if machine.checksum == checksum: - return - chromeos_root = label.chromeos_root - if not chromeos_root: - chromeos_root = self.chromeos_root - image_chromeos_args = [image_chromeos.__file__, - "--chromeos_root=%s" % chromeos_root, - "--image=%s" % label.chromeos_image, - "--image_args=%s" % label.image_args, - "--remote=%s" % machine.name] - if label.board: - image_chromeos_args.append("--board=%s" % label.board) - - # Currently can't image two machines at once. - # So have to serialized on this lock. - ce = command_executer.GetCommandExecuter() - with self.image_lock: - retval = ce.RunCommand(" ".join(["python"] + image_chromeos_args)) - if retval: - raise Exception("Could not image machine: '%s'." % machine.name) - else: - self.num_reimages += 1 - machine.checksum = checksum - machine.image = label.chromeos_image - - return retval - - def ComputeCommonCheckSum(self, label): - for machine in self.GetMachines(label): - if machine.machine_checksum: - self.machine_checksum[label.name] = machine.machine_checksum - break - - def ComputeCommonCheckSumString(self, label): - for machine in self.GetMachines(label): - if machine.checksum_string: - self.machine_checksum_string[label.name] = machine.checksum_string - break - - def _TryToLockMachine(self, cros_machine): - with self._lock: - assert cros_machine, "Machine can't be None" - for m in self._machines: - if m.name == cros_machine.name: - return - if self.no_lock: - locked = True - else: - locked = lock_machine.Machine(cros_machine.name).Lock(True, sys.argv[0]) - if locked: - self._machines.append(cros_machine) - ce = command_executer.GetCommandExecuter() - command = "cat %s" % CHECKSUM_FILE - ret, out, _ = ce.CrosRunCommand( - command, return_output=True, chromeos_root=self.chromeos_root, - machine=cros_machine.name) - if ret == 0: - cros_machine.checksum = out.strip() - else: - logger.GetLogger().LogOutput("Couldn't lock: %s" % cros_machine.name) - - # This is called from single threaded mode. - def AddMachine(self, machine_name): - with self._lock: - for m in self._all_machines: - assert m.name != machine_name, "Tried to double-add %s" % machine_name - cm = CrosMachine(machine_name, self.chromeos_root) - assert cm.machine_checksum, ("Could not find checksum for machine %s" % - machine_name) - self._all_machines.append(cm) - - def AreAllMachineSame(self, label): - checksums = [m.machine_checksum for m in self.GetMachines(label)] - return len(set(checksums)) == 1 - - def AcquireMachine(self, chromeos_image, label): - image_checksum = ImageChecksummer().Checksum(label) - machines = self.GetMachines(label) - check_interval_time = 120 - with self._lock: - # Lazily external lock machines - while self.acquire_timeout >= 0: - for m in machines: - new_machine = m not in self._all_machines - self._TryToLockMachine(m) - if new_machine: - m.released_time = time.time() - if not self.AreAllMachineSame(label): - logger.GetLogger().LogFatal("-- not all the machine are identical") - if self.GetAvailableMachines(label): - break - else: - sleep_time = max(1, min(self.acquire_timeout, check_interval_time)) - time.sleep(sleep_time) - self.acquire_timeout -= sleep_time - - if self.acquire_timeout < 0: - machine_names = [] - for machine in machines: - machine_names.append(machine.name) - logger.GetLogger().LogFatal("Could not acquire any of the " - "following machines: '%s'" - % ", ".join(machine_names)) - -### for m in self._machines: -### if (m.locked and time.time() - m.released_time < 10 and -### m.checksum == image_checksum): -### return None - for m in [machine for machine in self.GetAvailableMachines(label) - if not machine.locked]: - if m.checksum == image_checksum: - m.locked = True - m.autotest_run = threading.current_thread() - return m - for m in [machine for machine in self.GetAvailableMachines(label) - if not machine.locked]: - if not m.checksum: - m.locked = True - m.autotest_run = threading.current_thread() - return m - # This logic ensures that threads waiting on a machine will get a machine - # with a checksum equal to their image over other threads. This saves time - # when crosperf initially assigns the machines to threads by minimizing - # the number of re-images. - # TODO(asharif): If we centralize the thread-scheduler, we wont need this - # code and can implement minimal reimaging code more cleanly. - for m in [machine for machine in self.GetAvailableMachines(label) - if not machine.locked]: - if time.time() - m.released_time > 20: - m.locked = True - m.autotest_run = threading.current_thread() - return m - return None - - def GetAvailableMachines(self, label=None): - if not label: - return self._machines - return [m for m in self._machines if m.name in label.remote] - - def GetMachines(self, label=None): - if not label: - return self._all_machines - return [m for m in self._all_machines if m.name in label.remote] - - def ReleaseMachine(self, machine): - with self._lock: - for m in self._machines: - if machine.name == m.name: - assert m.locked == True, "Tried to double-release %s" % m.name - m.released_time = time.time() - m.locked = False - m.status = "Available" - break - - def Cleanup(self): - with self._lock: - # Unlock all machines. - for m in self._machines: - if not self.no_lock: - res = lock_machine.Machine(m.name).Unlock(True) - if not res: - logger.GetLogger().LogError("Could not unlock machine: '%s'." - % m.name) - - def __str__(self): - with self._lock: - l = ["MachineManager Status:"] - for m in self._machines: - l.append(str(m)) - return "\n".join(l) - - def AsString(self): - with self._lock: - stringify_fmt = "%-30s %-10s %-4s %-25s %-32s" - header = stringify_fmt % ("Machine", "Thread", "Lock", "Status", - "Checksum") - table = [header] - for m in self._machines: - if m.autotest_run: - autotest_name = m.autotest_run.name - autotest_status = m.autotest_run.timeline.GetLastEvent() - else: - autotest_name = "" - autotest_status = "" - - try: - machine_string = stringify_fmt % (m.name, - autotest_name, - m.locked, - autotest_status, - m.checksum) - except Exception: - machine_string = "" - table.append(machine_string) - return "Machine Status:\n%s" % "\n".join(table) - - def GetAllCPUInfo(self, labels): - """Get cpuinfo for labels, merge them if their cpuinfo are the same.""" - dic = {} - for label in labels: - for machine in self._all_machines: - if machine.name in label.remote: - if machine.cpuinfo not in dic: - dic[machine.cpuinfo] = [label.name] - else: - dic[machine.cpuinfo].append(label.name) - break - output = "" - for key, v in dic.items(): - output += " ".join(v) - output += "\n-------------------\n" - output += key - output += "\n\n\n" - return output - - -class MockCrosMachine(CrosMachine): - def __init__(self, name, chromeos_root): - self.name = name - self.image = None - self.checksum = None - self.locked = False - self.released_time = time.time() - self.autotest_run = None - self.chromeos_root = chromeos_root - self.checksum_string = re.sub("\d", "", name) - #In test, we assume "lumpy1", "lumpy2" are the same machine. - self.machine_checksum = self._GetMD5Checksum(self.checksum_string) - - -class MockMachineManager(MachineManager): - - def __init__(self, chromeos_root, acquire_timeout): - super(MockMachineManager, self).__init__(chromeos_root, acquire_timeout) - - def _TryToLockMachine(self, cros_machine): - self._machines.append(cros_machine) - cros_machine.checksum = "" - - def AddMachine(self, machine_name): - with self._lock: - for m in self._all_machines: - assert m.name != machine_name, "Tried to double-add %s" % machine_name - cm = MockCrosMachine(machine_name, self.chromeos_root) - assert cm.machine_checksum, ("Could not find checksum for machine %s" % - machine_name) - self._all_machines.append(cm) - - def AcquireMachine(self, chromeos_image, label): - for machine in self._all_machines: - if not machine.locked: - machine.locked = True - return machine - return None - - def ImageMachine(self, machine_name, label): - return 0 - - def ReleaseMachine(self, machine): - machine.locked = False - - def GetMachines(self, label): - return self._all_machines - - def GetAvailableMachines(self, label): - return self._all_machines diff --git a/v14/crosperf/machine_manager_unittest.py b/v14/crosperf/machine_manager_unittest.py deleted file mode 100755 index 84266d5e..00000000 --- a/v14/crosperf/machine_manager_unittest.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/python - -# Copyright 2012 Google Inc. All Rights Reserved. - -"""Unittest for machine_manager.""" -import unittest - -import label -import machine_manager - - -class MyMachineManager(machine_manager.MachineManager): - - def __init__(self, chromeos_root): - super(MyMachineManager, self).__init__(chromeos_root, 0) - - def _TryToLockMachine(self, cros_machine): - self._machines.append(cros_machine) - cros_machine.checksum = "" - - def AddMachine(self, machine_name): - with self._lock: - for m in self._all_machines: - assert m.name != machine_name, "Tried to double-add %s" % machine_name - cm = machine_manager.MockCrosMachine(machine_name, self.chromeos_root) - assert cm.machine_checksum, ("Could not find checksum for machine %s" % - machine_name) - self._all_machines.append(cm) - -CHROMEOS_ROOT = "/tmp/chromeos-root" -MACHINE_NAMES = ["lumpy1", "lumpy2", "lumpy3", "daisy1", "daisy2"] -LABEL_LUMPY = label.MockLabel("lumpy", "image", CHROMEOS_ROOT, "lumpy", - ["lumpy1", "lumpy2", "lumpy3", "lumpy4"], - "", "", "") -LABEL_MIX = label.MockLabel("mix", "image", CHROMEOS_ROOT, "mix", - ["daisy1", "daisy2", "lumpy3", "lumpy4"], - "", "", "") - - -class MachineManagerTest(unittest.TestCase): - - def testAreAllMachineSame(self): - manager = MyMachineManager(CHROMEOS_ROOT) - for m in MACHINE_NAMES: - manager.AddMachine(m) - self.assertEqual(manager.AreAllMachineSame(LABEL_LUMPY), True) - self.assertEqual(manager.AreAllMachineSame(LABEL_MIX), False) - - def testGetMachines(self): - manager = MyMachineManager(CHROMEOS_ROOT) - for m in MACHINE_NAMES: - manager.AddMachine(m) - names = [m.name for m in manager.GetMachines(LABEL_LUMPY)] - self.assertEqual(names, ["lumpy1", "lumpy2", "lumpy3"]) - - def testGetAvailableMachines(self): - manager = MyMachineManager(CHROMEOS_ROOT) - for m in MACHINE_NAMES: - manager.AddMachine(m) - for m in manager._all_machines: - if int(m.name[-1]) % 2: - manager._TryToLockMachine(m) - names = [m.name for m in manager.GetAvailableMachines(LABEL_LUMPY)] - self.assertEqual(names, ["lumpy1", "lumpy3"]) - -if __name__ == "__main__": - unittest.main() diff --git a/v14/crosperf/perf_table.py b/v14/crosperf/perf_table.py deleted file mode 100644 index 3c8b88b8..00000000 --- a/v14/crosperf/perf_table.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/python -# -# Copyright 2012 Google Inc. All Rights Reserved. -"""Parse perf report data for tabulator.""" - -import os - -from utils import perf_diff - - -def ParsePerfReport(perf_file): - """It should return a dict.""" - - return {"cycles": {"foo": 10, "bar": 20}, - "cache_miss": {"foo": 20, "bar": 10}} - - -class PerfTable(object): - """The class to generate dicts for tabulator.""" - - def __init__(self, experiment, label_names): - self._experiment = experiment - self._label_names = label_names - self.perf_data = {} - self.GenerateData() - - # {benchmark:{perf_event1:[[{func1:number, func2:number, - # rows_to_show: number} - # {func1: number, func2: number - # rows_to_show: number}]], ...}, - # benchmark2:...} - # The rows_to_show is temp data recording how many - # rows have over 1% running time. - self.row_info = {} - self.GetRowsToShow() - - def GenerateData(self): - for label in self._label_names: - for benchmark in self._experiment.benchmarks: - for i in range(1, benchmark.iterations+1): - dir_name = label + benchmark.name + str(i) - dir_name = filter(str.isalnum, dir_name) - perf_file = os.path.join(self._experiment.results_directory, - dir_name, - "perf.data.report.0") - self.ReadPerfReport(perf_file, label, benchmark.name, i - 1) - - def ReadPerfReport(self, perf_file, label, benchmark_name, iteration): - """Add the data from one run to the dict.""" - if os.path.isfile(perf_file): - perf_of_run = perf_diff.GetPerfDictFromReport(perf_file) - else: - perf_of_run = {} - if benchmark_name not in self.perf_data: - self.perf_data[benchmark_name] = {} - for event in perf_of_run: - self.perf_data[benchmark_name][event] = [] - ben_data = self.perf_data[benchmark_name] - - label_index = self._label_names.index(label) - for event in ben_data: - while len(ben_data[event]) <= label_index: - ben_data[event].append([]) - data_for_label = ben_data[event][label_index] - while len(data_for_label) <= iteration: - data_for_label.append({}) - if perf_of_run: - data_for_label[iteration] = perf_of_run[event] - else: - data_for_label[iteration] = {} - - def GetRowsToShow(self): - for benchmark in self.perf_data: - if benchmark not in self.row_info: - self.row_info[benchmark] = {} - for event in self.perf_data[benchmark]: - rows = 0 - for run in self.perf_data[benchmark][event]: - for iteration in run: - if perf_diff.ROWS_TO_SHOW in iteration: - rows = max(iteration[perf_diff.ROWS_TO_SHOW], rows) - # delete the temp data which stores how many rows of - # the perf data have over 1% running time. - del iteration[perf_diff.ROWS_TO_SHOW] - self.row_info[benchmark][event] = rows diff --git a/v14/crosperf/results_cache.py b/v14/crosperf/results_cache.py deleted file mode 100644 index 0357275d..00000000 --- a/v14/crosperf/results_cache.py +++ /dev/null @@ -1,417 +0,0 @@ -#!/usr/bin/python -# -# Copyright 2011 Google Inc. All Rights Reserved. - -import getpass -import glob -import hashlib -import os -import pickle -import re -import tempfile - -from utils import command_executer -from utils import misc - -from image_checksummer import ImageChecksummer - -SCRATCH_BASE = "/home/%s/cros_scratch" -SCRATCH_DIR = SCRATCH_BASE % getpass.getuser() -RESULTS_FILE = "results.txt" -MACHINE_FILE = "machine.txt" -AUTOTEST_TARBALL = "autotest.tbz2" -PERF_RESULTS_FILE = "perf-results.txt" - - -class Result(object): - """ This class manages what exactly is stored inside the cache without knowing - what the key of the cache is. For runs with perf, it stores perf.data, - perf.report, etc. The key generation is handled by the ResultsCache class. - """ - - def __init__(self, chromeos_root, logger, label_name): - self._chromeos_root = chromeos_root - self._logger = logger - self._ce = command_executer.GetCommandExecuter(self._logger) - self._temp_dir = None - self.label_name = label_name - - def _CopyFilesTo(self, dest_dir, files_to_copy): - file_index = 0 - for file_to_copy in files_to_copy: - if not os.path.isdir(dest_dir): - command = "mkdir -p %s" % dest_dir - self._ce.RunCommand(command) - dest_file = os.path.join(dest_dir, - ("%s.%s" % (os.path.basename(file_to_copy), - file_index))) - ret = self._ce.CopyFiles(file_to_copy, - dest_file, - recursive=False) - if ret: - raise Exception("Could not copy results file: %s" % file_to_copy) - - def CopyResultsTo(self, dest_dir): - self._CopyFilesTo(dest_dir, self.perf_data_files) - self._CopyFilesTo(dest_dir, self.perf_report_files) - - def _GetKeyvals(self): - results_in_chroot = os.path.join(self._chromeos_root, - "chroot", "tmp") - if not self._temp_dir: - self._temp_dir = tempfile.mkdtemp(dir=results_in_chroot) - command = "cp -r {0}/* {1}".format(self.results_dir, self._temp_dir) - self._ce.RunCommand(command) - - command = ("python generate_test_report --no-color --csv %s" % - (os.path.join("/tmp", os.path.basename(self._temp_dir)))) - [_, out, _] = self._ce.ChrootRunCommand(self._chromeos_root, - command, - return_output=True) - keyvals_dict = {} - for line in out.splitlines(): - tokens = re.split("=|,", line) - key = tokens[-2] - if key.startswith(self.results_dir): - key = key[len(self.results_dir) + 1:] - value = tokens[-1] - keyvals_dict[key] = value - - return keyvals_dict - - def _GetResultsDir(self): - mo = re.search(r"Results placed in (\S+)", self.out) - if mo: - result = mo.group(1) - return result - raise Exception("Could not find results directory.") - - def _FindFilesInResultsDir(self, find_args): - command = "find %s %s" % (self.results_dir, - find_args) - ret, out, _ = self._ce.RunCommand(command, return_output=True) - if ret: - raise Exception("Could not run find command!") - return out - - def _GetPerfDataFiles(self): - return self._FindFilesInResultsDir("-name perf.data").splitlines() - - def _GetPerfReportFiles(self): - return self._FindFilesInResultsDir("-name perf.data.report").splitlines() - - def _GeneratePerfReportFiles(self): - perf_report_files = [] - for perf_data_file in self.perf_data_files: - # Generate a perf.report and store it side-by-side with the perf.data - # file. - chroot_perf_data_file = misc.GetInsideChrootPath(self._chromeos_root, - perf_data_file) - perf_report_file = "%s.report" % perf_data_file - if os.path.exists(perf_report_file): - raise Exception("Perf report file already exists: %s" % - perf_report_file) - chroot_perf_report_file = misc.GetInsideChrootPath(self._chromeos_root, - perf_report_file) - command = ("/usr/sbin/perf report " - "-n " - "--symfs /build/%s " - "--vmlinux /build/%s/usr/lib/debug/boot/vmlinux " - "--kallsyms /build/%s/boot/System.map-* " - "-i %s --stdio " - "> %s" % - (self._board, - self._board, - self._board, - chroot_perf_data_file, - chroot_perf_report_file)) - self._ce.ChrootRunCommand(self._chromeos_root, - command) - - # Add a keyval to the dictionary for the events captured. - perf_report_files.append( - misc.GetOutsideChrootPath(self._chromeos_root, - chroot_perf_report_file)) - return perf_report_files - - def _GatherPerfResults(self): - report_id = 0 - for perf_report_file in self.perf_report_files: - with open(perf_report_file, "r") as f: - report_contents = f.read() - for group in re.findall(r"Events: (\S+) (\S+)", report_contents): - num_events = group[0] - event_name = group[1] - key = "perf_%s_%s" % (report_id, event_name) - value = str(misc.UnitToNumber(num_events)) - self.keyvals[key] = value - - def _PopulateFromRun(self, board, out, err, retval): - self._board = board - self.out = out - self.err = err - self.retval = retval - self.chroot_results_dir = self._GetResultsDir() - self.results_dir = misc.GetOutsideChrootPath(self._chromeos_root, - self.chroot_results_dir) - self.perf_data_files = self._GetPerfDataFiles() - # Include all perf.report data in table. - self.perf_report_files = self._GeneratePerfReportFiles() - # TODO(asharif): Do something similar with perf stat. - - # Grab keyvals from the directory. - self._ProcessResults() - - def _ProcessResults(self): - # Note that this function doesn't know anything about whether there is a - # cache hit or miss. It should process results agnostic of the cache hit - # state. - self.keyvals = self._GetKeyvals() - self.keyvals["retval"] = self.retval - # Generate report from all perf.data files. - # Now parse all perf report files and include them in keyvals. - self._GatherPerfResults() - - def _PopulateFromCacheDir(self, cache_dir): - # Read in everything from the cache directory. - with open(os.path.join(cache_dir, RESULTS_FILE), "r") as f: - self.out = pickle.load(f) - self.err = pickle.load(f) - self.retval = pickle.load(f) - - # Untar the tarball to a temporary directory - self._temp_dir = tempfile.mkdtemp(dir=os.path.join(self._chromeos_root, - "chroot", "tmp")) - - command = ("cd %s && tar xf %s" % - (self._temp_dir, - os.path.join(cache_dir, AUTOTEST_TARBALL))) - ret = self._ce.RunCommand(command) - if ret: - raise Exception("Could not untar cached tarball") - self.results_dir = self._temp_dir - self.perf_data_files = self._GetPerfDataFiles() - self.perf_report_files = self._GetPerfReportFiles() - self._ProcessResults() - - def CleanUp(self, rm_chroot_tmp): - if rm_chroot_tmp: - command = "rm -rf %s" % self.results_dir - self._ce.RunCommand(command) - if self._temp_dir: - command = "rm -rf %s" % self._temp_dir - self._ce.RunCommand(command) - - def StoreToCacheDir(self, cache_dir, machine_manager): - # Create the dir if it doesn't exist. - temp_dir = tempfile.mkdtemp() - - # Store to the temp directory. - with open(os.path.join(temp_dir, RESULTS_FILE), "w") as f: - pickle.dump(self.out, f) - pickle.dump(self.err, f) - pickle.dump(self.retval, f) - - tarball = os.path.join(temp_dir, AUTOTEST_TARBALL) - command = ("cd %s && " - "tar " - "--exclude=var/spool " - "--exclude=var/log " - "-cjf %s ." % (self.results_dir, tarball)) - ret = self._ce.RunCommand(command) - if ret: - raise Exception("Couldn't store autotest output directory.") - # Store machine info. - # TODO(asharif): Make machine_manager a singleton, and don't pass it into - # this function. - with open(os.path.join(temp_dir, MACHINE_FILE), "w") as f: - f.write(machine_manager.machine_checksum_string[self.label_name]) - - if os.path.exists(cache_dir): - command = "rm -rf {0}".format(cache_dir) - self._ce.RunCommand(command) - - command = "mkdir -p {0} && ".format(os.path.dirname(cache_dir)) - command += "mv {0} {1}".format(temp_dir, cache_dir) - ret = self._ce.RunCommand(command) - if ret: - command = "rm -rf {0}".format(temp_dir) - self._ce.RunCommand(command) - raise Exception("Could not move dir %s to dir %s" % - (temp_dir, cache_dir)) - - @classmethod - def CreateFromRun(cls, logger, chromeos_root, board, label_name, - out, err, retval): - result = cls(chromeos_root, logger, label_name) - result._PopulateFromRun(board, out, err, retval) - return result - - @classmethod - def CreateFromCacheHit(cls, chromeos_root, logger, cache_dir, label_name): - result = cls(chromeos_root, logger, label_name) - try: - result._PopulateFromCacheDir(cache_dir) - except Exception as e: - logger.LogError("Exception while using cache: %s" % e) - return None - return result - - -class CacheConditions(object): - # Cache hit only if the result file exists. - CACHE_FILE_EXISTS = 0 - - # Cache hit if the checksum of cpuinfo and totalmem of - # the cached result and the new run match. - MACHINES_MATCH = 1 - - # Cache hit if the image checksum of the cached result and the new run match. - CHECKSUMS_MATCH = 2 - - # Cache hit only if the cached result was successful - RUN_SUCCEEDED = 3 - - # Never a cache hit. - FALSE = 4 - - # Cache hit if the image path matches the cached image path. - IMAGE_PATH_MATCH = 5 - - # Cache hit if the uuid of hard disk mataches the cached one - - SAME_MACHINE_MATCH = 6 - - -class ResultsCache(object): - """ This class manages the key of the cached runs without worrying about what - is exactly stored (value). The value generation is handled by the Results - class. - """ - CACHE_VERSION = 6 - - def Init(self, chromeos_image, chromeos_root, autotest_name, iteration, - autotest_args, machine_manager, board, cache_conditions, - logger_to_use, label, share_users): - self.chromeos_image = chromeos_image - self.chromeos_root = chromeos_root - self.autotest_name = autotest_name - self.iteration = iteration - self.autotest_args = autotest_args, - self.board = board - self.cache_conditions = cache_conditions - self.machine_manager = machine_manager - self._logger = logger_to_use - self._ce = command_executer.GetCommandExecuter(self._logger) - self.label = label - self.share_users = share_users - - def _GetCacheDirForRead(self): - matching_dirs = [] - for glob_path in self._FormCacheDir(self._GetCacheKeyList(True)): - matching_dirs += glob.glob(glob_path) - - if matching_dirs: - # Cache file found. - return matching_dirs[0] - else: - return None - - def _GetCacheDirForWrite(self): - return self._FormCacheDir(self._GetCacheKeyList(False))[0] - - def _FormCacheDir(self, list_of_strings): - cache_key = " ".join(list_of_strings) - cache_dir = misc.GetFilenameFromString(cache_key) - if self.label.cache_dir: - cache_home = os.path.abspath(os.path.expanduser(self.label.cache_dir)) - cache_path = [os.path.join(cache_home, cache_dir)] - else: - cache_path = [os.path.join(SCRATCH_DIR, cache_dir)] - - for i in [x.strip() for x in self.share_users.split(",")]: - path = SCRATCH_BASE % i - cache_path.append(os.path.join(path, cache_dir)) - - return cache_path - - def _GetCacheKeyList(self, read): - if read and CacheConditions.MACHINES_MATCH not in self.cache_conditions: - machine_checksum = "*" - else: - machine_checksum = self.machine_manager.machine_checksum[self.label.name] - if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions: - checksum = "*" - else: - checksum = ImageChecksummer().Checksum(self.label) - - if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions: - image_path_checksum = "*" - else: - image_path_checksum = hashlib.md5(self.chromeos_image).hexdigest() - - if read and CacheConditions.SAME_MACHINE_MATCH not in self.cache_conditions: - machine_id_checksum = "*" - else: - for machine in self.machine_manager.GetMachines(self.label): - if machine.name == self.label.remote[0]: - machine_id_checksum = machine.machine_id_checksum - break - - autotest_args_checksum = hashlib.md5( - "".join(self.autotest_args)).hexdigest() - return (image_path_checksum, - self.autotest_name, str(self.iteration), - autotest_args_checksum, - checksum, - machine_checksum, - machine_id_checksum, - str(self.CACHE_VERSION)) - - def ReadResult(self): - if CacheConditions.FALSE in self.cache_conditions: - return None - cache_dir = self._GetCacheDirForRead() - - if not cache_dir: - return None - - if not os.path.isdir(cache_dir): - return None - - self._logger.LogOutput("Trying to read from cache dir: %s" % cache_dir) - - result = Result.CreateFromCacheHit(self.chromeos_root, - self._logger, cache_dir, self.label.name) - - if not result: - return None - - if (result.retval == 0 or - CacheConditions.RUN_SUCCEEDED not in self.cache_conditions): - return result - - return None - - def StoreResult(self, result): - cache_dir = self._GetCacheDirForWrite() - result.StoreToCacheDir(cache_dir, self.machine_manager) - - -class MockResultsCache(ResultsCache): - def Init(self, *args): - pass - - def ReadResult(self): - return None - - def StoreResult(self, result): - pass - - -class MockResult(Result): - def _PopulateFromRun(self, out, err, retval): - self.out = out - self.err = err - self.retval = retval - diff --git a/v14/crosperf/results_organizer.py b/v14/crosperf/results_organizer.py deleted file mode 100644 index 2e5c9296..00000000 --- a/v14/crosperf/results_organizer.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/python - -# Copyright 2012 Google Inc. All Rights Reserved. -"""Parse data from benchmark_runs for tabulator.""" -import re - - -class ResultOrganizer(object): - """Create a dict from benchmark_runs. - - The structure of the output dict is as follows: - {"benchmark_1":[ - [{"key1":"v1", "key2":"v2"},{"key1":"v1", "key2","v2"}] - #one label - [] - #the other label - ] - "benchmark_2": - [ - ]}. - """ - key_filter = ["milliseconds_", - "retval", - "iterations", - "ms_", - "score_"] - - def __init__(self, benchmark_runs, labels, benchmarks=None): - self.result = {} - self.labels = [] - self.prog = re.compile(r"(\w+)\{(\d+)\}") - self.benchmarks = benchmarks - if not self.benchmarks: - self.benchmarks = [] - for label in labels: - self.labels.append(label.name) - for benchmark_run in benchmark_runs: - benchmark_name = benchmark_run.benchmark.name - if benchmark_name not in self.result: - self.result[benchmark_name] = [] - while len(self.result[benchmark_name]) < len(labels): - self.result[benchmark_name].append([]) - label_index = self.labels.index(benchmark_run.label.name) - cur_table = self.result[benchmark_name][label_index] - index = benchmark_run.iteration - 1 - while index >= len(cur_table): - cur_table.append({}) - cur_dict = cur_table[index] - if not benchmark_run.result: - continue - benchmark = benchmark_run.benchmark - key_filter_on = (benchmark.key_results_only and - "PyAutoPerfTest" in benchmark.name + benchmark.autotest_name and - "perf." not in benchmark.autotest_args) - for autotest_key in benchmark_run.result.keyvals: - if (key_filter_on and - not any([key for key in self.key_filter if key in autotest_key]) - ): - continue - result_value = benchmark_run.result.keyvals[autotest_key] - cur_dict[autotest_key] = result_value - self._DuplicatePass() - - def _DuplicatePass(self): - for bench, data in self.result.items(): - max_dup = self._GetMaxDup(data) - if not max_dup: - continue - for label in data: - index = data.index(label) - data[index] = self._GetNonDupLabel(max_dup, label) - self._AdjustIteration(max_dup, bench) - - def _GetMaxDup(self, data): - """Find the maximum i inside ABCD{i}.""" - max_dup = 0 - for label in data: - for run in label: - for key in run: - if re.match(self.prog, key): - max_dup = max(max_dup, - int(re.search(self.prog, key).group(2))) - return max_dup - - def _GetNonDupLabel(self, max_dup, label): - """Create new list for the runs of the same label.""" - new_label = [] - for run in label: - start_index = len(new_label) - new_label.append(dict(run)) - for i in range(max_dup): - new_label.append({}) - new_run = new_label[start_index] - for key, value in new_run.items(): - if re.match(self.prog, key): - new_key = re.search(self.prog, key).group(1) - index = int(re.search(self.prog, key).group(2)) - new_label[start_index+index][new_key] = str(value) - del new_run[key] - return new_label - - def _AdjustIteration(self, max_dup, bench): - """Adjust the interation numbers if the have keys like ABCD{i}.""" - for benchmark in self.benchmarks: - if benchmark.name == bench: - if not benchmark.iteration_adjusted: - benchmark.iteration_adjusted = True - benchmark.iterations *= (max_dup +1) diff --git a/v14/crosperf/results_report.py b/v14/crosperf/results_report.py deleted file mode 100644 index 61c67d5b..00000000 --- a/v14/crosperf/results_report.py +++ /dev/null @@ -1,502 +0,0 @@ -#!/usr/bin/python - -# Copyright 2011 Google Inc. All Rights Reserved. - -from utils.tabulator import * - -from column_chart import ColumnChart -from results_organizer import ResultOrganizer -from perf_table import PerfTable - - -class ResultsReport(object): - MAX_COLOR_CODE = 255 - PERF_ROWS = 5 - - def __init__(self, experiment): - self.experiment = experiment - self.benchmark_runs = experiment.benchmark_runs - self.labels = experiment.labels - self.benchmarks = experiment.benchmarks - self.baseline = self.labels[0] - - def _SortByLabel(self, runs): - labels = {} - for benchmark_run in runs: - if benchmark_run.label_name not in labels: - labels[benchmark_run.label_name] = [] - labels[benchmark_run.label_name].append(benchmark_run) - return labels - - def GetFullTables(self, perf=False): - columns = [Column(NonEmptyCountResult(), - Format(), - "Completed"), - Column(RawResult(), - Format()), - Column(MinResult(), - Format()), - Column(MaxResult(), - Format()), - Column(AmeanResult(), - Format()), - Column(StdResult(), - Format(), "StdDev"), - Column(CoeffVarResult(), - CoeffVarFormat(), "StdDev/Mean"), - Column(GmeanRatioResult(), - RatioFormat(), "GmeanSpeedup"), - Column(PValueResult(), - PValueFormat(), "p-value") - ] - if not perf: - return self._GetTables(self.labels, self.benchmark_runs, columns) - return self. _GetPerfTables(self.labels, columns) - - def GetSummaryTables(self, perf=False): - columns = [Column(NonEmptyCountResult(), - Format(), - "Completed"), - Column(AmeanResult(), - Format()), - Column(StdResult(), - Format(), "StdDev"), - Column(CoeffVarResult(), - CoeffVarFormat(), "StdDev/Mean"), - Column(GmeanRatioResult(), - RatioFormat(), "GmeanSpeedup"), - Column(GmeanRatioResult(), - ColorBoxFormat(), " "), - Column(PValueResult(), - PValueFormat(), "p-value") - ] - if not perf: - return self._GetTables(self.labels, self.benchmark_runs, columns) - return self. _GetPerfTables(self.labels, columns) - - def _ParseColumn(self, columns, iteration): - new_column = [] - for column in columns: - if column.result.__class__.__name__ != "RawResult": - #TODO(asharif): tabulator should support full table natively. - new_column.append(column) - else: - for i in range(iteration): - cc = Column(LiteralResult(i), Format(), str(i+1)) - new_column.append(cc) - return new_column - - def _AreAllRunsEmpty(self, runs): - for label in runs: - for dictionary in label: - if dictionary: - return False - return True - - def _GetTableHeader(self, benchmark): - benchmark_info = ("Benchmark: {0}; Iterations: {1}" - .format(benchmark.name, benchmark.iterations)) - cell = Cell() - cell.string_value = benchmark_info - cell.header = True - return [[cell]] - - def _GetTables(self, labels, benchmark_runs, columns): - tables = [] - ro = ResultOrganizer(benchmark_runs, labels, self.benchmarks) - result = ro.result - label_name = ro.labels - for item in result: - runs = result[item] - for benchmark in self.benchmarks: - if benchmark.name == item: - break - ben_table = self._GetTableHeader(benchmark) - - if self._AreAllRunsEmpty(runs): - cell = Cell() - cell.string_value = ("This benchmark contains no result." - " Is the benchmark name valid?") - cell_table = [[cell]] - else: - tg = TableGenerator(runs, label_name) - table = tg.GetTable() - parsed_columns = self._ParseColumn(columns, benchmark.iterations) - tf = TableFormatter(table, parsed_columns) - cell_table = tf.GetCellTable() - tables.append(ben_table) - tables.append(cell_table) - return tables - - def _GetPerfTables(self, labels, columns): - tables = [] - label_names = [label.name for label in labels] - p_table = PerfTable(self.experiment, label_names) - - if not p_table.perf_data: - return tables - - for benchmark in p_table.perf_data: - ben = None - for ben in self.benchmarks: - if ben.name == benchmark: - break - - ben_table = self._GetTableHeader(ben) - tables.append(ben_table) - benchmark_data = p_table.perf_data[benchmark] - row_info = p_table.row_info[benchmark] - table = [] - for event in benchmark_data: - tg = TableGenerator(benchmark_data[event], label_names, - sort=TableGenerator.SORT_BY_VALUES_DESC) - table = tg.GetTable(max(self.PERF_ROWS, row_info[event])) - parsed_columns = self._ParseColumn(columns, ben.iterations) - tf = TableFormatter(table, parsed_columns) - tf.GenerateCellTable() - tf.AddColumnName() - tf.AddLabelName() - tf.AddHeader(str(event)) - table = tf.GetCellTable(headers=False) - tables.append(table) - return tables - - def PrintTables(self, tables, out_to): - output = "" - if not tables: - return output - for table in tables: - if out_to == "HTML": - tp = TablePrinter(table, TablePrinter.HTML) - elif out_to == "PLAIN": - tp = TablePrinter(table, TablePrinter.PLAIN) - elif out_to == "CONSOLE": - tp = TablePrinter(table, TablePrinter.CONSOLE) - elif out_to == "TSV": - tp = TablePrinter(table, TablePrinter.TSV) - elif out_to == "EMAIL": - tp = TablePrinter(table, TablePrinter.EMAIL) - else: - pass - output += tp.Print() - return output - - -class TextResultsReport(ResultsReport): - TEXT = """ -=========================================== -Results report for: '%s' -=========================================== - -------------------------------------------- -Summary -------------------------------------------- -%s - - -Number re-images: %s - -------------------------------------------- -Benchmark Run Status -------------------------------------------- -%s - - -------------------------------------------- -Perf Data -------------------------------------------- -%s - - - -Experiment File -------------------------------------------- -%s - - -CPUInfo -------------------------------------------- -%s -=========================================== -""" - - def __init__(self, experiment, email=False): - super(TextResultsReport, self).__init__(experiment) - self.email = email - - def GetStatusTable(self): - """Generate the status table by the tabulator.""" - table = [["", ""]] - columns = [Column(LiteralResult(iteration=0), Format(), "Status"), - Column(LiteralResult(iteration=1), Format(), "Failing Reason")] - - for benchmark_run in self.benchmark_runs: - status = [benchmark_run.name, [benchmark_run.timeline.GetLastEvent(), - benchmark_run.failure_reason]] - table.append(status) - tf = TableFormatter(table, columns) - cell_table = tf.GetCellTable() - return [cell_table] - - def GetReport(self): - """Generate the report for email and console.""" - status_table = self.GetStatusTable() - summary_table = self.GetSummaryTables() - full_table = self.GetFullTables() - perf_table = self.GetSummaryTables(perf=True) - if not perf_table: - perf_table = None - if not self.email: - return self.TEXT % (self.experiment.name, - self.PrintTables(summary_table, "CONSOLE"), - self.experiment.machine_manager.num_reimages, - self.PrintTables(status_table, "CONSOLE"), - self.PrintTables(perf_table, "CONSOLE"), - self.experiment.experiment_file, - self.experiment.machine_manager.GetAllCPUInfo( - self.experiment.labels)) - - return self.TEXT % (self.experiment.name, - self.PrintTables(summary_table, "EMAIL"), - self.experiment.machine_manager.num_reimages, - self.PrintTables(status_table, "EMAIL"), - self.PrintTables(perf_table, "EMAIL"), - self.experiment.experiment_file, - self.experiment.machine_manager.GetAllCPUInfo( - self.experiment.labels)) - - -class HTMLResultsReport(ResultsReport): - - HTML = """ -<html> - <head> - <style type="text/css"> - -body { - font-family: "Lucida Sans Unicode", "Lucida Grande", Sans-Serif; - font-size: 12px; -} - -pre { - margin: 10px; - color: #039; - font-size: 14px; -} - -.chart { - display: inline; -} - -.hidden { - visibility: hidden; -} - -.results-section { - border: 1px solid #b9c9fe; - margin: 10px; -} - -.results-section-title { - background-color: #b9c9fe; - color: #039; - padding: 7px; - font-size: 14px; - width: 200px; -} - -.results-section-content { - margin: 10px; - padding: 10px; - overflow:auto; -} - -#box-table-a { - font-size: 12px; - width: 480px; - text-align: left; - border-collapse: collapse; -} - -#box-table-a th { - padding: 6px; - background: #b9c9fe; - border-right: 1px solid #fff; - border-bottom: 1px solid #fff; - color: #039; - text-align: center; -} - -#box-table-a td { - padding: 4px; - background: #e8edff; - border-bottom: 1px solid #fff; - border-right: 1px solid #fff; - color: #669; - border-top: 1px solid transparent; -} - -#box-table-a tr:hover td { - background: #d0dafd; - color: #339; -} - - </style> - <script type='text/javascript' src='https://www.google.com/jsapi'></script> - <script type='text/javascript'> - google.load('visualization', '1', {packages:['corechart']}); - google.setOnLoadCallback(init); - function init() { - switchTab('summary', 'html'); - %s - switchTab('full', 'html'); - drawTable(); - } - function drawTable() { - %s - } - function switchTab(table, tab) { - document.getElementById(table + '-html').style.display = 'none'; - document.getElementById(table + '-text').style.display = 'none'; - document.getElementById(table + '-tsv').style.display = 'none'; - document.getElementById(table + '-' + tab).style.display = 'block'; - } - </script> - </head> - - <body> - <div class='results-section'> - <div class='results-section-title'>Summary Table</div> - <div class='results-section-content'> - <div id='summary-html'>%s</div> - <div id='summary-text'><pre>%s</pre></div> - <div id='summary-tsv'><pre>%s</pre></div> - </div> - %s - </div> - %s - <div class='results-section'> - <div class='results-section-title'>Charts</div> - <div class='results-section-content'>%s</div> - </div> - <div class='results-section'> - <div class='results-section-title'>Full Table</div> - <div class='results-section-content'> - <div id='full-html'>%s</div> - <div id='full-text'><pre>%s</pre></div> - <div id='full-tsv'><pre>%s</pre></div> - </div> - %s - </div> - <div class='results-section'> - <div class='results-section-title'>Experiment File</div> - <div class='results-section-content'> - <pre>%s</pre> - </div> - </div> - </body> -</html> -""" - - PERF_HTML = """ - <div class='results-section'> - <div class='results-section-title'>Perf Table</div> - <div class='results-section-content'> - <div id='perf-html'>%s</div> - <div id='perf-text'><pre>%s</pre></div> - <div id='perf-tsv'><pre>%s</pre></div> - </div> - %s - </div> -""" - - def __init__(self, experiment): - super(HTMLResultsReport, self).__init__(experiment) - - def _GetTabMenuHTML(self, table): - return """ -<div class='tab-menu'> - <a href="javascript:switchTab('%s', 'html')">HTML</a> - <a href="javascript:switchTab('%s', 'text')">Text</a> - <a href="javascript:switchTab('%s', 'tsv')">TSV</a> -</div>""" % (table, table, table) - - def GetReport(self): - chart_javascript = "" - charts = self._GetCharts(self.labels, self.benchmark_runs) - for chart in charts: - chart_javascript += chart.GetJavascript() - chart_divs = "" - for chart in charts: - chart_divs += chart.GetDiv() - - summary_table = self.GetSummaryTables() - full_table = self.GetFullTables() - perf_table = self.GetSummaryTables(perf=True) - if perf_table: - perf_html = self.PERF_HTML % ( - self.PrintTables(perf_table, "HTML"), - self.PrintTables(perf_table, "PLAIN"), - self.PrintTables(perf_table, "TSV"), - self._GetTabMenuHTML("perf") - ) - perf_init = "switchTab('perf', 'html');" - else: - perf_html = "" - perf_init = "" - - return self.HTML % (perf_init, - chart_javascript, - self.PrintTables(summary_table, "HTML"), - self.PrintTables(summary_table, "PLAIN"), - self.PrintTables(summary_table, "TSV"), - self._GetTabMenuHTML("summary"), - perf_html, - chart_divs, - self.PrintTables(full_table, "HTML"), - self.PrintTables(full_table, "PLAIN"), - self.PrintTables(full_table, "TSV"), - self._GetTabMenuHTML("full"), - self.experiment.experiment_file) - - def _GetCharts(self, labels, benchmark_runs): - charts = [] - ro = ResultOrganizer(benchmark_runs, labels) - result = ro.result - for item in result: - runs = result[item] - tg = TableGenerator(runs, ro.labels) - table = tg.GetTable() - columns = [Column(AmeanResult(), - Format()), - Column(MinResult(), - Format()), - Column(MaxResult(), - Format()) - ] - tf = TableFormatter(table, columns) - data_table = tf.GetCellTable() - - for i in range(2, len(data_table)): - cur_row_data = data_table[i] - autotest_key = cur_row_data[0].string_value - title = "{0}: {1}".format(item, autotest_key.replace("/", "")) - chart = ColumnChart(title, 300, 200) - chart.AddColumn("Label", "string") - chart.AddColumn("Average", "number") - chart.AddColumn("Min", "number") - chart.AddColumn("Max", "number") - chart.AddSeries("Min", "line", "black") - chart.AddSeries("Max", "line", "black") - cur_index = 1 - for label in ro.labels: - chart.AddRow([label, cur_row_data[cur_index].value, - cur_row_data[cur_index + 1].value, - cur_row_data[cur_index + 2].value]) - if isinstance(cur_row_data[cur_index].value, str): - chart = None - break - cur_index += 3 - if chart: - charts.append(chart) - return charts diff --git a/v14/crosperf/results_sorter.py b/v14/crosperf/results_sorter.py deleted file mode 100644 index 985a91fb..00000000 --- a/v14/crosperf/results_sorter.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/python - -# Copyright 2011 Google Inc. All Rights Reserved. - - -class ResultSorter(object): - def __init__(self, benchmark_runs): - self.table = {} - for benchmark_run in benchmark_runs: - benchmark_name = benchmark_run.benchmark_name - label_name = benchmark_run.label_name - if not benchmark_run.result: - continue - for autotest_key in benchmark_run.result.keyvals: - result_tuple = (benchmark_name, autotest_key, label_name) - if result_tuple not in self.table: - self.table[result_tuple] = [] - - cell = self.table[result_tuple] - index = benchmark_run.iteration - 1 - while index >= len(cell): - cell.append(None) - - result_value = benchmark_run.result.keyvals[autotest_key] - try: - result_value = float(result_value) - except ValueError: - pass - - cell[index] = result_value - - self.autotest_keys = {} - for benchmark_run in benchmark_runs: - benchmark_name = benchmark_run.benchmark_name - if benchmark_name not in self.autotest_keys: - self.autotest_keys[benchmark_name] = {} - if not benchmark_run.result: - continue - for autotest_key in benchmark_run.result.keyvals: - self.autotest_keys[benchmark_name][autotest_key] = True - - def GetAutotestKeys(self, benchmark_name): - return self.autotest_keys[benchmark_name].keys() - - def GetResults(self, benchmark_name, autotest_key, label_name): - try: - return self.table[(benchmark_name, autotest_key, label_name)] - except KeyError: - return [] diff --git a/v14/crosperf/run_tests.sh b/v14/crosperf/run_tests.sh deleted file mode 100755 index 2a35cc73..00000000 --- a/v14/crosperf/run_tests.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# -# Copyright 2011 Google Inc. All Rights Reserved. -# Author: raymes@google.com (Raymes Khoury) - -export PYTHONPATH+=":.." -for test in $(find -name \*test.py); do - echo RUNNING: ${test} - if ! ./${test} ; then - echo "Test Failed!" - exit 1 - fi -done diff --git a/v14/crosperf/settings.py b/v14/crosperf/settings.py deleted file mode 100644 index e407a143..00000000 --- a/v14/crosperf/settings.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/python - -# Copyright 2011 Google Inc. All Rights Reserved. - - -class Settings(object): - """Class representing settings (a set of fields) from an experiment file.""" - - def __init__(self, name, settings_type): - self.name = name - self.settings_type = settings_type - self.fields = {} - self.parent = None - - def SetParentSettings(self, settings): - """Set the parent settings which these settings can inherit from.""" - self.parent = settings - - def AddField(self, field): - name = field.name - if name in self.fields: - raise Exception("Field %s defined previously." % name) - self.fields[name] = field - - def SetField(self, name, value, append=False): - if name not in self.fields: - raise Exception("'%s' is not a valid field in '%s' settings" - % (name, self.settings_type)) - if append: - self.fields[name].Append(value) - else: - self.fields[name].Set(value) - - def GetField(self, name): - """Get the value of a field with a given name.""" - if name not in self.fields: - raise Exception("Field '%s' not a valid field in '%s' settings." % - (name, self.name)) - field = self.fields[name] - if not field.assigned and field.required: - raise Exception("Required field '%s' not defined in '%s' settings." % - (name, self.name)) - return self.fields[name].Get() - - def Inherit(self): - """Inherit any unset values from the parent settings.""" - for name in self.fields: - if (not self.fields[name].assigned and self.parent and - name in self.parent.fields and self.parent.fields[name].assigned): - self.fields[name].Set(self.parent.GetField(name), parse=False) - - def Override(self, settings): - """Override settings with settings from a different object.""" - for name in settings.fields: - if name in self.fields and settings.fields[name].assigned: - self.fields[name].Set(settings.GetField(name), parse=False) - - def Validate(self): - """Check that all required fields have been set.""" - for name in self.fields: - if not self.fields[name].assigned and self.fields[name].required: - raise Exception("Field %s is invalid." % name) diff --git a/v14/crosperf/settings_factory.py b/v14/crosperf/settings_factory.py deleted file mode 100644 index 924bc114..00000000 --- a/v14/crosperf/settings_factory.py +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/python - -# Copyright 2011 Google Inc. All Rights Reserved. -"""Setting files for global, benchmark and labels.""" - -from field import BooleanField -from field import FloatField -from field import IntegerField -from field import ListField -from field import TextField -from settings import Settings - - -class BenchmarkSettings(Settings): - def __init__(self, name): - super(BenchmarkSettings, self).__init__(name, "benchmark") - self.AddField(TextField("autotest_name", - description="The name of the autotest to run." - "Defaults to the name of the benchmark.")) - self.AddField(TextField("autotest_args", - description="Arguments to be passed to the " - "autotest.")) - self.AddField(IntegerField("iterations", default=1, - description="Number of iterations to run the " - "autotest.")) - self.AddField(FloatField("outlier_range", default=0.2, - description="The percentage of highest/lowest " - "values to omit when computing the average.")) - self.AddField(BooleanField("rm_chroot_tmp", default=False, - description="Whether remove the run_remote_test" - "result in the chroot")) - self.AddField(BooleanField("key_results_only", default=True, - description="Whether only show the key results" - "of pyautoperf")) - self.AddField(TextField("perf_args", default="", - description="The optional profile command. It " - "enables perf commands to record perforamance " - "related counters. It must start with perf " - "command record or stat followed by arguments.")) - - -class LabelSettings(Settings): - def __init__(self, name): - super(LabelSettings, self).__init__(name, "label") - self.AddField(TextField("chromeos_image", required=True, - description="The path to the image to run tests " - "on.")) - self.AddField(TextField("chromeos_root", - description="The path to a chromeos checkout which " - "contains a src/scripts directory. Defaults to " - "the chromeos checkout which contains the " - "chromeos_image.")) - self.AddField(TextField("md5sum", default="", - description="The md5sum of this image")) - self.AddField(TextField("board", required=True, description="The target " - "board for running experiments on, e.g. x86-alex.")) - self.AddField(ListField("remote", description= - "A comma-separated list of ip's of chromeos" - "devices to run experiments on.")) - self.AddField(TextField("image_args", required=False, - default="", - description="Extra arguments to pass to " - "image_chromeos.py.")) - self.AddField(TextField("cache_dir", default="", - description="The cache dir for this image.")) - - -class GlobalSettings(Settings): - def __init__(self, name): - super(GlobalSettings, self).__init__(name, "global") - self.AddField(TextField("name", - description="The name of the experiment. Just an " - "identifier.")) - self.AddField(TextField("board", description="The target " - "board for running experiments on, e.g. x86-alex.")) - self.AddField(ListField("remote", - description="A comma-separated list of ip's of " - "chromeos devices to run experiments on.")) - self.AddField(BooleanField("rerun_if_failed", description="Whether to " - "re-run failed autotest runs or not.", - default=False)) - self.AddField(BooleanField("rm_chroot_tmp", default=False, - description="Whether remove the run_remote_test" - "result in the chroot")) - self.AddField(ListField("email", description="Space-seperated" - "list of email addresses to send email to.")) - self.AddField(BooleanField("rerun", description="Whether to ignore the " - "cache and for autotests to be re-run.", - default=False)) - self.AddField(BooleanField("same_specs", default=True, - description="Ensure cached runs are run on the " - "same kind of devices which are specified as a " - "remote.")) - self.AddField(BooleanField("same_machine", default=False, - description="Ensure cached runs are run on the " - "exact the same remote")) - self.AddField(IntegerField("iterations", default=1, - description="Number of iterations to run all " - "autotests.")) - self.AddField(TextField("chromeos_root", - description="The path to a chromeos checkout which " - "contains a src/scripts directory. Defaults to " - "the chromeos checkout which contains the " - "chromeos_image.")) - self.AddField(BooleanField("key_results_only", default=True, - description="Whether only show the key results" - "of pyautoperf")) - self.AddField(IntegerField("acquire_timeout", default=0, - description="Number of seconds to wait for " - "machine before exit if all the machines in " - "the experiment file are busy. Default is 0")) - self.AddField(TextField("perf_args", default="", - description="The optional profile command. It " - "enables perf commands to record perforamance " - "related counters. It must start with perf " - "command record or stat followed by arguments.")) - self.AddField(TextField("cache_dir", default="", - description="The abs path of cache dir. " - "Default is /home/$(whoami)/cros_scratch.")) - self.AddField(BooleanField("no_email", default=False, - description="Whether to disable the email to " - "user after crosperf finishes.")) - self.AddField(TextField("share_users", default="", - description="Who's cache data you want to " - "use. It accepts multiple users seperated by \",\"")) - - -class SettingsFactory(object): - """Factory class for building different types of Settings objects. - - This factory is currently hardcoded to produce settings for ChromeOS - experiment files. The idea is that in the future, other types - of settings could be produced. - """ - - def GetSettings(self, name, settings_type): - if settings_type == "label" or not settings_type: - return LabelSettings(name) - if settings_type == "global": - return GlobalSettings(name) - if settings_type == "benchmark": - return BenchmarkSettings(name) - - raise Exception("Invalid settings type: '%s'." % settings_type) diff --git a/v14/crosperf/test_flag.py b/v14/crosperf/test_flag.py deleted file mode 100644 index 613138b2..00000000 --- a/v14/crosperf/test_flag.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/python -# -# Copyright 2011 Google Inc. All Rights Reserved. - -"""A global variable for testing.""" - - -_is_test = [False] - - -def SetTestMode(flag): - _is_test[0] = flag - - -def GetTestMode(): - return _is_test[0] diff --git a/v14/image_chromeos.py b/v14/image_chromeos.py deleted file mode 100755 index d5d5f7fe..00000000 --- a/v14/image_chromeos.py +++ /dev/null @@ -1,320 +0,0 @@ -#!/usr/bin/python -# -# Copyright 2011 Google Inc. All Rights Reserved. - -"""Script to image a ChromeOS device. - -This script images a remote ChromeOS device with a specific image." -""" - -__author__ = "asharif@google.com (Ahmad Sharif)" - -import filecmp -import glob -import optparse -import os -import re -import shutil -import sys -import tempfile -import time - -from utils import command_executer -from utils import logger -from utils import misc -from utils.file_utils import FileUtils - -checksum_file = "/usr/local/osimage_checksum_file" -lock_file = "/tmp/image_chromeos_lock/image_chromeos_lock" - -def Usage(parser, message): - print "ERROR: " + message - parser.print_help() - sys.exit(0) - - -def DoImage(argv): - """Build ChromeOS.""" - - # Common initializations - cmd_executer = command_executer.GetCommandExecuter() - l = logger.GetLogger() - - parser = optparse.OptionParser() - parser.add_option("-c", "--chromeos_root", dest="chromeos_root", - help="Target directory for ChromeOS installation.") - parser.add_option("-r", "--remote", dest="remote", - help="Target device.") - parser.add_option("-i", "--image", dest="image", - help="Image binary file.") - parser.add_option("-b", "--board", dest="board", - help="Target board override.") - parser.add_option("-f", "--force", dest="force", - action="store_true", - default=False, - help="Force an image even if it is non-test.") - parser.add_option("-a", - "--image_args", - dest="image_args") - - - options = parser.parse_args(argv[1:])[0] - - if options.chromeos_root is None: - Usage(parser, "--chromeos_root must be set") - - if options.remote is None: - Usage(parser, "--remote must be set") - - options.chromeos_root = os.path.expanduser(options.chromeos_root) - - if options.board is None: - board = cmd_executer.CrosLearnBoard(options.chromeos_root, options.remote) - else: - board = options.board - - if options.image is None: - images_dir = misc.GetImageDir(options.chromeos_root, board) - image = os.path.join(images_dir, - "latest", - "chromiumos_test_image.bin") - if not os.path.exists(image): - image = os.path.join(images_dir, - "latest", - "chromiumos_image.bin") - else: - image = options.image - image = os.path.expanduser(image) - - image = os.path.realpath(image) - - if not os.path.exists(image): - Usage(parser, "Image file: " + image + " does not exist!") - - image_checksum = FileUtils().Md5File(image) - - command = "cat " + checksum_file - retval, device_checksum, err = cmd_executer.CrosRunCommand(command, - return_output=True, - chromeos_root=options.chromeos_root, - machine=options.remote) - - device_checksum = device_checksum.strip() - image_checksum = str(image_checksum) - - l.LogOutput("Image checksum: " + image_checksum) - l.LogOutput("Device checksum: " + device_checksum) - - if image_checksum != device_checksum: - [found, located_image] = LocateOrCopyImage(options.chromeos_root, - image, - board=board) - - l.LogOutput("Checksums do not match. Re-imaging...") - - is_test_image = IsImageModdedForTest(options.chromeos_root, - located_image) - - if not is_test_image and not options.force: - logger.GetLogger().LogFatal("Have to pass --force to image a non-test " - "image!") - - # If the device has /tmp mounted as noexec, image_to_live.sh can fail. - command = "mount -o remount,rw,exec /tmp" - cmd_executer.CrosRunCommand(command, - chromeos_root=options.chromeos_root, - machine=options.remote) - - real_src_dir = os.path.join(os.path.realpath(options.chromeos_root), - "src") - if located_image.find(real_src_dir) != 0: - raise Exception("Located image: %s not in chromeos_root: %s" % - (located_image, options.chromeos_root)) - chroot_image = os.path.join( - "..", - located_image[len(real_src_dir):].lstrip("/")) - cros_image_to_target_args = ["--remote=%s" % options.remote, - "--board=%s" % board, - "--from=%s" % os.path.dirname(chroot_image), - "--image-name=%s" % - os.path.basename(located_image)] - - command = ("./bin/cros_image_to_target.py %s" % - " ".join(cros_image_to_target_args)) - if options.image_args: - command += " %s" % options.image_args - - # Workaround for crosbug.com/35684. - os.chmod(misc.GetChromeOSKeyFile(options.chromeos_root), 0600) - retval = cmd_executer.ChrootRunCommand(options.chromeos_root, - command) - if found == False: - temp_dir = os.path.dirname(located_image) - l.LogOutput("Deleting temp image dir: %s" % temp_dir) - shutil.rmtree(temp_dir) - - logger.GetLogger().LogFatalIf(retval, "Image command failed") - - # Unfortunately cros_image_to_target.py sometimes returns early when the - # machine isn't fully up yet. - retval = EnsureMachineUp(options.chromeos_root, options.remote) - - command = "echo %s > %s && chmod -w %s" % (image_checksum, checksum_file, - checksum_file) - retval = cmd_executer.CrosRunCommand(command, - chromeos_root=options.chromeos_root, - machine=options.remote) - logger.GetLogger().LogFatalIf(retval, "Writing checksum failed.") - - successfully_imaged = VerifyChromeChecksum(options.chromeos_root, - image, - options.remote) - logger.GetLogger().LogFatalIf(not successfully_imaged, - "Image verification failed!") - TryRemountPartitionAsRW(options.chromeos_root, options.remote) - else: - l.LogOutput("Checksums match. Skipping reimage") - return retval - - -def LocateOrCopyImage(chromeos_root, image, board=None): - l = logger.GetLogger() - if board is None: - board_glob = "*" - else: - board_glob = board - - chromeos_root_realpath = os.path.realpath(chromeos_root) - image = os.path.realpath(image) - - if image.startswith("%s/" % chromeos_root_realpath): - return [True, image] - - # First search within the existing build dirs for any matching files. - images_glob = ("%s/src/build/images/%s/*/*.bin" % - (chromeos_root_realpath, - board_glob)) - images_list = glob.glob(images_glob) - for potential_image in images_list: - if filecmp.cmp(potential_image, image): - l.LogOutput("Found matching image %s in chromeos_root." % potential_image) - return [True, potential_image] - # We did not find an image. Copy it in the src dir and return the copied file. - if board is None: - board = "" - base_dir = ("%s/src/build/images/%s" % - (chromeos_root_realpath, - board)) - if not os.path.isdir(base_dir): - os.makedirs(base_dir) - temp_dir = tempfile.mkdtemp(prefix="%s/tmp" % base_dir) - new_image = "%s/%s" % (temp_dir, os.path.basename(image)) - l.LogOutput("No matching image found. Copying %s to %s" % - (image, new_image)) - shutil.copyfile(image, new_image) - return [False, new_image] - - -def GetImageMountCommand(chromeos_root, image, rootfs_mp, stateful_mp): - image_dir = os.path.dirname(image) - image_file = os.path.basename(image) - mount_command = ("cd %s/src/scripts &&" - "./mount_gpt_image.sh --from=%s --image=%s" - " --safe --read_only" - " --rootfs_mountpt=%s" - " --stateful_mountpt=%s" % - (chromeos_root, image_dir, image_file, rootfs_mp, - stateful_mp)) - return mount_command - - -def MountImage(chromeos_root, image, rootfs_mp, stateful_mp, unmount=False): - cmd_executer = command_executer.GetCommandExecuter() - command = GetImageMountCommand(chromeos_root, image, rootfs_mp, stateful_mp) - if unmount: - command = "%s --unmount" % command - retval = cmd_executer.RunCommand(command) - logger.GetLogger().LogFatalIf(retval, "Mount/unmount command failed!") - return retval - - -def IsImageModdedForTest(chromeos_root, image): - rootfs_mp = tempfile.mkdtemp() - stateful_mp = tempfile.mkdtemp() - MountImage(chromeos_root, image, rootfs_mp, stateful_mp) - lsb_release_file = os.path.join(rootfs_mp, "etc/lsb-release") - lsb_release_contents = open(lsb_release_file).read() - is_test_image = re.search("test", lsb_release_contents, re.IGNORECASE) - MountImage(chromeos_root, image, rootfs_mp, stateful_mp, unmount=True) - return is_test_image - - -def VerifyChromeChecksum(chromeos_root, image, remote): - cmd_executer = command_executer.GetCommandExecuter() - rootfs_mp = tempfile.mkdtemp() - stateful_mp = tempfile.mkdtemp() - MountImage(chromeos_root, image, rootfs_mp, stateful_mp) - image_chrome_checksum = FileUtils().Md5File("%s/opt/google/chrome/chrome" % - rootfs_mp) - MountImage(chromeos_root, image, rootfs_mp, stateful_mp, unmount=True) - - command = "md5sum /opt/google/chrome/chrome" - [r, o, e] = cmd_executer.CrosRunCommand(command, - return_output=True, - chromeos_root=chromeos_root, - machine=remote) - device_chrome_checksum = o.split()[0] - if image_chrome_checksum.strip() == device_chrome_checksum.strip(): - return True - else: - return False - -# Remount partition as writable. -# TODO: auto-detect if an image is built using --noenable_rootfs_verification. -def TryRemountPartitionAsRW(chromeos_root, remote): - l = logger.GetLogger() - cmd_executer = command_executer.GetCommandExecuter() - command = "sudo mount -o remount,rw /" - retval = cmd_executer.CrosRunCommand(\ - command, chromeos_root=chromeos_root, machine=remote, terminated_timeout=10) - if retval: - ## Safely ignore. - l.LogWarning("Failed to remount partition as rw, " - "probably the image was not built with " - "\"--noenable_rootfs_verification\", " - "you can safely ignore this.") - else: - l.LogOutput("Re-mounted partition as writable.") - - -def EnsureMachineUp(chromeos_root, remote): - l = logger.GetLogger() - cmd_executer = command_executer.GetCommandExecuter() - timeout = 600 - magic = "abcdefghijklmnopqrstuvwxyz" - command = "echo %s" % magic - start_time = time.time() - while True: - current_time = time.time() - if current_time - start_time > timeout: - l.LogError("Timeout of %ss reached. Machine still not up. Aborting." % - timeout) - return False - retval = cmd_executer.CrosRunCommand(command, - chromeos_root=chromeos_root, - machine=remote) - if not retval: - return True - - -def Main(argv): - misc.AcquireLock(lock_file) - try: - return DoImage(argv) - finally: - misc.ReleaseLock(lock_file) - - -if __name__ == "__main__": - retval = Main(sys.argv) - sys.exit(retval) diff --git a/v14/utils/__init__.py b/v14/utils/__init__.py deleted file mode 100644 index e69de29b..00000000 --- a/v14/utils/__init__.py +++ /dev/null diff --git a/v14/utils/colortrans.py b/v14/utils/colortrans.py deleted file mode 100644 index 37e91572..00000000 --- a/v14/utils/colortrans.py +++ /dev/null @@ -1,376 +0,0 @@ -#! /usr/bin/env python - -""" Convert values between RGB hex codes and xterm-256 color codes. - -Nice long listing of all 256 colors and their codes. Useful for -developing console color themes, or even script output schemes. - -Resources: -* http://en.wikipedia.org/wiki/8-bit_color -* http://en.wikipedia.org/wiki/ANSI_escape_code -* /usr/share/X11/rgb.txt - -I'm not sure where this script was inspired from. I think I must have -written it from scratch, though it's been several years now. -""" - -__author__ = 'Micah Elliott http://MicahElliott.com' -__version__ = '0.1' -__copyright__ = 'Copyright (C) 2011 Micah Elliott. All rights reserved.' -__license__ = 'WTFPL http://sam.zoy.org/wtfpl/' - -#--------------------------------------------------------------------- - -import sys, re - -CLUT = [ # color look-up table -# 8-bit, RGB hex - - # Primary 3-bit (8 colors). Unique representation! - ('00', '000000'), - ('01', '800000'), - ('02', '008000'), - ('03', '808000'), - ('04', '000080'), - ('05', '800080'), - ('06', '008080'), - ('07', 'c0c0c0'), - - # Equivalent "bright" versions of original 8 colors. - ('08', '808080'), - ('09', 'ff0000'), - ('10', '00ff00'), - ('11', 'ffff00'), - ('12', '0000ff'), - ('13', 'ff00ff'), - ('14', '00ffff'), - ('15', 'ffffff'), - - # Strictly ascending. - ('16', '000000'), - ('17', '00005f'), - ('18', '000087'), - ('19', '0000af'), - ('20', '0000d7'), - ('21', '0000ff'), - ('22', '005f00'), - ('23', '005f5f'), - ('24', '005f87'), - ('25', '005faf'), - ('26', '005fd7'), - ('27', '005fff'), - ('28', '008700'), - ('29', '00875f'), - ('30', '008787'), - ('31', '0087af'), - ('32', '0087d7'), - ('33', '0087ff'), - ('34', '00af00'), - ('35', '00af5f'), - ('36', '00af87'), - ('37', '00afaf'), - ('38', '00afd7'), - ('39', '00afff'), - ('40', '00d700'), - ('41', '00d75f'), - ('42', '00d787'), - ('43', '00d7af'), - ('44', '00d7d7'), - ('45', '00d7ff'), - ('46', '00ff00'), - ('47', '00ff5f'), - ('48', '00ff87'), - ('49', '00ffaf'), - ('50', '00ffd7'), - ('51', '00ffff'), - ('52', '5f0000'), - ('53', '5f005f'), - ('54', '5f0087'), - ('55', '5f00af'), - ('56', '5f00d7'), - ('57', '5f00ff'), - ('58', '5f5f00'), - ('59', '5f5f5f'), - ('60', '5f5f87'), - ('61', '5f5faf'), - ('62', '5f5fd7'), - ('63', '5f5fff'), - ('64', '5f8700'), - ('65', '5f875f'), - ('66', '5f8787'), - ('67', '5f87af'), - ('68', '5f87d7'), - ('69', '5f87ff'), - ('70', '5faf00'), - ('71', '5faf5f'), - ('72', '5faf87'), - ('73', '5fafaf'), - ('74', '5fafd7'), - ('75', '5fafff'), - ('76', '5fd700'), - ('77', '5fd75f'), - ('78', '5fd787'), - ('79', '5fd7af'), - ('80', '5fd7d7'), - ('81', '5fd7ff'), - ('82', '5fff00'), - ('83', '5fff5f'), - ('84', '5fff87'), - ('85', '5fffaf'), - ('86', '5fffd7'), - ('87', '5fffff'), - ('88', '870000'), - ('89', '87005f'), - ('90', '870087'), - ('91', '8700af'), - ('92', '8700d7'), - ('93', '8700ff'), - ('94', '875f00'), - ('95', '875f5f'), - ('96', '875f87'), - ('97', '875faf'), - ('98', '875fd7'), - ('99', '875fff'), - ('100', '878700'), - ('101', '87875f'), - ('102', '878787'), - ('103', '8787af'), - ('104', '8787d7'), - ('105', '8787ff'), - ('106', '87af00'), - ('107', '87af5f'), - ('108', '87af87'), - ('109', '87afaf'), - ('110', '87afd7'), - ('111', '87afff'), - ('112', '87d700'), - ('113', '87d75f'), - ('114', '87d787'), - ('115', '87d7af'), - ('116', '87d7d7'), - ('117', '87d7ff'), - ('118', '87ff00'), - ('119', '87ff5f'), - ('120', '87ff87'), - ('121', '87ffaf'), - ('122', '87ffd7'), - ('123', '87ffff'), - ('124', 'af0000'), - ('125', 'af005f'), - ('126', 'af0087'), - ('127', 'af00af'), - ('128', 'af00d7'), - ('129', 'af00ff'), - ('130', 'af5f00'), - ('131', 'af5f5f'), - ('132', 'af5f87'), - ('133', 'af5faf'), - ('134', 'af5fd7'), - ('135', 'af5fff'), - ('136', 'af8700'), - ('137', 'af875f'), - ('138', 'af8787'), - ('139', 'af87af'), - ('140', 'af87d7'), - ('141', 'af87ff'), - ('142', 'afaf00'), - ('143', 'afaf5f'), - ('144', 'afaf87'), - ('145', 'afafaf'), - ('146', 'afafd7'), - ('147', 'afafff'), - ('148', 'afd700'), - ('149', 'afd75f'), - ('150', 'afd787'), - ('151', 'afd7af'), - ('152', 'afd7d7'), - ('153', 'afd7ff'), - ('154', 'afff00'), - ('155', 'afff5f'), - ('156', 'afff87'), - ('157', 'afffaf'), - ('158', 'afffd7'), - ('159', 'afffff'), - ('160', 'd70000'), - ('161', 'd7005f'), - ('162', 'd70087'), - ('163', 'd700af'), - ('164', 'd700d7'), - ('165', 'd700ff'), - ('166', 'd75f00'), - ('167', 'd75f5f'), - ('168', 'd75f87'), - ('169', 'd75faf'), - ('170', 'd75fd7'), - ('171', 'd75fff'), - ('172', 'd78700'), - ('173', 'd7875f'), - ('174', 'd78787'), - ('175', 'd787af'), - ('176', 'd787d7'), - ('177', 'd787ff'), - ('178', 'd7af00'), - ('179', 'd7af5f'), - ('180', 'd7af87'), - ('181', 'd7afaf'), - ('182', 'd7afd7'), - ('183', 'd7afff'), - ('184', 'd7d700'), - ('185', 'd7d75f'), - ('186', 'd7d787'), - ('187', 'd7d7af'), - ('188', 'd7d7d7'), - ('189', 'd7d7ff'), - ('190', 'd7ff00'), - ('191', 'd7ff5f'), - ('192', 'd7ff87'), - ('193', 'd7ffaf'), - ('194', 'd7ffd7'), - ('195', 'd7ffff'), - ('196', 'ff0000'), - ('197', 'ff005f'), - ('198', 'ff0087'), - ('199', 'ff00af'), - ('200', 'ff00d7'), - ('201', 'ff00ff'), - ('202', 'ff5f00'), - ('203', 'ff5f5f'), - ('204', 'ff5f87'), - ('205', 'ff5faf'), - ('206', 'ff5fd7'), - ('207', 'ff5fff'), - ('208', 'ff8700'), - ('209', 'ff875f'), - ('210', 'ff8787'), - ('211', 'ff87af'), - ('212', 'ff87d7'), - ('213', 'ff87ff'), - ('214', 'ffaf00'), - ('215', 'ffaf5f'), - ('216', 'ffaf87'), - ('217', 'ffafaf'), - ('218', 'ffafd7'), - ('219', 'ffafff'), - ('220', 'ffd700'), - ('221', 'ffd75f'), - ('222', 'ffd787'), - ('223', 'ffd7af'), - ('224', 'ffd7d7'), - ('225', 'ffd7ff'), - ('226', 'ffff00'), - ('227', 'ffff5f'), - ('228', 'ffff87'), - ('229', 'ffffaf'), - ('230', 'ffffd7'), - ('231', 'ffffff'), - - # Gray-scale range. - ('232', '080808'), - ('233', '121212'), - ('234', '1c1c1c'), - ('235', '262626'), - ('236', '303030'), - ('237', '3a3a3a'), - ('238', '444444'), - ('239', '4e4e4e'), - ('240', '585858'), - ('241', '626262'), - ('242', '6c6c6c'), - ('243', '767676'), - ('244', '808080'), - ('245', '8a8a8a'), - ('246', '949494'), - ('247', '9e9e9e'), - ('248', 'a8a8a8'), - ('249', 'b2b2b2'), - ('250', 'bcbcbc'), - ('251', 'c6c6c6'), - ('252', 'd0d0d0'), - ('253', 'dadada'), - ('254', 'e4e4e4'), - ('255', 'eeeeee'), -] - -def _str2hex(hexstr): - return int(hexstr, 16) - -def _strip_hash(rgb): - # Strip leading `#` if exists. - if rgb.startswith('#'): - rgb = rgb.lstrip('#') - return rgb - -def _create_dicts(): - short2rgb_dict = dict(CLUT) - rgb2short_dict = {} - for k, v in short2rgb_dict.items(): - rgb2short_dict[v] = k - return rgb2short_dict, short2rgb_dict - -def short2rgb(short): - return SHORT2RGB_DICT[short] - -def print_all(): - """ Print all 256 xterm color codes. - """ - for short, rgb in CLUT: - sys.stdout.write('\033[48;5;%sm%s:%s' % (short, short, rgb)) - sys.stdout.write("\033[0m ") - sys.stdout.write('\033[38;5;%sm%s:%s' % (short, short, rgb)) - sys.stdout.write("\033[0m\n") - print "Printed all codes." - print "You can translate a hex or 0-255 code by providing an argument." - -def rgb2short(rgb): - """ Find the closest xterm-256 approximation to the given RGB value. - @param rgb: Hex code representing an RGB value, eg, 'abcdef' - @returns: String between 0 and 255, compatible with xterm. - >>> rgb2short('123456') - ('23', '005f5f') - >>> rgb2short('ffffff') - ('231', 'ffffff') - >>> rgb2short('0DADD6') # vimeo logo - ('38', '00afd7') - """ - rgb = _strip_hash(rgb) - incs = (0x00, 0x5f, 0x87, 0xaf, 0xd7, 0xff) - # Break 6-char RGB code into 3 integer vals. - parts = [ int(h, 16) for h in re.split(r'(..)(..)(..)', rgb)[1:4] ] - res = [] - for part in parts: - i = 0 - while i < len(incs)-1: - s, b = incs[i], incs[i+1] # smaller, bigger - if s <= part <= b: - s1 = abs(s - part) - b1 = abs(b - part) - if s1 < b1: closest = s - else: closest = b - res.append(closest) - break - i += 1 - #print '***', res - res = ''.join([ ('%02.x' % i) for i in res ]) - equiv = RGB2SHORT_DICT[ res ] - #print '***', res, equiv - return equiv, res - -RGB2SHORT_DICT, SHORT2RGB_DICT = _create_dicts() - -#--------------------------------------------------------------------- - -if __name__ == '__main__': - import doctest - doctest.testmod() - if len(sys.argv) == 1: - print_all() - raise SystemExit - arg = sys.argv[1] - if len(arg) < 4 and int(arg) < 256: - rgb = short2rgb(arg) - sys.stdout.write('xterm color \033[38;5;%sm%s\033[0m -> RGB exact \033[38;5;%sm%s\033[0m' % (arg, arg, arg, rgb)) - sys.stdout.write("\033[0m\n") - else: - short, rgb = rgb2short(arg) - sys.stdout.write('RGB %s -> xterm color approx \033[38;5;%sm%s (%s)' % (arg, short, short, rgb)) - sys.stdout.write("\033[0m\n") diff --git a/v14/utils/command_executer.py b/v14/utils/command_executer.py deleted file mode 100644 index 0aedc47c..00000000 --- a/v14/utils/command_executer.py +++ /dev/null @@ -1,338 +0,0 @@ -#!/usr/bin/python -# -# Copyright 2011 Google Inc. All Rights Reserved. -# - -import getpass -import os -import pty -import re -import select -import subprocess -import tempfile -import time - -import logger -import misc - -mock_default = False - - -def InitCommandExecuter(mock=False): - global mock_default - # Whether to default to a mock command executer or not - mock_default = mock - - -def GetCommandExecuter(logger_to_set=None, mock=False): - # If the default is a mock executer, always return one. - if mock_default or mock: - return MockCommandExecuter(logger_to_set) - else: - return CommandExecuter(logger_to_set) - - -class CommandExecuter: - def __init__(self, logger_to_set=None): - if logger_to_set is not None: - self.logger = logger_to_set - else: - self.logger = logger.GetLogger() - - def RunCommand(self, cmd, return_output=False, machine=None, - username=None, command_terminator=None, - command_timeout=None, - terminated_timeout=10, - print_to_console=True): - """Run a command.""" - - cmd = str(cmd) - - self.logger.LogCmd(cmd, machine, username, print_to_console) - if command_terminator and command_terminator.IsTerminated(): - self.logger.LogError("Command was terminated!", print_to_console) - if return_output: - return [1, "", ""] - else: - return 1 - - if machine is not None: - user = "" - if username is not None: - user = username + "@" - cmd = "ssh -t -t %s%s -- '%s'" % (user, machine, cmd) - - pty_fds = pty.openpty() - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - stdin=pty_fds[0], shell=True) - - full_stdout = "" - full_stderr = "" - - # Pull output from pipes, send it to file/stdout/string - out = err = None - pipes = [p.stdout, p.stderr] - - my_poll = select.poll() - my_poll.register(p.stdout, select.POLLIN) - my_poll.register(p.stderr, select.POLLIN) - - terminated_time = None - started_time = time.time() - - while len(pipes): - if command_terminator and command_terminator.IsTerminated(): - self.RunCommand("sudo kill -9 " + str(p.pid), - print_to_console=print_to_console) - wait = p.wait() - self.logger.LogError("Command was terminated!", print_to_console) - if return_output: - return (p.wait, full_stdout, full_stderr) - else: - return wait - - l=my_poll.poll(100) - for (fd, evt) in l: - if fd == p.stdout.fileno(): - out = os.read(p.stdout.fileno(), 16384) - if return_output: - full_stdout += out - self.logger.LogCommandOutput(out, print_to_console) - if out == "": - pipes.remove(p.stdout) - my_poll.unregister(p.stdout) - if fd == p.stderr.fileno(): - err = os.read(p.stderr.fileno(), 16384) - if return_output: - full_stderr += err - self.logger.LogCommandError(err, print_to_console) - if err == "": - pipes.remove(p.stderr) - my_poll.unregister(p.stderr) - - if p.poll() is not None: - if terminated_time is None: - terminated_time = time.time() - elif (terminated_timeout is not None and - time.time() - terminated_time > terminated_timeout): - m = ("Timeout of %s seconds reached since process termination." - % terminated_timeout) - self.logger.LogWarning(m, print_to_console) - break - - if (command_timeout is not None and - time.time() - started_time > command_timeout): - m = ("Timeout of %s seconds reached since process started." - % command_timeout) - self.logger.LogWarning(m, print_to_console) - self.RunCommand("kill %d || sudo kill %d || sudo kill -9 %d" % - (p.pid, p.pid, p.pid), - print_to_console=print_to_console) - break - - if out == err == "": - break - - p.wait() - os.close(pty_fds[0]) - os.close(pty_fds[1]) - if return_output: - return (p.returncode, full_stdout, full_stderr) - return p.returncode - - def RemoteAccessInitCommand(self, chromeos_root, machine): - command = "" - command += "\nset -- --remote=" + machine - command += "\n. " + chromeos_root + "/src/scripts/common.sh" - command += "\n. " + chromeos_root + "/src/scripts/remote_access.sh" - command += "\nTMP=$(mktemp -d)" - command += "\nFLAGS \"$@\" || exit 1" - command += "\nremote_access_init" - return command - - def WriteToTempShFile(self, contents): - handle, command_file = tempfile.mkstemp(prefix=os.uname()[1], - suffix=".sh") - os.write(handle, "#!/bin/bash\n") - os.write(handle, contents) - os.close(handle) - return command_file - - - def CrosLearnBoard(self, chromeos_root, machine): - command = self.RemoteAccessInitCommand(chromeos_root, machine) - command += "\nlearn_board" - command += "\necho ${FLAGS_board}" - retval, output, err = self.RunCommand(command, True) - self.logger.LogFatalIf(retval, "learn_board command failed") - return output.split()[-1] - - def CrosRunCommand(self, cmd, return_output=False, machine=None, - username=None, command_terminator=None, chromeos_root=None, - command_timeout=None, - terminated_timeout=10, - print_to_console=True): - """Run a command on a chromeos box""" - self.logger.LogCmd(cmd, print_to_console) - self.logger.LogFatalIf(not machine, "No machine provided!") - self.logger.LogFatalIf(not chromeos_root, "chromeos_root not given!") - chromeos_root = os.path.expanduser(chromeos_root) - - # Write all commands to a file. - command_file = self.WriteToTempShFile(cmd) - retval = self.CopyFiles(command_file, command_file, - dest_machine=machine, - command_terminator=command_terminator, - chromeos_root=chromeos_root, - dest_cros=True, - recursive=False, - print_to_console=print_to_console) - if retval: - self.logger.LogError("Could not run remote command on machine." - " Is the machine up?") - return retval - - command = self.RemoteAccessInitCommand(chromeos_root, machine) - command += "\nremote_sh bash %s" % command_file - command += "\necho \"$REMOTE_OUT\"" - retval = self.RunCommand(command, return_output, - command_terminator=command_terminator, - command_timeout=command_timeout, - terminated_timeout=terminated_timeout, - print_to_console=print_to_console) - if return_output: - connect_signature = ("Initiating first contact with remote host\n" + - "Connection OK\n") - connect_signature_re = re.compile(connect_signature) - modded_return = [] - for r in retval: - modded_return.append(r) - modded_return[1] = connect_signature_re.sub("", modded_return[1]) - return modded_return - return retval - - def ChrootRunCommand(self, chromeos_root, command, return_output=False, - command_terminator=None, command_timeout=None, - terminated_timeout=10, - print_to_console=True, - cros_sdk_options=""): - self.logger.LogCmd(command, print_to_console) - - handle, command_file = tempfile.mkstemp(dir=os.path.join(chromeos_root, - "src/scripts"), - suffix=".sh", - prefix="in_chroot_cmd") - os.write(handle, "#!/bin/bash\n") - os.write(handle, command) - os.close(handle) - - os.chmod(command_file, 0777) - - command = "cd %s; cros_sdk %s -- ./%s" % (chromeos_root, cros_sdk_options, - os.path.basename(command_file)) - ret = self.RunCommand(command, return_output, - command_terminator=command_terminator, - command_timeout=command_timeout, - terminated_timeout=terminated_timeout, - print_to_console=print_to_console) - os.remove(command_file) - return ret - - - def RunCommands(self, cmdlist, return_output=False, machine=None, - username=None, command_terminator=None): - cmd = " ;\n" .join(cmdlist) - return self.RunCommand(cmd, return_output, machine, username, - command_terminator) - - def CopyFiles(self, src, dest, src_machine=None, dest_machine=None, - src_user=None, dest_user=None, recursive=True, - command_terminator=None, - chromeos_root=None, src_cros=False, dest_cros=False, - print_to_console=True): - src = os.path.expanduser(src) - dest = os.path.expanduser(dest) - - if recursive: - src = src + "/" - dest = dest + "/" - - if src_cros == True or dest_cros == True: - self.logger.LogFatalIf(not (src_cros ^ dest_cros), "Only one of src_cros " - "and desc_cros can be non-null.") - self.logger.LogFatalIf(not chromeos_root, "chromeos_root not given!") - if src_cros == True: - cros_machine = src_machine - else: - cros_machine = dest_machine - - command = self.RemoteAccessInitCommand(chromeos_root, cros_machine) - src_parent, src_child = misc.GetRoot(src) - dest_parent, dest_child = misc.GetRoot(dest) - ssh_command = ("ssh -p ${FLAGS_ssh_port}" + - " -o StrictHostKeyChecking=no" + - " -o UserKnownHostsFile=$(mktemp)" + - " -i $TMP_PRIVATE_KEY") - rsync_prefix = "\nrsync -r -e \"%s\" " % ssh_command - if dest_cros == True: - command += rsync_prefix + "%s root@%s:%s" % (src, dest_machine, dest) - return self.RunCommand(command, - machine=src_machine, - username=src_user, - command_terminator=command_terminator, - print_to_console=print_to_console) - else: - command += rsync_prefix + "root@%s:%s %s" % (src_machine, src, dest) - return self.RunCommand(command, - machine=dest_machine, - username=dest_user, - command_terminator=command_terminator, - print_to_console=print_to_console) - - - if dest_machine == src_machine: - command = ("rsync -a %s %s" % - (src, - dest)) - else: - if src_machine is None: - src_machine = os.uname()[1] - src_user = getpass.getuser() - command = ("rsync -a %s@%s:%s %s" % - (src_user, src_machine, src, - dest)) - return self.RunCommand(command, - machine=dest_machine, - username=dest_user, - command_terminator=command_terminator, - print_to_console=print_to_console) - - -class MockCommandExecuter(CommandExecuter): - def __init__(self, logger_to_set=None): - if logger is not None: - self.logger = logger_to_set - else: - self.logger = logger.GetLogger() - - def RunCommand(self, cmd, return_output=False, machine=None, username=None, - command_terminator=None): - cmd = str(cmd) - if machine is None: - machine = "localhost" - if username is None: - username = "current" - logger.GetLogger().LogCmd("(Mock) " + cmd, machine, username) - return 0 - - -class CommandTerminator: - def __init__(self): - self.terminated = False - - def Terminate(self): - self.terminated = True - - def IsTerminated(self): - return self.terminated diff --git a/v14/utils/constants.py b/v14/utils/constants.py deleted file mode 100644 index 33875d66..00000000 --- a/v14/utils/constants.py +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/python2.6 -# -# Copyright 2010 Google Inc. All Rights Reserved. - -"""Generic constants used accross modules. -""" - -__author__ = "shenhan@google.com (Han Shen)" - -mounted_toolchain_root='/usr/local/toolchain_root' diff --git a/v14/utils/email_sender.py b/v14/utils/email_sender.py deleted file mode 100644 index f8c0d62c..00000000 --- a/v14/utils/email_sender.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/python - -# Copyright 2011 Google Inc. All Rights Reserved. - -from email import Encoders -from email.MIMEBase import MIMEBase -from email.MIMEMultipart import MIMEMultipart -from email.MIMEText import MIMEText -import getpass -import os -import smtplib -import sys - - -class EmailSender(object): - class Attachment(object): - def __init__(self, name, content): - self.name = name - self.content = content - - def SendEmail(self, - email_to, - subject, - text_to_send, - email_cc=None, - email_bcc=None, - email_from=None, - msg_type="plain", - attachments=None): - # Email summary to the current user. - msg = MIMEMultipart() - - if not email_from: - email_from = os.path.basename(__file__) - - msg["To"] = ",".join(email_to) - msg["Subject"] = subject - - if email_from: - msg["From"] = email_from - if email_cc: - msg["CC"] = ",".join(email_cc) - email_to += email_cc - if email_bcc: - msg["BCC"] = ",".join(email_bcc) - email_to += email_bcc - - msg.attach(MIMEText(text_to_send, msg_type)) - if attachments: - for attachment in attachments: - part = MIMEBase("application", "octet-stream") - part.set_payload(attachment.content) - Encoders.encode_base64(part) - part.add_header("Content-Disposition", "attachment; filename=\"%s\"" % - attachment.name) - msg.attach(part) - - # Send the message via our own SMTP server, but don't include the - # envelope header. - s = smtplib.SMTP("localhost") - s.sendmail(email_from, email_to, msg.as_string()) - s.quit() diff --git a/v14/utils/file_utils.py b/v14/utils/file_utils.py deleted file mode 100644 index 24809333..00000000 --- a/v14/utils/file_utils.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/python - -# Copyright 2011 Google Inc. All Rights Reserved. - -import errno -import hashlib -import os -import shutil -import command_executer - - -class FileUtils(object): - """Utilities for operations on files.""" - _instance = None - DRY_RUN = False - - @classmethod - def Configure(cls, dry_run): - cls.DRY_RUN = dry_run - - def __new__(cls, *args, **kwargs): - if not cls._instance: - if cls.DRY_RUN: - cls._instance = super(FileUtils, cls).__new__(MockFileUtils, *args, - **kwargs) - else: - cls._instance = super(FileUtils, cls).__new__(cls, *args, - **kwargs) - return cls._instance - - def Md5File(self, filename, block_size=2 ** 10): - command = "md5sum %s" % filename - ce = command_executer.GetCommandExecuter() - ret, out, err = ce.RunCommand(command, return_output=True) - if ret: - raise Exception("Could not run md5sum on: %s" % filename) - - return out.strip().split()[0] - - def CanonicalizeChromeOSRoot(self, chromeos_root): - chromeos_root = os.path.expanduser(chromeos_root) - if os.path.isfile(os.path.join(chromeos_root, - "src/scripts/run_remote_tests.sh")): - return chromeos_root - else: - return None - - def ChromeOSRootFromImage(self, chromeos_image): - chromeos_root = os.path.join(os.path.dirname(chromeos_image), - "../../../../..") - return self.CanonicalizeChromeOSRoot(chromeos_root) - - def MkDirP(self, path): - try: - os.makedirs(path) - except OSError as exc: - if exc.errno == errno.EEXIST: - pass - else: - raise - - def RmDir(self, path): - shutil.rmtree(path, ignore_errors=True) - - def WriteFile(self, path, contents): - with open(path, "wb") as f: - f.write(contents) - - -class MockFileUtils(FileUtils): - """Mock class for file utilities.""" - - def Md5File(self, filename, block_size=2 ** 10): - return "d41d8cd98f00b204e9800998ecf8427e" - - def CanonicalizeChromeOSRoot(self, chromeos_root): - return "/tmp/chromeos_root" - - def ChromeOSRootFromImage(self, chromeos_image): - return "/tmp/chromeos_root" - - def RmDir(self, path): - pass - - def MkDirP(self, path): - pass - - def WriteFile(self, path, contents): - pass diff --git a/v14/utils/html_tools.py b/v14/utils/html_tools.py deleted file mode 100644 index c90a57b5..00000000 --- a/v14/utils/html_tools.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/python2.6 -# -# Copyright 2010 Google Inc. All Rights Reserved. -# - - -def GetPageHeader(page_title): - return """<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" -"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> -<html> -<head> -<style type="text/css"> -table -{ -border-collapse:collapse; -} -table, td, th -{ -border:1px solid black; -} -</style> -<script type="text/javascript"> -function displayRow(id){ - var row = document.getElementById("group_"+id); - if (row.style.display == '') row.style.display = 'none'; - else row.style.display = ''; - } -</script> -<title>%s</title> -</head> -<body> - -""" % page_title - - -def GetListHeader(): - return "<ul>" - - -def GetListItem(text): - return "<li>%s</li>" % text - - -def GetListFooter(): - return "</ul>" - - -def GetList(items): - return "<ul>%s</ul>" % "".join(["<li>%s</li>" % item for item in items]) - - -def GetParagraph(text): - return "<p>%s</p>" % text - - -def GetFooter(): - return "</body>\n</html>" - - -def GetHeader(text, h=1): - return "<h%s>%s</h%s>" % (h, text, h) - - -def GetTableHeader(headers): - row = "".join(["<th>%s</th>" % header for header in headers]) - return "<table><tr>%s</tr>" % row - - -def GetTableFooter(): - return "</table>" - - -def FormatLineBreaks(text): - return text.replace("\n", "<br/>") - - -def GetTableCell(text): - return "<td>%s</td>" % FormatLineBreaks(str(text)) - - -def GetTableRow(columns): - return "<tr>%s</tr>" % "\n".join([GetTableCell(column) for column in columns]) - - -def GetTable(headers, rows): - table = [GetTableHeader(headers)] - table.extend([GetTableRow(row) for row in rows]) - table.append(GetTableFooter()) - return "\n".join(table) - - -def GetLink(link, text): - return "<a href='%s'>%s</a>" % (link, text) diff --git a/v14/utils/logger.py b/v14/utils/logger.py deleted file mode 100644 index 99c679ab..00000000 --- a/v14/utils/logger.py +++ /dev/null @@ -1,195 +0,0 @@ -#!/usr/bin/python2.6 -# -# Copyright 2010 Google Inc. All Rights Reserved. - -# System modules -import os.path -import sys -import traceback - -#TODO(yunlian@google.com): Use GetRoot from misc -def GetRoot(scr_name): - """Break up pathname into (dir+name).""" - abs_path = os.path.abspath(scr_name) - return (os.path.dirname(abs_path), os.path.basename(abs_path)) - - -class Logger(object): - """Logging helper class.""" - - MAX_LOG_FILES = 10 - - def __init__ (self, rootdir, basefilename, print_console, subdir="logs"): - logdir = os.path.join(rootdir, subdir) - basename = os.path.join(logdir, basefilename) - - try: - os.makedirs(logdir) - except OSError: - pass - # print "Warning: Logs directory '%s' already exists." % logdir - - self.print_console = print_console - - self._CreateLogFileHandles(basename) - - self._WriteTo(self.cmdfd, " ".join(sys.argv), True) - - def _AddSuffix(self, basename, suffix): - return "%s%s" % (basename, suffix) - - def _FindSuffix(self, basename): - timestamps = [] - found_suffix = None - for i in range(self.MAX_LOG_FILES): - suffix = str(i) - suffixed_basename = self._AddSuffix(basename, suffix) - cmd_file = "%s.cmd" % suffixed_basename - if not os.path.exists(cmd_file): - found_suffix = suffix - break - timestamps.append(os.stat(cmd_file).st_mtime) - - if found_suffix: - return found_suffix - - # Try to pick the oldest file with the suffix and return that one. - suffix = str(timestamps.index(min(timestamps))) - # print ("Warning: Overwriting log file: %s" % - # self._AddSuffix(basename, suffix)) - return suffix - - def _CreateLogFileHandle(self, name): - fd = None - try: - fd = open(name, "w") - except IOError: - print "Warning: could not open %s for writing." % name - return fd - - def _CreateLogFileHandles(self, basename): - suffix = self._FindSuffix(basename) - suffixed_basename = self._AddSuffix(basename, suffix) - - self.cmdfd = self._CreateLogFileHandle("%s.cmd" % suffixed_basename) - self.stdout = self._CreateLogFileHandle("%s.out" % suffixed_basename) - self.stderr = self._CreateLogFileHandle("%s.err" % suffixed_basename) - - self._CreateLogFileSymlinks(basename, suffixed_basename) - - # Symlink unsuffixed basename to currently suffixed one. - def _CreateLogFileSymlinks(self, basename, suffixed_basename): - try: - for extension in ["cmd", "out", "err"]: - src_file = "%s.%s" % (os.path.basename(suffixed_basename), extension) - dest_file = "%s.%s" % (basename, extension) - if os.path.exists(dest_file): - os.remove(dest_file) - os.symlink(src_file, dest_file) - except Exception as ex: - print "Exception while creating symlinks: %s" % str(ex) - - def _WriteTo(self, fd, msg, flush): - if fd: - fd.write(msg) - if flush: - fd.flush() - - def _LogMsg(self, file_fd, term_fd, msg, flush=True): - if file_fd: - self._WriteTo(file_fd, msg, flush) - if self.print_console: - self._WriteTo(term_fd, msg, flush) - - def _GetStdout(self, print_to_console): - if print_to_console: - return sys.stdout - return None - - def _GetStderr(self, print_to_console): - if print_to_console: - return sys.stderr - return None - - def LogCmd(self, cmd, machine="", user=None, print_to_console=True): - if user: - host = "%s@%s" % (user, machine) - else: - host = machine - - self._LogMsg(self.cmdfd, self._GetStdout(print_to_console), - "CMD (%s): %s\n" % (host, cmd)) - - def LogFatal(self, msg, print_to_console=True): - self._LogMsg(self.stderr, self._GetStderr(print_to_console), - "FATAL: %s\n" % msg) - self._LogMsg(self.stderr, self._GetStderr(print_to_console), - "\n".join(traceback.format_stack())) - sys.exit(1) - - def LogError(self, msg, print_to_console=True): - self._LogMsg(self.stderr, self._GetStderr(print_to_console), - "ERROR: %s\n" % msg) - - def LogWarning(self, msg, print_to_console=True): - self._LogMsg(self.stderr, self._GetStderr(print_to_console), - "WARNING: %s\n" % msg) - - def LogOutput(self, msg, print_to_console=True): - self._LogMsg(self.stdout, self._GetStdout(print_to_console), - "OUTPUT: %s\n" % msg) - - def LogFatalIf(self, condition, msg): - if condition: - self.LogFatal(msg) - - def LogErrorIf(self, condition, msg): - if condition: - self.LogError(msg) - - def LogWarningIf(self, condition, msg): - if condition: - self.LogWarning(msg) - - def LogCommandOutput(self, msg, print_to_console=True): - self._LogMsg(self.stdout, self._GetStdout(print_to_console), - msg, flush=False) - - def LogCommandError(self, msg, print_to_console=True): - self._LogMsg(self.stderr, self._GetStderr(print_to_console), - msg, flush=False) - - def Flush(self): - self.cmdfd.flush() - self.stdout.flush() - self.stderr.flush() - -main_logger = None - - -def InitLogger(script_name, log_dir, print_console=True): - """Initialize a global logger. To be called only once.""" - global main_logger - assert not main_logger, "The logger has already been initialized" - rootdir, basefilename = GetRoot(script_name) - if not log_dir: - log_dir = rootdir - main_logger = Logger(log_dir, basefilename, print_console) - - -def GetLogger(log_dir=""): - if not main_logger: - InitLogger(sys.argv[0], log_dir) - return main_logger - - -def HandleUncaughtExceptions(fun): - """Catches all exceptions that would go outside decorated fun scope.""" - - def _Interceptor(*args, **kwargs): - try: - return fun(*args, **kwargs) - except StandardError: - GetLogger().LogFatal("Uncaught exception:\n%s" % traceback.format_exc()) - - return _Interceptor diff --git a/v14/utils/misc.py b/v14/utils/misc.py deleted file mode 100644 index 175a6e07..00000000 --- a/v14/utils/misc.py +++ /dev/null @@ -1,327 +0,0 @@ -#!/usr/bin/python -# -# Copyright 2010 Google Inc. All Rights Reserved. - -"""Utilities for toolchain build.""" - -__author__ = "asharif@google.com (Ahmad Sharif)" - -from contextlib import contextmanager -import os -import re -import shutil -import sys -import time - -import lock_machine - -import command_executer -import logger - - -def GetChromeOSVersionFromLSBVersion(lsb_version): - """Get Chromeos version from Lsb version.""" - ce = command_executer.GetCommandExecuter() - command = "git ls-remote http://git.chromium.org/chromiumos/manifest.git" - ret, out, _ = ce.RunCommand(command, return_output=True, - print_to_console=False) - assert ret == 0, "Command %s failed" % command - lower = [] - for line in out.splitlines(): - mo = re.search(r"refs/heads/release-R(\d+)-(\d+)\.B", line) - if mo: - revision = int(mo.group(1)) - build = int(mo.group(2)) - lsb_build = int(lsb_version.split(".")[0]) - if lsb_build > build: - lower.append(revision) - lower = sorted(lower) - if lower: - return "R%d-%s" % (lower[-1] + 1, lsb_version) - else: - return "Unknown" - - -def ApplySubs(string, *substitutions): - for pattern, replacement in substitutions: - string = re.sub(pattern, replacement, string) - return string - - -def UnitToNumber(unit_num, base=1000): - """Convert a number with unit to float.""" - unit_dict = {"kilo": base, - "mega": base**2, - "giga": base**3} - unit_num = unit_num.lower() - mo = re.search(r"(\d*)(.+)?", unit_num) - number = mo.group(1) - unit = mo.group(2) - if not unit: - return float(number) - for k, v in unit_dict.items(): - if k.startswith(unit): - return float(number) * v - raise Exception("Unit: %s not found in byte: %s!" % - (unit, - unit_num)) - - -def GetFilenameFromString(string): - return ApplySubs(string, - (r"/", "__"), - (r"\s", "_"), - (r"[\^\$=\"\\\?]", ""), - ) - - -def GetRoot(scr_name): - """Break up pathname into (dir+name).""" - abs_path = os.path.abspath(scr_name) - return (os.path.dirname(abs_path), os.path.basename(abs_path)) - - -def GetChromeOSKeyFile(chromeos_root): - return os.path.join(chromeos_root, - "src", - "scripts", - "mod_for_test_scripts", - "ssh_keys", - "testing_rsa") - - -def GetChrootPath(chromeos_root): - return os.path.join(chromeos_root, - "chroot") - - -def GetInsideChrootPath(chromeos_root, file_path): - if not file_path.startswith(GetChrootPath(chromeos_root)): - raise Exception("File: %s doesn't seem to be in the chroot: %s" % - (file_path, - chromeos_root)) - return file_path[len(GetChrootPath(chromeos_root)):] - - -def GetOutsideChrootPath(chromeos_root, file_path): - return os.path.join(GetChrootPath(chromeos_root), - file_path.lstrip("/")) - - -def FormatQuotedCommand(command): - return ApplySubs(command, - ("\"", "\\\"")) - - -def FormatCommands(commands): - return ApplySubs(str(commands), - ("&&", "&&\n"), - (";", ";\n"), - (r"\n+\s*", "\n")) - - -def GetImageDir(chromeos_root, board): - return os.path.join(chromeos_root, - "src", - "build", - "images", - board) - - -def LabelLatestImage(chromeos_root, board, label): - image_dir = GetImageDir(chromeos_root, board) - latest_image_dir = os.path.join(image_dir, "latest") - latest_image_dir = os.path.realpath(latest_image_dir) - latest_image_dir = os.path.basename(latest_image_dir) - with WorkingDirectory(image_dir): - command = "ln -sf -T %s %s" % (latest_image_dir, label) - ce = command_executer.GetCommandExecuter() - return ce.RunCommand(command) - - -def DoesLabelExist(chromeos_root, board, label): - image_label = os.path.join(GetImageDir(chromeos_root, board), - label) - return os.path.exists(image_label) - - -def GetBuildPackagesCommand(board, usepkg=False, debug=False): - if usepkg: - usepkg_flag = "--usepkg" - else: - usepkg_flag = "--nousepkg" - if debug: - withdebug_flag = '--withdebug' - else: - withdebug_flag = '--nowithdebug' - return ("./build_packages %s --withdev --withtest --withautotest " - "--skip_toolchain_update %s --board=%s" % - (usepkg_flag, withdebug_flag, board)) - - -def GetBuildImageCommand(board, dev=False): - dev_args = "" - if dev: - dev_args = "--noenable_rootfs_verification --disk_layout=2gb-rootfs" - return "./build_image --board=%s %s test" % (board, dev_args) - -def GetSetupBoardCommand(board, gcc_version=None, binutils_version=None, - usepkg=None, force=None): - """Get setup_board command.""" - options = [] - - if gcc_version: - options.append("--gcc_version=%s" % gcc_version) - - if binutils_version: - options.append("--binutils_version=%s" % binutils_version) - - if usepkg: - options.append("--usepkg") - else: - options.append("--nousepkg") - - if force: - options.append("--force") - - return "./setup_board --board=%s %s" % (board, " ".join(options)) - - -def CanonicalizePath(path): - path = os.path.expanduser(path) - path = os.path.realpath(path) - return path - - -def GetCtargetFromBoard(board, chromeos_root): - """Get Ctarget from board.""" - base_board = board.split("_")[0] - command = ("source " - "../platform/dev/toolchain_utils.sh; get_ctarget_from_board %s" % - base_board) - ce = command_executer.GetCommandExecuter() - ret, out, _ = ce.ChrootRunCommand(chromeos_root, - command, - return_output=True) - if ret != 0: - raise ValueError("Board %s is invalid!" % board) - # Remove ANSI escape sequences. - out = StripANSIEscapeSequences(out) - return out.strip() - - -def StripANSIEscapeSequences(string): - string = re.sub(r"\x1b\[[0-9]*[a-zA-Z]", "", string) - return string - - -def GetChromeSrcDir(): - return "var/cache/distfiles/target/chrome-src/src" - - -def GetEnvStringFromDict(env_dict): - return " ".join(["%s=\"%s\"" % var for var in env_dict.items()]) - - -def MergeEnvStringWithDict(env_string, env_dict, prepend=True): - """Merge env string with dict.""" - if not env_string.strip(): - return GetEnvStringFromDict(env_dict) - override_env_list = [] - ce = command_executer.GetCommandExecuter() - for k, v in env_dict.items(): - v = v.strip("\"'") - if prepend: - new_env = "%s=\"%s $%s\"" % (k, v, k) - else: - new_env = "%s=\"$%s %s\"" % (k, k, v) - command = "; ".join([env_string, new_env, "echo $%s" % k]) - ret, out, _ = ce.RunCommand(command, return_output=True) - override_env_list.append("%s=%r" % (k, out.strip())) - ret = env_string + " " + " ".join(override_env_list) - return ret.strip() - - -def GetAllImages(chromeos_root, board): - ce = command_executer.GetCommandExecuter() - command = ("find %s/src/build/images/%s -name chromiumos_test_image.bin" % - (chromeos_root, board)) - ret, out, _ = ce.RunCommand(command, return_output=True) - assert ret == 0, "Could not run command: %s" % command - return out.splitlines() - - -def AcquireLock(lock_file, timeout=1200): - """Acquire a lock with timeout.""" - start_time = time.time() - locked = False - abs_path = os.path.abspath(lock_file) - dir_path = os.path.dirname(abs_path) - sleep_time = min(10, timeout/10.0) - reason = "pid: {0}, commad: {1}".format(os.getpid(), - sys.argv[0]) - if not os.path.exists(dir_path): - try: - with lock_machine.FileCreationMask(0002): - os.makedirs(dir_path) - except OSError: - print "Cannot create dir {0}, exiting...".format(dir_path) - exit(0) - while True: - locked = (lock_machine.Lock(lock_file).NonBlockingLock(True, reason)) - if locked: - break - time.sleep(sleep_time) - if time.time() - start_time > timeout: - logger.GetLogger().LogWarning( - "Could not acquire lock on this file: {0} within {1} seconds." - "Manually remove the file if you think the lock is stale" - .format(abs_path, timeout)) - break - return locked - - -def ReleaseLock(lock_file): - lock_file = os.path.abspath(lock_file) - ret = lock_machine.Lock(lock_file).Unlock(True) - assert ret, ("Could not unlock {0}," - "Please remove it manually".format(lock_file)) - - -def IsFloat(text): - if text is None: - return False - try: - float(text) - return True - except ValueError: - return False - -def RemoveChromeBrowserObjectFiles(chromeos_root, board): - """ Remove any object files from all the posible locations """ - out_dir = os.path.join( - GetChrootPath(chromeos_root), - "var/cache/chromeos-chrome/chrome-src/src/out_%s" % board) - if os.path.exists(out_dir): - shutil.rmtree(out_dir) - logger.GetLogger().LogCmd("rm -rf %s" % out_dir) - out_dir = os.path.join( - GetChrootPath(chromeos_root), - "var/cache/chromeos-chrome/chrome-src-internal/src/out_%s" % board) - if os.path.exists(out_dir): - shutil.rmtree(out_dir) - logger.GetLogger().LogCmd("rm -rf %s" % out_dir) - -@contextmanager -def WorkingDirectory(new_dir): - """Get the working directory.""" - old_dir = os.getcwd() - if old_dir != new_dir: - msg = "cd %s" % new_dir - logger.GetLogger().LogCmd(msg) - os.chdir(new_dir) - yield new_dir - if old_dir != new_dir: - msg = "cd %s" % old_dir - logger.GetLogger().LogCmd(msg) - os.chdir(old_dir) diff --git a/v14/utils/misc_test.py b/v14/utils/misc_test.py deleted file mode 100644 index e234332f..00000000 --- a/v14/utils/misc_test.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2012 Google Inc. All Rights Reserved. - -"""Tests for misc.""" - -__author__ = 'asharif@google.com (Ahmad Sharif)' - -# System modules -import re -import unittest - -# Local modules -import misc - - -class UtilsTest(unittest.TestCase): - def testGetFilenameFromString(self): - string = 'a /b=c"d^$?\\' - filename = misc.GetFilenameFromString(string) - self.assertTrue(filename == 'a___bcd') - - def testPrependMergeEnv(self): - var = 'USE' - use_flags = 'hello 123' - added_use_flags = 'bla bla' - env_string = '%s=%r' % (var, use_flags) - new_env_string = misc.MergeEnvStringWithDict(env_string, - {var: added_use_flags}) - expected_new_env = '%s=%r' % (var, ' '.join([added_use_flags, use_flags])) - self.assertTrue(new_env_string == ' '.join([env_string, expected_new_env])) - - def testGetChromeOSVersionFromLSBVersion(self): - versions_dict = {"2630.0.0": "22", - "2030.0.0": "19"} - f = misc.GetChromeOSVersionFromLSBVersion - for k, v in versions_dict.items(): - self.assertTrue(f(k) == "R%s-%s" % (v, k)) - - def testPostpendMergeEnv(self): - var = 'USE' - use_flags = 'hello 123' - added_use_flags = 'bla bla' - env_string = '%s=%r' % (var, use_flags) - new_env_string = misc.MergeEnvStringWithDict(env_string, - {var: added_use_flags}, - False) - expected_new_env = '%s=%r' % (var, ' '.join([use_flags, added_use_flags])) - self.assertTrue(new_env_string == ' '.join([env_string, expected_new_env])) - -if __name__ == '__main__': - unittest.main() diff --git a/v14/utils/perf_diff.py b/v14/utils/perf_diff.py deleted file mode 100755 index d56eba16..00000000 --- a/v14/utils/perf_diff.py +++ /dev/null @@ -1,324 +0,0 @@ -#!/usr/bin/python -# Copyright 2012 Google Inc. All Rights Reserved. - -"""One-line documentation for perf_diff module. - -A detailed description of perf_diff. -""" - -__author__ = "asharif@google.com (Ahmad Sharif)" - -import optparse -import re -import sys - -import misc -import tabulator - -ROWS_TO_SHOW = "Rows_to_show_in_the_perf_table" - -def GetPerfDictFromReport(report_file): - output = {} - perf_report = PerfReport(report_file) - for k, v in perf_report.sections.items(): - if k not in output: - output[k] = {} - output[k][ROWS_TO_SHOW] = 0 - for function in v.functions: - out_key = "%s" % (function.name) - output[k][out_key] = function.count - if function.percent > 1: - output[k][ROWS_TO_SHOW] += 1 - return output - - -def _SortDictionaryByValue(d): - l = [(k, v) for (k, v) in d.iteritems()] - - def GetFloat(x): - if misc.IsFloat(x): - return float(x) - else: - return x - - sorted_l = sorted(l, - key=lambda x: GetFloat(x[1])) - sorted_l.reverse() - return [f[0] for f in sorted_l] - - -class Tabulator(object): - def __init__(self, all_dicts): - self._all_dicts = all_dicts - - def PrintTable(self): - for dicts in self._all_dicts: - self.PrintTableHelper(dicts) - - def PrintTableHelper(self, dicts): - """Transfrom dicts to tables.""" - fields = {} - for d in dicts: - for f in d.keys(): - if f not in fields: - fields[f] = d[f] - else: - fields[f] = max(fields[f], d[f]) - table = [] - header = ["name"] - for i in range(len(dicts)): - header.append(i) - - table.append(header) - - sorted_fields = _SortDictionaryByValue(fields) - - for f in sorted_fields: - row = [f] - for d in dicts: - if f in d: - row.append(d[f]) - else: - row.append("0") - table.append(row) - - print tabulator.GetSimpleTable(table) - - -class Function(object): - def __init__(self): - self.count = 0 - self.name = "" - self.percent = 0 - - -class Section(object): - def __init__(self, contents): - self.raw_contents = contents - self._ParseSection() - - def _ParseSection(self): - matches = re.findall(r"Events: (\w+)\s+(.*)", self.raw_contents) - assert len(matches) <= 1, "More than one event found in 1 section" - if not matches: - return - match = matches[0] - self.name = match[1] - self.count = misc.UnitToNumber(match[0]) - - self.functions = [] - for line in self.raw_contents.splitlines(): - if not line.strip(): - continue - if "%" not in line: - continue - if not line.startswith("#"): - fields = [f for f in line.split(" ") if f] - function = Function() - function.percent = float(fields[0].strip("%")) - function.count = int(fields[1]) - function.name = " ".join(fields[2:]) - self.functions.append(function) - - -class PerfReport(object): - """Get report from raw report.""" - - def __init__(self, perf_file): - self.perf_file = perf_file - self._ReadFile() - self.sections = {} - self.metadata = {} - self._section_contents = [] - self._section_header = "" - self._SplitSections() - self._ParseSections() - self._ParseSectionHeader() - - def _ParseSectionHeader(self): - """Parse a header of a perf report file.""" - # The "captured on" field is inaccurate - this actually refers to when the - # report was generated, not when the data was captured. - for line in self._section_header.splitlines(): - line = line[2:] - if ":" in line: - key, val = line.strip().split(":", 1) - key = key.strip() - val = val.strip() - self.metadata[key] = val - - def _ReadFile(self): - self._perf_contents = open(self.perf_file).read() - - def _ParseSections(self): - self.event_counts = {} - self.sections = {} - for section_content in self._section_contents: - section = Section(section_content) - section.name = self._GetHumanReadableName(section.name) - self.sections[section.name] = section - - # TODO(asharif): Do this better. - def _GetHumanReadableName(self, section_name): - if not "raw" in section_name: - return section_name - raw_number = section_name.strip().split(" ")[-1] - for line in self._section_header.splitlines(): - if raw_number in line: - name = line.strip().split(" ")[5] - return name - - def _SplitSections(self): - self._section_contents = [] - indices = [m.start() for m in re.finditer("# Events:", self._perf_contents)] - indices.append(len(self._perf_contents)) - for i in range(len(indices) - 1): - section_content = self._perf_contents[indices[i]:indices[i+1]] - self._section_contents.append(section_content) - self._section_header = "" - if indices: - self._section_header = self._perf_contents[0:indices[0]] - - -class PerfDiffer(object): - """Perf differ class.""" - - def __init__(self, reports, num_symbols, common_only): - self._reports = reports - self._num_symbols = num_symbols - self._common_only = common_only - self._common_function_names = {} - - def DoDiff(self): - """The function that does the diff.""" - section_names = self._FindAllSections() - - filename_dicts = [] - summary_dicts = [] - for report in self._reports: - d = {} - filename_dicts.append({"file": report.perf_file}) - for section_name in section_names: - if section_name in report.sections: - d[section_name] = report.sections[section_name].count - summary_dicts.append(d) - - all_dicts = [filename_dicts, summary_dicts] - - for section_name in section_names: - function_names = self._GetTopFunctions(section_name, - self._num_symbols) - self._FindCommonFunctions(section_name) - dicts = [] - for report in self._reports: - d = {} - if section_name in report.sections: - section = report.sections[section_name] - - # Get a common scaling factor for this report. - common_scaling_factor = self._GetCommonScalingFactor(section) - - for function in section.functions: - if function.name in function_names: - key = "%s %s" % (section.name, function.name) - d[key] = function.count - # Compute a factor to scale the function count by in common_only - # mode. - if self._common_only and ( - function.name in self._common_function_names[section.name]): - d[key + " scaled"] = common_scaling_factor * function.count - dicts.append(d) - - all_dicts.append(dicts) - - mytabulator = Tabulator(all_dicts) - mytabulator.PrintTable() - - def _FindAllSections(self): - sections = {} - for report in self._reports: - for section in report.sections.values(): - if section.name not in sections: - sections[section.name] = section.count - else: - sections[section.name] = max(sections[section.name], - section.count) - return _SortDictionaryByValue(sections) - - def _GetCommonScalingFactor(self, section): - unique_count = self._GetCount( - section, - lambda x: x in self._common_function_names[section.name]) - return 100.0/unique_count - - def _GetCount(self, section, filter_fun=None): - total_count = 0 - for function in section.functions: - if not filter_fun or filter_fun(function.name): - total_count += int(function.count) - return total_count - - def _FindCommonFunctions(self, section_name): - function_names_list = [] - for report in self._reports: - if section_name in report.sections: - section = report.sections[section_name] - function_names = [f.name for f in section.functions] - function_names_list.append(function_names) - - self._common_function_names[section_name] = ( - reduce(set.intersection, map(set, function_names_list))) - - def _GetTopFunctions(self, section_name, num_functions): - all_functions = {} - for report in self._reports: - if section_name in report.sections: - section = report.sections[section_name] - for f in section.functions[:num_functions]: - if f.name in all_functions: - all_functions[f.name] = max(all_functions[f.name], f.count) - else: - all_functions[f.name] = f.count - # FIXME(asharif): Don't really need to sort these... - return _SortDictionaryByValue(all_functions) - - def _GetFunctionsDict(self, section, function_names): - d = {} - for function in section.functions: - if function.name in function_names: - d[function.name] = function.count - return d - - -def Main(argv): - """The entry of the main.""" - parser = optparse.OptionParser() - parser.add_option("-n", - "--num_symbols", - dest="num_symbols", - default="5", - help="The number of symbols to show.") - parser.add_option("-c", - "--common_only", - dest="common_only", - action="store_true", - default=False, - help="Diff common symbols only.") - - options, args = parser.parse_args(argv) - - try: - reports = [] - for report in args[1:]: - report = PerfReport(report) - reports.append(report) - pd = PerfDiffer(reports, int(options.num_symbols), options.common_only) - pd.DoDiff() - finally: - pass - - return 0 - - -if __name__ == "__main__": - sys.exit(Main(sys.argv)) diff --git a/v14/utils/pstat.py b/v14/utils/pstat.py deleted file mode 100644 index dae681e6..00000000 --- a/v14/utils/pstat.py +++ /dev/null @@ -1,1068 +0,0 @@ -# Copyright (c) 1999-2007 Gary Strangman; All Rights Reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -# Comments and/or additions are welcome (send e-mail to: -# strang@nmr.mgh.harvard.edu). -# -""" -pstat.py module - -################################################# -####### Written by: Gary Strangman ########### -####### Last modified: Dec 18, 2007 ########### -################################################# - -This module provides some useful list and array manipulation routines -modeled after those found in the |Stat package by Gary Perlman, plus a -number of other useful list/file manipulation functions. The list-based -functions include: - - abut (source,*args) - simpleabut (source, addon) - colex (listoflists,cnums) - collapse (listoflists,keepcols,collapsecols,fcn1=None,fcn2=None,cfcn=None) - dm (listoflists,criterion) - flat (l) - linexand (listoflists,columnlist,valuelist) - linexor (listoflists,columnlist,valuelist) - linedelimited (inlist,delimiter) - lineincols (inlist,colsize) - lineincustcols (inlist,colsizes) - list2string (inlist) - makelol(inlist) - makestr(x) - printcc (lst,extra=2) - printincols (listoflists,colsize) - pl (listoflists) - printl(listoflists) - replace (lst,oldval,newval) - recode (inlist,listmap,cols='all') - remap (listoflists,criterion) - roundlist (inlist,num_digits_to_round_floats_to) - sortby(listoflists,sortcols) - unique (inlist) - duplicates(inlist) - writedelimited (listoflists, delimiter, file, writetype='w') - -Some of these functions have alternate versions which are defined only if -Numeric (NumPy) can be imported. These functions are generally named as -above, with an 'a' prefix. - - aabut (source, *args) - acolex (a,indices,axis=1) - acollapse (a,keepcols,collapsecols,sterr=0,ns=0) - adm (a,criterion) - alinexand (a,columnlist,valuelist) - alinexor (a,columnlist,valuelist) - areplace (a,oldval,newval) - arecode (a,listmap,col='all') - arowcompare (row1, row2) - arowsame (row1, row2) - asortrows(a,axis=0) - aunique(inarray) - aduplicates(inarray) - -Currently, the code is all but completely un-optimized. In many cases, the -array versions of functions amount simply to aliases to built-in array -functions/methods. Their inclusion here is for function name consistency. -""" - -## CHANGE LOG: -## ========== -## 07-11-26 ... edited to work with numpy -## 01-11-15 ... changed list2string() to accept a delimiter -## 01-06-29 ... converted exec()'s to eval()'s to make compatible with Py2.1 -## 01-05-31 ... added duplicates() and aduplicates() functions -## 00-12-28 ... license made GPL, docstring and import requirements -## 99-11-01 ... changed version to 0.3 -## 99-08-30 ... removed get, getstrings, put, aget, aput (into io.py) -## 03/27/99 ... added areplace function, made replace fcn recursive -## 12/31/98 ... added writefc function for ouput to fixed column sizes -## 12/07/98 ... fixed import problem (failed on collapse() fcn) -## added __version__ variable (now 0.2) -## 12/05/98 ... updated doc-strings -## added features to collapse() function -## added flat() function for lists -## fixed a broken asortrows() -## 11/16/98 ... fixed minor bug in aput for 1D arrays -## -## 11/08/98 ... fixed aput to output large arrays correctly - -import stats # required 3rd party module -import string, copy -from types import * - -__version__ = 0.4 - -###=========================== LIST FUNCTIONS ========================== -### -### Here are the list functions, DEFINED FOR ALL SYSTEMS. -### Array functions (for NumPy-enabled computers) appear below. -### - -def abut (source,*args): - """ -Like the |Stat abut command. It concatenates two lists side-by-side -and returns the result. '2D' lists are also accomodated for either argument -(source or addon). CAUTION: If one list is shorter, it will be repeated -until it is as long as the longest list. If this behavior is not desired, -use pstat.simpleabut(). - -Usage: abut(source, args) where args=any # of lists -Returns: a list of lists as long as the LONGEST list past, source on the - 'left', lists in <args> attached consecutively on the 'right' -""" - - if type(source) not in [ListType,TupleType]: - source = [source] - for addon in args: - if type(addon) not in [ListType,TupleType]: - addon = [addon] - if len(addon) < len(source): # is source list longer? - if len(source) % len(addon) == 0: # are they integer multiples? - repeats = len(source)/len(addon) # repeat addon n times - origadd = copy.deepcopy(addon) - for i in range(repeats-1): - addon = addon + origadd - else: - repeats = len(source)/len(addon)+1 # repeat addon x times, - origadd = copy.deepcopy(addon) # x is NOT an integer - for i in range(repeats-1): - addon = addon + origadd - addon = addon[0:len(source)] - elif len(source) < len(addon): # is addon list longer? - if len(addon) % len(source) == 0: # are they integer multiples? - repeats = len(addon)/len(source) # repeat source n times - origsour = copy.deepcopy(source) - for i in range(repeats-1): - source = source + origsour - else: - repeats = len(addon)/len(source)+1 # repeat source x times, - origsour = copy.deepcopy(source) # x is NOT an integer - for i in range(repeats-1): - source = source + origsour - source = source[0:len(addon)] - - source = simpleabut(source,addon) - return source - - -def simpleabut (source, addon): - """ -Concatenates two lists as columns and returns the result. '2D' lists -are also accomodated for either argument (source or addon). This DOES NOT -repeat either list to make the 2 lists of equal length. Beware of list pairs -with different lengths ... the resulting list will be the length of the -FIRST list passed. - -Usage: simpleabut(source,addon) where source, addon=list (or list-of-lists) -Returns: a list of lists as long as source, with source on the 'left' and - addon on the 'right' -""" - if type(source) not in [ListType,TupleType]: - source = [source] - if type(addon) not in [ListType,TupleType]: - addon = [addon] - minlen = min(len(source),len(addon)) - list = copy.deepcopy(source) # start abut process - if type(source[0]) not in [ListType,TupleType]: - if type(addon[0]) not in [ListType,TupleType]: - for i in range(minlen): - list[i] = [source[i]] + [addon[i]] # source/addon = column - else: - for i in range(minlen): - list[i] = [source[i]] + addon[i] # addon=list-of-lists - else: - if type(addon[0]) not in [ListType,TupleType]: - for i in range(minlen): - list[i] = source[i] + [addon[i]] # source=list-of-lists - else: - for i in range(minlen): - list[i] = source[i] + addon[i] # source/addon = list-of-lists - source = list - return source - - -def colex (listoflists,cnums): - """ -Extracts from listoflists the columns specified in the list 'cnums' -(cnums can be an integer, a sequence of integers, or a string-expression that -corresponds to a slice operation on the variable x ... e.g., 'x[3:]' will colex -columns 3 onward from the listoflists). - -Usage: colex (listoflists,cnums) -Returns: a list-of-lists corresponding to the columns from listoflists - specified by cnums, in the order the column numbers appear in cnums -""" - global index - column = 0 - if type(cnums) in [ListType,TupleType]: # if multiple columns to get - index = cnums[0] - column = map(lambda x: x[index], listoflists) - for col in cnums[1:]: - index = col - column = abut(column,map(lambda x: x[index], listoflists)) - elif type(cnums) == StringType: # if an 'x[3:]' type expr. - evalstring = 'map(lambda x: x'+cnums+', listoflists)' - column = eval(evalstring) - else: # else it's just 1 col to get - index = cnums - column = map(lambda x: x[index], listoflists) - return column - - -def collapse (listoflists,keepcols,collapsecols,fcn1=None,fcn2=None,cfcn=None): - """ -Averages data in collapsecol, keeping all unique items in keepcols -(using unique, which keeps unique LISTS of column numbers), retaining the -unique sets of values in keepcols, the mean for each. Setting fcn1 -and/or fcn2 to point to a function rather than None (e.g., stats.sterr, len) -will append those results (e.g., the sterr, N) after each calculated mean. -cfcn is the collapse function to apply (defaults to mean, defined here in the -pstat module to avoid circular imports with stats.py, but harmonicmean or -others could be passed). - -Usage: collapse (listoflists,keepcols,collapsecols,fcn1=None,fcn2=None,cfcn=None) -Returns: a list of lists with all unique permutations of entries appearing in - columns ("conditions") specified by keepcols, abutted with the result of - cfcn (if cfcn=None, defaults to the mean) of each column specified by - collapsecols. -""" - def collmean (inlist): - s = 0 - for item in inlist: - s = s + item - return s/float(len(inlist)) - - if type(keepcols) not in [ListType,TupleType]: - keepcols = [keepcols] - if type(collapsecols) not in [ListType,TupleType]: - collapsecols = [collapsecols] - if cfcn == None: - cfcn = collmean - if keepcols == []: - means = [0]*len(collapsecols) - for i in range(len(collapsecols)): - avgcol = colex(listoflists,collapsecols[i]) - means[i] = cfcn(avgcol) - if fcn1: - try: - test = fcn1(avgcol) - except: - test = 'N/A' - means[i] = [means[i], test] - if fcn2: - try: - test = fcn2(avgcol) - except: - test = 'N/A' - try: - means[i] = means[i] + [len(avgcol)] - except TypeError: - means[i] = [means[i],len(avgcol)] - return means - else: - values = colex(listoflists,keepcols) - uniques = unique(values) - uniques.sort() - newlist = [] - if type(keepcols) not in [ListType,TupleType]: keepcols = [keepcols] - for item in uniques: - if type(item) not in [ListType,TupleType]: item =[item] - tmprows = linexand(listoflists,keepcols,item) - for col in collapsecols: - avgcol = colex(tmprows,col) - item.append(cfcn(avgcol)) - if fcn1 <> None: - try: - test = fcn1(avgcol) - except: - test = 'N/A' - item.append(test) - if fcn2 <> None: - try: - test = fcn2(avgcol) - except: - test = 'N/A' - item.append(test) - newlist.append(item) - return newlist - - -def dm (listoflists,criterion): - """ -Returns rows from the passed list of lists that meet the criteria in -the passed criterion expression (a string as a function of x; e.g., 'x[3]>=9' -will return all rows where the 4th column>=9 and "x[2]=='N'" will return rows -with column 2 equal to the string 'N'). - -Usage: dm (listoflists, criterion) -Returns: rows from listoflists that meet the specified criterion. -""" - function = 'filter(lambda x: '+criterion+',listoflists)' - lines = eval(function) - return lines - - -def flat(l): - """ -Returns the flattened version of a '2D' list. List-correlate to the a.ravel()() -method of NumPy arrays. - -Usage: flat(l) -""" - newl = [] - for i in range(len(l)): - for j in range(len(l[i])): - newl.append(l[i][j]) - return newl - - -def linexand (listoflists,columnlist,valuelist): - """ -Returns the rows of a list of lists where col (from columnlist) = val -(from valuelist) for EVERY pair of values (columnlist[i],valuelists[i]). -len(columnlist) must equal len(valuelist). - -Usage: linexand (listoflists,columnlist,valuelist) -Returns: the rows of listoflists where columnlist[i]=valuelist[i] for ALL i -""" - if type(columnlist) not in [ListType,TupleType]: - columnlist = [columnlist] - if type(valuelist) not in [ListType,TupleType]: - valuelist = [valuelist] - criterion = '' - for i in range(len(columnlist)): - if type(valuelist[i])==StringType: - critval = '\'' + valuelist[i] + '\'' - else: - critval = str(valuelist[i]) - criterion = criterion + ' x['+str(columnlist[i])+']=='+critval+' and' - criterion = criterion[0:-3] # remove the "and" after the last crit - function = 'filter(lambda x: '+criterion+',listoflists)' - lines = eval(function) - return lines - - -def linexor (listoflists,columnlist,valuelist): - """ -Returns the rows of a list of lists where col (from columnlist) = val -(from valuelist) for ANY pair of values (colunmlist[i],valuelist[i[). -One value is required for each column in columnlist. If only one value -exists for columnlist but multiple values appear in valuelist, the -valuelist values are all assumed to pertain to the same column. - -Usage: linexor (listoflists,columnlist,valuelist) -Returns: the rows of listoflists where columnlist[i]=valuelist[i] for ANY i -""" - if type(columnlist) not in [ListType,TupleType]: - columnlist = [columnlist] - if type(valuelist) not in [ListType,TupleType]: - valuelist = [valuelist] - criterion = '' - if len(columnlist) == 1 and len(valuelist) > 1: - columnlist = columnlist*len(valuelist) - for i in range(len(columnlist)): # build an exec string - if type(valuelist[i])==StringType: - critval = '\'' + valuelist[i] + '\'' - else: - critval = str(valuelist[i]) - criterion = criterion + ' x['+str(columnlist[i])+']=='+critval+' or' - criterion = criterion[0:-2] # remove the "or" after the last crit - function = 'filter(lambda x: '+criterion+',listoflists)' - lines = eval(function) - return lines - - -def linedelimited (inlist,delimiter): - """ -Returns a string composed of elements in inlist, with each element -separated by 'delimiter.' Used by function writedelimited. Use '\t' -for tab-delimiting. - -Usage: linedelimited (inlist,delimiter) -""" - outstr = '' - for item in inlist: - if type(item) <> StringType: - item = str(item) - outstr = outstr + item + delimiter - outstr = outstr[0:-1] - return outstr - - -def lineincols (inlist,colsize): - """ -Returns a string composed of elements in inlist, with each element -right-aligned in columns of (fixed) colsize. - -Usage: lineincols (inlist,colsize) where colsize is an integer -""" - outstr = '' - for item in inlist: - if type(item) <> StringType: - item = str(item) - size = len(item) - if size <= colsize: - for i in range(colsize-size): - outstr = outstr + ' ' - outstr = outstr + item - else: - outstr = outstr + item[0:colsize+1] - return outstr - - -def lineincustcols (inlist,colsizes): - """ -Returns a string composed of elements in inlist, with each element -right-aligned in a column of width specified by a sequence colsizes. The -length of colsizes must be greater than or equal to the number of columns -in inlist. - -Usage: lineincustcols (inlist,colsizes) -Returns: formatted string created from inlist -""" - outstr = '' - for i in range(len(inlist)): - if type(inlist[i]) <> StringType: - item = str(inlist[i]) - else: - item = inlist[i] - size = len(item) - if size <= colsizes[i]: - for j in range(colsizes[i]-size): - outstr = outstr + ' ' - outstr = outstr + item - else: - outstr = outstr + item[0:colsizes[i]+1] - return outstr - - -def list2string (inlist,delimit=' '): - """ -Converts a 1D list to a single long string for file output, using -the string.join function. - -Usage: list2string (inlist,delimit=' ') -Returns: the string created from inlist -""" - stringlist = map(makestr,inlist) - return string.join(stringlist,delimit) - - -def makelol(inlist): - """ -Converts a 1D list to a 2D list (i.e., a list-of-lists). Useful when you -want to use put() to write a 1D list one item per line in the file. - -Usage: makelol(inlist) -Returns: if l = [1,2,'hi'] then returns [[1],[2],['hi']] etc. -""" - x = [] - for item in inlist: - x.append([item]) - return x - - -def makestr (x): - if type(x) <> StringType: - x = str(x) - return x - - -def printcc (lst,extra=2): - """ -Prints a list of lists in columns, customized by the max size of items -within the columns (max size of items in col, plus 'extra' number of spaces). -Use 'dashes' or '\\n' in the list-of-lists to print dashes or blank lines, -respectively. - -Usage: printcc (lst,extra=2) -Returns: None -""" - if type(lst[0]) not in [ListType,TupleType]: - lst = [lst] - rowstokill = [] - list2print = copy.deepcopy(lst) - for i in range(len(lst)): - if lst[i] == ['\n'] or lst[i]=='\n' or lst[i]=='dashes' or lst[i]=='' or lst[i]==['']: - rowstokill = rowstokill + [i] - rowstokill.reverse() # delete blank rows from the end - for row in rowstokill: - del list2print[row] - maxsize = [0]*len(list2print[0]) - for col in range(len(list2print[0])): - items = colex(list2print,col) - items = map(makestr,items) - maxsize[col] = max(map(len,items)) + extra - for row in lst: - if row == ['\n'] or row == '\n' or row == '' or row == ['']: - print - elif row == ['dashes'] or row == 'dashes': - dashes = [0]*len(maxsize) - for j in range(len(maxsize)): - dashes[j] = '-'*(maxsize[j]-2) - print lineincustcols(dashes,maxsize) - else: - print lineincustcols(row,maxsize) - return None - - -def printincols (listoflists,colsize): - """ -Prints a list of lists in columns of (fixed) colsize width, where -colsize is an integer. - -Usage: printincols (listoflists,colsize) -Returns: None -""" - for row in listoflists: - print lineincols(row,colsize) - return None - - -def pl (listoflists): - """ -Prints a list of lists, 1 list (row) at a time. - -Usage: pl(listoflists) -Returns: None -""" - for row in listoflists: - if row[-1] == '\n': - print row, - else: - print row - return None - - -def printl(listoflists): - """Alias for pl.""" - pl(listoflists) - return - - -def replace (inlst,oldval,newval): - """ -Replaces all occurrences of 'oldval' with 'newval', recursively. - -Usage: replace (inlst,oldval,newval) -""" - lst = inlst*1 - for i in range(len(lst)): - if type(lst[i]) not in [ListType,TupleType]: - if lst[i]==oldval: lst[i]=newval - else: - lst[i] = replace(lst[i],oldval,newval) - return lst - - -def recode (inlist,listmap,cols=None): - """ -Changes the values in a list to a new set of values (useful when -you need to recode data from (e.g.) strings to numbers. cols defaults -to None (meaning all columns are recoded). - -Usage: recode (inlist,listmap,cols=None) cols=recode cols, listmap=2D list -Returns: inlist with the appropriate values replaced with new ones -""" - lst = copy.deepcopy(inlist) - if cols != None: - if type(cols) not in [ListType,TupleType]: - cols = [cols] - for col in cols: - for row in range(len(lst)): - try: - idx = colex(listmap,0).index(lst[row][col]) - lst[row][col] = listmap[idx][1] - except ValueError: - pass - else: - for row in range(len(lst)): - for col in range(len(lst)): - try: - idx = colex(listmap,0).index(lst[row][col]) - lst[row][col] = listmap[idx][1] - except ValueError: - pass - return lst - - -def remap (listoflists,criterion): - """ -Remaps values in a given column of a 2D list (listoflists). This requires -a criterion as a function of 'x' so that the result of the following is -returned ... map(lambda x: 'criterion',listoflists). - -Usage: remap(listoflists,criterion) criterion=string -Returns: remapped version of listoflists -""" - function = 'map(lambda x: '+criterion+',listoflists)' - lines = eval(function) - return lines - - -def roundlist (inlist,digits): - """ -Goes through each element in a 1D or 2D inlist, and applies the following -function to all elements of FloatType ... round(element,digits). - -Usage: roundlist(inlist,digits) -Returns: list with rounded floats -""" - if type(inlist[0]) in [IntType, FloatType]: - inlist = [inlist] - l = inlist*1 - for i in range(len(l)): - for j in range(len(l[i])): - if type(l[i][j])==FloatType: - l[i][j] = round(l[i][j],digits) - return l - - -def sortby(listoflists,sortcols): - """ -Sorts a list of lists on the column(s) specified in the sequence -sortcols. - -Usage: sortby(listoflists,sortcols) -Returns: sorted list, unchanged column ordering -""" - newlist = abut(colex(listoflists,sortcols),listoflists) - newlist.sort() - try: - numcols = len(sortcols) - except TypeError: - numcols = 1 - crit = '[' + str(numcols) + ':]' - newlist = colex(newlist,crit) - return newlist - - -def unique (inlist): - """ -Returns all unique items in the passed list. If the a list-of-lists -is passed, unique LISTS are found (i.e., items in the first dimension are -compared). - -Usage: unique (inlist) -Returns: the unique elements (or rows) in inlist -""" - uniques = [] - for item in inlist: - if item not in uniques: - uniques.append(item) - return uniques - -def duplicates(inlist): - """ -Returns duplicate items in the FIRST dimension of the passed list. - -Usage: duplicates (inlist) -""" - dups = [] - for i in range(len(inlist)): - if inlist[i] in inlist[i+1:]: - dups.append(inlist[i]) - return dups - - -def nonrepeats(inlist): - """ -Returns items that are NOT duplicated in the first dim of the passed list. - -Usage: nonrepeats (inlist) -""" - nonrepeats = [] - for i in range(len(inlist)): - if inlist.count(inlist[i]) == 1: - nonrepeats.append(inlist[i]) - return nonrepeats - - -#=================== PSTAT ARRAY FUNCTIONS ===================== -#=================== PSTAT ARRAY FUNCTIONS ===================== -#=================== PSTAT ARRAY FUNCTIONS ===================== -#=================== PSTAT ARRAY FUNCTIONS ===================== -#=================== PSTAT ARRAY FUNCTIONS ===================== -#=================== PSTAT ARRAY FUNCTIONS ===================== -#=================== PSTAT ARRAY FUNCTIONS ===================== -#=================== PSTAT ARRAY FUNCTIONS ===================== -#=================== PSTAT ARRAY FUNCTIONS ===================== -#=================== PSTAT ARRAY FUNCTIONS ===================== -#=================== PSTAT ARRAY FUNCTIONS ===================== -#=================== PSTAT ARRAY FUNCTIONS ===================== -#=================== PSTAT ARRAY FUNCTIONS ===================== -#=================== PSTAT ARRAY FUNCTIONS ===================== -#=================== PSTAT ARRAY FUNCTIONS ===================== -#=================== PSTAT ARRAY FUNCTIONS ===================== - -try: # DEFINE THESE *ONLY* IF numpy IS AVAILABLE - import numpy as N - - def aabut (source, *args): - """ -Like the |Stat abut command. It concatenates two arrays column-wise -and returns the result. CAUTION: If one array is shorter, it will be -repeated until it is as long as the other. - -Usage: aabut (source, args) where args=any # of arrays -Returns: an array as long as the LONGEST array past, source appearing on the - 'left', arrays in <args> attached on the 'right'. -""" - if len(source.shape)==1: - width = 1 - source = N.resize(source,[source.shape[0],width]) - else: - width = source.shape[1] - for addon in args: - if len(addon.shape)==1: - width = 1 - addon = N.resize(addon,[source.shape[0],width]) - else: - width = source.shape[1] - if len(addon) < len(source): - addon = N.resize(addon,[source.shape[0],addon.shape[1]]) - elif len(source) < len(addon): - source = N.resize(source,[addon.shape[0],source.shape[1]]) - source = N.concatenate((source,addon),1) - return source - - - def acolex (a,indices,axis=1): - """ -Extracts specified indices (a list) from passed array, along passed -axis (column extraction is default). BEWARE: A 1D array is presumed to be a -column-array (and that the whole array will be returned as a column). - -Usage: acolex (a,indices,axis=1) -Returns: the columns of a specified by indices -""" - if type(indices) not in [ListType,TupleType,N.ndarray]: - indices = [indices] - if len(N.shape(a)) == 1: - cols = N.resize(a,[a.shape[0],1]) - else: -# print a[:3] - cols = N.take(a,indices,axis) -# print cols[:3] - return cols - - - def acollapse (a,keepcols,collapsecols,fcn1=None,fcn2=None,cfcn=None): - """ -Averages data in collapsecol, keeping all unique items in keepcols -(using unique, which keeps unique LISTS of column numbers), retaining -the unique sets of values in keepcols, the mean for each. If stderror or -N of the mean are desired, set either or both parameters to 1. - -Usage: acollapse (a,keepcols,collapsecols,fcn1=None,fcn2=None,cfcn=None) -Returns: unique 'conditions' specified by the contents of columns specified - by keepcols, abutted with the mean(s) of column(s) specified by - collapsecols -""" - def acollmean (inarray): - return N.sum(N.ravel(inarray)) - - if type(keepcols) not in [ListType,TupleType,N.ndarray]: - keepcols = [keepcols] - if type(collapsecols) not in [ListType,TupleType,N.ndarray]: - collapsecols = [collapsecols] - - if cfcn == None: - cfcn = acollmean - if keepcols == []: - avgcol = acolex(a,collapsecols) - means = N.sum(avgcol)/float(len(avgcol)) - if fcn1<>None: - try: - test = fcn1(avgcol) - except: - test = N.array(['N/A']*len(means)) - means = aabut(means,test) - if fcn2<>None: - try: - test = fcn2(avgcol) - except: - test = N.array(['N/A']*len(means)) - means = aabut(means,test) - return means - else: - if type(keepcols) not in [ListType,TupleType,N.ndarray]: - keepcols = [keepcols] - values = colex(a,keepcols) # so that "item" can be appended (below) - uniques = unique(values) # get a LIST, so .sort keeps rows intact - uniques.sort() - newlist = [] - for item in uniques: - if type(item) not in [ListType,TupleType,N.ndarray]: - item =[item] - tmprows = alinexand(a,keepcols,item) - for col in collapsecols: - avgcol = acolex(tmprows,col) - item.append(acollmean(avgcol)) - if fcn1<>None: - try: - test = fcn1(avgcol) - except: - test = 'N/A' - item.append(test) - if fcn2<>None: - try: - test = fcn2(avgcol) - except: - test = 'N/A' - item.append(test) - newlist.append(item) - try: - new_a = N.array(newlist) - except TypeError: - new_a = N.array(newlist,'O') - return new_a - - - def adm (a,criterion): - """ -Returns rows from the passed list of lists that meet the criteria in -the passed criterion expression (a string as a function of x). - -Usage: adm (a,criterion) where criterion is like 'x[2]==37' -""" - function = 'filter(lambda x: '+criterion+',a)' - lines = eval(function) - try: - lines = N.array(lines) - except: - lines = N.array(lines,dtype='O') - return lines - - - def isstring(x): - if type(x)==StringType: - return 1 - else: - return 0 - - - def alinexand (a,columnlist,valuelist): - """ -Returns the rows of an array where col (from columnlist) = val -(from valuelist). One value is required for each column in columnlist. - -Usage: alinexand (a,columnlist,valuelist) -Returns: the rows of a where columnlist[i]=valuelist[i] for ALL i -""" - if type(columnlist) not in [ListType,TupleType,N.ndarray]: - columnlist = [columnlist] - if type(valuelist) not in [ListType,TupleType,N.ndarray]: - valuelist = [valuelist] - criterion = '' - for i in range(len(columnlist)): - if type(valuelist[i])==StringType: - critval = '\'' + valuelist[i] + '\'' - else: - critval = str(valuelist[i]) - criterion = criterion + ' x['+str(columnlist[i])+']=='+critval+' and' - criterion = criterion[0:-3] # remove the "and" after the last crit - return adm(a,criterion) - - - def alinexor (a,columnlist,valuelist): - """ -Returns the rows of an array where col (from columnlist) = val (from -valuelist). One value is required for each column in columnlist. -The exception is if either columnlist or valuelist has only 1 value, -in which case that item will be expanded to match the length of the -other list. - -Usage: alinexor (a,columnlist,valuelist) -Returns: the rows of a where columnlist[i]=valuelist[i] for ANY i -""" - if type(columnlist) not in [ListType,TupleType,N.ndarray]: - columnlist = [columnlist] - if type(valuelist) not in [ListType,TupleType,N.ndarray]: - valuelist = [valuelist] - criterion = '' - if len(columnlist) == 1 and len(valuelist) > 1: - columnlist = columnlist*len(valuelist) - elif len(valuelist) == 1 and len(columnlist) > 1: - valuelist = valuelist*len(columnlist) - for i in range(len(columnlist)): - if type(valuelist[i])==StringType: - critval = '\'' + valuelist[i] + '\'' - else: - critval = str(valuelist[i]) - criterion = criterion + ' x['+str(columnlist[i])+']=='+critval+' or' - criterion = criterion[0:-2] # remove the "or" after the last crit - return adm(a,criterion) - - - def areplace (a,oldval,newval): - """ -Replaces all occurrences of oldval with newval in array a. - -Usage: areplace(a,oldval,newval) -""" - return N.where(a==oldval,newval,a) - - - def arecode (a,listmap,col='all'): - """ -Remaps the values in an array to a new set of values (useful when -you need to recode data from (e.g.) strings to numbers as most stats -packages require. Can work on SINGLE columns, or 'all' columns at once. -@@@BROKEN 2007-11-26 - -Usage: arecode (a,listmap,col='all') -Returns: a version of array a where listmap[i][0] = (instead) listmap[i][1] -""" - ashape = a.shape - if col == 'all': - work = a.ravel() - else: - work = acolex(a,col) - work = work.ravel() - for pair in listmap: - if type(pair[1]) == StringType or work.dtype.char=='O' or a.dtype.char=='O': - work = N.array(work,dtype='O') - a = N.array(a,dtype='O') - for i in range(len(work)): - if work[i]==pair[0]: - work[i] = pair[1] - if col == 'all': - return N.reshape(work,ashape) - else: - return N.concatenate([a[:,0:col],work[:,N.newaxis],a[:,col+1:]],1) - else: # must be a non-Object type array and replacement - work = N.where(work==pair[0],pair[1],work) - return N.concatenate([a[:,0:col],work[:,N.newaxis],a[:,col+1:]],1) - - - def arowcompare(row1, row2): - """ -Compares two rows from an array, regardless of whether it is an -array of numbers or of python objects (which requires the cmp function). -@@@PURPOSE? 2007-11-26 - -Usage: arowcompare(row1,row2) -Returns: an array of equal length containing 1s where the two rows had - identical elements and 0 otherwise -""" - return - if row1.dtype.char=='O' or row2.dtype=='O': - cmpvect = N.logical_not(abs(N.array(map(cmp,row1,row2)))) # cmp fcn gives -1,0,1 - else: - cmpvect = N.equal(row1,row2) - return cmpvect - - - def arowsame(row1, row2): - """ -Compares two rows from an array, regardless of whether it is an -array of numbers or of python objects (which requires the cmp function). - -Usage: arowsame(row1,row2) -Returns: 1 if the two rows are identical, 0 otherwise. -""" - cmpval = N.alltrue(arowcompare(row1,row2)) - return cmpval - - - def asortrows(a,axis=0): - """ -Sorts an array "by rows". This differs from the Numeric.sort() function, -which sorts elements WITHIN the given axis. Instead, this function keeps -the elements along the given axis intact, but shifts them 'up or down' -relative to one another. - -Usage: asortrows(a,axis=0) -Returns: sorted version of a -""" - return N.sort(a,axis=axis,kind='mergesort') - - - def aunique(inarray): - """ -Returns unique items in the FIRST dimension of the passed array. Only -works on arrays NOT including string items. - -Usage: aunique (inarray) -""" - uniques = N.array([inarray[0]]) - if len(uniques.shape) == 1: # IF IT'S A 1D ARRAY - for item in inarray[1:]: - if N.add.reduce(N.equal(uniques,item).ravel()) == 0: - try: - uniques = N.concatenate([uniques,N.array[N.newaxis,:]]) - except TypeError: - uniques = N.concatenate([uniques,N.array([item])]) - else: # IT MUST BE A 2+D ARRAY - if inarray.dtype.char != 'O': # not an Object array - for item in inarray[1:]: - if not N.sum(N.alltrue(N.equal(uniques,item),1)): - try: - uniques = N.concatenate( [uniques,item[N.newaxis,:]] ) - except TypeError: # the item to add isn't a list - uniques = N.concatenate([uniques,N.array([item])]) - else: - pass # this item is already in the uniques array - else: # must be an Object array, alltrue/equal functions don't work - for item in inarray[1:]: - newflag = 1 - for unq in uniques: # NOTE: cmp --> 0=same, -1=<, 1=> - test = N.sum(abs(N.array(map(cmp,item,unq)))) - if test == 0: # if item identical to any 1 row in uniques - newflag = 0 # then not a novel item to add - break - if newflag == 1: - try: - uniques = N.concatenate( [uniques,item[N.newaxis,:]] ) - except TypeError: # the item to add isn't a list - uniques = N.concatenate([uniques,N.array([item])]) - return uniques - - - def aduplicates(inarray): - """ -Returns duplicate items in the FIRST dimension of the passed array. Only -works on arrays NOT including string items. - -Usage: aunique (inarray) -""" - inarray = N.array(inarray) - if len(inarray.shape) == 1: # IF IT'S A 1D ARRAY - dups = [] - inarray = inarray.tolist() - for i in range(len(inarray)): - if inarray[i] in inarray[i+1:]: - dups.append(inarray[i]) - dups = aunique(dups) - else: # IT MUST BE A 2+D ARRAY - dups = [] - aslist = inarray.tolist() - for i in range(len(aslist)): - if aslist[i] in aslist[i+1:]: - dups.append(aslist[i]) - dups = unique(dups) - dups = N.array(dups) - return dups - -except ImportError: # IF NUMERIC ISN'T AVAILABLE, SKIP ALL arrayfuncs - pass diff --git a/v14/utils/stats.py b/v14/utils/stats.py deleted file mode 100644 index aceed824..00000000 --- a/v14/utils/stats.py +++ /dev/null @@ -1,4528 +0,0 @@ -# Copyright (c) 1999-2008 Gary Strangman; All Rights Reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -# Comments and/or additions are welcome (send e-mail to: -# strang@nmr.mgh.harvard.edu). -# -""" -stats.py module - -(Requires pstat.py module.) - -################################################# -####### Written by: Gary Strangman ########### -####### Last modified: Oct 31, 2008 ########### -################################################# - -A collection of basic statistical functions for python. The function -names appear below. - -IMPORTANT: There are really *3* sets of functions. The first set has an 'l' -prefix, which can be used with list or tuple arguments. The second set has -an 'a' prefix, which can accept NumPy array arguments. These latter -functions are defined only when NumPy is available on the system. The third -type has NO prefix (i.e., has the name that appears below). Functions of -this set are members of a "Dispatch" class, c/o David Ascher. This class -allows different functions to be called depending on the type of the passed -arguments. Thus, stats.mean is a member of the Dispatch class and -stats.mean(range(20)) will call stats.lmean(range(20)) while -stats.mean(Numeric.arange(20)) will call stats.amean(Numeric.arange(20)). -This is a handy way to keep consistent function names when different -argument types require different functions to be called. Having -implementated the Dispatch class, however, means that to get info on -a given function, you must use the REAL function name ... that is -"print stats.lmean.__doc__" or "print stats.amean.__doc__" work fine, -while "print stats.mean.__doc__" will print the doc for the Dispatch -class. NUMPY FUNCTIONS ('a' prefix) generally have more argument options -but should otherwise be consistent with the corresponding list functions. - -Disclaimers: The function list is obviously incomplete and, worse, the -functions are not optimized. All functions have been tested (some more -so than others), but they are far from bulletproof. Thus, as with any -free software, no warranty or guarantee is expressed or implied. :-) A -few extra functions that don't appear in the list below can be found by -interested treasure-hunters. These functions don't necessarily have -both list and array versions but were deemed useful - -CENTRAL TENDENCY: geometricmean - harmonicmean - mean - median - medianscore - mode - -MOMENTS: moment - variation - skew - kurtosis - skewtest (for Numpy arrays only) - kurtosistest (for Numpy arrays only) - normaltest (for Numpy arrays only) - -ALTERED VERSIONS: tmean (for Numpy arrays only) - tvar (for Numpy arrays only) - tmin (for Numpy arrays only) - tmax (for Numpy arrays only) - tstdev (for Numpy arrays only) - tsem (for Numpy arrays only) - describe - -FREQUENCY STATS: itemfreq - scoreatpercentile - percentileofscore - histogram - cumfreq - relfreq - -VARIABILITY: obrientransform - samplevar - samplestdev - signaltonoise (for Numpy arrays only) - var - stdev - sterr - sem - z - zs - zmap (for Numpy arrays only) - -TRIMMING FCNS: threshold (for Numpy arrays only) - trimboth - trim1 - round (round all vals to 'n' decimals; Numpy only) - -CORRELATION FCNS: covariance (for Numpy arrays only) - correlation (for Numpy arrays only) - paired - pearsonr - spearmanr - pointbiserialr - kendalltau - linregress - -INFERENTIAL STATS: ttest_1samp - ttest_ind - ttest_rel - chisquare - ks_2samp - mannwhitneyu - ranksums - wilcoxont - kruskalwallish - friedmanchisquare - -PROBABILITY CALCS: chisqprob - erfcc - zprob - ksprob - fprob - betacf - gammln - betai - -ANOVA FUNCTIONS: F_oneway - F_value - -SUPPORT FUNCTIONS: writecc - incr - sign (for Numpy arrays only) - sum - cumsum - ss - summult - sumdiffsquared - square_of_sums - shellsort - rankdata - outputpairedstats - findwithin -""" -## CHANGE LOG: -## =========== -## 09-07-21 ... added capability for getting the 'proportion' out of l/amannwhitneyu (but comment-disabled) -## 08-10-31 ... fixed import LinearAlgebra bug before glm fcns -## 07-11-26 ... conversion for numpy started -## 07-05-16 ... added Lin's Concordance Correlation Coefficient (alincc) and acov -## 05-08-21 ... added "Dice's coefficient" -## 04-10-26 ... added ap2t(), an ugly fcn for converting p-vals to T-vals -## 04-04-03 ... added amasslinregress() function to do regression on N-D arrays -## 03-01-03 ... CHANGED VERSION TO 0.6 -## fixed atsem() to properly handle limits=None case -## improved histogram and median functions (estbinwidth) and -## fixed atvar() function (wrong answers for neg numbers?!?) -## 02-11-19 ... fixed attest_ind and attest_rel for div-by-zero Overflows -## 02-05-10 ... fixed lchisqprob indentation (failed when df=even) -## 00-12-28 ... removed aanova() to separate module, fixed licensing to -## match Python License, fixed doc string & imports -## 00-04-13 ... pulled all "global" statements, except from aanova() -## added/fixed lots of documentation, removed io.py dependency -## changed to version 0.5 -## 99-11-13 ... added asign() function -## 99-11-01 ... changed version to 0.4 ... enough incremental changes now -## 99-10-25 ... added acovariance and acorrelation functions -## 99-10-10 ... fixed askew/akurtosis to avoid divide-by-zero errors -## added aglm function (crude, but will be improved) -## 99-10-04 ... upgraded acumsum, ass, asummult, asamplevar, avar, etc. to -## all handle lists of 'dimension's and keepdims -## REMOVED ar0, ar2, ar3, ar4 and replaced them with around -## reinserted fixes for abetai to avoid math overflows -## 99-09-05 ... rewrote achisqprob/aerfcc/aksprob/afprob/abetacf/abetai to -## handle multi-dimensional arrays (whew!) -## 99-08-30 ... fixed l/amoment, l/askew, l/akurtosis per D'Agostino (1990) -## added anormaltest per same reference -## re-wrote azprob to calc arrays of probs all at once -## 99-08-22 ... edited attest_ind printing section so arrays could be rounded -## 99-08-19 ... fixed amean and aharmonicmean for non-error(!) overflow on -## short/byte arrays (mean of #s btw 100-300 = -150??) -## 99-08-09 ... fixed asum so that the None case works for Byte arrays -## 99-08-08 ... fixed 7/3 'improvement' to handle t-calcs on N-D arrays -## 99-07-03 ... improved attest_ind, attest_rel (zero-division errortrap) -## 99-06-24 ... fixed bug(?) in attest_ind (n1=a.shape[0]) -## 04/11/99 ... added asignaltonoise, athreshold functions, changed all -## max/min in array section to N.maximum/N.minimum, -## fixed square_of_sums to prevent integer overflow -## 04/10/99 ... !!! Changed function name ... sumsquared ==> square_of_sums -## 03/18/99 ... Added ar0, ar2, ar3 and ar4 rounding functions -## 02/28/99 ... Fixed aobrientransform to return an array rather than a list -## 01/15/99 ... Essentially ceased updating list-versions of functions (!!!) -## 01/13/99 ... CHANGED TO VERSION 0.3 -## fixed bug in a/lmannwhitneyu p-value calculation -## 12/31/98 ... fixed variable-name bug in ldescribe -## 12/19/98 ... fixed bug in findwithin (fcns needed pstat. prefix) -## 12/16/98 ... changed amedianscore to return float (not array) for 1 score -## 12/14/98 ... added atmin and atmax functions -## removed umath from import line (not needed) -## l/ageometricmean modified to reduce chance of overflows (take -## nth root first, then multiply) -## 12/07/98 ... added __version__variable (now 0.2) -## removed all 'stats.' from anova() fcn -## 12/06/98 ... changed those functions (except shellsort) that altered -## arguments in-place ... cumsum, ranksort, ... -## updated (and fixed some) doc-strings -## 12/01/98 ... added anova() function (requires NumPy) -## incorporated Dispatch class -## 11/12/98 ... added functionality to amean, aharmonicmean, ageometricmean -## added 'asum' function (added functionality to N.add.reduce) -## fixed both moment and amoment (two errors) -## changed name of skewness and askewness to skew and askew -## fixed (a)histogram (which sometimes counted points <lowerlimit) - -import pstat # required 3rd party module -import math, string, copy # required python modules -from types import * - -__version__ = 0.6 - -############# DISPATCH CODE ############## - - -class Dispatch: - """ -The Dispatch class, care of David Ascher, allows different functions to -be called depending on the argument types. This way, there can be one -function name regardless of the argument type. To access function doc -in stats.py module, prefix the function with an 'l' or 'a' for list or -array arguments, respectively. That is, print stats.lmean.__doc__ or -print stats.amean.__doc__ or whatever. -""" - - def __init__(self, *tuples): - self._dispatch = {} - for func, types in tuples: - for t in types: - if t in self._dispatch.keys(): - raise ValueError, "can't have two dispatches on "+str(t) - self._dispatch[t] = func - self._types = self._dispatch.keys() - - def __call__(self, arg1, *args, **kw): - if type(arg1) not in self._types: - raise TypeError, "don't know how to dispatch %s arguments" % type(arg1) - return apply(self._dispatch[type(arg1)], (arg1,) + args, kw) - - -########################################################################## -######################## LIST-BASED FUNCTIONS ######################## -########################################################################## - -### Define these regardless - -#################################### -####### CENTRAL TENDENCY ######### -#################################### - -def lgeometricmean (inlist): - """ -Calculates the geometric mean of the values in the passed list. -That is: n-th root of (x1 * x2 * ... * xn). Assumes a '1D' list. - -Usage: lgeometricmean(inlist) -""" - mult = 1.0 - one_over_n = 1.0/len(inlist) - for item in inlist: - mult = mult * pow(item,one_over_n) - return mult - - -def lharmonicmean (inlist): - """ -Calculates the harmonic mean of the values in the passed list. -That is: n / (1/x1 + 1/x2 + ... + 1/xn). Assumes a '1D' list. - -Usage: lharmonicmean(inlist) -""" - sum = 0 - for item in inlist: - sum = sum + 1.0/item - return len(inlist) / sum - - -def lmean (inlist): - """ -Returns the arithematic mean of the values in the passed list. -Assumes a '1D' list, but will function on the 1st dim of an array(!). - -Usage: lmean(inlist) -""" - sum = 0 - for item in inlist: - sum = sum + item - return sum/float(len(inlist)) - - -def lmedian (inlist,numbins=1000): - """ -Returns the computed median value of a list of numbers, given the -number of bins to use for the histogram (more bins brings the computed value -closer to the median score, default number of bins = 1000). See G.W. -Heiman's Basic Stats (1st Edition), or CRC Probability & Statistics. - -Usage: lmedian (inlist, numbins=1000) -""" - (hist, smallest, binsize, extras) = histogram(inlist,numbins,[min(inlist),max(inlist)]) # make histog - cumhist = cumsum(hist) # make cumulative histogram - for i in range(len(cumhist)): # get 1st(!) index holding 50%ile score - if cumhist[i]>=len(inlist)/2.0: - cfbin = i - break - LRL = smallest + binsize*cfbin # get lower read limit of that bin - cfbelow = cumhist[cfbin-1] - freq = float(hist[cfbin]) # frequency IN the 50%ile bin - median = LRL + ((len(inlist)/2.0 - cfbelow)/float(freq))*binsize # median formula - return median - - -def lmedianscore (inlist): - """ -Returns the 'middle' score of the passed list. If there is an even -number of scores, the mean of the 2 middle scores is returned. - -Usage: lmedianscore(inlist) -""" - - newlist = copy.deepcopy(inlist) - newlist.sort() - if len(newlist) % 2 == 0: # if even number of scores, average middle 2 - index = len(newlist)/2 # integer division correct - median = float(newlist[index] + newlist[index-1]) /2 - else: - index = len(newlist)/2 # int divsion gives mid value when count from 0 - median = newlist[index] - return median - - -def lmode(inlist): - """ -Returns a list of the modal (most common) score(s) in the passed -list. If there is more than one such score, all are returned. The -bin-count for the mode(s) is also returned. - -Usage: lmode(inlist) -Returns: bin-count for mode(s), a list of modal value(s) -""" - - scores = pstat.unique(inlist) - scores.sort() - freq = [] - for item in scores: - freq.append(inlist.count(item)) - maxfreq = max(freq) - mode = [] - stillmore = 1 - while stillmore: - try: - indx = freq.index(maxfreq) - mode.append(scores[indx]) - del freq[indx] - del scores[indx] - except ValueError: - stillmore=0 - return maxfreq, mode - - -#################################### -############ MOMENTS ############# -#################################### - -def lmoment(inlist,moment=1): - """ -Calculates the nth moment about the mean for a sample (defaults to -the 1st moment). Used to calculate coefficients of skewness and kurtosis. - -Usage: lmoment(inlist,moment=1) -Returns: appropriate moment (r) from ... 1/n * SUM((inlist(i)-mean)**r) -""" - if moment == 1: - return 0.0 - else: - mn = mean(inlist) - n = len(inlist) - s = 0 - for x in inlist: - s = s + (x-mn)**moment - return s/float(n) - - -def lvariation(inlist): - """ -Returns the coefficient of variation, as defined in CRC Standard -Probability and Statistics, p.6. - -Usage: lvariation(inlist) -""" - return 100.0*samplestdev(inlist)/float(mean(inlist)) - - -def lskew(inlist): - """ -Returns the skewness of a distribution, as defined in Numerical -Recipies (alternate defn in CRC Standard Probability and Statistics, p.6.) - -Usage: lskew(inlist) -""" - return moment(inlist,3)/pow(moment(inlist,2),1.5) - - -def lkurtosis(inlist): - """ -Returns the kurtosis of a distribution, as defined in Numerical -Recipies (alternate defn in CRC Standard Probability and Statistics, p.6.) - -Usage: lkurtosis(inlist) -""" - return moment(inlist,4)/pow(moment(inlist,2),2.0) - - -def ldescribe(inlist): - """ -Returns some descriptive statistics of the passed list (assumed to be 1D). - -Usage: ldescribe(inlist) -Returns: n, mean, standard deviation, skew, kurtosis -""" - n = len(inlist) - mm = (min(inlist),max(inlist)) - m = mean(inlist) - sd = stdev(inlist) - sk = skew(inlist) - kurt = kurtosis(inlist) - return n, mm, m, sd, sk, kurt - - -#################################### -####### FREQUENCY STATS ########## -#################################### - -def litemfreq(inlist): - """ -Returns a list of pairs. Each pair consists of one of the scores in inlist -and it's frequency count. Assumes a 1D list is passed. - -Usage: litemfreq(inlist) -Returns: a 2D frequency table (col [0:n-1]=scores, col n=frequencies) -""" - scores = pstat.unique(inlist) - scores.sort() - freq = [] - for item in scores: - freq.append(inlist.count(item)) - return pstat.abut(scores, freq) - - -def lscoreatpercentile (inlist, percent): - """ -Returns the score at a given percentile relative to the distribution -given by inlist. - -Usage: lscoreatpercentile(inlist,percent) -""" - if percent > 1: - print "\nDividing percent>1 by 100 in lscoreatpercentile().\n" - percent = percent / 100.0 - targetcf = percent*len(inlist) - h, lrl, binsize, extras = histogram(inlist) - cumhist = cumsum(copy.deepcopy(h)) - for i in range(len(cumhist)): - if cumhist[i] >= targetcf: - break - score = binsize * ((targetcf - cumhist[i-1]) / float(h[i])) + (lrl+binsize*i) - return score - - -def lpercentileofscore (inlist, score,histbins=10,defaultlimits=None): - """ -Returns the percentile value of a score relative to the distribution -given by inlist. Formula depends on the values used to histogram the data(!). - -Usage: lpercentileofscore(inlist,score,histbins=10,defaultlimits=None) -""" - - h, lrl, binsize, extras = histogram(inlist,histbins,defaultlimits) - cumhist = cumsum(copy.deepcopy(h)) - i = int((score - lrl)/float(binsize)) - pct = (cumhist[i-1]+((score-(lrl+binsize*i))/float(binsize))*h[i])/float(len(inlist)) * 100 - return pct - - -def lhistogram (inlist,numbins=10,defaultreallimits=None,printextras=0): - """ -Returns (i) a list of histogram bin counts, (ii) the smallest value -of the histogram binning, and (iii) the bin width (the last 2 are not -necessarily integers). Default number of bins is 10. If no sequence object -is given for defaultreallimits, the routine picks (usually non-pretty) bins -spanning all the numbers in the inlist. - -Usage: lhistogram (inlist, numbins=10, defaultreallimits=None,suppressoutput=0) -Returns: list of bin values, lowerreallimit, binsize, extrapoints -""" - if (defaultreallimits <> None): - if type(defaultreallimits) not in [ListType,TupleType] or len(defaultreallimits)==1: # only one limit given, assumed to be lower one & upper is calc'd - lowerreallimit = defaultreallimits - upperreallimit = 1.000001 * max(inlist) - else: # assume both limits given - lowerreallimit = defaultreallimits[0] - upperreallimit = defaultreallimits[1] - binsize = (upperreallimit-lowerreallimit)/float(numbins) - else: # no limits given for histogram, both must be calc'd - estbinwidth=(max(inlist)-min(inlist))/float(numbins) +1e-6 #1=>cover all - binsize = ((max(inlist)-min(inlist)+estbinwidth))/float(numbins) - lowerreallimit = min(inlist) - binsize/2 #lower real limit,1st bin - bins = [0]*(numbins) - extrapoints = 0 - for num in inlist: - try: - if (num-lowerreallimit) < 0: - extrapoints = extrapoints + 1 - else: - bintoincrement = int((num-lowerreallimit)/float(binsize)) - bins[bintoincrement] = bins[bintoincrement] + 1 - except: - extrapoints = extrapoints + 1 - if (extrapoints > 0 and printextras == 1): - print '\nPoints outside given histogram range =',extrapoints - return (bins, lowerreallimit, binsize, extrapoints) - - -def lcumfreq(inlist,numbins=10,defaultreallimits=None): - """ -Returns a cumulative frequency histogram, using the histogram function. - -Usage: lcumfreq(inlist,numbins=10,defaultreallimits=None) -Returns: list of cumfreq bin values, lowerreallimit, binsize, extrapoints -""" - h,l,b,e = histogram(inlist,numbins,defaultreallimits) - cumhist = cumsum(copy.deepcopy(h)) - return cumhist,l,b,e - - -def lrelfreq(inlist,numbins=10,defaultreallimits=None): - """ -Returns a relative frequency histogram, using the histogram function. - -Usage: lrelfreq(inlist,numbins=10,defaultreallimits=None) -Returns: list of cumfreq bin values, lowerreallimit, binsize, extrapoints -""" - h,l,b,e = histogram(inlist,numbins,defaultreallimits) - for i in range(len(h)): - h[i] = h[i]/float(len(inlist)) - return h,l,b,e - - -#################################### -##### VARIABILITY FUNCTIONS ###### -#################################### - -def lobrientransform(*args): - """ -Computes a transform on input data (any number of columns). Used to -test for homogeneity of variance prior to running one-way stats. From -Maxwell and Delaney, p.112. - -Usage: lobrientransform(*args) -Returns: transformed data for use in an ANOVA -""" - TINY = 1e-10 - k = len(args) - n = [0.0]*k - v = [0.0]*k - m = [0.0]*k - nargs = [] - for i in range(k): - nargs.append(copy.deepcopy(args[i])) - n[i] = float(len(nargs[i])) - v[i] = var(nargs[i]) - m[i] = mean(nargs[i]) - for j in range(k): - for i in range(n[j]): - t1 = (n[j]-1.5)*n[j]*(nargs[j][i]-m[j])**2 - t2 = 0.5*v[j]*(n[j]-1.0) - t3 = (n[j]-1.0)*(n[j]-2.0) - nargs[j][i] = (t1-t2) / float(t3) - check = 1 - for j in range(k): - if v[j] - mean(nargs[j]) > TINY: - check = 0 - if check <> 1: - raise ValueError, 'Problem in obrientransform.' - else: - return nargs - - -def lsamplevar (inlist): - """ -Returns the variance of the values in the passed list using -N for the denominator (i.e., DESCRIBES the sample variance only). - -Usage: lsamplevar(inlist) -""" - n = len(inlist) - mn = mean(inlist) - deviations = [] - for item in inlist: - deviations.append(item-mn) - return ss(deviations)/float(n) - - -def lsamplestdev (inlist): - """ -Returns the standard deviation of the values in the passed list using -N for the denominator (i.e., DESCRIBES the sample stdev only). - -Usage: lsamplestdev(inlist) -""" - return math.sqrt(samplevar(inlist)) - - -def lcov (x,y, keepdims=0): - """ -Returns the estimated covariance of the values in the passed -array (i.e., N-1). Dimension can equal None (ravel array first), an -integer (the dimension over which to operate), or a sequence (operate -over multiple dimensions). Set keepdims=1 to return an array with the -same number of dimensions as inarray. - -Usage: lcov(x,y,keepdims=0) -""" - - n = len(x) - xmn = mean(x) - ymn = mean(y) - xdeviations = [0]*len(x) - ydeviations = [0]*len(y) - for i in range(len(x)): - xdeviations[i] = x[i] - xmn - ydeviations[i] = y[i] - ymn - ss = 0.0 - for i in range(len(xdeviations)): - ss = ss + xdeviations[i]*ydeviations[i] - return ss/float(n-1) - - -def lvar (inlist): - """ -Returns the variance of the values in the passed list using N-1 -for the denominator (i.e., for estimating population variance). - -Usage: lvar(inlist) -""" - n = len(inlist) - mn = mean(inlist) - deviations = [0]*len(inlist) - for i in range(len(inlist)): - deviations[i] = inlist[i] - mn - return ss(deviations)/float(n-1) - - -def lstdev (inlist): - """ -Returns the standard deviation of the values in the passed list -using N-1 in the denominator (i.e., to estimate population stdev). - -Usage: lstdev(inlist) -""" - return math.sqrt(var(inlist)) - - -def lsterr(inlist): - """ -Returns the standard error of the values in the passed list using N-1 -in the denominator (i.e., to estimate population standard error). - -Usage: lsterr(inlist) -""" - return stdev(inlist) / float(math.sqrt(len(inlist))) - - -def lsem (inlist): - """ -Returns the estimated standard error of the mean (sx-bar) of the -values in the passed list. sem = stdev / sqrt(n) - -Usage: lsem(inlist) -""" - sd = stdev(inlist) - n = len(inlist) - return sd/math.sqrt(n) - - -def lz (inlist, score): - """ -Returns the z-score for a given input score, given that score and the -list from which that score came. Not appropriate for population calculations. - -Usage: lz(inlist, score) -""" - z = (score-mean(inlist))/samplestdev(inlist) - return z - - -def lzs (inlist): - """ -Returns a list of z-scores, one for each score in the passed list. - -Usage: lzs(inlist) -""" - zscores = [] - for item in inlist: - zscores.append(z(inlist,item)) - return zscores - - -#################################### -####### TRIMMING FUNCTIONS ####### -#################################### - -def ltrimboth (l,proportiontocut): - """ -Slices off the passed proportion of items from BOTH ends of the passed -list (i.e., with proportiontocut=0.1, slices 'leftmost' 10% AND 'rightmost' -10% of scores. Assumes list is sorted by magnitude. Slices off LESS if -proportion results in a non-integer slice index (i.e., conservatively -slices off proportiontocut). - -Usage: ltrimboth (l,proportiontocut) -Returns: trimmed version of list l -""" - lowercut = int(proportiontocut*len(l)) - uppercut = len(l) - lowercut - return l[lowercut:uppercut] - - -def ltrim1 (l,proportiontocut,tail='right'): - """ -Slices off the passed proportion of items from ONE end of the passed -list (i.e., if proportiontocut=0.1, slices off 'leftmost' or 'rightmost' -10% of scores). Slices off LESS if proportion results in a non-integer -slice index (i.e., conservatively slices off proportiontocut). - -Usage: ltrim1 (l,proportiontocut,tail='right') or set tail='left' -Returns: trimmed version of list l -""" - if tail == 'right': - lowercut = 0 - uppercut = len(l) - int(proportiontocut*len(l)) - elif tail == 'left': - lowercut = int(proportiontocut*len(l)) - uppercut = len(l) - return l[lowercut:uppercut] - - -#################################### -##### CORRELATION FUNCTIONS ###### -#################################### - -def lpaired(x,y): - """ -Interactively determines the type of data and then runs the -appropriated statistic for paired group data. - -Usage: lpaired(x,y) -Returns: appropriate statistic name, value, and probability -""" - samples = '' - while samples not in ['i','r','I','R','c','C']: - print '\nIndependent or related samples, or correlation (i,r,c): ', - samples = raw_input() - - if samples in ['i','I','r','R']: - print '\nComparing variances ...', -# USE O'BRIEN'S TEST FOR HOMOGENEITY OF VARIANCE, Maxwell & delaney, p.112 - r = obrientransform(x,y) - f,p = F_oneway(pstat.colex(r,0),pstat.colex(r,1)) - if p<0.05: - vartype='unequal, p='+str(round(p,4)) - else: - vartype='equal' - print vartype - if samples in ['i','I']: - if vartype[0]=='e': - t,p = ttest_ind(x,y,0) - print '\nIndependent samples t-test: ', round(t,4),round(p,4) - else: - if len(x)>20 or len(y)>20: - z,p = ranksums(x,y) - print '\nRank Sums test (NONparametric, n>20): ', round(z,4),round(p,4) - else: - u,p = mannwhitneyu(x,y) - print '\nMann-Whitney U-test (NONparametric, ns<20): ', round(u,4),round(p,4) - - else: # RELATED SAMPLES - if vartype[0]=='e': - t,p = ttest_rel(x,y,0) - print '\nRelated samples t-test: ', round(t,4),round(p,4) - else: - t,p = ranksums(x,y) - print '\nWilcoxon T-test (NONparametric): ', round(t,4),round(p,4) - else: # CORRELATION ANALYSIS - corrtype = '' - while corrtype not in ['c','C','r','R','d','D']: - print '\nIs the data Continuous, Ranked, or Dichotomous (c,r,d): ', - corrtype = raw_input() - if corrtype in ['c','C']: - m,b,r,p,see = linregress(x,y) - print '\nLinear regression for continuous variables ...' - lol = [['Slope','Intercept','r','Prob','SEestimate'],[round(m,4),round(b,4),round(r,4),round(p,4),round(see,4)]] - pstat.printcc(lol) - elif corrtype in ['r','R']: - r,p = spearmanr(x,y) - print '\nCorrelation for ranked variables ...' - print "Spearman's r: ",round(r,4),round(p,4) - else: # DICHOTOMOUS - r,p = pointbiserialr(x,y) - print '\nAssuming x contains a dichotomous variable ...' - print 'Point Biserial r: ',round(r,4),round(p,4) - print '\n\n' - return None - - -def lpearsonr(x,y): - """ -Calculates a Pearson correlation coefficient and the associated -probability value. Taken from Heiman's Basic Statistics for the Behav. -Sci (2nd), p.195. - -Usage: lpearsonr(x,y) where x and y are equal-length lists -Returns: Pearson's r value, two-tailed p-value -""" - TINY = 1.0e-30 - if len(x) <> len(y): - raise ValueError, 'Input values not paired in pearsonr. Aborting.' - n = len(x) - x = map(float,x) - y = map(float,y) - xmean = mean(x) - ymean = mean(y) - r_num = n*(summult(x,y)) - sum(x)*sum(y) - r_den = math.sqrt((n*ss(x) - square_of_sums(x))*(n*ss(y)-square_of_sums(y))) - r = (r_num / r_den) # denominator already a float - df = n-2 - t = r*math.sqrt(df/((1.0-r+TINY)*(1.0+r+TINY))) - prob = betai(0.5*df,0.5,df/float(df+t*t)) - return r, prob - - -def llincc(x,y): - """ -Calculates Lin's concordance correlation coefficient. - -Usage: alincc(x,y) where x, y are equal-length arrays -Returns: Lin's CC -""" - covar = lcov(x,y)*(len(x)-1)/float(len(x)) # correct denom to n - xvar = lvar(x)*(len(x)-1)/float(len(x)) # correct denom to n - yvar = lvar(y)*(len(y)-1)/float(len(y)) # correct denom to n - lincc = (2 * covar) / ((xvar+yvar) +((amean(x)-amean(y))**2)) - return lincc - - -def lspearmanr(x,y): - """ -Calculates a Spearman rank-order correlation coefficient. Taken -from Heiman's Basic Statistics for the Behav. Sci (1st), p.192. - -Usage: lspearmanr(x,y) where x and y are equal-length lists -Returns: Spearman's r, two-tailed p-value -""" - TINY = 1e-30 - if len(x) <> len(y): - raise ValueError, 'Input values not paired in spearmanr. Aborting.' - n = len(x) - rankx = rankdata(x) - ranky = rankdata(y) - dsq = sumdiffsquared(rankx,ranky) - rs = 1 - 6*dsq / float(n*(n**2-1)) - t = rs * math.sqrt((n-2) / ((rs+1.0)*(1.0-rs))) - df = n-2 - probrs = betai(0.5*df,0.5,df/(df+t*t)) # t already a float -# probability values for rs are from part 2 of the spearman function in -# Numerical Recipies, p.510. They are close to tables, but not exact. (?) - return rs, probrs - - -def lpointbiserialr(x,y): - """ -Calculates a point-biserial correlation coefficient and the associated -probability value. Taken from Heiman's Basic Statistics for the Behav. -Sci (1st), p.194. - -Usage: lpointbiserialr(x,y) where x,y are equal-length lists -Returns: Point-biserial r, two-tailed p-value -""" - TINY = 1e-30 - if len(x) <> len(y): - raise ValueError, 'INPUT VALUES NOT PAIRED IN pointbiserialr. ABORTING.' - data = pstat.abut(x,y) - categories = pstat.unique(x) - if len(categories) <> 2: - raise ValueError, "Exactly 2 categories required for pointbiserialr()." - else: # there are 2 categories, continue - codemap = pstat.abut(categories,range(2)) - recoded = pstat.recode(data,codemap,0) - x = pstat.linexand(data,0,categories[0]) - y = pstat.linexand(data,0,categories[1]) - xmean = mean(pstat.colex(x,1)) - ymean = mean(pstat.colex(y,1)) - n = len(data) - adjust = math.sqrt((len(x)/float(n))*(len(y)/float(n))) - rpb = (ymean - xmean)/samplestdev(pstat.colex(data,1))*adjust - df = n-2 - t = rpb*math.sqrt(df/((1.0-rpb+TINY)*(1.0+rpb+TINY))) - prob = betai(0.5*df,0.5,df/(df+t*t)) # t already a float - return rpb, prob - - -def lkendalltau(x,y): - """ -Calculates Kendall's tau ... correlation of ordinal data. Adapted -from function kendl1 in Numerical Recipies. Needs good test-routine.@@@ - -Usage: lkendalltau(x,y) -Returns: Kendall's tau, two-tailed p-value -""" - n1 = 0 - n2 = 0 - iss = 0 - for j in range(len(x)-1): - for k in range(j,len(y)): - a1 = x[j] - x[k] - a2 = y[j] - y[k] - aa = a1 * a2 - if (aa): # neither list has a tie - n1 = n1 + 1 - n2 = n2 + 1 - if aa > 0: - iss = iss + 1 - else: - iss = iss -1 - else: - if (a1): - n1 = n1 + 1 - else: - n2 = n2 + 1 - tau = iss / math.sqrt(n1*n2) - svar = (4.0*len(x)+10.0) / (9.0*len(x)*(len(x)-1)) - z = tau / math.sqrt(svar) - prob = erfcc(abs(z)/1.4142136) - return tau, prob - - -def llinregress(x,y): - """ -Calculates a regression line on x,y pairs. - -Usage: llinregress(x,y) x,y are equal-length lists of x-y coordinates -Returns: slope, intercept, r, two-tailed prob, sterr-of-estimate -""" - TINY = 1.0e-20 - if len(x) <> len(y): - raise ValueError, 'Input values not paired in linregress. Aborting.' - n = len(x) - x = map(float,x) - y = map(float,y) - xmean = mean(x) - ymean = mean(y) - r_num = float(n*(summult(x,y)) - sum(x)*sum(y)) - r_den = math.sqrt((n*ss(x) - square_of_sums(x))*(n*ss(y)-square_of_sums(y))) - r = r_num / r_den - z = 0.5*math.log((1.0+r+TINY)/(1.0-r+TINY)) - df = n-2 - t = r*math.sqrt(df/((1.0-r+TINY)*(1.0+r+TINY))) - prob = betai(0.5*df,0.5,df/(df+t*t)) - slope = r_num / float(n*ss(x) - square_of_sums(x)) - intercept = ymean - slope*xmean - sterrest = math.sqrt(1-r*r)*samplestdev(y) - return slope, intercept, r, prob, sterrest - - -#################################### -##### INFERENTIAL STATISTICS ##### -#################################### - -def lttest_1samp(a,popmean,printit=0,name='Sample',writemode='a'): - """ -Calculates the t-obtained for the independent samples T-test on ONE group -of scores a, given a population mean. If printit=1, results are printed -to the screen. If printit='filename', the results are output to 'filename' -using the given writemode (default=append). Returns t-value, and prob. - -Usage: lttest_1samp(a,popmean,Name='Sample',printit=0,writemode='a') -Returns: t-value, two-tailed prob -""" - x = mean(a) - v = var(a) - n = len(a) - df = n-1 - svar = ((n-1)*v)/float(df) - t = (x-popmean)/math.sqrt(svar*(1.0/n)) - prob = betai(0.5*df,0.5,float(df)/(df+t*t)) - - if printit <> 0: - statname = 'Single-sample T-test.' - outputpairedstats(printit,writemode, - 'Population','--',popmean,0,0,0, - name,n,x,v,min(a),max(a), - statname,t,prob) - return t,prob - - -def lttest_ind (a, b, printit=0, name1='Samp1', name2='Samp2', writemode='a'): - """ -Calculates the t-obtained T-test on TWO INDEPENDENT samples of -scores a, and b. From Numerical Recipies, p.483. If printit=1, results -are printed to the screen. If printit='filename', the results are output -to 'filename' using the given writemode (default=append). Returns t-value, -and prob. - -Usage: lttest_ind(a,b,printit=0,name1='Samp1',name2='Samp2',writemode='a') -Returns: t-value, two-tailed prob -""" - x1 = mean(a) - x2 = mean(b) - v1 = stdev(a)**2 - v2 = stdev(b)**2 - n1 = len(a) - n2 = len(b) - df = n1+n2-2 - svar = ((n1-1)*v1+(n2-1)*v2)/float(df) - if not svar: - svar = 1.0e-26 - t = (x1-x2)/math.sqrt(svar*(1.0/n1 + 1.0/n2)) - prob = betai(0.5*df,0.5,df/(df+t*t)) - - if printit <> 0: - statname = 'Independent samples T-test.' - outputpairedstats(printit,writemode, - name1,n1,x1,v1,min(a),max(a), - name2,n2,x2,v2,min(b),max(b), - statname,t,prob) - return t,prob - - -def lttest_rel (a,b,printit=0,name1='Sample1',name2='Sample2',writemode='a'): - """ -Calculates the t-obtained T-test on TWO RELATED samples of scores, -a and b. From Numerical Recipies, p.483. If printit=1, results are -printed to the screen. If printit='filename', the results are output to -'filename' using the given writemode (default=append). Returns t-value, -and prob. - -Usage: lttest_rel(a,b,printit=0,name1='Sample1',name2='Sample2',writemode='a') -Returns: t-value, two-tailed prob -""" - if len(a)<>len(b): - raise ValueError, 'Unequal length lists in ttest_rel.' - x1 = mean(a) - x2 = mean(b) - v1 = var(a) - v2 = var(b) - n = len(a) - cov = 0 - for i in range(len(a)): - cov = cov + (a[i]-x1) * (b[i]-x2) - df = n-1 - cov = cov / float(df) - sd = math.sqrt((v1+v2 - 2.0*cov)/float(n)) - t = (x1-x2)/sd - prob = betai(0.5*df,0.5,df/(df+t*t)) - - if printit <> 0: - statname = 'Related samples T-test.' - outputpairedstats(printit,writemode, - name1,n,x1,v1,min(a),max(a), - name2,n,x2,v2,min(b),max(b), - statname,t,prob) - return t, prob - - -def lchisquare(f_obs,f_exp=None): - """ -Calculates a one-way chi square for list of observed frequencies and returns -the result. If no expected frequencies are given, the total N is assumed to -be equally distributed across all groups. - -Usage: lchisquare(f_obs, f_exp=None) f_obs = list of observed cell freq. -Returns: chisquare-statistic, associated p-value -""" - k = len(f_obs) # number of groups - if f_exp == None: - f_exp = [sum(f_obs)/float(k)] * len(f_obs) # create k bins with = freq. - chisq = 0 - for i in range(len(f_obs)): - chisq = chisq + (f_obs[i]-f_exp[i])**2 / float(f_exp[i]) - return chisq, chisqprob(chisq, k-1) - - -def lks_2samp (data1,data2): - """ -Computes the Kolmogorov-Smirnof statistic on 2 samples. From -Numerical Recipies in C, page 493. - -Usage: lks_2samp(data1,data2) data1&2 are lists of values for 2 conditions -Returns: KS D-value, associated p-value -""" - j1 = 0 - j2 = 0 - fn1 = 0.0 - fn2 = 0.0 - n1 = len(data1) - n2 = len(data2) - en1 = n1 - en2 = n2 - d = 0.0 - data1.sort() - data2.sort() - while j1 < n1 and j2 < n2: - d1=data1[j1] - d2=data2[j2] - if d1 <= d2: - fn1 = (j1)/float(en1) - j1 = j1 + 1 - if d2 <= d1: - fn2 = (j2)/float(en2) - j2 = j2 + 1 - dt = (fn2-fn1) - if math.fabs(dt) > math.fabs(d): - d = dt - try: - en = math.sqrt(en1*en2/float(en1+en2)) - prob = ksprob((en+0.12+0.11/en)*abs(d)) - except: - prob = 1.0 - return d, prob - - -def lmannwhitneyu(x,y): - """ -Calculates a Mann-Whitney U statistic on the provided scores and -returns the result. Use only when the n in each condition is < 20 and -you have 2 independent samples of ranks. NOTE: Mann-Whitney U is -significant if the u-obtained is LESS THAN or equal to the critical -value of U found in the tables. Equivalent to Kruskal-Wallis H with -just 2 groups. - -Usage: lmannwhitneyu(data) -Returns: u-statistic, one-tailed p-value (i.e., p(z(U))) -""" - n1 = len(x) - n2 = len(y) - ranked = rankdata(x+y) - rankx = ranked[0:n1] # get the x-ranks - ranky = ranked[n1:] # the rest are y-ranks - u1 = n1*n2 + (n1*(n1+1))/2.0 - sum(rankx) # calc U for x - u2 = n1*n2 - u1 # remainder is U for y - bigu = max(u1,u2) - smallu = min(u1,u2) - proportion = bigu/float(n1*n2) - T = math.sqrt(tiecorrect(ranked)) # correction factor for tied scores - if T == 0: - raise ValueError, 'All numbers are identical in lmannwhitneyu' - sd = math.sqrt(T*n1*n2*(n1+n2+1)/12.0) - z = abs((bigu-n1*n2/2.0) / sd) # normal approximation for prob calc - return smallu, 1.0 - zprob(z) #, proportion - - -def ltiecorrect(rankvals): - """ -Corrects for ties in Mann Whitney U and Kruskal Wallis H tests. See -Siegel, S. (1956) Nonparametric Statistics for the Behavioral Sciences. -New York: McGraw-Hill. Code adapted from |Stat rankind.c code. - -Usage: ltiecorrect(rankvals) -Returns: T correction factor for U or H -""" - sorted,posn = shellsort(rankvals) - n = len(sorted) - T = 0.0 - i = 0 - while (i<n-1): - if sorted[i] == sorted[i+1]: - nties = 1 - while (i<n-1) and (sorted[i] == sorted[i+1]): - nties = nties +1 - i = i +1 - T = T + nties**3 - nties - i = i+1 - T = T / float(n**3-n) - return 1.0 - T - - -def lranksums(x,y): - """ -Calculates the rank sums statistic on the provided scores and -returns the result. Use only when the n in each condition is > 20 and you -have 2 independent samples of ranks. - -Usage: lranksums(x,y) -Returns: a z-statistic, two-tailed p-value -""" - n1 = len(x) - n2 = len(y) - alldata = x+y - ranked = rankdata(alldata) - x = ranked[:n1] - y = ranked[n1:] - s = sum(x) - expected = n1*(n1+n2+1) / 2.0 - z = (s - expected) / math.sqrt(n1*n2*(n1+n2+1)/12.0) - prob = 2*(1.0 -zprob(abs(z))) - return z, prob - - -def lwilcoxont(x,y): - """ -Calculates the Wilcoxon T-test for related samples and returns the -result. A non-parametric T-test. - -Usage: lwilcoxont(x,y) -Returns: a t-statistic, two-tail probability estimate -""" - if len(x) <> len(y): - raise ValueError, 'Unequal N in wilcoxont. Aborting.' - d=[] - for i in range(len(x)): - diff = x[i] - y[i] - if diff <> 0: - d.append(diff) - count = len(d) - absd = map(abs,d) - absranked = rankdata(absd) - r_plus = 0.0 - r_minus = 0.0 - for i in range(len(absd)): - if d[i] < 0: - r_minus = r_minus + absranked[i] - else: - r_plus = r_plus + absranked[i] - wt = min(r_plus, r_minus) - mn = count * (count+1) * 0.25 - se = math.sqrt(count*(count+1)*(2.0*count+1.0)/24.0) - z = math.fabs(wt-mn) / se - prob = 2*(1.0 -zprob(abs(z))) - return wt, prob - - -def lkruskalwallish(*args): - """ -The Kruskal-Wallis H-test is a non-parametric ANOVA for 3 or more -groups, requiring at least 5 subjects in each group. This function -calculates the Kruskal-Wallis H-test for 3 or more independent samples -and returns the result. - -Usage: lkruskalwallish(*args) -Returns: H-statistic (corrected for ties), associated p-value -""" - args = list(args) - n = [0]*len(args) - all = [] - n = map(len,args) - for i in range(len(args)): - all = all + args[i] - ranked = rankdata(all) - T = tiecorrect(ranked) - for i in range(len(args)): - args[i] = ranked[0:n[i]] - del ranked[0:n[i]] - rsums = [] - for i in range(len(args)): - rsums.append(sum(args[i])**2) - rsums[i] = rsums[i] / float(n[i]) - ssbn = sum(rsums) - totaln = sum(n) - h = 12.0 / (totaln*(totaln+1)) * ssbn - 3*(totaln+1) - df = len(args) - 1 - if T == 0: - raise ValueError, 'All numbers are identical in lkruskalwallish' - h = h / float(T) - return h, chisqprob(h,df) - - -def lfriedmanchisquare(*args): - """ -Friedman Chi-Square is a non-parametric, one-way within-subjects -ANOVA. This function calculates the Friedman Chi-square test for repeated -measures and returns the result, along with the associated probability -value. It assumes 3 or more repeated measures. Only 3 levels requires a -minimum of 10 subjects in the study. Four levels requires 5 subjects per -level(??). - -Usage: lfriedmanchisquare(*args) -Returns: chi-square statistic, associated p-value -""" - k = len(args) - if k < 3: - raise ValueError, 'Less than 3 levels. Friedman test not appropriate.' - n = len(args[0]) - data = apply(pstat.abut,tuple(args)) - for i in range(len(data)): - data[i] = rankdata(data[i]) - ssbn = 0 - for i in range(k): - ssbn = ssbn + sum(args[i])**2 - chisq = 12.0 / (k*n*(k+1)) * ssbn - 3*n*(k+1) - return chisq, chisqprob(chisq,k-1) - - -#################################### -#### PROBABILITY CALCULATIONS #### -#################################### - -def lchisqprob(chisq,df): - """ -Returns the (1-tailed) probability value associated with the provided -chi-square value and df. Adapted from chisq.c in Gary Perlman's |Stat. - -Usage: lchisqprob(chisq,df) -""" - BIG = 20.0 - def ex(x): - BIG = 20.0 - if x < -BIG: - return 0.0 - else: - return math.exp(x) - - if chisq <=0 or df < 1: - return 1.0 - a = 0.5 * chisq - if df%2 == 0: - even = 1 - else: - even = 0 - if df > 1: - y = ex(-a) - if even: - s = y - else: - s = 2.0 * zprob(-math.sqrt(chisq)) - if (df > 2): - chisq = 0.5 * (df - 1.0) - if even: - z = 1.0 - else: - z = 0.5 - if a > BIG: - if even: - e = 0.0 - else: - e = math.log(math.sqrt(math.pi)) - c = math.log(a) - while (z <= chisq): - e = math.log(z) + e - s = s + ex(c*z-a-e) - z = z + 1.0 - return s - else: - if even: - e = 1.0 - else: - e = 1.0 / math.sqrt(math.pi) / math.sqrt(a) - c = 0.0 - while (z <= chisq): - e = e * (a/float(z)) - c = c + e - z = z + 1.0 - return (c*y+s) - else: - return s - - -def lerfcc(x): - """ -Returns the complementary error function erfc(x) with fractional -error everywhere less than 1.2e-7. Adapted from Numerical Recipies. - -Usage: lerfcc(x) -""" - z = abs(x) - t = 1.0 / (1.0+0.5*z) - ans = t * math.exp(-z*z-1.26551223 + t*(1.00002368+t*(0.37409196+t*(0.09678418+t*(-0.18628806+t*(0.27886807+t*(-1.13520398+t*(1.48851587+t*(-0.82215223+t*0.17087277))))))))) - if x >= 0: - return ans - else: - return 2.0 - ans - - -def lzprob(z): - """ -Returns the area under the normal curve 'to the left of' the given z value. -Thus, - for z<0, zprob(z) = 1-tail probability - for z>0, 1.0-zprob(z) = 1-tail probability - for any z, 2.0*(1.0-zprob(abs(z))) = 2-tail probability -Adapted from z.c in Gary Perlman's |Stat. - -Usage: lzprob(z) -""" - Z_MAX = 6.0 # maximum meaningful z-value - if z == 0.0: - x = 0.0 - else: - y = 0.5 * math.fabs(z) - if y >= (Z_MAX*0.5): - x = 1.0 - elif (y < 1.0): - w = y*y - x = ((((((((0.000124818987 * w - -0.001075204047) * w +0.005198775019) * w - -0.019198292004) * w +0.059054035642) * w - -0.151968751364) * w +0.319152932694) * w - -0.531923007300) * w +0.797884560593) * y * 2.0 - else: - y = y - 2.0 - x = (((((((((((((-0.000045255659 * y - +0.000152529290) * y -0.000019538132) * y - -0.000676904986) * y +0.001390604284) * y - -0.000794620820) * y -0.002034254874) * y - +0.006549791214) * y -0.010557625006) * y - +0.011630447319) * y -0.009279453341) * y - +0.005353579108) * y -0.002141268741) * y - +0.000535310849) * y +0.999936657524 - if z > 0.0: - prob = ((x+1.0)*0.5) - else: - prob = ((1.0-x)*0.5) - return prob - - -def lksprob(alam): - """ -Computes a Kolmolgorov-Smirnov t-test significance level. Adapted from -Numerical Recipies. - -Usage: lksprob(alam) -""" - fac = 2.0 - sum = 0.0 - termbf = 0.0 - a2 = -2.0*alam*alam - for j in range(1,201): - term = fac*math.exp(a2*j*j) - sum = sum + term - if math.fabs(term) <= (0.001*termbf) or math.fabs(term) < (1.0e-8*sum): - return sum - fac = -fac - termbf = math.fabs(term) - return 1.0 # Get here only if fails to converge; was 0.0!! - - -def lfprob (dfnum, dfden, F): - """ -Returns the (1-tailed) significance level (p-value) of an F -statistic given the degrees of freedom for the numerator (dfR-dfF) and -the degrees of freedom for the denominator (dfF). - -Usage: lfprob(dfnum, dfden, F) where usually dfnum=dfbn, dfden=dfwn -""" - p = betai(0.5*dfden, 0.5*dfnum, dfden/float(dfden+dfnum*F)) - return p - - -def lbetacf(a,b,x): - """ -This function evaluates the continued fraction form of the incomplete -Beta function, betai. (Adapted from: Numerical Recipies in C.) - -Usage: lbetacf(a,b,x) -""" - ITMAX = 200 - EPS = 3.0e-7 - - bm = az = am = 1.0 - qab = a+b - qap = a+1.0 - qam = a-1.0 - bz = 1.0-qab*x/qap - for i in range(ITMAX+1): - em = float(i+1) - tem = em + em - d = em*(b-em)*x/((qam+tem)*(a+tem)) - ap = az + d*am - bp = bz+d*bm - d = -(a+em)*(qab+em)*x/((qap+tem)*(a+tem)) - app = ap+d*az - bpp = bp+d*bz - aold = az - am = ap/bpp - bm = bp/bpp - az = app/bpp - bz = 1.0 - if (abs(az-aold)<(EPS*abs(az))): - return az - print 'a or b too big, or ITMAX too small in Betacf.' - - -def lgammln(xx): - """ -Returns the gamma function of xx. - Gamma(z) = Integral(0,infinity) of t^(z-1)exp(-t) dt. -(Adapted from: Numerical Recipies in C.) - -Usage: lgammln(xx) -""" - - coeff = [76.18009173, -86.50532033, 24.01409822, -1.231739516, - 0.120858003e-2, -0.536382e-5] - x = xx - 1.0 - tmp = x + 5.5 - tmp = tmp - (x+0.5)*math.log(tmp) - ser = 1.0 - for j in range(len(coeff)): - x = x + 1 - ser = ser + coeff[j]/x - return -tmp + math.log(2.50662827465*ser) - - -def lbetai(a,b,x): - """ -Returns the incomplete beta function: - - I-sub-x(a,b) = 1/B(a,b)*(Integral(0,x) of t^(a-1)(1-t)^(b-1) dt) - -where a,b>0 and B(a,b) = G(a)*G(b)/(G(a+b)) where G(a) is the gamma -function of a. The continued fraction formulation is implemented here, -using the betacf function. (Adapted from: Numerical Recipies in C.) - -Usage: lbetai(a,b,x) -""" - if (x<0.0 or x>1.0): - raise ValueError, 'Bad x in lbetai' - if (x==0.0 or x==1.0): - bt = 0.0 - else: - bt = math.exp(gammln(a+b)-gammln(a)-gammln(b)+a*math.log(x)+b* - math.log(1.0-x)) - if (x<(a+1.0)/(a+b+2.0)): - return bt*betacf(a,b,x)/float(a) - else: - return 1.0-bt*betacf(b,a,1.0-x)/float(b) - - -#################################### -####### ANOVA CALCULATIONS ####### -#################################### - -def lF_oneway(*lists): - """ -Performs a 1-way ANOVA, returning an F-value and probability given -any number of groups. From Heiman, pp.394-7. - -Usage: F_oneway(*lists) where *lists is any number of lists, one per - treatment group -Returns: F value, one-tailed p-value -""" - a = len(lists) # ANOVA on 'a' groups, each in it's own list - means = [0]*a - vars = [0]*a - ns = [0]*a - alldata = [] - tmp = map(N.array,lists) - means = map(amean,tmp) - vars = map(avar,tmp) - ns = map(len,lists) - for i in range(len(lists)): - alldata = alldata + lists[i] - alldata = N.array(alldata) - bign = len(alldata) - sstot = ass(alldata)-(asquare_of_sums(alldata)/float(bign)) - ssbn = 0 - for list in lists: - ssbn = ssbn + asquare_of_sums(N.array(list))/float(len(list)) - ssbn = ssbn - (asquare_of_sums(alldata)/float(bign)) - sswn = sstot-ssbn - dfbn = a-1 - dfwn = bign - a - msb = ssbn/float(dfbn) - msw = sswn/float(dfwn) - f = msb/msw - prob = fprob(dfbn,dfwn,f) - return f, prob - - -def lF_value (ER,EF,dfnum,dfden): - """ -Returns an F-statistic given the following: - ER = error associated with the null hypothesis (the Restricted model) - EF = error associated with the alternate hypothesis (the Full model) - dfR-dfF = degrees of freedom of the numerator - dfF = degrees of freedom associated with the denominator/Full model - -Usage: lF_value(ER,EF,dfnum,dfden) -""" - return ((ER-EF)/float(dfnum) / (EF/float(dfden))) - - -#################################### -######## SUPPORT FUNCTIONS ####### -#################################### - -def writecc (listoflists,file,writetype='w',extra=2): - """ -Writes a list of lists to a file in columns, customized by the max -size of items within the columns (max size of items in col, +2 characters) -to specified file. File-overwrite is the default. - -Usage: writecc (listoflists,file,writetype='w',extra=2) -Returns: None -""" - if type(listoflists[0]) not in [ListType,TupleType]: - listoflists = [listoflists] - outfile = open(file,writetype) - rowstokill = [] - list2print = copy.deepcopy(listoflists) - for i in range(len(listoflists)): - if listoflists[i] == ['\n'] or listoflists[i]=='\n' or listoflists[i]=='dashes': - rowstokill = rowstokill + [i] - rowstokill.reverse() - for row in rowstokill: - del list2print[row] - maxsize = [0]*len(list2print[0]) - for col in range(len(list2print[0])): - items = pstat.colex(list2print,col) - items = map(pstat.makestr,items) - maxsize[col] = max(map(len,items)) + extra - for row in listoflists: - if row == ['\n'] or row == '\n': - outfile.write('\n') - elif row == ['dashes'] or row == 'dashes': - dashes = [0]*len(maxsize) - for j in range(len(maxsize)): - dashes[j] = '-'*(maxsize[j]-2) - outfile.write(pstat.lineincustcols(dashes,maxsize)) - else: - outfile.write(pstat.lineincustcols(row,maxsize)) - outfile.write('\n') - outfile.close() - return None - - -def lincr(l,cap): # to increment a list up to a max-list of 'cap' - """ -Simulate a counting system from an n-dimensional list. - -Usage: lincr(l,cap) l=list to increment, cap=max values for each list pos'n -Returns: next set of values for list l, OR -1 (if overflow) -""" - l[0] = l[0] + 1 # e.g., [0,0,0] --> [2,4,3] (=cap) - for i in range(len(l)): - if l[i] > cap[i] and i < len(l)-1: # if carryover AND not done - l[i] = 0 - l[i+1] = l[i+1] + 1 - elif l[i] > cap[i] and i == len(l)-1: # overflow past last column, must be finished - l = -1 - return l - - -def lsum (inlist): - """ -Returns the sum of the items in the passed list. - -Usage: lsum(inlist) -""" - s = 0 - for item in inlist: - s = s + item - return s - - -def lcumsum (inlist): - """ -Returns a list consisting of the cumulative sum of the items in the -passed list. - -Usage: lcumsum(inlist) -""" - newlist = copy.deepcopy(inlist) - for i in range(1,len(newlist)): - newlist[i] = newlist[i] + newlist[i-1] - return newlist - - -def lss(inlist): - """ -Squares each value in the passed list, adds up these squares and -returns the result. - -Usage: lss(inlist) -""" - ss = 0 - for item in inlist: - ss = ss + item*item - return ss - - -def lsummult (list1,list2): - """ -Multiplies elements in list1 and list2, element by element, and -returns the sum of all resulting multiplications. Must provide equal -length lists. - -Usage: lsummult(list1,list2) -""" - if len(list1) <> len(list2): - raise ValueError, "Lists not equal length in summult." - s = 0 - for item1,item2 in pstat.abut(list1,list2): - s = s + item1*item2 - return s - - -def lsumdiffsquared(x,y): - """ -Takes pairwise differences of the values in lists x and y, squares -these differences, and returns the sum of these squares. - -Usage: lsumdiffsquared(x,y) -Returns: sum[(x[i]-y[i])**2] -""" - sds = 0 - for i in range(len(x)): - sds = sds + (x[i]-y[i])**2 - return sds - - -def lsquare_of_sums(inlist): - """ -Adds the values in the passed list, squares the sum, and returns -the result. - -Usage: lsquare_of_sums(inlist) -Returns: sum(inlist[i])**2 -""" - s = sum(inlist) - return float(s)*s - - -def lshellsort(inlist): - """ -Shellsort algorithm. Sorts a 1D-list. - -Usage: lshellsort(inlist) -Returns: sorted-inlist, sorting-index-vector (for original list) -""" - n = len(inlist) - svec = copy.deepcopy(inlist) - ivec = range(n) - gap = n/2 # integer division needed - while gap >0: - for i in range(gap,n): - for j in range(i-gap,-1,-gap): - while j>=0 and svec[j]>svec[j+gap]: - temp = svec[j] - svec[j] = svec[j+gap] - svec[j+gap] = temp - itemp = ivec[j] - ivec[j] = ivec[j+gap] - ivec[j+gap] = itemp - gap = gap / 2 # integer division needed -# svec is now sorted inlist, and ivec has the order svec[i] = vec[ivec[i]] - return svec, ivec - - -def lrankdata(inlist): - """ -Ranks the data in inlist, dealing with ties appropritely. Assumes -a 1D inlist. Adapted from Gary Perlman's |Stat ranksort. - -Usage: lrankdata(inlist) -Returns: a list of length equal to inlist, containing rank scores -""" - n = len(inlist) - svec, ivec = shellsort(inlist) - sumranks = 0 - dupcount = 0 - newlist = [0]*n - for i in range(n): - sumranks = sumranks + i - dupcount = dupcount + 1 - if i==n-1 or svec[i] <> svec[i+1]: - averank = sumranks / float(dupcount) + 1 - for j in range(i-dupcount+1,i+1): - newlist[ivec[j]] = averank - sumranks = 0 - dupcount = 0 - return newlist - - -def outputpairedstats(fname,writemode,name1,n1,m1,se1,min1,max1,name2,n2,m2,se2,min2,max2,statname,stat,prob): - """ -Prints or write to a file stats for two groups, using the name, n, -mean, sterr, min and max for each group, as well as the statistic name, -its value, and the associated p-value. - -Usage: outputpairedstats(fname,writemode, - name1,n1,mean1,stderr1,min1,max1, - name2,n2,mean2,stderr2,min2,max2, - statname,stat,prob) -Returns: None -""" - suffix = '' # for *s after the p-value - try: - x = prob.shape - prob = prob[0] - except: - pass - if prob < 0.001: suffix = ' ***' - elif prob < 0.01: suffix = ' **' - elif prob < 0.05: suffix = ' *' - title = [['Name','N','Mean','SD','Min','Max']] - lofl = title+[[name1,n1,round(m1,3),round(math.sqrt(se1),3),min1,max1], - [name2,n2,round(m2,3),round(math.sqrt(se2),3),min2,max2]] - if type(fname)<>StringType or len(fname)==0: - print - print statname - print - pstat.printcc(lofl) - print - try: - if stat.shape == (): - stat = stat[0] - if prob.shape == (): - prob = prob[0] - except: - pass - print 'Test statistic = ',round(stat,3),' p = ',round(prob,3),suffix - print - else: - file = open(fname,writemode) - file.write('\n'+statname+'\n\n') - file.close() - writecc(lofl,fname,'a') - file = open(fname,'a') - try: - if stat.shape == (): - stat = stat[0] - if prob.shape == (): - prob = prob[0] - except: - pass - file.write(pstat.list2string(['\nTest statistic = ',round(stat,4),' p = ',round(prob,4),suffix,'\n\n'])) - file.close() - return None - - -def lfindwithin (data): - """ -Returns an integer representing a binary vector, where 1=within- -subject factor, 0=between. Input equals the entire data 2D list (i.e., -column 0=random factor, column -1=measured values (those two are skipped). -Note: input data is in |Stat format ... a list of lists ("2D list") with -one row per measured value, first column=subject identifier, last column= -score, one in-between column per factor (these columns contain level -designations on each factor). See also stats.anova.__doc__. - -Usage: lfindwithin(data) data in |Stat format -""" - - numfact = len(data[0])-1 - withinvec = 0 - for col in range(1,numfact): - examplelevel = pstat.unique(pstat.colex(data,col))[0] - rows = pstat.linexand(data,col,examplelevel) # get 1 level of this factor - factsubjs = pstat.unique(pstat.colex(rows,0)) - allsubjs = pstat.unique(pstat.colex(data,0)) - if len(factsubjs) == len(allsubjs): # fewer Ss than scores on this factor? - withinvec = withinvec + (1 << col) - return withinvec - - -######################################################### -######################################################### -####### DISPATCH LISTS AND TUPLES TO ABOVE FCNS ######### -######################################################### -######################################################### - -## CENTRAL TENDENCY: -geometricmean = Dispatch ( (lgeometricmean, (ListType, TupleType)), ) -harmonicmean = Dispatch ( (lharmonicmean, (ListType, TupleType)), ) -mean = Dispatch ( (lmean, (ListType, TupleType)), ) -median = Dispatch ( (lmedian, (ListType, TupleType)), ) -medianscore = Dispatch ( (lmedianscore, (ListType, TupleType)), ) -mode = Dispatch ( (lmode, (ListType, TupleType)), ) - -## MOMENTS: -moment = Dispatch ( (lmoment, (ListType, TupleType)), ) -variation = Dispatch ( (lvariation, (ListType, TupleType)), ) -skew = Dispatch ( (lskew, (ListType, TupleType)), ) -kurtosis = Dispatch ( (lkurtosis, (ListType, TupleType)), ) -describe = Dispatch ( (ldescribe, (ListType, TupleType)), ) - -## FREQUENCY STATISTICS: -itemfreq = Dispatch ( (litemfreq, (ListType, TupleType)), ) -scoreatpercentile = Dispatch ( (lscoreatpercentile, (ListType, TupleType)), ) -percentileofscore = Dispatch ( (lpercentileofscore, (ListType, TupleType)), ) -histogram = Dispatch ( (lhistogram, (ListType, TupleType)), ) -cumfreq = Dispatch ( (lcumfreq, (ListType, TupleType)), ) -relfreq = Dispatch ( (lrelfreq, (ListType, TupleType)), ) - -## VARIABILITY: -obrientransform = Dispatch ( (lobrientransform, (ListType, TupleType)), ) -samplevar = Dispatch ( (lsamplevar, (ListType, TupleType)), ) -samplestdev = Dispatch ( (lsamplestdev, (ListType, TupleType)), ) -var = Dispatch ( (lvar, (ListType, TupleType)), ) -stdev = Dispatch ( (lstdev, (ListType, TupleType)), ) -sterr = Dispatch ( (lsterr, (ListType, TupleType)), ) -sem = Dispatch ( (lsem, (ListType, TupleType)), ) -z = Dispatch ( (lz, (ListType, TupleType)), ) -zs = Dispatch ( (lzs, (ListType, TupleType)), ) - -## TRIMMING FCNS: -trimboth = Dispatch ( (ltrimboth, (ListType, TupleType)), ) -trim1 = Dispatch ( (ltrim1, (ListType, TupleType)), ) - -## CORRELATION FCNS: -paired = Dispatch ( (lpaired, (ListType, TupleType)), ) -pearsonr = Dispatch ( (lpearsonr, (ListType, TupleType)), ) -spearmanr = Dispatch ( (lspearmanr, (ListType, TupleType)), ) -pointbiserialr = Dispatch ( (lpointbiserialr, (ListType, TupleType)), ) -kendalltau = Dispatch ( (lkendalltau, (ListType, TupleType)), ) -linregress = Dispatch ( (llinregress, (ListType, TupleType)), ) - -## INFERENTIAL STATS: -ttest_1samp = Dispatch ( (lttest_1samp, (ListType, TupleType)), ) -ttest_ind = Dispatch ( (lttest_ind, (ListType, TupleType)), ) -ttest_rel = Dispatch ( (lttest_rel, (ListType, TupleType)), ) -chisquare = Dispatch ( (lchisquare, (ListType, TupleType)), ) -ks_2samp = Dispatch ( (lks_2samp, (ListType, TupleType)), ) -mannwhitneyu = Dispatch ( (lmannwhitneyu, (ListType, TupleType)), ) -ranksums = Dispatch ( (lranksums, (ListType, TupleType)), ) -tiecorrect = Dispatch ( (ltiecorrect, (ListType, TupleType)), ) -wilcoxont = Dispatch ( (lwilcoxont, (ListType, TupleType)), ) -kruskalwallish = Dispatch ( (lkruskalwallish, (ListType, TupleType)), ) -friedmanchisquare = Dispatch ( (lfriedmanchisquare, (ListType, TupleType)), ) - -## PROBABILITY CALCS: -chisqprob = Dispatch ( (lchisqprob, (IntType, FloatType)), ) -zprob = Dispatch ( (lzprob, (IntType, FloatType)), ) -ksprob = Dispatch ( (lksprob, (IntType, FloatType)), ) -fprob = Dispatch ( (lfprob, (IntType, FloatType)), ) -betacf = Dispatch ( (lbetacf, (IntType, FloatType)), ) -betai = Dispatch ( (lbetai, (IntType, FloatType)), ) -erfcc = Dispatch ( (lerfcc, (IntType, FloatType)), ) -gammln = Dispatch ( (lgammln, (IntType, FloatType)), ) - -## ANOVA FUNCTIONS: -F_oneway = Dispatch ( (lF_oneway, (ListType, TupleType)), ) -F_value = Dispatch ( (lF_value, (ListType, TupleType)), ) - -## SUPPORT FUNCTIONS: -incr = Dispatch ( (lincr, (ListType, TupleType)), ) -sum = Dispatch ( (lsum, (ListType, TupleType)), ) -cumsum = Dispatch ( (lcumsum, (ListType, TupleType)), ) -ss = Dispatch ( (lss, (ListType, TupleType)), ) -summult = Dispatch ( (lsummult, (ListType, TupleType)), ) -square_of_sums = Dispatch ( (lsquare_of_sums, (ListType, TupleType)), ) -sumdiffsquared = Dispatch ( (lsumdiffsquared, (ListType, TupleType)), ) -shellsort = Dispatch ( (lshellsort, (ListType, TupleType)), ) -rankdata = Dispatch ( (lrankdata, (ListType, TupleType)), ) -findwithin = Dispatch ( (lfindwithin, (ListType, TupleType)), ) - - -#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS =============== -#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS =============== -#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS =============== -#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS =============== -#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS =============== -#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS =============== -#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS =============== -#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS =============== -#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS =============== -#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS =============== -#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS =============== -#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS =============== -#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS =============== -#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS =============== -#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS =============== -#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS =============== -#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS =============== -#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS =============== -#============= THE ARRAY-VERSION OF THE STATS FUNCTIONS =============== - -try: # DEFINE THESE *ONLY* IF NUMERIC IS AVAILABLE - import numpy as N - import numpy.linalg as LA - - -##################################### -######## ACENTRAL TENDENCY ######## -##################################### - - def ageometricmean (inarray,dimension=None,keepdims=0): - """ -Calculates the geometric mean of the values in the passed array. -That is: n-th root of (x1 * x2 * ... * xn). Defaults to ALL values in -the passed array. Use dimension=None to flatten array first. REMEMBER: if -dimension=0, it collapses over dimension 0 ('rows' in a 2D array) only, and -if dimension is a sequence, it collapses over all specified dimensions. If -keepdims is set to 1, the resulting array will have as many dimensions as -inarray, with only 1 'level' per dim that was collapsed over. - -Usage: ageometricmean(inarray,dimension=None,keepdims=0) -Returns: geometric mean computed over dim(s) listed in dimension -""" - inarray = N.array(inarray,N.float_) - if dimension == None: - inarray = N.ravel(inarray) - size = len(inarray) - mult = N.power(inarray,1.0/size) - mult = N.multiply.reduce(mult) - elif type(dimension) in [IntType,FloatType]: - size = inarray.shape[dimension] - mult = N.power(inarray,1.0/size) - mult = N.multiply.reduce(mult,dimension) - if keepdims == 1: - shp = list(inarray.shape) - shp[dimension] = 1 - sum = N.reshape(sum,shp) - else: # must be a SEQUENCE of dims to average over - dims = list(dimension) - dims.sort() - dims.reverse() - size = N.array(N.multiply.reduce(N.take(inarray.shape,dims)),N.float_) - mult = N.power(inarray,1.0/size) - for dim in dims: - mult = N.multiply.reduce(mult,dim) - if keepdims == 1: - shp = list(inarray.shape) - for dim in dims: - shp[dim] = 1 - mult = N.reshape(mult,shp) - return mult - - - def aharmonicmean (inarray,dimension=None,keepdims=0): - """ -Calculates the harmonic mean of the values in the passed array. -That is: n / (1/x1 + 1/x2 + ... + 1/xn). Defaults to ALL values in -the passed array. Use dimension=None to flatten array first. REMEMBER: if -dimension=0, it collapses over dimension 0 ('rows' in a 2D array) only, and -if dimension is a sequence, it collapses over all specified dimensions. If -keepdims is set to 1, the resulting array will have as many dimensions as -inarray, with only 1 'level' per dim that was collapsed over. - -Usage: aharmonicmean(inarray,dimension=None,keepdims=0) -Returns: harmonic mean computed over dim(s) in dimension -""" - inarray = inarray.astype(N.float_) - if dimension == None: - inarray = N.ravel(inarray) - size = len(inarray) - s = N.add.reduce(1.0 / inarray) - elif type(dimension) in [IntType,FloatType]: - size = float(inarray.shape[dimension]) - s = N.add.reduce(1.0/inarray, dimension) - if keepdims == 1: - shp = list(inarray.shape) - shp[dimension] = 1 - s = N.reshape(s,shp) - else: # must be a SEQUENCE of dims to average over - dims = list(dimension) - dims.sort() - nondims = [] - for i in range(len(inarray.shape)): - if i not in dims: - nondims.append(i) - tinarray = N.transpose(inarray,nondims+dims) # put keep-dims first - idx = [0] *len(nondims) - if idx == []: - size = len(N.ravel(inarray)) - s = asum(1.0 / inarray) - if keepdims == 1: - s = N.reshape([s],N.ones(len(inarray.shape))) - else: - idx[0] = -1 - loopcap = N.array(tinarray.shape[0:len(nondims)]) -1 - s = N.zeros(loopcap+1,N.float_) - while incr(idx,loopcap) <> -1: - s[idx] = asum(1.0/tinarray[idx]) - size = N.multiply.reduce(N.take(inarray.shape,dims)) - if keepdims == 1: - shp = list(inarray.shape) - for dim in dims: - shp[dim] = 1 - s = N.reshape(s,shp) - return size / s - - - def amean (inarray,dimension=None,keepdims=0): - """ -Calculates the arithmatic mean of the values in the passed array. -That is: 1/n * (x1 + x2 + ... + xn). Defaults to ALL values in the -passed array. Use dimension=None to flatten array first. REMEMBER: if -dimension=0, it collapses over dimension 0 ('rows' in a 2D array) only, and -if dimension is a sequence, it collapses over all specified dimensions. If -keepdims is set to 1, the resulting array will have as many dimensions as -inarray, with only 1 'level' per dim that was collapsed over. - -Usage: amean(inarray,dimension=None,keepdims=0) -Returns: arithematic mean calculated over dim(s) in dimension -""" - if inarray.dtype in [N.int_, N.short,N.ubyte]: - inarray = inarray.astype(N.float_) - if dimension == None: - inarray = N.ravel(inarray) - sum = N.add.reduce(inarray) - denom = float(len(inarray)) - elif type(dimension) in [IntType,FloatType]: - sum = asum(inarray,dimension) - denom = float(inarray.shape[dimension]) - if keepdims == 1: - shp = list(inarray.shape) - shp[dimension] = 1 - sum = N.reshape(sum,shp) - else: # must be a TUPLE of dims to average over - dims = list(dimension) - dims.sort() - dims.reverse() - sum = inarray *1.0 - for dim in dims: - sum = N.add.reduce(sum,dim) - denom = N.array(N.multiply.reduce(N.take(inarray.shape,dims)),N.float_) - if keepdims == 1: - shp = list(inarray.shape) - for dim in dims: - shp[dim] = 1 - sum = N.reshape(sum,shp) - return sum/denom - - - def amedian (inarray,numbins=1000): - """ -Calculates the COMPUTED median value of an array of numbers, given the -number of bins to use for the histogram (more bins approaches finding the -precise median value of the array; default number of bins = 1000). From -G.W. Heiman's Basic Stats, or CRC Probability & Statistics. -NOTE: THIS ROUTINE ALWAYS uses the entire passed array (flattens it first). - -Usage: amedian(inarray,numbins=1000) -Returns: median calculated over ALL values in inarray -""" - inarray = N.ravel(inarray) - (hist, smallest, binsize, extras) = ahistogram(inarray,numbins,[min(inarray),max(inarray)]) - cumhist = N.cumsum(hist) # make cumulative histogram - otherbins = N.greater_equal(cumhist,len(inarray)/2.0) - otherbins = list(otherbins) # list of 0/1s, 1s start at median bin - cfbin = otherbins.index(1) # get 1st(!) index holding 50%ile score - LRL = smallest + binsize*cfbin # get lower read limit of that bin - cfbelow = N.add.reduce(hist[0:cfbin]) # cum. freq. below bin - freq = hist[cfbin] # frequency IN the 50%ile bin - median = LRL + ((len(inarray)/2.0-cfbelow)/float(freq))*binsize # MEDIAN - return median - - - def amedianscore (inarray,dimension=None): - """ -Returns the 'middle' score of the passed array. If there is an even -number of scores, the mean of the 2 middle scores is returned. Can function -with 1D arrays, or on the FIRST dimension of 2D arrays (i.e., dimension can -be None, to pre-flatten the array, or else dimension must equal 0). - -Usage: amedianscore(inarray,dimension=None) -Returns: 'middle' score of the array, or the mean of the 2 middle scores -""" - if dimension == None: - inarray = N.ravel(inarray) - dimension = 0 - inarray = N.sort(inarray,dimension) - if inarray.shape[dimension] % 2 == 0: # if even number of elements - indx = inarray.shape[dimension]/2 # integer division correct - median = N.asarray(inarray[indx]+inarray[indx-1]) / 2.0 - else: - indx = inarray.shape[dimension] / 2 # integer division correct - median = N.take(inarray,[indx],dimension) - if median.shape == (1,): - median = median[0] - return median - - - def amode(a, dimension=None): - """ -Returns an array of the modal (most common) score in the passed array. -If there is more than one such score, ONLY THE FIRST is returned. -The bin-count for the modal values is also returned. Operates on whole -array (dimension=None), or on a given dimension. - -Usage: amode(a, dimension=None) -Returns: array of bin-counts for mode(s), array of corresponding modal values -""" - - if dimension == None: - a = N.ravel(a) - dimension = 0 - scores = pstat.aunique(N.ravel(a)) # get ALL unique values - testshape = list(a.shape) - testshape[dimension] = 1 - oldmostfreq = N.zeros(testshape) - oldcounts = N.zeros(testshape) - for score in scores: - template = N.equal(a,score) - counts = asum(template,dimension,1) - mostfrequent = N.where(counts>oldcounts,score,oldmostfreq) - oldcounts = N.where(counts>oldcounts,counts,oldcounts) - oldmostfreq = mostfrequent - return oldcounts, mostfrequent - - - def atmean(a,limits=None,inclusive=(1,1)): - """ -Returns the arithmetic mean of all values in an array, ignoring values -strictly outside the sequence passed to 'limits'. Note: either limit -in the sequence, or the value of limits itself, can be set to None. The -inclusive list/tuple determines whether the lower and upper limiting bounds -(respectively) are open/exclusive (0) or closed/inclusive (1). - -Usage: atmean(a,limits=None,inclusive=(1,1)) -""" - if a.dtype in [N.int_, N.short,N.ubyte]: - a = a.astype(N.float_) - if limits == None: - return mean(a) - assert type(limits) in [ListType,TupleType,N.ndarray], "Wrong type for limits in atmean" - if inclusive[0]: lowerfcn = N.greater_equal - else: lowerfcn = N.greater - if inclusive[1]: upperfcn = N.less_equal - else: upperfcn = N.less - if limits[0] > N.maximum.reduce(N.ravel(a)) or limits[1] < N.minimum.reduce(N.ravel(a)): - raise ValueError, "No array values within given limits (atmean)." - elif limits[0]==None and limits[1]<>None: - mask = upperfcn(a,limits[1]) - elif limits[0]<>None and limits[1]==None: - mask = lowerfcn(a,limits[0]) - elif limits[0]<>None and limits[1]<>None: - mask = lowerfcn(a,limits[0])*upperfcn(a,limits[1]) - s = float(N.add.reduce(N.ravel(a*mask))) - n = float(N.add.reduce(N.ravel(mask))) - return s/n - - - def atvar(a,limits=None,inclusive=(1,1)): - """ -Returns the sample variance of values in an array, (i.e., using N-1), -ignoring values strictly outside the sequence passed to 'limits'. -Note: either limit in the sequence, or the value of limits itself, -can be set to None. The inclusive list/tuple determines whether the lower -and upper limiting bounds (respectively) are open/exclusive (0) or -closed/inclusive (1). ASSUMES A FLAT ARRAY (OR ELSE PREFLATTENS). - -Usage: atvar(a,limits=None,inclusive=(1,1)) -""" - a = a.astype(N.float_) - if limits == None or limits == [None,None]: - return avar(a) - assert type(limits) in [ListType,TupleType,N.ndarray], "Wrong type for limits in atvar" - if inclusive[0]: lowerfcn = N.greater_equal - else: lowerfcn = N.greater - if inclusive[1]: upperfcn = N.less_equal - else: upperfcn = N.less - if limits[0] > N.maximum.reduce(N.ravel(a)) or limits[1] < N.minimum.reduce(N.ravel(a)): - raise ValueError, "No array values within given limits (atvar)." - elif limits[0]==None and limits[1]<>None: - mask = upperfcn(a,limits[1]) - elif limits[0]<>None and limits[1]==None: - mask = lowerfcn(a,limits[0]) - elif limits[0]<>None and limits[1]<>None: - mask = lowerfcn(a,limits[0])*upperfcn(a,limits[1]) - - a = N.compress(mask,a) # squish out excluded values - return avar(a) - - - def atmin(a,lowerlimit=None,dimension=None,inclusive=1): - """ -Returns the minimum value of a, along dimension, including only values less -than (or equal to, if inclusive=1) lowerlimit. If the limit is set to None, -all values in the array are used. - -Usage: atmin(a,lowerlimit=None,dimension=None,inclusive=1) -""" - if inclusive: lowerfcn = N.greater - else: lowerfcn = N.greater_equal - if dimension == None: - a = N.ravel(a) - dimension = 0 - if lowerlimit == None: - lowerlimit = N.minimum.reduce(N.ravel(a))-11 - biggest = N.maximum.reduce(N.ravel(a)) - ta = N.where(lowerfcn(a,lowerlimit),a,biggest) - return N.minimum.reduce(ta,dimension) - - - def atmax(a,upperlimit,dimension=None,inclusive=1): - """ -Returns the maximum value of a, along dimension, including only values greater -than (or equal to, if inclusive=1) upperlimit. If the limit is set to None, -a limit larger than the max value in the array is used. - -Usage: atmax(a,upperlimit,dimension=None,inclusive=1) -""" - if inclusive: upperfcn = N.less - else: upperfcn = N.less_equal - if dimension == None: - a = N.ravel(a) - dimension = 0 - if upperlimit == None: - upperlimit = N.maximum.reduce(N.ravel(a))+1 - smallest = N.minimum.reduce(N.ravel(a)) - ta = N.where(upperfcn(a,upperlimit),a,smallest) - return N.maximum.reduce(ta,dimension) - - - def atstdev(a,limits=None,inclusive=(1,1)): - """ -Returns the standard deviation of all values in an array, ignoring values -strictly outside the sequence passed to 'limits'. Note: either limit -in the sequence, or the value of limits itself, can be set to None. The -inclusive list/tuple determines whether the lower and upper limiting bounds -(respectively) are open/exclusive (0) or closed/inclusive (1). - -Usage: atstdev(a,limits=None,inclusive=(1,1)) -""" - return N.sqrt(tvar(a,limits,inclusive)) - - - def atsem(a,limits=None,inclusive=(1,1)): - """ -Returns the standard error of the mean for the values in an array, -(i.e., using N for the denominator), ignoring values strictly outside -the sequence passed to 'limits'. Note: either limit in the sequence, -or the value of limits itself, can be set to None. The inclusive list/tuple -determines whether the lower and upper limiting bounds (respectively) are -open/exclusive (0) or closed/inclusive (1). - -Usage: atsem(a,limits=None,inclusive=(1,1)) -""" - sd = tstdev(a,limits,inclusive) - if limits == None or limits == [None,None]: - n = float(len(N.ravel(a))) - limits = [min(a)-1, max(a)+1] - assert type(limits) in [ListType,TupleType,N.ndarray], "Wrong type for limits in atsem" - if inclusive[0]: lowerfcn = N.greater_equal - else: lowerfcn = N.greater - if inclusive[1]: upperfcn = N.less_equal - else: upperfcn = N.less - if limits[0] > N.maximum.reduce(N.ravel(a)) or limits[1] < N.minimum.reduce(N.ravel(a)): - raise ValueError, "No array values within given limits (atsem)." - elif limits[0]==None and limits[1]<>None: - mask = upperfcn(a,limits[1]) - elif limits[0]<>None and limits[1]==None: - mask = lowerfcn(a,limits[0]) - elif limits[0]<>None and limits[1]<>None: - mask = lowerfcn(a,limits[0])*upperfcn(a,limits[1]) - term1 = N.add.reduce(N.ravel(a*a*mask)) - n = float(N.add.reduce(N.ravel(mask))) - return sd/math.sqrt(n) - - -##################################### -############ AMOMENTS ############# -##################################### - - def amoment(a,moment=1,dimension=None): - """ -Calculates the nth moment about the mean for a sample (defaults to the -1st moment). Generally used to calculate coefficients of skewness and -kurtosis. Dimension can equal None (ravel array first), an integer -(the dimension over which to operate), or a sequence (operate over -multiple dimensions). - -Usage: amoment(a,moment=1,dimension=None) -Returns: appropriate moment along given dimension -""" - if dimension == None: - a = N.ravel(a) - dimension = 0 - if moment == 1: - return 0.0 - else: - mn = amean(a,dimension,1) # 1=keepdims - s = N.power((a-mn),moment) - return amean(s,dimension) - - - def avariation(a,dimension=None): - """ -Returns the coefficient of variation, as defined in CRC Standard -Probability and Statistics, p.6. Dimension can equal None (ravel array -first), an integer (the dimension over which to operate), or a -sequence (operate over multiple dimensions). - -Usage: avariation(a,dimension=None) -""" - return 100.0*asamplestdev(a,dimension)/amean(a,dimension) - - - def askew(a,dimension=None): - """ -Returns the skewness of a distribution (normal ==> 0.0; >0 means extra -weight in left tail). Use askewtest() to see if it's close enough. -Dimension can equal None (ravel array first), an integer (the -dimension over which to operate), or a sequence (operate over multiple -dimensions). - -Usage: askew(a, dimension=None) -Returns: skew of vals in a along dimension, returning ZERO where all vals equal -""" - denom = N.power(amoment(a,2,dimension),1.5) - zero = N.equal(denom,0) - if type(denom) == N.ndarray and asum(zero) <> 0: - print "Number of zeros in askew: ",asum(zero) - denom = denom + zero # prevent divide-by-zero - return N.where(zero, 0, amoment(a,3,dimension)/denom) - - - def akurtosis(a,dimension=None): - """ -Returns the kurtosis of a distribution (normal ==> 3.0; >3 means -heavier in the tails, and usually more peaked). Use akurtosistest() -to see if it's close enough. Dimension can equal None (ravel array -first), an integer (the dimension over which to operate), or a -sequence (operate over multiple dimensions). - -Usage: akurtosis(a,dimension=None) -Returns: kurtosis of values in a along dimension, and ZERO where all vals equal -""" - denom = N.power(amoment(a,2,dimension),2) - zero = N.equal(denom,0) - if type(denom) == N.ndarray and asum(zero) <> 0: - print "Number of zeros in akurtosis: ",asum(zero) - denom = denom + zero # prevent divide-by-zero - return N.where(zero,0,amoment(a,4,dimension)/denom) - - - def adescribe(inarray,dimension=None): - """ -Returns several descriptive statistics of the passed array. Dimension -can equal None (ravel array first), an integer (the dimension over -which to operate), or a sequence (operate over multiple dimensions). - -Usage: adescribe(inarray,dimension=None) -Returns: n, (min,max), mean, standard deviation, skew, kurtosis -""" - if dimension == None: - inarray = N.ravel(inarray) - dimension = 0 - n = inarray.shape[dimension] - mm = (N.minimum.reduce(inarray),N.maximum.reduce(inarray)) - m = amean(inarray,dimension) - sd = astdev(inarray,dimension) - skew = askew(inarray,dimension) - kurt = akurtosis(inarray,dimension) - return n, mm, m, sd, skew, kurt - - -##################################### -######## NORMALITY TESTS ########## -##################################### - - def askewtest(a,dimension=None): - """ -Tests whether the skew is significantly different from a normal -distribution. Dimension can equal None (ravel array first), an -integer (the dimension over which to operate), or a sequence (operate -over multiple dimensions). - -Usage: askewtest(a,dimension=None) -Returns: z-score and 2-tail z-probability -""" - if dimension == None: - a = N.ravel(a) - dimension = 0 - b2 = askew(a,dimension) - n = float(a.shape[dimension]) - y = b2 * N.sqrt(((n+1)*(n+3)) / (6.0*(n-2)) ) - beta2 = ( 3.0*(n*n+27*n-70)*(n+1)*(n+3) ) / ( (n-2.0)*(n+5)*(n+7)*(n+9) ) - W2 = -1 + N.sqrt(2*(beta2-1)) - delta = 1/N.sqrt(N.log(N.sqrt(W2))) - alpha = N.sqrt(2/(W2-1)) - y = N.where(y==0,1,y) - Z = delta*N.log(y/alpha + N.sqrt((y/alpha)**2+1)) - return Z, (1.0-zprob(Z))*2 - - - def akurtosistest(a,dimension=None): - """ -Tests whether a dataset has normal kurtosis (i.e., -kurtosis=3(n-1)/(n+1)) Valid only for n>20. Dimension can equal None -(ravel array first), an integer (the dimension over which to operate), -or a sequence (operate over multiple dimensions). - -Usage: akurtosistest(a,dimension=None) -Returns: z-score and 2-tail z-probability, returns 0 for bad pixels -""" - if dimension == None: - a = N.ravel(a) - dimension = 0 - n = float(a.shape[dimension]) - if n<20: - print "akurtosistest only valid for n>=20 ... continuing anyway, n=",n - b2 = akurtosis(a,dimension) - E = 3.0*(n-1) /(n+1) - varb2 = 24.0*n*(n-2)*(n-3) / ((n+1)*(n+1)*(n+3)*(n+5)) - x = (b2-E)/N.sqrt(varb2) - sqrtbeta1 = 6.0*(n*n-5*n+2)/((n+7)*(n+9)) * N.sqrt((6.0*(n+3)*(n+5))/ - (n*(n-2)*(n-3))) - A = 6.0 + 8.0/sqrtbeta1 *(2.0/sqrtbeta1 + N.sqrt(1+4.0/(sqrtbeta1**2))) - term1 = 1 -2/(9.0*A) - denom = 1 +x*N.sqrt(2/(A-4.0)) - denom = N.where(N.less(denom,0), 99, denom) - term2 = N.where(N.equal(denom,0), term1, N.power((1-2.0/A)/denom,1/3.0)) - Z = ( term1 - term2 ) / N.sqrt(2/(9.0*A)) - Z = N.where(N.equal(denom,99), 0, Z) - return Z, (1.0-zprob(Z))*2 - - - def anormaltest(a,dimension=None): - """ -Tests whether skew and/OR kurtosis of dataset differs from normal -curve. Can operate over multiple dimensions. Dimension can equal -None (ravel array first), an integer (the dimension over which to -operate), or a sequence (operate over multiple dimensions). - -Usage: anormaltest(a,dimension=None) -Returns: z-score and 2-tail probability -""" - if dimension == None: - a = N.ravel(a) - dimension = 0 - s,p = askewtest(a,dimension) - k,p = akurtosistest(a,dimension) - k2 = N.power(s,2) + N.power(k,2) - return k2, achisqprob(k2,2) - - -##################################### -###### AFREQUENCY FUNCTIONS ####### -##################################### - - def aitemfreq(a): - """ -Returns a 2D array of item frequencies. Column 1 contains item values, -column 2 contains their respective counts. Assumes a 1D array is passed. -@@@sorting OK? - -Usage: aitemfreq(a) -Returns: a 2D frequency table (col [0:n-1]=scores, col n=frequencies) -""" - scores = pstat.aunique(a) - scores = N.sort(scores) - freq = N.zeros(len(scores)) - for i in range(len(scores)): - freq[i] = N.add.reduce(N.equal(a,scores[i])) - return N.array(pstat.aabut(scores, freq)) - - - def ascoreatpercentile (inarray, percent): - """ -Usage: ascoreatpercentile(inarray,percent) 0<percent<100 -Returns: score at given percentile, relative to inarray distribution -""" - percent = percent / 100.0 - targetcf = percent*len(inarray) - h, lrl, binsize, extras = histogram(inarray) - cumhist = cumsum(h*1) - for i in range(len(cumhist)): - if cumhist[i] >= targetcf: - break - score = binsize * ((targetcf - cumhist[i-1]) / float(h[i])) + (lrl+binsize*i) - return score - - - def apercentileofscore (inarray,score,histbins=10,defaultlimits=None): - """ -Note: result of this function depends on the values used to histogram -the data(!). - -Usage: apercentileofscore(inarray,score,histbins=10,defaultlimits=None) -Returns: percentile-position of score (0-100) relative to inarray -""" - h, lrl, binsize, extras = histogram(inarray,histbins,defaultlimits) - cumhist = cumsum(h*1) - i = int((score - lrl)/float(binsize)) - pct = (cumhist[i-1]+((score-(lrl+binsize*i))/float(binsize))*h[i])/float(len(inarray)) * 100 - return pct - - - def ahistogram (inarray,numbins=10,defaultlimits=None,printextras=1): - """ -Returns (i) an array of histogram bin counts, (ii) the smallest value -of the histogram binning, and (iii) the bin width (the last 2 are not -necessarily integers). Default number of bins is 10. Defaultlimits -can be None (the routine picks bins spanning all the numbers in the -inarray) or a 2-sequence (lowerlimit, upperlimit). Returns all of the -following: array of bin values, lowerreallimit, binsize, extrapoints. - -Usage: ahistogram(inarray,numbins=10,defaultlimits=None,printextras=1) -Returns: (array of bin counts, bin-minimum, min-width, #-points-outside-range) -""" - inarray = N.ravel(inarray) # flatten any >1D arrays - if (defaultlimits <> None): - lowerreallimit = defaultlimits[0] - upperreallimit = defaultlimits[1] - binsize = (upperreallimit-lowerreallimit) / float(numbins) - else: - Min = N.minimum.reduce(inarray) - Max = N.maximum.reduce(inarray) - estbinwidth = float(Max - Min)/float(numbins) + 1e-6 - binsize = (Max-Min+estbinwidth)/float(numbins) - lowerreallimit = Min - binsize/2.0 #lower real limit,1st bin - bins = N.zeros(numbins) - extrapoints = 0 - for num in inarray: - try: - if (num-lowerreallimit) < 0: - extrapoints = extrapoints + 1 - else: - bintoincrement = int((num-lowerreallimit) / float(binsize)) - bins[bintoincrement] = bins[bintoincrement] + 1 - except: # point outside lower/upper limits - extrapoints = extrapoints + 1 - if (extrapoints > 0 and printextras == 1): - print '\nPoints outside given histogram range =',extrapoints - return (bins, lowerreallimit, binsize, extrapoints) - - - def acumfreq(a,numbins=10,defaultreallimits=None): - """ -Returns a cumulative frequency histogram, using the histogram function. -Defaultreallimits can be None (use all data), or a 2-sequence containing -lower and upper limits on values to include. - -Usage: acumfreq(a,numbins=10,defaultreallimits=None) -Returns: array of cumfreq bin values, lowerreallimit, binsize, extrapoints -""" - h,l,b,e = histogram(a,numbins,defaultreallimits) - cumhist = cumsum(h*1) - return cumhist,l,b,e - - - def arelfreq(a,numbins=10,defaultreallimits=None): - """ -Returns a relative frequency histogram, using the histogram function. -Defaultreallimits can be None (use all data), or a 2-sequence containing -lower and upper limits on values to include. - -Usage: arelfreq(a,numbins=10,defaultreallimits=None) -Returns: array of cumfreq bin values, lowerreallimit, binsize, extrapoints -""" - h,l,b,e = histogram(a,numbins,defaultreallimits) - h = N.array(h/float(a.shape[0])) - return h,l,b,e - - -##################################### -###### AVARIABILITY FUNCTIONS ##### -##################################### - - def aobrientransform(*args): - """ -Computes a transform on input data (any number of columns). Used to -test for homogeneity of variance prior to running one-way stats. Each -array in *args is one level of a factor. If an F_oneway() run on the -transformed data and found significant, variances are unequal. From -Maxwell and Delaney, p.112. - -Usage: aobrientransform(*args) *args = 1D arrays, one per level of factor -Returns: transformed data for use in an ANOVA -""" - TINY = 1e-10 - k = len(args) - n = N.zeros(k,N.float_) - v = N.zeros(k,N.float_) - m = N.zeros(k,N.float_) - nargs = [] - for i in range(k): - nargs.append(args[i].astype(N.float_)) - n[i] = float(len(nargs[i])) - v[i] = var(nargs[i]) - m[i] = mean(nargs[i]) - for j in range(k): - for i in range(n[j]): - t1 = (n[j]-1.5)*n[j]*(nargs[j][i]-m[j])**2 - t2 = 0.5*v[j]*(n[j]-1.0) - t3 = (n[j]-1.0)*(n[j]-2.0) - nargs[j][i] = (t1-t2) / float(t3) - check = 1 - for j in range(k): - if v[j] - mean(nargs[j]) > TINY: - check = 0 - if check <> 1: - raise ValueError, 'Lack of convergence in obrientransform.' - else: - return N.array(nargs) - - - def asamplevar (inarray,dimension=None,keepdims=0): - """ -Returns the sample standard deviation of the values in the passed -array (i.e., using N). Dimension can equal None (ravel array first), -an integer (the dimension over which to operate), or a sequence -(operate over multiple dimensions). Set keepdims=1 to return an array -with the same number of dimensions as inarray. - -Usage: asamplevar(inarray,dimension=None,keepdims=0) -""" - if dimension == None: - inarray = N.ravel(inarray) - dimension = 0 - if dimension == 1: - mn = amean(inarray,dimension)[:,N.NewAxis] - else: - mn = amean(inarray,dimension,keepdims=1) - deviations = inarray - mn - if type(dimension) == ListType: - n = 1 - for d in dimension: - n = n*inarray.shape[d] - else: - n = inarray.shape[dimension] - svar = ass(deviations,dimension,keepdims) / float(n) - return svar - - - def asamplestdev (inarray, dimension=None, keepdims=0): - """ -Returns the sample standard deviation of the values in the passed -array (i.e., using N). Dimension can equal None (ravel array first), -an integer (the dimension over which to operate), or a sequence -(operate over multiple dimensions). Set keepdims=1 to return an array -with the same number of dimensions as inarray. - -Usage: asamplestdev(inarray,dimension=None,keepdims=0) -""" - return N.sqrt(asamplevar(inarray,dimension,keepdims)) - - - def asignaltonoise(instack,dimension=0): - """ -Calculates signal-to-noise. Dimension can equal None (ravel array -first), an integer (the dimension over which to operate), or a -sequence (operate over multiple dimensions). - -Usage: asignaltonoise(instack,dimension=0): -Returns: array containing the value of (mean/stdev) along dimension, - or 0 when stdev=0 -""" - m = mean(instack,dimension) - sd = stdev(instack,dimension) - return N.where(sd==0,0,m/sd) - - - def acov (x,y, dimension=None,keepdims=0): - """ -Returns the estimated covariance of the values in the passed -array (i.e., N-1). Dimension can equal None (ravel array first), an -integer (the dimension over which to operate), or a sequence (operate -over multiple dimensions). Set keepdims=1 to return an array with the -same number of dimensions as inarray. - -Usage: acov(x,y,dimension=None,keepdims=0) -""" - if dimension == None: - x = N.ravel(x) - y = N.ravel(y) - dimension = 0 - xmn = amean(x,dimension,1) # keepdims - xdeviations = x - xmn - ymn = amean(y,dimension,1) # keepdims - ydeviations = y - ymn - if type(dimension) == ListType: - n = 1 - for d in dimension: - n = n*x.shape[d] - else: - n = x.shape[dimension] - covar = N.sum(xdeviations*ydeviations)/float(n-1) - return covar - - - def avar (inarray, dimension=None,keepdims=0): - """ -Returns the estimated population variance of the values in the passed -array (i.e., N-1). Dimension can equal None (ravel array first), an -integer (the dimension over which to operate), or a sequence (operate -over multiple dimensions). Set keepdims=1 to return an array with the -same number of dimensions as inarray. - -Usage: avar(inarray,dimension=None,keepdims=0) -""" - if dimension == None: - inarray = N.ravel(inarray) - dimension = 0 - mn = amean(inarray,dimension,1) - deviations = inarray - mn - if type(dimension) == ListType: - n = 1 - for d in dimension: - n = n*inarray.shape[d] - else: - n = inarray.shape[dimension] - var = ass(deviations,dimension,keepdims)/float(n-1) - return var - - - def astdev (inarray, dimension=None, keepdims=0): - """ -Returns the estimated population standard deviation of the values in -the passed array (i.e., N-1). Dimension can equal None (ravel array -first), an integer (the dimension over which to operate), or a -sequence (operate over multiple dimensions). Set keepdims=1 to return -an array with the same number of dimensions as inarray. - -Usage: astdev(inarray,dimension=None,keepdims=0) -""" - return N.sqrt(avar(inarray,dimension,keepdims)) - - - def asterr (inarray, dimension=None, keepdims=0): - """ -Returns the estimated population standard error of the values in the -passed array (i.e., N-1). Dimension can equal None (ravel array -first), an integer (the dimension over which to operate), or a -sequence (operate over multiple dimensions). Set keepdims=1 to return -an array with the same number of dimensions as inarray. - -Usage: asterr(inarray,dimension=None,keepdims=0) -""" - if dimension == None: - inarray = N.ravel(inarray) - dimension = 0 - return astdev(inarray,dimension,keepdims) / float(N.sqrt(inarray.shape[dimension])) - - - def asem (inarray, dimension=None, keepdims=0): - """ -Returns the standard error of the mean (i.e., using N) of the values -in the passed array. Dimension can equal None (ravel array first), an -integer (the dimension over which to operate), or a sequence (operate -over multiple dimensions). Set keepdims=1 to return an array with the -same number of dimensions as inarray. - -Usage: asem(inarray,dimension=None, keepdims=0) -""" - if dimension == None: - inarray = N.ravel(inarray) - dimension = 0 - if type(dimension) == ListType: - n = 1 - for d in dimension: - n = n*inarray.shape[d] - else: - n = inarray.shape[dimension] - s = asamplestdev(inarray,dimension,keepdims) / N.sqrt(n-1) - return s - - - def az (a, score): - """ -Returns the z-score of a given input score, given thearray from which -that score came. Not appropriate for population calculations, nor for -arrays > 1D. - -Usage: az(a, score) -""" - z = (score-amean(a)) / asamplestdev(a) - return z - - - def azs (a): - """ -Returns a 1D array of z-scores, one for each score in the passed array, -computed relative to the passed array. - -Usage: azs(a) -""" - zscores = [] - for item in a: - zscores.append(z(a,item)) - return N.array(zscores) - - - def azmap (scores, compare, dimension=0): - """ -Returns an array of z-scores the shape of scores (e.g., [x,y]), compared to -array passed to compare (e.g., [time,x,y]). Assumes collapsing over dim 0 -of the compare array. - -Usage: azs(scores, compare, dimension=0) -""" - mns = amean(compare,dimension) - sstd = asamplestdev(compare,0) - return (scores - mns) / sstd - - -##################################### -####### ATRIMMING FUNCTIONS ####### -##################################### - -## deleted around() as it's in numpy now - - def athreshold(a,threshmin=None,threshmax=None,newval=0): - """ -Like Numeric.clip() except that values <threshmid or >threshmax are replaced -by newval instead of by threshmin/threshmax (respectively). - -Usage: athreshold(a,threshmin=None,threshmax=None,newval=0) -Returns: a, with values <threshmin or >threshmax replaced with newval -""" - mask = N.zeros(a.shape) - if threshmin <> None: - mask = mask + N.where(a<threshmin,1,0) - if threshmax <> None: - mask = mask + N.where(a>threshmax,1,0) - mask = N.clip(mask,0,1) - return N.where(mask,newval,a) - - - def atrimboth (a,proportiontocut): - """ -Slices off the passed proportion of items from BOTH ends of the passed -array (i.e., with proportiontocut=0.1, slices 'leftmost' 10% AND -'rightmost' 10% of scores. You must pre-sort the array if you want -"proper" trimming. Slices off LESS if proportion results in a -non-integer slice index (i.e., conservatively slices off -proportiontocut). - -Usage: atrimboth (a,proportiontocut) -Returns: trimmed version of array a -""" - lowercut = int(proportiontocut*len(a)) - uppercut = len(a) - lowercut - return a[lowercut:uppercut] - - - def atrim1 (a,proportiontocut,tail='right'): - """ -Slices off the passed proportion of items from ONE end of the passed -array (i.e., if proportiontocut=0.1, slices off 'leftmost' or 'rightmost' -10% of scores). Slices off LESS if proportion results in a non-integer -slice index (i.e., conservatively slices off proportiontocut). - -Usage: atrim1(a,proportiontocut,tail='right') or set tail='left' -Returns: trimmed version of array a -""" - if string.lower(tail) == 'right': - lowercut = 0 - uppercut = len(a) - int(proportiontocut*len(a)) - elif string.lower(tail) == 'left': - lowercut = int(proportiontocut*len(a)) - uppercut = len(a) - return a[lowercut:uppercut] - - -##################################### -##### ACORRELATION FUNCTIONS ###### -##################################### - - def acovariance(X): - """ -Computes the covariance matrix of a matrix X. Requires a 2D matrix input. - -Usage: acovariance(X) -Returns: covariance matrix of X -""" - if len(X.shape) <> 2: - raise TypeError, "acovariance requires 2D matrices" - n = X.shape[0] - mX = amean(X,0) - return N.dot(N.transpose(X),X) / float(n) - N.multiply.outer(mX,mX) - - - def acorrelation(X): - """ -Computes the correlation matrix of a matrix X. Requires a 2D matrix input. - -Usage: acorrelation(X) -Returns: correlation matrix of X -""" - C = acovariance(X) - V = N.diagonal(C) - return C / N.sqrt(N.multiply.outer(V,V)) - - - def apaired(x,y): - """ -Interactively determines the type of data in x and y, and then runs the -appropriated statistic for paired group data. - -Usage: apaired(x,y) x,y = the two arrays of values to be compared -Returns: appropriate statistic name, value, and probability -""" - samples = '' - while samples not in ['i','r','I','R','c','C']: - print '\nIndependent or related samples, or correlation (i,r,c): ', - samples = raw_input() - - if samples in ['i','I','r','R']: - print '\nComparing variances ...', -# USE O'BRIEN'S TEST FOR HOMOGENEITY OF VARIANCE, Maxwell & delaney, p.112 - r = obrientransform(x,y) - f,p = F_oneway(pstat.colex(r,0),pstat.colex(r,1)) - if p<0.05: - vartype='unequal, p='+str(round(p,4)) - else: - vartype='equal' - print vartype - if samples in ['i','I']: - if vartype[0]=='e': - t,p = ttest_ind(x,y,None,0) - print '\nIndependent samples t-test: ', round(t,4),round(p,4) - else: - if len(x)>20 or len(y)>20: - z,p = ranksums(x,y) - print '\nRank Sums test (NONparametric, n>20): ', round(z,4),round(p,4) - else: - u,p = mannwhitneyu(x,y) - print '\nMann-Whitney U-test (NONparametric, ns<20): ', round(u,4),round(p,4) - - else: # RELATED SAMPLES - if vartype[0]=='e': - t,p = ttest_rel(x,y,0) - print '\nRelated samples t-test: ', round(t,4),round(p,4) - else: - t,p = ranksums(x,y) - print '\nWilcoxon T-test (NONparametric): ', round(t,4),round(p,4) - else: # CORRELATION ANALYSIS - corrtype = '' - while corrtype not in ['c','C','r','R','d','D']: - print '\nIs the data Continuous, Ranked, or Dichotomous (c,r,d): ', - corrtype = raw_input() - if corrtype in ['c','C']: - m,b,r,p,see = linregress(x,y) - print '\nLinear regression for continuous variables ...' - lol = [['Slope','Intercept','r','Prob','SEestimate'],[round(m,4),round(b,4),round(r,4),round(p,4),round(see,4)]] - pstat.printcc(lol) - elif corrtype in ['r','R']: - r,p = spearmanr(x,y) - print '\nCorrelation for ranked variables ...' - print "Spearman's r: ",round(r,4),round(p,4) - else: # DICHOTOMOUS - r,p = pointbiserialr(x,y) - print '\nAssuming x contains a dichotomous variable ...' - print 'Point Biserial r: ',round(r,4),round(p,4) - print '\n\n' - return None - - - def dices(x,y): - """ -Calculates Dice's coefficient ... (2*number of common terms)/(number of terms in x + -number of terms in y). Returns a value between 0 (orthogonal) and 1. - -Usage: dices(x,y) -""" - import sets - x = sets.Set(x) - y = sets.Set(y) - common = len(x.intersection(y)) - total = float(len(x) + len(y)) - return 2*common/total - - - def icc(x,y=None,verbose=0): - """ -Calculates intraclass correlation coefficients using simple, Type I sums of squares. -If only one variable is passed, assumed it's an Nx2 matrix - -Usage: icc(x,y=None,verbose=0) -Returns: icc rho, prob ####PROB IS A GUESS BASED ON PEARSON -""" - TINY = 1.0e-20 - if y: - all = N.concatenate([x,y],0) - else: - all = x+0 - x = all[:,0] - y = all[:,1] - totalss = ass(all-mean(all)) - pairmeans = (x+y)/2. - withinss = ass(x-pairmeans) + ass(y-pairmeans) - withindf = float(len(x)) - betwdf = float(len(x)-1) - withinms = withinss / withindf - betweenms = (totalss-withinss) / betwdf - rho = (betweenms-withinms)/(withinms+betweenms) - t = rho*math.sqrt(betwdf/((1.0-rho+TINY)*(1.0+rho+TINY))) - prob = abetai(0.5*betwdf,0.5,betwdf/(betwdf+t*t),verbose) - return rho, prob - - - def alincc(x,y): - """ -Calculates Lin's concordance correlation coefficient. - -Usage: alincc(x,y) where x, y are equal-length arrays -Returns: Lin's CC -""" - x = N.ravel(x) - y = N.ravel(y) - covar = acov(x,y)*(len(x)-1)/float(len(x)) # correct denom to n - xvar = avar(x)*(len(x)-1)/float(len(x)) # correct denom to n - yvar = avar(y)*(len(y)-1)/float(len(y)) # correct denom to n - lincc = (2 * covar) / ((xvar+yvar) +((amean(x)-amean(y))**2)) - return lincc - - - def apearsonr(x,y,verbose=1): - """ -Calculates a Pearson correlation coefficient and returns p. Taken -from Heiman's Basic Statistics for the Behav. Sci (2nd), p.195. - -Usage: apearsonr(x,y,verbose=1) where x,y are equal length arrays -Returns: Pearson's r, two-tailed p-value -""" - TINY = 1.0e-20 - n = len(x) - xmean = amean(x) - ymean = amean(y) - r_num = n*(N.add.reduce(x*y)) - N.add.reduce(x)*N.add.reduce(y) - r_den = math.sqrt((n*ass(x) - asquare_of_sums(x))*(n*ass(y)-asquare_of_sums(y))) - r = (r_num / r_den) - df = n-2 - t = r*math.sqrt(df/((1.0-r+TINY)*(1.0+r+TINY))) - prob = abetai(0.5*df,0.5,df/(df+t*t),verbose) - return r,prob - - - def aspearmanr(x,y): - """ -Calculates a Spearman rank-order correlation coefficient. Taken -from Heiman's Basic Statistics for the Behav. Sci (1st), p.192. - -Usage: aspearmanr(x,y) where x,y are equal-length arrays -Returns: Spearman's r, two-tailed p-value -""" - TINY = 1e-30 - n = len(x) - rankx = rankdata(x) - ranky = rankdata(y) - dsq = N.add.reduce((rankx-ranky)**2) - rs = 1 - 6*dsq / float(n*(n**2-1)) - t = rs * math.sqrt((n-2) / ((rs+1.0)*(1.0-rs))) - df = n-2 - probrs = abetai(0.5*df,0.5,df/(df+t*t)) -# probability values for rs are from part 2 of the spearman function in -# Numerical Recipies, p.510. They close to tables, but not exact.(?) - return rs, probrs - - - def apointbiserialr(x,y): - """ -Calculates a point-biserial correlation coefficient and the associated -probability value. Taken from Heiman's Basic Statistics for the Behav. -Sci (1st), p.194. - -Usage: apointbiserialr(x,y) where x,y are equal length arrays -Returns: Point-biserial r, two-tailed p-value -""" - TINY = 1e-30 - categories = pstat.aunique(x) - data = pstat.aabut(x,y) - if len(categories) <> 2: - raise ValueError, "Exactly 2 categories required (in x) for pointbiserialr()." - else: # there are 2 categories, continue - codemap = pstat.aabut(categories,N.arange(2)) - recoded = pstat.arecode(data,codemap,0) - x = pstat.alinexand(data,0,categories[0]) - y = pstat.alinexand(data,0,categories[1]) - xmean = amean(pstat.acolex(x,1)) - ymean = amean(pstat.acolex(y,1)) - n = len(data) - adjust = math.sqrt((len(x)/float(n))*(len(y)/float(n))) - rpb = (ymean - xmean)/asamplestdev(pstat.acolex(data,1))*adjust - df = n-2 - t = rpb*math.sqrt(df/((1.0-rpb+TINY)*(1.0+rpb+TINY))) - prob = abetai(0.5*df,0.5,df/(df+t*t)) - return rpb, prob - - - def akendalltau(x,y): - """ -Calculates Kendall's tau ... correlation of ordinal data. Adapted -from function kendl1 in Numerical Recipies. Needs good test-cases.@@@ - -Usage: akendalltau(x,y) -Returns: Kendall's tau, two-tailed p-value -""" - n1 = 0 - n2 = 0 - iss = 0 - for j in range(len(x)-1): - for k in range(j,len(y)): - a1 = x[j] - x[k] - a2 = y[j] - y[k] - aa = a1 * a2 - if (aa): # neither array has a tie - n1 = n1 + 1 - n2 = n2 + 1 - if aa > 0: - iss = iss + 1 - else: - iss = iss -1 - else: - if (a1): - n1 = n1 + 1 - else: - n2 = n2 + 1 - tau = iss / math.sqrt(n1*n2) - svar = (4.0*len(x)+10.0) / (9.0*len(x)*(len(x)-1)) - z = tau / math.sqrt(svar) - prob = erfcc(abs(z)/1.4142136) - return tau, prob - - - def alinregress(*args): - """ -Calculates a regression line on two arrays, x and y, corresponding to x,y -pairs. If a single 2D array is passed, alinregress finds dim with 2 levels -and splits data into x,y pairs along that dim. - -Usage: alinregress(*args) args=2 equal-length arrays, or one 2D array -Returns: slope, intercept, r, two-tailed prob, sterr-of-the-estimate, n -""" - TINY = 1.0e-20 - if len(args) == 1: # more than 1D array? - args = args[0] - if len(args) == 2: - x = args[0] - y = args[1] - else: - x = args[:,0] - y = args[:,1] - else: - x = args[0] - y = args[1] - n = len(x) - xmean = amean(x) - ymean = amean(y) - r_num = n*(N.add.reduce(x*y)) - N.add.reduce(x)*N.add.reduce(y) - r_den = math.sqrt((n*ass(x) - asquare_of_sums(x))*(n*ass(y)-asquare_of_sums(y))) - r = r_num / r_den - z = 0.5*math.log((1.0+r+TINY)/(1.0-r+TINY)) - df = n-2 - t = r*math.sqrt(df/((1.0-r+TINY)*(1.0+r+TINY))) - prob = abetai(0.5*df,0.5,df/(df+t*t)) - slope = r_num / (float(n)*ass(x) - asquare_of_sums(x)) - intercept = ymean - slope*xmean - sterrest = math.sqrt(1-r*r)*asamplestdev(y) - return slope, intercept, r, prob, sterrest, n - - def amasslinregress(*args): - """ -Calculates a regression line on one 1D array (x) and one N-D array (y). - -Returns: slope, intercept, r, two-tailed prob, sterr-of-the-estimate, n -""" - TINY = 1.0e-20 - if len(args) == 1: # more than 1D array? - args = args[0] - if len(args) == 2: - x = N.ravel(args[0]) - y = args[1] - else: - x = N.ravel(args[:,0]) - y = args[:,1] - else: - x = args[0] - y = args[1] - x = x.astype(N.float_) - y = y.astype(N.float_) - n = len(x) - xmean = amean(x) - ymean = amean(y,0) - shp = N.ones(len(y.shape)) - shp[0] = len(x) - x.shape = shp - print x.shape, y.shape - r_num = n*(N.add.reduce(x*y,0)) - N.add.reduce(x)*N.add.reduce(y,0) - r_den = N.sqrt((n*ass(x) - asquare_of_sums(x))*(n*ass(y,0)-asquare_of_sums(y,0))) - zerodivproblem = N.equal(r_den,0) - r_den = N.where(zerodivproblem,1,r_den) # avoid zero-division in 1st place - r = r_num / r_den # need to do this nicely for matrix division - r = N.where(zerodivproblem,0.0,r) - z = 0.5*N.log((1.0+r+TINY)/(1.0-r+TINY)) - df = n-2 - t = r*N.sqrt(df/((1.0-r+TINY)*(1.0+r+TINY))) - prob = abetai(0.5*df,0.5,df/(df+t*t)) - - ss = float(n)*ass(x)-asquare_of_sums(x) - s_den = N.where(ss==0,1,ss) # avoid zero-division in 1st place - slope = r_num / s_den - intercept = ymean - slope*xmean - sterrest = N.sqrt(1-r*r)*asamplestdev(y,0) - return slope, intercept, r, prob, sterrest, n - - -##################################### -##### AINFERENTIAL STATISTICS ##### -##################################### - - def attest_1samp(a,popmean,printit=0,name='Sample',writemode='a'): - """ -Calculates the t-obtained for the independent samples T-test on ONE group -of scores a, given a population mean. If printit=1, results are printed -to the screen. If printit='filename', the results are output to 'filename' -using the given writemode (default=append). Returns t-value, and prob. - -Usage: attest_1samp(a,popmean,Name='Sample',printit=0,writemode='a') -Returns: t-value, two-tailed prob -""" - if type(a) != N.ndarray: - a = N.array(a) - x = amean(a) - v = avar(a) - n = len(a) - df = n-1 - svar = ((n-1)*v) / float(df) - t = (x-popmean)/math.sqrt(svar*(1.0/n)) - prob = abetai(0.5*df,0.5,df/(df+t*t)) - - if printit <> 0: - statname = 'Single-sample T-test.' - outputpairedstats(printit,writemode, - 'Population','--',popmean,0,0,0, - name,n,x,v,N.minimum.reduce(N.ravel(a)), - N.maximum.reduce(N.ravel(a)), - statname,t,prob) - return t,prob - - - def attest_ind (a, b, dimension=None, printit=0, name1='Samp1', name2='Samp2',writemode='a'): - """ -Calculates the t-obtained T-test on TWO INDEPENDENT samples of scores -a, and b. From Numerical Recipies, p.483. If printit=1, results are -printed to the screen. If printit='filename', the results are output -to 'filename' using the given writemode (default=append). Dimension -can equal None (ravel array first), or an integer (the dimension over -which to operate on a and b). - -Usage: attest_ind (a,b,dimension=None,printit=0, - Name1='Samp1',Name2='Samp2',writemode='a') -Returns: t-value, two-tailed p-value -""" - if dimension == None: - a = N.ravel(a) - b = N.ravel(b) - dimension = 0 - x1 = amean(a,dimension) - x2 = amean(b,dimension) - v1 = avar(a,dimension) - v2 = avar(b,dimension) - n1 = a.shape[dimension] - n2 = b.shape[dimension] - df = n1+n2-2 - svar = ((n1-1)*v1+(n2-1)*v2) / float(df) - zerodivproblem = N.equal(svar,0) - svar = N.where(zerodivproblem,1,svar) # avoid zero-division in 1st place - t = (x1-x2)/N.sqrt(svar*(1.0/n1 + 1.0/n2)) # N-D COMPUTATION HERE!!!!!! - t = N.where(zerodivproblem,1.0,t) # replace NaN/wrong t-values with 1.0 - probs = abetai(0.5*df,0.5,float(df)/(df+t*t)) - - if type(t) == N.ndarray: - probs = N.reshape(probs,t.shape) - if probs.shape == (1,): - probs = probs[0] - - if printit <> 0: - if type(t) == N.ndarray: - t = t[0] - if type(probs) == N.ndarray: - probs = probs[0] - statname = 'Independent samples T-test.' - outputpairedstats(printit,writemode, - name1,n1,x1,v1,N.minimum.reduce(N.ravel(a)), - N.maximum.reduce(N.ravel(a)), - name2,n2,x2,v2,N.minimum.reduce(N.ravel(b)), - N.maximum.reduce(N.ravel(b)), - statname,t,probs) - return - return t, probs - - def ap2t(pval,df): - """ -Tries to compute a t-value from a p-value (or pval array) and associated df. -SLOW for large numbers of elements(!) as it re-computes p-values 20 times -(smaller step-sizes) at which point it decides it's done. Keeps the signs -of the input array. Returns 1000 (or -1000) if t>100. - -Usage: ap2t(pval,df) -Returns: an array of t-values with the shape of pval - """ - pval = N.array(pval) - signs = N.sign(pval) - pval = abs(pval) - t = N.ones(pval.shape,N.float_)*50 - step = N.ones(pval.shape,N.float_)*25 - print "Initial ap2t() prob calc" - prob = abetai(0.5*df,0.5,float(df)/(df+t*t)) - print 'ap2t() iter: ', - for i in range(10): - print i,' ', - t = N.where(pval<prob,t+step,t-step) - prob = abetai(0.5*df,0.5,float(df)/(df+t*t)) - step = step/2 - print - # since this is an ugly hack, we get ugly boundaries - t = N.where(t>99.9,1000,t) # hit upper-boundary - t = t+signs - return t #, prob, pval - - - def attest_rel (a,b,dimension=None,printit=0,name1='Samp1',name2='Samp2',writemode='a'): - """ -Calculates the t-obtained T-test on TWO RELATED samples of scores, a -and b. From Numerical Recipies, p.483. If printit=1, results are -printed to the screen. If printit='filename', the results are output -to 'filename' using the given writemode (default=append). Dimension -can equal None (ravel array first), or an integer (the dimension over -which to operate on a and b). - -Usage: attest_rel(a,b,dimension=None,printit=0, - name1='Samp1',name2='Samp2',writemode='a') -Returns: t-value, two-tailed p-value -""" - if dimension == None: - a = N.ravel(a) - b = N.ravel(b) - dimension = 0 - if len(a)<>len(b): - raise ValueError, 'Unequal length arrays.' - x1 = amean(a,dimension) - x2 = amean(b,dimension) - v1 = avar(a,dimension) - v2 = avar(b,dimension) - n = a.shape[dimension] - df = float(n-1) - d = (a-b).astype('d') - - denom = N.sqrt((n*N.add.reduce(d*d,dimension) - N.add.reduce(d,dimension)**2) /df) - zerodivproblem = N.equal(denom,0) - denom = N.where(zerodivproblem,1,denom) # avoid zero-division in 1st place - t = N.add.reduce(d,dimension) / denom # N-D COMPUTATION HERE!!!!!! - t = N.where(zerodivproblem,1.0,t) # replace NaN/wrong t-values with 1.0 - probs = abetai(0.5*df,0.5,float(df)/(df+t*t)) - if type(t) == N.ndarray: - probs = N.reshape(probs,t.shape) - if probs.shape == (1,): - probs = probs[0] - - if printit <> 0: - statname = 'Related samples T-test.' - outputpairedstats(printit,writemode, - name1,n,x1,v1,N.minimum.reduce(N.ravel(a)), - N.maximum.reduce(N.ravel(a)), - name2,n,x2,v2,N.minimum.reduce(N.ravel(b)), - N.maximum.reduce(N.ravel(b)), - statname,t,probs) - return - return t, probs - - - def achisquare(f_obs,f_exp=None): - """ -Calculates a one-way chi square for array of observed frequencies and returns -the result. If no expected frequencies are given, the total N is assumed to -be equally distributed across all groups. -@@@NOT RIGHT?? - -Usage: achisquare(f_obs, f_exp=None) f_obs = array of observed cell freq. -Returns: chisquare-statistic, associated p-value -""" - - k = len(f_obs) - if f_exp == None: - f_exp = N.array([sum(f_obs)/float(k)] * len(f_obs),N.float_) - f_exp = f_exp.astype(N.float_) - chisq = N.add.reduce((f_obs-f_exp)**2 / f_exp) - return chisq, achisqprob(chisq, k-1) - - - def aks_2samp (data1,data2): - """ -Computes the Kolmogorov-Smirnof statistic on 2 samples. Modified from -Numerical Recipies in C, page 493. Returns KS D-value, prob. Not ufunc- -like. - -Usage: aks_2samp(data1,data2) where data1 and data2 are 1D arrays -Returns: KS D-value, p-value -""" - j1 = 0 # N.zeros(data1.shape[1:]) TRIED TO MAKE THIS UFUNC-LIKE - j2 = 0 # N.zeros(data2.shape[1:]) - fn1 = 0.0 # N.zeros(data1.shape[1:],N.float_) - fn2 = 0.0 # N.zeros(data2.shape[1:],N.float_) - n1 = data1.shape[0] - n2 = data2.shape[0] - en1 = n1*1 - en2 = n2*1 - d = N.zeros(data1.shape[1:],N.float_) - data1 = N.sort(data1,0) - data2 = N.sort(data2,0) - while j1 < n1 and j2 < n2: - d1=data1[j1] - d2=data2[j2] - if d1 <= d2: - fn1 = (j1)/float(en1) - j1 = j1 + 1 - if d2 <= d1: - fn2 = (j2)/float(en2) - j2 = j2 + 1 - dt = (fn2-fn1) - if abs(dt) > abs(d): - d = dt -# try: - en = math.sqrt(en1*en2/float(en1+en2)) - prob = aksprob((en+0.12+0.11/en)*N.fabs(d)) -# except: -# prob = 1.0 - return d, prob - - - def amannwhitneyu(x,y): - """ -Calculates a Mann-Whitney U statistic on the provided scores and -returns the result. Use only when the n in each condition is < 20 and -you have 2 independent samples of ranks. REMEMBER: Mann-Whitney U is -significant if the u-obtained is LESS THAN or equal to the critical -value of U. - -Usage: amannwhitneyu(x,y) where x,y are arrays of values for 2 conditions -Returns: u-statistic, one-tailed p-value (i.e., p(z(U))) -""" - n1 = len(x) - n2 = len(y) - ranked = rankdata(N.concatenate((x,y))) - rankx = ranked[0:n1] # get the x-ranks - ranky = ranked[n1:] # the rest are y-ranks - u1 = n1*n2 + (n1*(n1+1))/2.0 - sum(rankx) # calc U for x - u2 = n1*n2 - u1 # remainder is U for y - bigu = max(u1,u2) - smallu = min(u1,u2) - proportion = bigu/float(n1*n2) - T = math.sqrt(tiecorrect(ranked)) # correction factor for tied scores - if T == 0: - raise ValueError, 'All numbers are identical in amannwhitneyu' - sd = math.sqrt(T*n1*n2*(n1+n2+1)/12.0) - z = abs((bigu-n1*n2/2.0) / sd) # normal approximation for prob calc - return smallu, 1.0 - azprob(z), proportion - - - def atiecorrect(rankvals): - """ -Tie-corrector for ties in Mann Whitney U and Kruskal Wallis H tests. -See Siegel, S. (1956) Nonparametric Statistics for the Behavioral -Sciences. New York: McGraw-Hill. Code adapted from |Stat rankind.c -code. - -Usage: atiecorrect(rankvals) -Returns: T correction factor for U or H -""" - sorted,posn = ashellsort(N.array(rankvals)) - n = len(sorted) - T = 0.0 - i = 0 - while (i<n-1): - if sorted[i] == sorted[i+1]: - nties = 1 - while (i<n-1) and (sorted[i] == sorted[i+1]): - nties = nties +1 - i = i +1 - T = T + nties**3 - nties - i = i+1 - T = T / float(n**3-n) - return 1.0 - T - - - def aranksums(x,y): - """ -Calculates the rank sums statistic on the provided scores and returns -the result. - -Usage: aranksums(x,y) where x,y are arrays of values for 2 conditions -Returns: z-statistic, two-tailed p-value -""" - n1 = len(x) - n2 = len(y) - alldata = N.concatenate((x,y)) - ranked = arankdata(alldata) - x = ranked[:n1] - y = ranked[n1:] - s = sum(x) - expected = n1*(n1+n2+1) / 2.0 - z = (s - expected) / math.sqrt(n1*n2*(n1+n2+1)/12.0) - prob = 2*(1.0 - azprob(abs(z))) - return z, prob - - - def awilcoxont(x,y): - """ -Calculates the Wilcoxon T-test for related samples and returns the -result. A non-parametric T-test. - -Usage: awilcoxont(x,y) where x,y are equal-length arrays for 2 conditions -Returns: t-statistic, two-tailed p-value -""" - if len(x) <> len(y): - raise ValueError, 'Unequal N in awilcoxont. Aborting.' - d = x-y - d = N.compress(N.not_equal(d,0),d) # Keep all non-zero differences - count = len(d) - absd = abs(d) - absranked = arankdata(absd) - r_plus = 0.0 - r_minus = 0.0 - for i in range(len(absd)): - if d[i] < 0: - r_minus = r_minus + absranked[i] - else: - r_plus = r_plus + absranked[i] - wt = min(r_plus, r_minus) - mn = count * (count+1) * 0.25 - se = math.sqrt(count*(count+1)*(2.0*count+1.0)/24.0) - z = math.fabs(wt-mn) / se - z = math.fabs(wt-mn) / se - prob = 2*(1.0 -zprob(abs(z))) - return wt, prob - - - def akruskalwallish(*args): - """ -The Kruskal-Wallis H-test is a non-parametric ANOVA for 3 or more -groups, requiring at least 5 subjects in each group. This function -calculates the Kruskal-Wallis H and associated p-value for 3 or more -independent samples. - -Usage: akruskalwallish(*args) args are separate arrays for 3+ conditions -Returns: H-statistic (corrected for ties), associated p-value -""" - assert len(args) == 3, "Need at least 3 groups in stats.akruskalwallish()" - args = list(args) - n = [0]*len(args) - n = map(len,args) - all = [] - for i in range(len(args)): - all = all + args[i].tolist() - ranked = rankdata(all) - T = tiecorrect(ranked) - for i in range(len(args)): - args[i] = ranked[0:n[i]] - del ranked[0:n[i]] - rsums = [] - for i in range(len(args)): - rsums.append(sum(args[i])**2) - rsums[i] = rsums[i] / float(n[i]) - ssbn = sum(rsums) - totaln = sum(n) - h = 12.0 / (totaln*(totaln+1)) * ssbn - 3*(totaln+1) - df = len(args) - 1 - if T == 0: - raise ValueError, 'All numbers are identical in akruskalwallish' - h = h / float(T) - return h, chisqprob(h,df) - - - def afriedmanchisquare(*args): - """ -Friedman Chi-Square is a non-parametric, one-way within-subjects -ANOVA. This function calculates the Friedman Chi-square test for -repeated measures and returns the result, along with the associated -probability value. It assumes 3 or more repeated measures. Only 3 -levels requires a minimum of 10 subjects in the study. Four levels -requires 5 subjects per level(??). - -Usage: afriedmanchisquare(*args) args are separate arrays for 2+ conditions -Returns: chi-square statistic, associated p-value -""" - k = len(args) - if k < 3: - raise ValueError, '\nLess than 3 levels. Friedman test not appropriate.\n' - n = len(args[0]) - data = apply(pstat.aabut,args) - data = data.astype(N.float_) - for i in range(len(data)): - data[i] = arankdata(data[i]) - ssbn = asum(asum(args,1)**2) - chisq = 12.0 / (k*n*(k+1)) * ssbn - 3*n*(k+1) - return chisq, achisqprob(chisq,k-1) - - -##################################### -#### APROBABILITY CALCULATIONS #### -##################################### - - def achisqprob(chisq,df): - """ -Returns the (1-tail) probability value associated with the provided chi-square -value and df. Heavily modified from chisq.c in Gary Perlman's |Stat. Can -handle multiple dimensions. - -Usage: achisqprob(chisq,df) chisq=chisquare stat., df=degrees of freedom -""" - BIG = 200.0 - def ex(x): - BIG = 200.0 - exponents = N.where(N.less(x,-BIG),-BIG,x) - return N.exp(exponents) - - if type(chisq) == N.ndarray: - arrayflag = 1 - else: - arrayflag = 0 - chisq = N.array([chisq]) - if df < 1: - return N.ones(chisq.shape,N.float) - probs = N.zeros(chisq.shape,N.float_) - probs = N.where(N.less_equal(chisq,0),1.0,probs) # set prob=1 for chisq<0 - a = 0.5 * chisq - if df > 1: - y = ex(-a) - if df%2 == 0: - even = 1 - s = y*1 - s2 = s*1 - else: - even = 0 - s = 2.0 * azprob(-N.sqrt(chisq)) - s2 = s*1 - if (df > 2): - chisq = 0.5 * (df - 1.0) - if even: - z = N.ones(probs.shape,N.float_) - else: - z = 0.5 *N.ones(probs.shape,N.float_) - if even: - e = N.zeros(probs.shape,N.float_) - else: - e = N.log(N.sqrt(N.pi)) *N.ones(probs.shape,N.float_) - c = N.log(a) - mask = N.zeros(probs.shape) - a_big = N.greater(a,BIG) - a_big_frozen = -1 *N.ones(probs.shape,N.float_) - totalelements = N.multiply.reduce(N.array(probs.shape)) - while asum(mask)<>totalelements: - e = N.log(z) + e - s = s + ex(c*z-a-e) - z = z + 1.0 -# print z, e, s - newmask = N.greater(z,chisq) - a_big_frozen = N.where(newmask*N.equal(mask,0)*a_big, s, a_big_frozen) - mask = N.clip(newmask+mask,0,1) - if even: - z = N.ones(probs.shape,N.float_) - e = N.ones(probs.shape,N.float_) - else: - z = 0.5 *N.ones(probs.shape,N.float_) - e = 1.0 / N.sqrt(N.pi) / N.sqrt(a) * N.ones(probs.shape,N.float_) - c = 0.0 - mask = N.zeros(probs.shape) - a_notbig_frozen = -1 *N.ones(probs.shape,N.float_) - while asum(mask)<>totalelements: - e = e * (a/z.astype(N.float_)) - c = c + e - z = z + 1.0 -# print '#2', z, e, c, s, c*y+s2 - newmask = N.greater(z,chisq) - a_notbig_frozen = N.where(newmask*N.equal(mask,0)*(1-a_big), - c*y+s2, a_notbig_frozen) - mask = N.clip(newmask+mask,0,1) - probs = N.where(N.equal(probs,1),1, - N.where(N.greater(a,BIG),a_big_frozen,a_notbig_frozen)) - return probs - else: - return s - - - def aerfcc(x): - """ -Returns the complementary error function erfc(x) with fractional error -everywhere less than 1.2e-7. Adapted from Numerical Recipies. Can -handle multiple dimensions. - -Usage: aerfcc(x) -""" - z = abs(x) - t = 1.0 / (1.0+0.5*z) - ans = t * N.exp(-z*z-1.26551223 + t*(1.00002368+t*(0.37409196+t*(0.09678418+t*(-0.18628806+t*(0.27886807+t*(-1.13520398+t*(1.48851587+t*(-0.82215223+t*0.17087277))))))))) - return N.where(N.greater_equal(x,0), ans, 2.0-ans) - - - def azprob(z): - """ -Returns the area under the normal curve 'to the left of' the given z value. -Thus, - for z<0, zprob(z) = 1-tail probability - for z>0, 1.0-zprob(z) = 1-tail probability - for any z, 2.0*(1.0-zprob(abs(z))) = 2-tail probability -Adapted from z.c in Gary Perlman's |Stat. Can handle multiple dimensions. - -Usage: azprob(z) where z is a z-value -""" - def yfunc(y): - x = (((((((((((((-0.000045255659 * y - +0.000152529290) * y -0.000019538132) * y - -0.000676904986) * y +0.001390604284) * y - -0.000794620820) * y -0.002034254874) * y - +0.006549791214) * y -0.010557625006) * y - +0.011630447319) * y -0.009279453341) * y - +0.005353579108) * y -0.002141268741) * y - +0.000535310849) * y +0.999936657524 - return x - - def wfunc(w): - x = ((((((((0.000124818987 * w - -0.001075204047) * w +0.005198775019) * w - -0.019198292004) * w +0.059054035642) * w - -0.151968751364) * w +0.319152932694) * w - -0.531923007300) * w +0.797884560593) * N.sqrt(w) * 2.0 - return x - - Z_MAX = 6.0 # maximum meaningful z-value - x = N.zeros(z.shape,N.float_) # initialize - y = 0.5 * N.fabs(z) - x = N.where(N.less(y,1.0),wfunc(y*y),yfunc(y-2.0)) # get x's - x = N.where(N.greater(y,Z_MAX*0.5),1.0,x) # kill those with big Z - prob = N.where(N.greater(z,0),(x+1)*0.5,(1-x)*0.5) - return prob - - - def aksprob(alam): - """ -Returns the probability value for a K-S statistic computed via ks_2samp. -Adapted from Numerical Recipies. Can handle multiple dimensions. - -Usage: aksprob(alam) -""" - if type(alam) == N.ndarray: - frozen = -1 *N.ones(alam.shape,N.float64) - alam = alam.astype(N.float64) - arrayflag = 1 - else: - frozen = N.array(-1.) - alam = N.array(alam,N.float64) - arrayflag = 1 - mask = N.zeros(alam.shape) - fac = 2.0 *N.ones(alam.shape,N.float_) - sum = N.zeros(alam.shape,N.float_) - termbf = N.zeros(alam.shape,N.float_) - a2 = N.array(-2.0*alam*alam,N.float64) - totalelements = N.multiply.reduce(N.array(mask.shape)) - for j in range(1,201): - if asum(mask) == totalelements: - break - exponents = (a2*j*j) - overflowmask = N.less(exponents,-746) - frozen = N.where(overflowmask,0,frozen) - mask = mask+overflowmask - term = fac*N.exp(exponents) - sum = sum + term - newmask = N.where(N.less_equal(abs(term),(0.001*termbf)) + - N.less(abs(term),1.0e-8*sum), 1, 0) - frozen = N.where(newmask*N.equal(mask,0), sum, frozen) - mask = N.clip(mask+newmask,0,1) - fac = -fac - termbf = abs(term) - if arrayflag: - return N.where(N.equal(frozen,-1), 1.0, frozen) # 1.0 if doesn't converge - else: - return N.where(N.equal(frozen,-1), 1.0, frozen)[0] # 1.0 if doesn't converge - - - def afprob (dfnum, dfden, F): - """ -Returns the 1-tailed significance level (p-value) of an F statistic -given the degrees of freedom for the numerator (dfR-dfF) and the degrees -of freedom for the denominator (dfF). Can handle multiple dims for F. - -Usage: afprob(dfnum, dfden, F) where usually dfnum=dfbn, dfden=dfwn -""" - if type(F) == N.ndarray: - return abetai(0.5*dfden, 0.5*dfnum, dfden/(1.0*dfden+dfnum*F)) - else: - return abetai(0.5*dfden, 0.5*dfnum, dfden/float(dfden+dfnum*F)) - - - def abetacf(a,b,x,verbose=1): - """ -Evaluates the continued fraction form of the incomplete Beta function, -betai. (Adapted from: Numerical Recipies in C.) Can handle multiple -dimensions for x. - -Usage: abetacf(a,b,x,verbose=1) -""" - ITMAX = 200 - EPS = 3.0e-7 - - arrayflag = 1 - if type(x) == N.ndarray: - frozen = N.ones(x.shape,N.float_) *-1 #start out w/ -1s, should replace all - else: - arrayflag = 0 - frozen = N.array([-1]) - x = N.array([x]) - mask = N.zeros(x.shape) - bm = az = am = 1.0 - qab = a+b - qap = a+1.0 - qam = a-1.0 - bz = 1.0-qab*x/qap - for i in range(ITMAX+1): - if N.sum(N.ravel(N.equal(frozen,-1)))==0: - break - em = float(i+1) - tem = em + em - d = em*(b-em)*x/((qam+tem)*(a+tem)) - ap = az + d*am - bp = bz+d*bm - d = -(a+em)*(qab+em)*x/((qap+tem)*(a+tem)) - app = ap+d*az - bpp = bp+d*bz - aold = az*1 - am = ap/bpp - bm = bp/bpp - az = app/bpp - bz = 1.0 - newmask = N.less(abs(az-aold),EPS*abs(az)) - frozen = N.where(newmask*N.equal(mask,0), az, frozen) - mask = N.clip(mask+newmask,0,1) - noconverge = asum(N.equal(frozen,-1)) - if noconverge <> 0 and verbose: - print 'a or b too big, or ITMAX too small in Betacf for ',noconverge,' elements' - if arrayflag: - return frozen - else: - return frozen[0] - - - def agammln(xx): - """ -Returns the gamma function of xx. - Gamma(z) = Integral(0,infinity) of t^(z-1)exp(-t) dt. -Adapted from: Numerical Recipies in C. Can handle multiple dims ... but -probably doesn't normally have to. - -Usage: agammln(xx) -""" - coeff = [76.18009173, -86.50532033, 24.01409822, -1.231739516, - 0.120858003e-2, -0.536382e-5] - x = xx - 1.0 - tmp = x + 5.5 - tmp = tmp - (x+0.5)*N.log(tmp) - ser = 1.0 - for j in range(len(coeff)): - x = x + 1 - ser = ser + coeff[j]/x - return -tmp + N.log(2.50662827465*ser) - - - def abetai(a,b,x,verbose=1): - """ -Returns the incomplete beta function: - - I-sub-x(a,b) = 1/B(a,b)*(Integral(0,x) of t^(a-1)(1-t)^(b-1) dt) - -where a,b>0 and B(a,b) = G(a)*G(b)/(G(a+b)) where G(a) is the gamma -function of a. The continued fraction formulation is implemented -here, using the betacf function. (Adapted from: Numerical Recipies in -C.) Can handle multiple dimensions. - -Usage: abetai(a,b,x,verbose=1) -""" - TINY = 1e-15 - if type(a) == N.ndarray: - if asum(N.less(x,0)+N.greater(x,1)) <> 0: - raise ValueError, 'Bad x in abetai' - x = N.where(N.equal(x,0),TINY,x) - x = N.where(N.equal(x,1.0),1-TINY,x) - - bt = N.where(N.equal(x,0)+N.equal(x,1), 0, -1) - exponents = ( gammln(a+b)-gammln(a)-gammln(b)+a*N.log(x)+b* - N.log(1.0-x) ) - # 746 (below) is the MAX POSSIBLE BEFORE OVERFLOW - exponents = N.where(N.less(exponents,-740),-740,exponents) - bt = N.exp(exponents) - if type(x) == N.ndarray: - ans = N.where(N.less(x,(a+1)/(a+b+2.0)), - bt*abetacf(a,b,x,verbose)/float(a), - 1.0-bt*abetacf(b,a,1.0-x,verbose)/float(b)) - else: - if x<(a+1)/(a+b+2.0): - ans = bt*abetacf(a,b,x,verbose)/float(a) - else: - ans = 1.0-bt*abetacf(b,a,1.0-x,verbose)/float(b) - return ans - - -##################################### -####### AANOVA CALCULATIONS ####### -##################################### - - import numpy.linalg, operator - LA = numpy.linalg - - def aglm(data,para): - """ -Calculates a linear model fit ... anova/ancova/lin-regress/t-test/etc. Taken -from: - Peterson et al. Statistical limitations in functional neuroimaging - I. Non-inferential methods and statistical models. Phil Trans Royal Soc - Lond B 354: 1239-1260. - -Usage: aglm(data,para) -Returns: statistic, p-value ??? -""" - if len(para) <> len(data): - print "data and para must be same length in aglm" - return - n = len(para) - p = pstat.aunique(para) - x = N.zeros((n,len(p))) # design matrix - for l in range(len(p)): - x[:,l] = N.equal(para,p[l]) - b = N.dot(N.dot(LA.inv(N.dot(N.transpose(x),x)), # i.e., b=inv(X'X)X'Y - N.transpose(x)), - data) - diffs = (data - N.dot(x,b)) - s_sq = 1./(n-len(p)) * N.dot(N.transpose(diffs), diffs) - - if len(p) == 2: # ttest_ind - c = N.array([1,-1]) - df = n-2 - fact = asum(1.0/asum(x,0)) # i.e., 1/n1 + 1/n2 + 1/n3 ... - t = N.dot(c,b) / N.sqrt(s_sq*fact) - probs = abetai(0.5*df,0.5,float(df)/(df+t*t)) - return t, probs - - - def aF_oneway(*args): - """ -Performs a 1-way ANOVA, returning an F-value and probability given -any number of groups. From Heiman, pp.394-7. - -Usage: aF_oneway (*args) where *args is 2 or more arrays, one per - treatment group -Returns: f-value, probability -""" - na = len(args) # ANOVA on 'na' groups, each in it's own array - means = [0]*na - vars = [0]*na - ns = [0]*na - alldata = [] - tmp = map(N.array,args) - means = map(amean,tmp) - vars = map(avar,tmp) - ns = map(len,args) - alldata = N.concatenate(args) - bign = len(alldata) - sstot = ass(alldata)-(asquare_of_sums(alldata)/float(bign)) - ssbn = 0 - for a in args: - ssbn = ssbn + asquare_of_sums(N.array(a))/float(len(a)) - ssbn = ssbn - (asquare_of_sums(alldata)/float(bign)) - sswn = sstot-ssbn - dfbn = na-1 - dfwn = bign - na - msb = ssbn/float(dfbn) - msw = sswn/float(dfwn) - f = msb/msw - prob = fprob(dfbn,dfwn,f) - return f, prob - - - def aF_value (ER,EF,dfR,dfF): - """ -Returns an F-statistic given the following: - ER = error associated with the null hypothesis (the Restricted model) - EF = error associated with the alternate hypothesis (the Full model) - dfR = degrees of freedom the Restricted model - dfF = degrees of freedom associated with the Restricted model -""" - return ((ER-EF)/float(dfR-dfF) / (EF/float(dfF))) - - - def outputfstats(Enum, Eden, dfnum, dfden, f, prob): - Enum = round(Enum,3) - Eden = round(Eden,3) - dfnum = round(Enum,3) - dfden = round(dfden,3) - f = round(f,3) - prob = round(prob,3) - suffix = '' # for *s after the p-value - if prob < 0.001: suffix = ' ***' - elif prob < 0.01: suffix = ' **' - elif prob < 0.05: suffix = ' *' - title = [['EF/ER','DF','Mean Square','F-value','prob','']] - lofl = title+[[Enum, dfnum, round(Enum/float(dfnum),3), f, prob, suffix], - [Eden, dfden, round(Eden/float(dfden),3),'','','']] - pstat.printcc(lofl) - return - - - def F_value_multivariate(ER, EF, dfnum, dfden): - """ -Returns an F-statistic given the following: - ER = error associated with the null hypothesis (the Restricted model) - EF = error associated with the alternate hypothesis (the Full model) - dfR = degrees of freedom the Restricted model - dfF = degrees of freedom associated with the Restricted model -where ER and EF are matrices from a multivariate F calculation. -""" - if type(ER) in [IntType, FloatType]: - ER = N.array([[ER]]) - if type(EF) in [IntType, FloatType]: - EF = N.array([[EF]]) - n_um = (LA.det(ER) - LA.det(EF)) / float(dfnum) - d_en = LA.det(EF) / float(dfden) - return n_um / d_en - - -##################################### -####### ASUPPORT FUNCTIONS ######## -##################################### - - def asign(a): - """ -Usage: asign(a) -Returns: array shape of a, with -1 where a<0 and +1 where a>=0 -""" - a = N.asarray(a) - if ((type(a) == type(1.4)) or (type(a) == type(1))): - return a-a-N.less(a,0)+N.greater(a,0) - else: - return N.zeros(N.shape(a))-N.less(a,0)+N.greater(a,0) - - - def asum (a, dimension=None,keepdims=0): - """ -An alternative to the Numeric.add.reduce function, which allows one to -(1) collapse over multiple dimensions at once, and/or (2) to retain -all dimensions in the original array (squashing one down to size. -Dimension can equal None (ravel array first), an integer (the -dimension over which to operate), or a sequence (operate over multiple -dimensions). If keepdims=1, the resulting array will have as many -dimensions as the input array. - -Usage: asum(a, dimension=None, keepdims=0) -Returns: array summed along 'dimension'(s), same _number_ of dims if keepdims=1 -""" - if type(a) == N.ndarray and a.dtype in [N.int_, N.short, N.ubyte]: - a = a.astype(N.float_) - if dimension == None: - s = N.sum(N.ravel(a)) - elif type(dimension) in [IntType,FloatType]: - s = N.add.reduce(a, dimension) - if keepdims == 1: - shp = list(a.shape) - shp[dimension] = 1 - s = N.reshape(s,shp) - else: # must be a SEQUENCE of dims to sum over - dims = list(dimension) - dims.sort() - dims.reverse() - s = a *1.0 - for dim in dims: - s = N.add.reduce(s,dim) - if keepdims == 1: - shp = list(a.shape) - for dim in dims: - shp[dim] = 1 - s = N.reshape(s,shp) - return s - - - def acumsum (a,dimension=None): - """ -Returns an array consisting of the cumulative sum of the items in the -passed array. Dimension can equal None (ravel array first), an -integer (the dimension over which to operate), or a sequence (operate -over multiple dimensions, but this last one just barely makes sense). - -Usage: acumsum(a,dimension=None) -""" - if dimension == None: - a = N.ravel(a) - dimension = 0 - if type(dimension) in [ListType, TupleType, N.ndarray]: - dimension = list(dimension) - dimension.sort() - dimension.reverse() - for d in dimension: - a = N.add.accumulate(a,d) - return a - else: - return N.add.accumulate(a,dimension) - - - def ass(inarray, dimension=None, keepdims=0): - """ -Squares each value in the passed array, adds these squares & returns -the result. Unfortunate function name. :-) Defaults to ALL values in -the array. Dimension can equal None (ravel array first), an integer -(the dimension over which to operate), or a sequence (operate over -multiple dimensions). Set keepdims=1 to maintain the original number -of dimensions. - -Usage: ass(inarray, dimension=None, keepdims=0) -Returns: sum-along-'dimension' for (inarray*inarray) -""" - if dimension == None: - inarray = N.ravel(inarray) - dimension = 0 - return asum(inarray*inarray,dimension,keepdims) - - - def asummult (array1,array2,dimension=None,keepdims=0): - """ -Multiplies elements in array1 and array2, element by element, and -returns the sum (along 'dimension') of all resulting multiplications. -Dimension can equal None (ravel array first), an integer (the -dimension over which to operate), or a sequence (operate over multiple -dimensions). A trivial function, but included for completeness. - -Usage: asummult(array1,array2,dimension=None,keepdims=0) -""" - if dimension == None: - array1 = N.ravel(array1) - array2 = N.ravel(array2) - dimension = 0 - return asum(array1*array2,dimension,keepdims) - - - def asquare_of_sums(inarray, dimension=None, keepdims=0): - """ -Adds the values in the passed array, squares that sum, and returns the -result. Dimension can equal None (ravel array first), an integer (the -dimension over which to operate), or a sequence (operate over multiple -dimensions). If keepdims=1, the returned array will have the same -NUMBER of dimensions as the original. - -Usage: asquare_of_sums(inarray, dimension=None, keepdims=0) -Returns: the square of the sum over dim(s) in dimension -""" - if dimension == None: - inarray = N.ravel(inarray) - dimension = 0 - s = asum(inarray,dimension,keepdims) - if type(s) == N.ndarray: - return s.astype(N.float_)*s - else: - return float(s)*s - - - def asumdiffsquared(a,b, dimension=None, keepdims=0): - """ -Takes pairwise differences of the values in arrays a and b, squares -these differences, and returns the sum of these squares. Dimension -can equal None (ravel array first), an integer (the dimension over -which to operate), or a sequence (operate over multiple dimensions). -keepdims=1 means the return shape = len(a.shape) = len(b.shape) - -Usage: asumdiffsquared(a,b) -Returns: sum[ravel(a-b)**2] -""" - if dimension == None: - inarray = N.ravel(a) - dimension = 0 - return asum((a-b)**2,dimension,keepdims) - - - def ashellsort(inarray): - """ -Shellsort algorithm. Sorts a 1D-array. - -Usage: ashellsort(inarray) -Returns: sorted-inarray, sorting-index-vector (for original array) -""" - n = len(inarray) - svec = inarray *1.0 - ivec = range(n) - gap = n/2 # integer division needed - while gap >0: - for i in range(gap,n): - for j in range(i-gap,-1,-gap): - while j>=0 and svec[j]>svec[j+gap]: - temp = svec[j] - svec[j] = svec[j+gap] - svec[j+gap] = temp - itemp = ivec[j] - ivec[j] = ivec[j+gap] - ivec[j+gap] = itemp - gap = gap / 2 # integer division needed -# svec is now sorted input vector, ivec has the order svec[i] = vec[ivec[i]] - return svec, ivec - - - def arankdata(inarray): - """ -Ranks the data in inarray, dealing with ties appropritely. Assumes -a 1D inarray. Adapted from Gary Perlman's |Stat ranksort. - -Usage: arankdata(inarray) -Returns: array of length equal to inarray, containing rank scores -""" - n = len(inarray) - svec, ivec = ashellsort(inarray) - sumranks = 0 - dupcount = 0 - newarray = N.zeros(n,N.float_) - for i in range(n): - sumranks = sumranks + i - dupcount = dupcount + 1 - if i==n-1 or svec[i] <> svec[i+1]: - averank = sumranks / float(dupcount) + 1 - for j in range(i-dupcount+1,i+1): - newarray[ivec[j]] = averank - sumranks = 0 - dupcount = 0 - return newarray - - - def afindwithin(data): - """ -Returns a binary vector, 1=within-subject factor, 0=between. Input -equals the entire data array (i.e., column 1=random factor, last -column = measured values. - -Usage: afindwithin(data) data in |Stat format -""" - numfact = len(data[0])-2 - withinvec = [0]*numfact - for col in range(1,numfact+1): - rows = pstat.linexand(data,col,pstat.unique(pstat.colex(data,1))[0]) # get 1 level of this factor - if len(pstat.unique(pstat.colex(rows,0))) < len(rows): # if fewer subjects than scores on this factor - withinvec[col-1] = 1 - return withinvec - - - ######################################################### - ######################################################### - ###### RE-DEFINE DISPATCHES TO INCLUDE ARRAYS ######### - ######################################################### - ######################################################### - -## CENTRAL TENDENCY: - geometricmean = Dispatch ( (lgeometricmean, (ListType, TupleType)), - (ageometricmean, (N.ndarray,)) ) - harmonicmean = Dispatch ( (lharmonicmean, (ListType, TupleType)), - (aharmonicmean, (N.ndarray,)) ) - mean = Dispatch ( (lmean, (ListType, TupleType)), - (amean, (N.ndarray,)) ) - median = Dispatch ( (lmedian, (ListType, TupleType)), - (amedian, (N.ndarray,)) ) - medianscore = Dispatch ( (lmedianscore, (ListType, TupleType)), - (amedianscore, (N.ndarray,)) ) - mode = Dispatch ( (lmode, (ListType, TupleType)), - (amode, (N.ndarray,)) ) - tmean = Dispatch ( (atmean, (N.ndarray,)) ) - tvar = Dispatch ( (atvar, (N.ndarray,)) ) - tstdev = Dispatch ( (atstdev, (N.ndarray,)) ) - tsem = Dispatch ( (atsem, (N.ndarray,)) ) - -## VARIATION: - moment = Dispatch ( (lmoment, (ListType, TupleType)), - (amoment, (N.ndarray,)) ) - variation = Dispatch ( (lvariation, (ListType, TupleType)), - (avariation, (N.ndarray,)) ) - skew = Dispatch ( (lskew, (ListType, TupleType)), - (askew, (N.ndarray,)) ) - kurtosis = Dispatch ( (lkurtosis, (ListType, TupleType)), - (akurtosis, (N.ndarray,)) ) - describe = Dispatch ( (ldescribe, (ListType, TupleType)), - (adescribe, (N.ndarray,)) ) - -## DISTRIBUTION TESTS - - skewtest = Dispatch ( (askewtest, (ListType, TupleType)), - (askewtest, (N.ndarray,)) ) - kurtosistest = Dispatch ( (akurtosistest, (ListType, TupleType)), - (akurtosistest, (N.ndarray,)) ) - normaltest = Dispatch ( (anormaltest, (ListType, TupleType)), - (anormaltest, (N.ndarray,)) ) - -## FREQUENCY STATS: - itemfreq = Dispatch ( (litemfreq, (ListType, TupleType)), - (aitemfreq, (N.ndarray,)) ) - scoreatpercentile = Dispatch ( (lscoreatpercentile, (ListType, TupleType)), - (ascoreatpercentile, (N.ndarray,)) ) - percentileofscore = Dispatch ( (lpercentileofscore, (ListType, TupleType)), - (apercentileofscore, (N.ndarray,)) ) - histogram = Dispatch ( (lhistogram, (ListType, TupleType)), - (ahistogram, (N.ndarray,)) ) - cumfreq = Dispatch ( (lcumfreq, (ListType, TupleType)), - (acumfreq, (N.ndarray,)) ) - relfreq = Dispatch ( (lrelfreq, (ListType, TupleType)), - (arelfreq, (N.ndarray,)) ) - -## VARIABILITY: - obrientransform = Dispatch ( (lobrientransform, (ListType, TupleType)), - (aobrientransform, (N.ndarray,)) ) - samplevar = Dispatch ( (lsamplevar, (ListType, TupleType)), - (asamplevar, (N.ndarray,)) ) - samplestdev = Dispatch ( (lsamplestdev, (ListType, TupleType)), - (asamplestdev, (N.ndarray,)) ) - signaltonoise = Dispatch( (asignaltonoise, (N.ndarray,)),) - var = Dispatch ( (lvar, (ListType, TupleType)), - (avar, (N.ndarray,)) ) - stdev = Dispatch ( (lstdev, (ListType, TupleType)), - (astdev, (N.ndarray,)) ) - sterr = Dispatch ( (lsterr, (ListType, TupleType)), - (asterr, (N.ndarray,)) ) - sem = Dispatch ( (lsem, (ListType, TupleType)), - (asem, (N.ndarray,)) ) - z = Dispatch ( (lz, (ListType, TupleType)), - (az, (N.ndarray,)) ) - zs = Dispatch ( (lzs, (ListType, TupleType)), - (azs, (N.ndarray,)) ) - -## TRIMMING FCNS: - threshold = Dispatch( (athreshold, (N.ndarray,)),) - trimboth = Dispatch ( (ltrimboth, (ListType, TupleType)), - (atrimboth, (N.ndarray,)) ) - trim1 = Dispatch ( (ltrim1, (ListType, TupleType)), - (atrim1, (N.ndarray,)) ) - -## CORRELATION FCNS: - paired = Dispatch ( (lpaired, (ListType, TupleType)), - (apaired, (N.ndarray,)) ) - lincc = Dispatch ( (llincc, (ListType, TupleType)), - (alincc, (N.ndarray,)) ) - pearsonr = Dispatch ( (lpearsonr, (ListType, TupleType)), - (apearsonr, (N.ndarray,)) ) - spearmanr = Dispatch ( (lspearmanr, (ListType, TupleType)), - (aspearmanr, (N.ndarray,)) ) - pointbiserialr = Dispatch ( (lpointbiserialr, (ListType, TupleType)), - (apointbiserialr, (N.ndarray,)) ) - kendalltau = Dispatch ( (lkendalltau, (ListType, TupleType)), - (akendalltau, (N.ndarray,)) ) - linregress = Dispatch ( (llinregress, (ListType, TupleType)), - (alinregress, (N.ndarray,)) ) - -## INFERENTIAL STATS: - ttest_1samp = Dispatch ( (lttest_1samp, (ListType, TupleType)), - (attest_1samp, (N.ndarray,)) ) - ttest_ind = Dispatch ( (lttest_ind, (ListType, TupleType)), - (attest_ind, (N.ndarray,)) ) - ttest_rel = Dispatch ( (lttest_rel, (ListType, TupleType)), - (attest_rel, (N.ndarray,)) ) - chisquare = Dispatch ( (lchisquare, (ListType, TupleType)), - (achisquare, (N.ndarray,)) ) - ks_2samp = Dispatch ( (lks_2samp, (ListType, TupleType)), - (aks_2samp, (N.ndarray,)) ) - mannwhitneyu = Dispatch ( (lmannwhitneyu, (ListType, TupleType)), - (amannwhitneyu, (N.ndarray,)) ) - tiecorrect = Dispatch ( (ltiecorrect, (ListType, TupleType)), - (atiecorrect, (N.ndarray,)) ) - ranksums = Dispatch ( (lranksums, (ListType, TupleType)), - (aranksums, (N.ndarray,)) ) - wilcoxont = Dispatch ( (lwilcoxont, (ListType, TupleType)), - (awilcoxont, (N.ndarray,)) ) - kruskalwallish = Dispatch ( (lkruskalwallish, (ListType, TupleType)), - (akruskalwallish, (N.ndarray,)) ) - friedmanchisquare = Dispatch ( (lfriedmanchisquare, (ListType, TupleType)), - (afriedmanchisquare, (N.ndarray,)) ) - -## PROBABILITY CALCS: - chisqprob = Dispatch ( (lchisqprob, (IntType, FloatType)), - (achisqprob, (N.ndarray,)) ) - zprob = Dispatch ( (lzprob, (IntType, FloatType)), - (azprob, (N.ndarray,)) ) - ksprob = Dispatch ( (lksprob, (IntType, FloatType)), - (aksprob, (N.ndarray,)) ) - fprob = Dispatch ( (lfprob, (IntType, FloatType)), - (afprob, (N.ndarray,)) ) - betacf = Dispatch ( (lbetacf, (IntType, FloatType)), - (abetacf, (N.ndarray,)) ) - betai = Dispatch ( (lbetai, (IntType, FloatType)), - (abetai, (N.ndarray,)) ) - erfcc = Dispatch ( (lerfcc, (IntType, FloatType)), - (aerfcc, (N.ndarray,)) ) - gammln = Dispatch ( (lgammln, (IntType, FloatType)), - (agammln, (N.ndarray,)) ) - -## ANOVA FUNCTIONS: - F_oneway = Dispatch ( (lF_oneway, (ListType, TupleType)), - (aF_oneway, (N.ndarray,)) ) - F_value = Dispatch ( (lF_value, (ListType, TupleType)), - (aF_value, (N.ndarray,)) ) - -## SUPPORT FUNCTIONS: - incr = Dispatch ( (lincr, (ListType, TupleType, N.ndarray)), ) - sum = Dispatch ( (lsum, (ListType, TupleType)), - (asum, (N.ndarray,)) ) - cumsum = Dispatch ( (lcumsum, (ListType, TupleType)), - (acumsum, (N.ndarray,)) ) - ss = Dispatch ( (lss, (ListType, TupleType)), - (ass, (N.ndarray,)) ) - summult = Dispatch ( (lsummult, (ListType, TupleType)), - (asummult, (N.ndarray,)) ) - square_of_sums = Dispatch ( (lsquare_of_sums, (ListType, TupleType)), - (asquare_of_sums, (N.ndarray,)) ) - sumdiffsquared = Dispatch ( (lsumdiffsquared, (ListType, TupleType)), - (asumdiffsquared, (N.ndarray,)) ) - shellsort = Dispatch ( (lshellsort, (ListType, TupleType)), - (ashellsort, (N.ndarray,)) ) - rankdata = Dispatch ( (lrankdata, (ListType, TupleType)), - (arankdata, (N.ndarray,)) ) - findwithin = Dispatch ( (lfindwithin, (ListType, TupleType)), - (afindwithin, (N.ndarray,)) ) - -###################### END OF NUMERIC FUNCTION BLOCK ##################### - -###################### END OF STATISTICAL FUNCTIONS ###################### - -except ImportError: - pass diff --git a/v14/utils/tabulator.py b/v14/utils/tabulator.py deleted file mode 100644 index f75419ab..00000000 --- a/v14/utils/tabulator.py +++ /dev/null @@ -1,1125 +0,0 @@ -#!/usr/bin/python - -# Copyright 2011 Google Inc. All Rights Reserved. - -"""Table generating, analyzing and printing functions. - -This defines several classes that are used to generate, analyze and print -tables. - -Example usage: - - from utils import tabulator - - data = [["benchmark1", "33", "44"],["benchmark2", "44", "33"]] - tabulator.GetSimpleTable(data) - -You could also use it to generate more complex tables with analysis such as -p-values, custom colors, etc. Tables are generated by TableGenerator and -analyzed/formatted by TableFormatter. TableFormatter can take in a list of -columns with custom result computation and coloring, and will compare values in -each row according to taht scheme. Here is a complex example on printing a -table: - - from utils import tabulator - - runs = [[{"k1": "10", "k2": "12", "k5": "40", "k6": "40", - "ms_1": "20", "k7": "FAIL", "k8": "PASS", "k9": "PASS", - "k10": "0"}, - {"k1": "13", "k2": "14", "k3": "15", "ms_1": "10", "k8": "PASS", - "k9": "FAIL", "k10": "0"}], - [{"k1": "50", "k2": "51", "k3": "52", "k4": "53", "k5": "35", "k6": - "45", "ms_1": "200", "ms_2": "20", "k7": "FAIL", "k8": "PASS", "k9": - "PASS"}]] - labels = ["vanilla", "modified"] - tg = TableGenerator(runs, labels, TableGenerator.SORT_BY_VALUES_DESC) - table = tg.GetTable() - columns = [Column(LiteralResult(), - Format(), - "Literal"), - Column(AmeanResult(), - Format()), - Column(StdResult(), - Format()), - Column(CoeffVarResult(), - CoeffVarFormat()), - Column(NonEmptyCountResult(), - Format()), - Column(AmeanRatioResult(), - PercentFormat()), - Column(AmeanRatioResult(), - RatioFormat()), - Column(GmeanRatioResult(), - RatioFormat()), - Column(PValueResult(), - PValueFormat()), - ] - tf = TableFormatter(table, columns) - cell_table = tf.GetCellTable() - tp = TablePrinter(cell_table, out_to) - print tp.Print() - -""" - - -import getpass -import math -import sys -import numpy - -import colortrans -from email_sender import EmailSender -import misc - - -def _AllFloat(values): - return all([misc.IsFloat(v) for v in values]) - - -def _GetFloats(values): - return [float(v) for v in values] - - -def _StripNone(results): - res = [] - for result in results: - if result is not None: - res.append(result) - return res - - -class TableGenerator(object): - """Creates a table from a list of list of dicts. - - The main public function is called GetTable(). - """ - SORT_BY_KEYS = 0 - SORT_BY_KEYS_DESC = 1 - SORT_BY_VALUES = 2 - SORT_BY_VALUES_DESC = 3 - - MISSING_VALUE = "x" - - def __init__(self, d, l, sort=SORT_BY_KEYS, key_name="keys"): - self._runs = d - self._labels = l - self._sort = sort - self._key_name = key_name - - def _AggregateKeys(self): - keys = set([]) - for run_list in self._runs: - for run in run_list: - keys = keys.union(run.keys()) - return keys - - def _GetHighestValue(self, key): - values = [] - for run_list in self._runs: - for run in run_list: - if key in run: - values.append(run[key]) - values = _StripNone(values) - if _AllFloat(values): - values = _GetFloats(values) - return max(values) - - def _GetLowestValue(self, key): - values = [] - for run_list in self._runs: - for run in run_list: - if key in run: - values.append(run[key]) - values = _StripNone(values) - if _AllFloat(values): - values = _GetFloats(values) - return min(values) - - def _SortKeys(self, keys): - if self._sort == self.SORT_BY_KEYS: - return sorted(keys) - elif self._sort == self.SORT_BY_VALUES: - # pylint: disable=unnecessary-lambda - return sorted(keys, key=lambda x: self._GetLowestValue(x)) - elif self._sort == self.SORT_BY_VALUES_DESC: - # pylint: disable=unnecessary-lambda - return sorted(keys, key=lambda x: self._GetHighestValue(x), reverse=True) - else: - assert 0, "Unimplemented sort %s" % self._sort - - def _GetKeys(self): - keys = self._AggregateKeys() - return self._SortKeys(keys) - - def GetTable(self, number_of_rows=sys.maxint): - """Returns a table from a list of list of dicts. - - The list of list of dicts is passed into the constructor of TableGenerator. - This method converts that into a canonical list of lists which represents a - table of values. - - Args: - number_of_rows: Maximum number of rows to return from the table. - Returns: - A list of lists which is the table. - - Example: - We have the following runs: - [[{"k1": "v1", "k2": "v2"}, {"k1": "v3"}], - [{"k1": "v4", "k4": "v5"}]] - and the following labels: - ["vanilla", "modified"] - it will return: - [["Key", "vanilla", "modified"] - ["k1", ["v1", "v3"], ["v4"]] - ["k2", ["v2"], []] - ["k4", [], ["v5"]]] - The returned table can then be processed further by other classes in this - module. - """ - keys = self._GetKeys() - header = [self._key_name] + self._labels - table = [header] - rows = 0 - for k in keys: - row = [k] - for run_list in self._runs: - v = [] - for run in run_list: - if k in run: - v.append(run[k]) - else: - v.append(None) - row.append(v) - table.append(row) - rows += 1 - if rows == number_of_rows: - break - return table - - -class Result(object): - """A class that respresents a single result. - - This single result is obtained by condensing the information from a list of - runs and a list of baseline runs. - """ - - def __init__(self): - pass - - def _AllStringsSame(self, values): - values_set = set(values) - return len(values_set) == 1 - - def NeedsBaseline(self): - return False - - # pylint: disable=unused-argument - def _Literal(self, cell, values, baseline_values): - cell.value = " ".join([str(v) for v in values]) - - def _ComputeFloat(self, cell, values, baseline_values): - self._Literal(cell, values, baseline_values) - - def _ComputeString(self, cell, values, baseline_values): - self._Literal(cell, values, baseline_values) - - def _InvertIfLowerIsBetter(self, cell): - pass - - def _GetGmean(self, values): - if not values: - return float("nan") - if any([v < 0 for v in values]): - return float ("nan") - if any([v == 0 for v in values]): - return 0.0 - log_list = [math.log(v) for v in values] - gmean_log = sum(log_list)/len(log_list) - return math.exp(gmean_log) - - def Compute(self, cell, values, baseline_values): - """Compute the result given a list of values and baseline values. - - Args: - cell: A cell data structure to populate. - values: List of values. - baseline_values: List of baseline values. Can be none if this is the - baseline itself. - """ - all_floats = True - values = _StripNone(values) - if not values: - cell.value = "" - return - if _AllFloat(values): - float_values = _GetFloats(values) - else: - all_floats = False - if baseline_values: - baseline_values = _StripNone(baseline_values) - if baseline_values: - if _AllFloat(baseline_values): - float_baseline_values = _GetFloats(baseline_values) - else: - all_floats = False - else: - if self.NeedsBaseline(): - cell.value = "" - return - float_baseline_values = None - if all_floats: - self._ComputeFloat(cell, float_values, float_baseline_values) - self._InvertIfLowerIsBetter(cell) - else: - self._ComputeString(cell, values, baseline_values) - - -class LiteralResult(Result): - def __init__(self, iteration=0): - super(LiteralResult, self).__init__() - self.iteration = iteration - - def Compute(self, cell, values, baseline_values): - try: - cell.value = values[self.iteration] - except IndexError: - cell.value = "-" - - -class NonEmptyCountResult(Result): - """A class that counts the number of non-empty results. - - The number of non-empty values will be stored in the cell. - """ - - def Compute(self, cell, values, baseline_values): - """Put the number of non-empty values in the cell result. - - Args: - cell: Put the result in cell.value. - values: A list of values for the row. - baseline_values: A list of baseline values for the row. - """ - cell.value = len(_StripNone(values)) - if not baseline_values: - return - base_value = len(_StripNone(baseline_values)) - if cell.value == base_value: - return - f = ColorBoxFormat() - len_values = len(values) - len_baseline_values = len(baseline_values) - tmp_cell = Cell() - tmp_cell.value = 1.0 + (float(cell.value - base_value) / - (max(len_values, len_baseline_values))) - f.Compute(tmp_cell) - cell.bgcolor = tmp_cell.bgcolor - - -class StringMeanResult(Result): - def _ComputeString(self, cell, values, baseline_values): - if self._AllStringsSame(values): - cell.value = str(values[0]) - else: - cell.value = "?" - - -class AmeanResult(StringMeanResult): - def _ComputeFloat(self, cell, values, baseline_values): - cell.value = numpy.mean(values) - - -class RawResult(Result): - pass - - -class MinResult(Result): - def _ComputeFloat(self, cell, values, baseline_values): - cell.value = min(values) - - def _ComputeString(self, cell, values, baseline_values): - if values: - cell.value = min(values) - else: - cell.value = "" - - -class MaxResult(Result): - def _ComputeFloat(self, cell, values, baseline_values): - cell.value = max(values) - - def _ComputeString(self, cell, values, baseline_values): - if values: - cell.value = max(values) - else: - cell.value = "" - - -class NumericalResult(Result): - def _ComputeString(self, cell, values, baseline_values): - cell.value = "?" - - -class StdResult(NumericalResult): - def _ComputeFloat(self, cell, values, baseline_values): - cell.value = numpy.std(values) - - -class CoeffVarResult(NumericalResult): - def _ComputeFloat(self, cell, values, baseline_values): - noise = numpy.abs(numpy.std(values)/numpy.mean(values)) - cell.value = noise - - -class ComparisonResult(Result): - def NeedsBaseline(self): - return True - - def _ComputeString(self, cell, values, baseline_values): - value = None - baseline_value = None - if self._AllStringsSame(values): - value = values[0] - if self._AllStringsSame(baseline_values): - baseline_value = baseline_values[0] - if value is not None and baseline_value is not None: - if value == baseline_value: - cell.value = "SAME" - else: - cell.value = "DIFFERENT" - else: - cell.value = "?" - - -class PValueResult(ComparisonResult): - def _ComputeFloat(self, cell, values, baseline_values): - if len(values) < 2 or len(baseline_values) < 2: - cell.value = float("nan") - return - import stats # pylint: disable=g-import-not-at-top - _, cell.value = stats.lttest_ind(values, baseline_values) - - def _ComputeString(self, cell, values, baseline_values): - return float("nan") - - -class KeyAwareComparisonResult(ComparisonResult): - def _IsLowerBetter(self, key): - lower_is_better_keys = ["milliseconds", "ms", "seconds", "KB", - "rdbytes", "wrbytes"] - return any([key.startswith(l + "_") for l in lower_is_better_keys]) - - def _InvertIfLowerIsBetter(self, cell): - if self._IsLowerBetter(cell.name): - if cell.value: - cell.value = 1.0/cell.value - - -class AmeanRatioResult(KeyAwareComparisonResult): - def _ComputeFloat(self, cell, values, baseline_values): - if numpy.mean(baseline_values) != 0: - cell.value = numpy.mean(values)/numpy.mean(baseline_values) - elif numpy.mean(values) != 0: - cell.value = 0.00 - # cell.value = 0 means the values and baseline_values have big difference - else: - cell.value = 1.00 - # no difference if both values and baseline_values are 0 - - -class GmeanRatioResult(KeyAwareComparisonResult): - def _ComputeFloat(self, cell, values, baseline_values): - if self._GetGmean(baseline_values) != 0: - cell.value = self._GetGmean(values)/self._GetGmean(baseline_values) - elif self._GetGmean(values) != 0: - cell.value = 0.00 - else: - cell.value = 1.00 - - -class Color(object): - """Class that represents color in RGBA format.""" - - def __init__(self, r=0, g=0, b=0, a=0): - self.r = r - self.g = g - self.b = b - self.a = a - - def __str__(self): - return "r: %s g: %s: b: %s: a: %s" % (self.r, self.g, self.b, self.a) - - def Round(self): - """Round RGBA values to the nearest integer.""" - self.r = int(self.r) - self.g = int(self.g) - self.b = int(self.b) - self.a = int(self.a) - - def GetRGB(self): - """Get a hex representation of the color.""" - return "%02x%02x%02x" % (self.r, self.g, self.b) - - @classmethod - def Lerp(cls, ratio, a, b): - """Perform linear interpolation between two colors. - - Args: - ratio: The ratio to use for linear polation. - a: The first color object (used when ratio is 0). - b: The second color object (used when ratio is 1). - - Returns: - Linearly interpolated color. - """ - ret = cls() - ret.r = (b.r - a.r)*ratio + a.r - ret.g = (b.g - a.g)*ratio + a.g - ret.b = (b.b - a.b)*ratio + a.b - ret.a = (b.a - a.a)*ratio + a.a - return ret - - -class Format(object): - """A class that represents the format of a column.""" - - def __init__(self): - pass - - def Compute(self, cell): - """Computes the attributes of a cell based on its value. - - Attributes typically are color, width, etc. - - Args: - cell: The cell whose attributes are to be populated. - """ - if cell.value is None: - cell.string_value = "" - if isinstance(cell.value, float): - self._ComputeFloat(cell) - else: - self._ComputeString(cell) - - def _ComputeFloat(self, cell): - cell.string_value = "{0:.2f}".format(cell.value) - - def _ComputeString(self, cell): - cell.string_value = str(cell.value) - - def _GetColor(self, value, low, mid, high, power=6, mid_value=1.0): - min_value = 0.0 - max_value = 2.0 - if math.isnan(value): - return mid - if value > mid_value: - value = max_value - mid_value/value - - return self._GetColorBetweenRange(value, min_value, mid_value, max_value, - low, mid, high, power) - - def _GetColorBetweenRange(self, - value, - min_value, mid_value, max_value, - low_color, mid_color, high_color, - power): - assert value <= max_value - assert value >= min_value - if value > mid_value: - value = (max_value - value)/(max_value - mid_value) - value **= power - ret = Color.Lerp(value, high_color, mid_color) - else: - value = (value - min_value)/(mid_value - min_value) - value **= power - ret = Color.Lerp(value, low_color, mid_color) - ret.Round() - return ret - - -class PValueFormat(Format): - def _ComputeFloat(self, cell): - cell.string_value = "%0.2f" % float(cell.value) - if float(cell.value) < 0.05: - cell.bgcolor = self._GetColor(cell.value, - Color(255, 255, 0, 0), - Color(255, 255, 255, 0), - Color(255, 255, 255, 0), - mid_value=0.05, - power=1) - - -class StorageFormat(Format): - """Format the cell as a storage number. - - Example: - If the cell contains a value of 1024, the string_value will be 1.0K. - """ - - def _ComputeFloat(self, cell): - base = 1024 - suffices = ["K", "M", "G"] - v = float(cell.value) - current = 0 - while v >= base**(current + 1) and current < len(suffices): - current += 1 - - if current: - divisor = base**current - cell.string_value = "%1.1f%s" % ((v/divisor), suffices[current - 1]) - else: - cell.string_value = str(cell.value) - - -class CoeffVarFormat(Format): - """Format the cell as a percent. - - Example: - If the cell contains a value of 1.5, the string_value will be +150%. - """ - - def _ComputeFloat(self, cell): - cell.string_value = "%1.1f%%" % (float(cell.value) * 100) - cell.color = self._GetColor(cell.value, - Color(0, 255, 0, 0), - Color(0, 0, 0, 0), - Color(255, 0, 0, 0), - mid_value=0.02, - power=1) - - -class PercentFormat(Format): - """Format the cell as a percent. - - Example: - If the cell contains a value of 1.5, the string_value will be +50%. - """ - - def _ComputeFloat(self, cell): - cell.string_value = "%+1.1f%%" % ((float(cell.value) - 1) * 100) - cell.color = self._GetColor(cell.value, - Color(255, 0, 0, 0), - Color(0, 0, 0, 0), - Color(0, 255, 0, 0)) - - -class RatioFormat(Format): - """Format the cell as a ratio. - - Example: - If the cell contains a value of 1.5642, the string_value will be 1.56. - """ - - def _ComputeFloat(self, cell): - cell.string_value = "%+1.1f%%" % ((cell.value - 1) * 100) - cell.color = self._GetColor(cell.value, - Color(255, 0, 0, 0), - Color(0, 0, 0, 0), - Color(0, 255, 0, 0)) - - -class ColorBoxFormat(Format): - """Format the cell as a color box. - - Example: - If the cell contains a value of 1.5, it will get a green color. - If the cell contains a value of 0.5, it will get a red color. - The intensity of the green/red will be determined by how much above or below - 1.0 the value is. - """ - - def _ComputeFloat(self, cell): - cell.string_value = "--" - bgcolor = self._GetColor(cell.value, - Color(255, 0, 0, 0), - Color(255, 255, 255, 0), - Color(0, 255, 0, 0)) - cell.bgcolor = bgcolor - cell.color = bgcolor - - -class Cell(object): - """A class to represent a cell in a table. - - Attributes: - value: The raw value of the cell. - color: The color of the cell. - bgcolor: The background color of the cell. - string_value: The string value of the cell. - suffix: A string suffix to be attached to the value when displaying. - prefix: A string prefix to be attached to the value when displaying. - color_row: Indicates whether the whole row is to inherit this cell's color. - bgcolor_row: Indicates whether the whole row is to inherit this cell's - bgcolor. - width: Optional specifier to make a column narrower than the usual width. - The usual width of a column is the max of all its cells widths. - colspan: Set the colspan of the cell in the HTML table, this is used for - table headers. Default value is 1. - name: the test name of the cell. - header: Whether this is a header in html. - """ - - def __init__(self): - self.value = None - self.color = None - self.bgcolor = None - self.string_value = None - self.suffix = None - self.prefix = None - # Entire row inherits this color. - self.color_row = False - self.bgcolor_row = False - self.width = None - self.colspan = 1 - self.name = None - self.header = False - - def __str__(self): - l = [] - l.append("value: %s" % self.value) - l.append("string_value: %s" % self.string_value) - return " ".join(l) - - -class Column(object): - """Class representing a column in a table. - - Attributes: - result: an object of the Result class. - fmt: an object of the Format class. - """ - - def __init__(self, result, fmt, name=""): - self.result = result - self.fmt = fmt - self.name = name - - -# Takes in: -# ["Key", "Label1", "Label2"] -# ["k", ["v", "v2"], [v3]] -# etc. -# Also takes in a format string. -# Returns a table like: -# ["Key", "Label1", "Label2"] -# ["k", avg("v", "v2"), stddev("v", "v2"), etc.]] -# according to format string -class TableFormatter(object): - """Class to convert a plain table into a cell-table. - - This class takes in a table generated by TableGenerator and a list of column - formats to apply to the table and returns a table of cells. - """ - - def __init__(self, table, columns): - """The constructor takes in a table and a list of columns. - - Args: - table: A list of lists of values. - columns: A list of column containing what to produce and how to format it. - """ - self._table = table - self._columns = columns - self._table_columns = [] - self._out_table = [] - - def GenerateCellTable(self): - row_index = 0 - - for row in self._table[1:]: - key = Cell() - key.string_value = str(row[0]) - out_row = [key] - baseline = None - for values in row[1:]: - for column in self._columns: - cell = Cell() - cell.name = key.string_value - if column.result.NeedsBaseline(): - if baseline is not None: - column.result.Compute(cell, values, baseline) - column.fmt.Compute(cell) - out_row.append(cell) - if not row_index: - self._table_columns.append(column) - else: - column.result.Compute(cell, values, baseline) - column.fmt.Compute(cell) - out_row.append(cell) - if not row_index: - self._table_columns.append(column) - - if baseline is None: - baseline = values - self._out_table.append(out_row) - row_index += 1 - - def AddColumnName(self): - """Generate Column name at the top of table.""" - key = Cell() - key.header = True - key.string_value = "Keys" - header = [key] - for column in self._table_columns: - cell = Cell() - cell.header = True - if column.name: - cell.string_value = column.name - else: - result_name = column.result.__class__.__name__ - format_name = column.fmt.__class__.__name__ - - cell.string_value = "%s %s" % (result_name.replace("Result", ""), - format_name.replace("Format", "")) - - header.append(cell) - - self._out_table = [header] + self._out_table - - def AddHeader(self, s): - """Put additional string on the top of the table.""" - cell = Cell() - cell.header = True - cell.string_value = str(s) - header = [cell] - colspan = max(1, max(len(row) for row in self._table)) - cell.colspan = colspan - self._out_table = [header] + self._out_table - - def AddLabelName(self): - """Put label on the top of the table.""" - top_header = [] - base_colspan = len([c for c in self._columns - if not c.result.NeedsBaseline()]) - compare_colspan = len(self._columns) - # The label is organized as follows - # "keys" label_base, label_comparison1, label_comparison2 - # The first cell has colspan 1, the second is base_colspan - # The others are compare_colspan - for label in self._table[0]: - cell = Cell() - cell.header = True - cell.string_value = str(label) - if top_header: - cell.colspan = base_colspan - if len(top_header) > 1: - cell.colspan = compare_colspan - top_header.append(cell) - self._out_table = [top_header] + self._out_table - - def _PrintOutTable(self): - o = "" - for row in self._out_table: - for cell in row: - o += str(cell) + " " - o += "\n" - print o - - def GetCellTable(self, headers=True): - """Function to return a table of cells. - - The table (list of lists) is converted into a table of cells by this - function. - Args: - headers: A boolean saying whether we want default headers - - Returns: - A table of cells with each cell having the properties and string values as - requiested by the columns passed in the constructor. - """ - # Generate the cell table, creating a list of dynamic columns on the fly. - if not self._out_table: - self.GenerateCellTable() - if headers: - self.AddColumnName() - self.AddLabelName() - return self._out_table - - -class TablePrinter(object): - """Class to print a cell table to the console, file or html.""" - PLAIN = 0 - CONSOLE = 1 - HTML = 2 - TSV = 3 - EMAIL = 4 - - def __init__(self, table, output_type): - """Constructor that stores the cell table and output type.""" - self._table = table - self._output_type = output_type - - # Compute whole-table properties like max-size, etc. - def _ComputeStyle(self): - self._row_styles = [] - for row in self._table: - row_style = Cell() - for cell in row: - if cell.color_row: - assert cell.color, "Cell color not set but color_row set!" - assert not row_style.color, "Multiple row_style.colors found!" - row_style.color = cell.color - if cell.bgcolor_row: - assert cell.bgcolor, "Cell bgcolor not set but bgcolor_row set!" - assert not row_style.bgcolor, "Multiple row_style.bgcolors found!" - row_style.bgcolor = cell.bgcolor - self._row_styles.append(row_style) - - self._column_styles = [] - if len(self._table) < 2: - return - - for i in range(max(len(row) for row in self._table)): - column_style = Cell() - for row in self._table: - if not any([cell.colspan != 1 for cell in row]): - column_style.width = max(column_style.width, - len(row[i].string_value)) - self._column_styles.append(column_style) - - def _GetBGColorFix(self, color): - if self._output_type == self.CONSOLE: - rgb = color.GetRGB() - prefix, _ = colortrans.rgb2short(rgb) - # pylint: disable=anomalous-backslash-in-string - prefix = "\033[48;5;%sm" % prefix - suffix = "\033[0m" - elif self._output_type in [self.EMAIL, self.HTML]: - rgb = color.GetRGB() - prefix = ("<FONT style=\"BACKGROUND-COLOR:#{0}\">" - .format(rgb)) - suffix = "</FONT>" - elif self._output_type in [self.PLAIN, self.TSV]: - prefix = "" - suffix = "" - return prefix, suffix - - def _GetColorFix(self, color): - if self._output_type == self.CONSOLE: - rgb = color.GetRGB() - prefix, _ = colortrans.rgb2short(rgb) - # pylint: disable=anomalous-backslash-in-string - prefix = "\033[38;5;%sm" % prefix - suffix = "\033[0m" - elif self._output_type in [self.EMAIL, self.HTML]: - rgb = color.GetRGB() - prefix = "<FONT COLOR=#{0}>".format(rgb) - suffix = "</FONT>" - elif self._output_type in [self.PLAIN, self.TSV]: - prefix = "" - suffix = "" - return prefix, suffix - - def Print(self): - """Print the table to a console, html, etc. - - Returns: - A string that contains the desired representation of the table. - """ - self._ComputeStyle() - return self._GetStringValue() - - def _GetCellValue(self, i, j): - cell = self._table[i][j] - out = cell.string_value - raw_width = len(out) - - if cell.color: - p, s = self._GetColorFix(cell.color) - out = "%s%s%s" % (p, out, s) - - if cell.bgcolor: - p, s = self._GetBGColorFix(cell.bgcolor) - out = "%s%s%s" % (p, out, s) - - if self._output_type in [self.PLAIN, self.CONSOLE, self.EMAIL]: - if cell.width: - width = cell.width - else: - if self._column_styles: - width = self._column_styles[j].width - else: - width = len(cell.string_value) - if cell.colspan > 1: - width = 0 - start = 0 - for k in range(j): - start += self._table[i][k].colspan - for k in range(cell.colspan): - width += self._column_styles[start + k].width - if width > raw_width: - padding = ("%" + str(width - raw_width) + "s") % "" - out = padding + out - - if self._output_type == self.HTML: - if cell.header: - tag = "th" - else: - tag = "td" - out = "<{0} colspan = \"{2}\"> {1} </{0}>".format(tag, out, cell.colspan) - - return out - - def _GetHorizontalSeparator(self): - if self._output_type in [self.CONSOLE, self.PLAIN, self.EMAIL]: - return " " - if self._output_type == self.HTML: - return "" - if self._output_type == self.TSV: - return "\t" - - def _GetVerticalSeparator(self): - if self._output_type in [self.PLAIN, self.CONSOLE, self.TSV, self.EMAIL]: - return "\n" - if self._output_type == self.HTML: - return "</tr>\n<tr>" - - def _GetPrefix(self): - if self._output_type in [self.PLAIN, self.CONSOLE, self.TSV, self.EMAIL]: - return "" - if self._output_type == self.HTML: - return "<p></p><table id=\"box-table-a\">\n<tr>" - - def _GetSuffix(self): - if self._output_type in [self.PLAIN, self.CONSOLE, self.TSV, self.EMAIL]: - return "" - if self._output_type == self.HTML: - return "</tr>\n</table>" - - def _GetStringValue(self): - o = "" - o += self._GetPrefix() - for i in range(len(self._table)): - row = self._table[i] - # Apply row color and bgcolor. - p = s = bgp = bgs = "" - if self._row_styles[i].bgcolor: - bgp, bgs = self._GetBGColorFix(self._row_styles[i].bgcolor) - if self._row_styles[i].color: - p, s = self._GetColorFix(self._row_styles[i].color) - o += p + bgp - for j in range(len(row)): - out = self._GetCellValue(i, j) - o += out + self._GetHorizontalSeparator() - o += s + bgs - o += self._GetVerticalSeparator() - o += self._GetSuffix() - return o - - -# Some common drivers -def GetSimpleTable(table, out_to=TablePrinter.CONSOLE): - """Prints a simple table. - - This is used by code that has a very simple list-of-lists and wants to produce - a table with ameans, a percentage ratio of ameans and a colorbox. - - Args: - table: a list of lists. - out_to: specify the fomat of output. Currently it supports HTML and CONSOLE. - - Returns: - A string version of the table that can be printed to the console. - - Example: - GetSimpleConsoleTable([["binary", "b1", "b2"],["size", "300", "400"]]) - will produce a colored table that can be printed to the console. - """ - columns = [ - Column(AmeanResult(), - Format()), - Column(AmeanRatioResult(), - PercentFormat()), - Column(AmeanRatioResult(), - ColorBoxFormat()), - ] - our_table = [table[0]] - for row in table[1:]: - our_row = [row[0]] - for v in row[1:]: - our_row.append([v]) - our_table.append(our_row) - - tf = TableFormatter(our_table, columns) - cell_table = tf.GetCellTable() - tp = TablePrinter(cell_table, out_to) - return tp.Print() - - -# pylint: disable=redefined-outer-name -def GetComplexTable(runs, labels, out_to=TablePrinter.CONSOLE): - """Prints a complex table. - - This can be used to generate a table with arithmetic mean, standard deviation, - coefficient of variation, p-values, etc. - - Args: - runs: A list of lists with data to tabulate. - labels: A list of labels that correspond to the runs. - out_to: specifies the format of the table (example CONSOLE or HTML). - - Returns: - A string table that can be printed to the console or put in an HTML file. - """ - tg = TableGenerator(runs, labels, TableGenerator.SORT_BY_VALUES_DESC) - table = tg.GetTable() - columns = [Column(LiteralResult(), - Format(), - "Literal"), - Column(AmeanResult(), - Format()), - Column(StdResult(), - Format()), - Column(CoeffVarResult(), - CoeffVarFormat()), - Column(NonEmptyCountResult(), - Format()), - Column(AmeanRatioResult(), - PercentFormat()), - Column(AmeanRatioResult(), - RatioFormat()), - Column(GmeanRatioResult(), - RatioFormat()), - Column(PValueResult(), - PValueFormat()), - ] - tf = TableFormatter(table, columns) - cell_table = tf.GetCellTable() - tp = TablePrinter(cell_table, out_to) - return tp.Print() - -if __name__ == "__main__": - # Run a few small tests here. - runs = [[{"k1": "10", "k2": "12", "k5": "40", "k6": "40", - "ms_1": "20", "k7": "FAIL", "k8": "PASS", "k9": "PASS", - "k10": "0"}, - {"k1": "13", "k2": "14", "k3": "15", "ms_1": "10", "k8": "PASS", - "k9": "FAIL", "k10": "0"}], - [{"k1": "50", "k2": "51", "k3": "52", "k4": "53", "k5": "35", "k6": - "45", "ms_1": "200", "ms_2": "20", "k7": "FAIL", "k8": "PASS", "k9": - "PASS"}]] - labels = ["vanilla", "modified"] - t = GetComplexTable(runs, labels, TablePrinter.CONSOLE) - print t - email = GetComplexTable(runs, labels, TablePrinter.EMAIL) - - runs = [[{"k1": "1"}, {"k1": "1.1"}, {"k1": "1.2"}], - [{"k1": "5"}, {"k1": "5.1"}, {"k1": "5.2"}]] - t = GetComplexTable(runs, labels, TablePrinter.CONSOLE) - print t - - simple_table = [ - ["binary", "b1", "b2", "b3"], - ["size", 100, 105, 108], - ["rodata", 100, 80, 70], - ["data", 100, 100, 100], - ["debug", 100, 140, 60], - ] - t = GetSimpleTable(simple_table) - print t - email += GetSimpleTable(simple_table, TablePrinter.HTML) - email_to = [getpass.getuser()] - email = "<pre style='font-size: 13px'>%s</pre>" % email - EmailSender().SendEmail(email_to, "SimpleTableTest", email, msg_type="html") diff --git a/v14/utils/tabulator_test.py b/v14/utils/tabulator_test.py deleted file mode 100644 index 16406107..00000000 --- a/v14/utils/tabulator_test.py +++ /dev/null @@ -1,135 +0,0 @@ -# Copyright 2012 Google Inc. All Rights Reserved. - -"""Tests for the tabulator module.""" - -__author__ = "asharif@google.com (Ahmad Sharif)" - -# System modules -import unittest - -# Local modules -import tabulator - - -class TabulatorTest(unittest.TestCase): - def testResult(self): - table = ["k1", ["1", "3"], ["55"]] - result = tabulator.Result() - cell = tabulator.Cell() - result.Compute(cell, table[2], table[1]) - expected = " ".join([str(float(v)) for v in table[2]]) - self.assertTrue(cell.value == expected) - - result = tabulator.AmeanResult() - cell = tabulator.Cell() - result.Compute(cell, table[2], table[1]) - self.assertTrue(cell.value == float(table[2][0])) - - def testStringMean(self): - smr = tabulator.StringMeanResult() - cell = tabulator.Cell() - value = "PASS" - values = [value for _ in range(3)] - smr.Compute(cell, values, None) - self.assertTrue(cell.value == value) - values.append("FAIL") - smr.Compute(cell, values, None) - self.assertTrue(cell.value == "?") - - def testStorageFormat(self): - sf = tabulator.StorageFormat() - cell = tabulator.Cell() - base = 1024.0 - cell.value = base - sf.Compute(cell) - self.assertTrue(cell.string_value == "1.0K") - cell.value = base**2 - sf.Compute(cell) - self.assertTrue(cell.string_value == "1.0M") - cell.value = base**3 - sf.Compute(cell) - self.assertTrue(cell.string_value == "1.0G") - - def testLerp(self): - c1 = tabulator.Color(0, 0, 0, 0) - c2 = tabulator.Color(255, 0, 0, 0) - c3 = tabulator.Color.Lerp(0.5, c1, c2) - self.assertTrue(c3.r == 127.5) - self.assertTrue(c3.g == 0) - self.assertTrue(c3.b == 0) - self.assertTrue(c3.a == 0) - c3.Round() - self.assertTrue(c3.r == 127) - - def testGmean(self): - a = [1.0e+308] * 3 - b = tabulator.Result()._GetGmean(a) - self.assertTrue(b >= 0.99e+308 and b <= 1.01e+308) - - def testTableGenerator(self): - runs = [[{"k1": "10", "k2": "12"}, - {"k1": "13", "k2": "14", "k3": "15"}], - [{"k1": "50", "k2": "51", "k3": "52", "k4": "53"}]] - labels = ["vanilla", "modified"] - tg = tabulator.TableGenerator(runs, labels) - table = tg.GetTable() - header = table.pop(0) - - self.assertTrue(header == ["keys", "vanilla", "modified"]) - row = table.pop(0) - self.assertTrue(row == ["k1", ["10", "13"], ["50"]]) - row = table.pop(0) - self.assertTrue(row == ["k2", ["12", "14"], ["51"]]) - row = table.pop(0) - self.assertTrue(row == ["k3", [None, "15"], ["52"]]) - row = table.pop(0) - self.assertTrue(row == ["k4", [None, None], ["53"]]) - - table = tg.GetTable() - columns = [ - tabulator.Column(tabulator.AmeanResult(), - tabulator.Format()), - tabulator.Column(tabulator.AmeanRatioResult(), - tabulator.PercentFormat()), - ] - tf = tabulator.TableFormatter(table, columns) - table = tf.GetCellTable() - self.assertTrue(table) - - def testColspan(self): - simple_table = [ - ["binary", "b1", "b2", "b3"], - ["size", 100, 105, 108], - ["rodata", 100, 80, 70], - ["data", 100, 100, 100], - ["debug", 100, 140, 60], - ] - columns = [ - tabulator.Column(tabulator.AmeanResult(), - tabulator.Format()), - tabulator.Column(tabulator.MinResult(), - tabulator.Format()), - tabulator.Column(tabulator.AmeanRatioResult(), - tabulator.PercentFormat()), - tabulator.Column(tabulator.AmeanRatioResult(), - tabulator.ColorBoxFormat()), - ] - our_table = [simple_table[0]] - for row in simple_table[1:]: - our_row = [row[0]] - for v in row[1:]: - our_row.append([v]) - our_table.append(our_row) - - tf = tabulator.TableFormatter(our_table, columns) - cell_table = tf.GetCellTable() - self.assertTrue(cell_table[0][0].colspan == 1) - self.assertTrue(cell_table[0][1].colspan == 2) - self.assertTrue(cell_table[0][2].colspan == 4) - self.assertTrue(cell_table[0][3].colspan == 4) - for row in cell_table[1:]: - for cell in row: - self.assertTrue(cell.colspan == 1) - -if __name__ == "__main__": - unittest.main() diff --git a/v14/utils/timeline.py b/v14/utils/timeline.py deleted file mode 100644 index fa75ba01..00000000 --- a/v14/utils/timeline.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/python2.6 -# -# Copyright 2012 Google Inc. All Rights Reserved. -# - -"""Tools for recording and reporting timeline of benchmark_run.""" - -__author__ = 'yunlian@google.com (Yunlian Jiang)' - -import time - - -class Event(object): - def __init__(self, name='', cur_time=0): - self.name = name - self.timestamp = cur_time - - -class Timeline(object): - """Use a dict to store the timeline.""" - - def __init__(self): - self.events = [] - - def Record(self, event): - for e in self.events: - assert e.name != event, ('The event {0} is already recorded.' - .format(event)) - cur_event = Event(name=event, cur_time=time.time()) - self.events.append(cur_event) - - def GetEvents(self): - return([e.name for e in self.events]) - - def GetEventDict(self): - tl = {} - for e in self.events: - tl[e.name] = e.timestamp - return tl - - def GetEventTime(self, event): - for e in self.events: - if e.name == event: - return e.timestamp - raise IndexError, 'The event {0} is not recorded'.format(event) - - def GetLastEventTime(self): - return self.events[-1].timestamp - - def GetLastEvent(self): - return self.events[-1].name diff --git a/v14/utils/timeline_test.py b/v14/utils/timeline_test.py deleted file mode 100644 index 1f4d178a..00000000 --- a/v14/utils/timeline_test.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2012 Google Inc. All Rights Reserved. - -"""Tests for time_line.py.""" - -__author__ = 'yunlian@google.com (Yunlian Jiang)' - -import time -import unittest - -import timeline - - -class TimeLineTest(unittest.TestCase): - - def testRecord(self): - tl = timeline.Timeline() - tl.Record('A') - t = time.time() - t1 = tl.events[0].timestamp - self.assertEqual(int(t1-t), 0) - self.assertRaises(AssertionError, tl.Record, 'A') - - def testGetEvents(self): - tl = timeline.Timeline() - tl.Record('A') - e = tl.GetEvents() - self.assertEqual(e, ['A']) - tl.Record('B') - e = tl.GetEvents() - self.assertEqual(e, ['A', 'B']) - - def testGetEventTime(self): - tl = timeline.Timeline() - tl.Record('A') - t = time.time() - t1 = tl.GetEventTime('A') - self.assertEqual(int(t1-t), 0) - self.assertRaises(IndexError, tl.GetEventTime, 'B') - - def testGetLastEventTime(self): - tl = timeline.Timeline() - self.assertRaises(IndexError, tl.GetLastEventTime) - tl.Record('A') - t = time.time() - t1 = tl.GetLastEventTime() - self.assertEqual(int(t1-t), 0) - time.sleep(2) - tl.Record('B') - t = time.time() - t1 = tl.GetLastEventTime() - self.assertEqual(int(t1-t), 0) - -if __name__ == '__main__': - unittest.main() |