From fd6b63bf0db9abab9b06ef993e2ed71442639386 Mon Sep 17 00:00:00 2001 From: Tai Kuo Date: Sat, 20 Jun 2020 09:35:31 +0800 Subject: Merge google/walt Merge from https://github.com/google/walt.git Bug: 149721303 Bug: 149721109 Test: ./gradlew build -x :app:lint Change-Id: I56e2f506782fbbe01e3bf2489fa3091226a2b5ca Signed-off-by: Tai Kuo --- android/WALT/app/build.gradle | 67 +- .../latency/walt/AccelerometerFragment.java | 377 +++ .../org/chromium/latency/walt/MainActivity.java | 5 + .../main/java/org/chromium/latency/walt/Utils.java | 32 + .../java/org/chromium/latency/walt/WaltDevice.java | 3 +- .../src/main/res/layout/fragment_accelerometer.xml | 78 + .../src/main/res/layout/fragment_front_page.xml | 25 + android/WALT/app/src/main/res/raw/walt.hex | 2671 ++++++++++---------- .../latency/walt/AccelerometerFragmentTest.java | 60 + .../java/org/chromium/latency/walt/UtilsTest.java | 30 + android/WALT/build.gradle | 4 +- android/WALT/gradle/wrapper/gradle-wrapper.jar | Bin 53637 -> 54329 bytes .../WALT/gradle/wrapper/gradle-wrapper.properties | 3 +- android/WALT/gradlew | 72 +- android/WALT/gradlew.bat | 14 +- 15 files changed, 2033 insertions(+), 1408 deletions(-) create mode 100644 android/WALT/app/src/main/java/org/chromium/latency/walt/AccelerometerFragment.java create mode 100644 android/WALT/app/src/main/res/layout/fragment_accelerometer.xml create mode 100644 android/WALT/app/src/test/java/org/chromium/latency/walt/AccelerometerFragmentTest.java (limited to 'android/WALT') diff --git a/android/WALT/app/build.gradle b/android/WALT/app/build.gradle index 531142e..ec99946 100644 --- a/android/WALT/app/build.gradle +++ b/android/WALT/app/build.gradle @@ -1,49 +1,50 @@ -apply plugin: 'com.android.model.application' +apply plugin: 'com.android.application' -model { - android { - compileSdkVersion 25 - buildToolsVersion "25.0.2" - defaultConfig { - applicationId "org.chromium.latency.walt" - minSdkVersion.apiLevel 17 - targetSdkVersion.apiLevel 23 - versionCode 8 - versionName "0.1.7" - } - ndk { - moduleName "sync_clock_jni" - CFlags.addAll "-I${project.rootDir}/app/src/main/jni".toString(), "-g", "-DUSE_LIBLOG", "-Werror" - ldLibs.addAll "OpenSLES", "log" +android { + compileSdkVersion 27 + buildToolsVersion '27.0.3' + + defaultConfig { + applicationId "org.chromium.latency.walt" + minSdkVersion 17 + targetSdkVersion 23 + versionCode 9 + versionName "0.1.9" + externalNativeBuild.ndkBuild { + arguments "APP_PLATFORM=android-14", "APP_ALLOW_MISSING_DEPS=true" } - buildTypes { - release { + } + + externalNativeBuild.ndkBuild { + path 'src/main/jni/Android.mk' + } + + buildTypes { + release { minifyEnabled false - proguardFiles.add(file("proguard-rules.pro")) - } - debug { - ndk { - debuggable true - } + } + debug { + ndk { + debuggable true } } } } dependencies { - compile 'com.android.support:appcompat-v7:25.1.0' - compile 'com.android.support:design:25.1.0' - compile 'com.android.support:preference-v7:25.1.0' - compile 'com.android.support:preference-v14:25.1.0' - compile 'com.github.PhilJay:MPAndroidChart:v3.0.1' - testCompile 'junit:junit:4.12' - testCompile 'org.mockito:mockito-core:1.10.19' - testCompile ('org.powermock:powermock-api-mockito:1.6.2') { + implementation 'com.android.support:appcompat-v7:27.1.1' + implementation 'com.android.support:design:27.1.1' + implementation 'com.android.support:preference-v7:27.1.1' + implementation 'com.android.support:preference-v14:27.1.1' + implementation 'com.github.PhilJay:MPAndroidChart:v3.0.1' + testImplementation 'junit:junit:4.12' + testImplementation 'org.mockito:mockito-core:1.10.19' + testImplementation ('org.powermock:powermock-api-mockito:1.6.2') { exclude module: 'hamcrest-core' exclude module: 'objenesis' } - testCompile ('org.powermock:powermock-module-junit4:1.6.2') { + testImplementation ('org.powermock:powermock-module-junit4:1.6.2') { exclude module: 'hamcrest-core' exclude module: 'objenesis' } diff --git a/android/WALT/app/src/main/java/org/chromium/latency/walt/AccelerometerFragment.java b/android/WALT/app/src/main/java/org/chromium/latency/walt/AccelerometerFragment.java new file mode 100644 index 0000000..3ed677c --- /dev/null +++ b/android/WALT/app/src/main/java/org/chromium/latency/walt/AccelerometerFragment.java @@ -0,0 +1,377 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.chromium.latency.walt; + +import android.content.BroadcastReceiver; +import android.content.Context; +import android.content.Intent; +import android.graphics.Color; +import android.hardware.Sensor; +import android.hardware.SensorEvent; +import android.hardware.SensorEventListener; +import android.hardware.SensorManager; +import android.os.Bundle; +import android.os.Handler; +import android.os.SystemClock; +import android.support.v4.app.Fragment; +import android.text.method.ScrollingMovementMethod; +import android.view.LayoutInflater; +import android.view.View; +import android.view.ViewGroup; +import android.widget.TextView; +import android.widget.Toast; + +import com.github.mikephil.charting.charts.ScatterChart; +import com.github.mikephil.charting.components.Description; +import com.github.mikephil.charting.data.Entry; +import com.github.mikephil.charting.data.ScatterData; +import com.github.mikephil.charting.data.ScatterDataSet; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.chromium.latency.walt.Utils.argmax; +import static org.chromium.latency.walt.Utils.interp; +import static org.chromium.latency.walt.Utils.max; +import static org.chromium.latency.walt.Utils.mean; +import static org.chromium.latency.walt.Utils.min; + +public class AccelerometerFragment extends Fragment implements + View.OnClickListener, SensorEventListener { + + private static final int MAX_TEST_LENGTH_MS = 10000; + private SimpleLogger logger; + private WaltDevice waltDevice; + private TextView logTextView; + private View startButton; + private ScatterChart latencyChart; + private View latencyChartLayout; + private StringBuilder accelerometerData; + private List phoneAccelerometerData = new ArrayList<>(); + private Handler handler = new Handler(); + private SensorManager sensorManager; + private Sensor accelerometer; + private double realTimeOffsetMs; + private boolean isTestRunning = false; + + Runnable finishAccelerometer = new Runnable() { + @Override + public void run() { + isTestRunning = false; + waltDevice.stopListener(); + waltDevice.clearTriggerHandler(); + calculateAndDrawLatencyChart(accelerometerData.toString()); + startButton.setEnabled(true); + accelerometerData = new StringBuilder(); + LogUploader.uploadIfAutoEnabled(getContext()); + } + }; + + private BroadcastReceiver logReceiver = new BroadcastReceiver() { + @Override + public void onReceive(Context context, Intent intent) { + String msg = intent.getStringExtra("message"); + AccelerometerFragment.this.appendLogText(msg); + } + }; + + private WaltDevice.TriggerHandler triggerHandler = new WaltDevice.TriggerHandler() { + @Override + public void onReceive(WaltDevice.TriggerMessage tmsg) { + logger.log("ERROR: Accelerometer trigger got a trigger message, " + + "this should never happen."); + } + + @Override + public void onReceiveRaw(String s) { + if (s.trim().equals("end")) { + // Remove the delayed callback and run it now + handler.removeCallbacks(finishAccelerometer); + handler.post(finishAccelerometer); + } else { + accelerometerData.append(s); + } + } + }; + + Runnable startAccelerometer = new Runnable() { + @Override + public void run() { + waltDevice.setTriggerHandler(triggerHandler); + try { + waltDevice.command(WaltDevice.CMD_ACCELEROMETER); + } catch (IOException e) { + logger.log("Error sending command CMD_ACCELEROMETER: " + e.getMessage()); + startButton.setEnabled(true); + return; + } + + logger.log("=== Accelerometer Test ===\n"); + isTestRunning = true; + handler.postDelayed(finishAccelerometer, MAX_TEST_LENGTH_MS); + } + }; + + public AccelerometerFragment() { + // Required empty public constructor + } + + static List getEntriesFromString(final String latencyString) { + List entries = new ArrayList<>(); + // "o" marks the start of the accelerometer data + int startIndex = latencyString.indexOf("o") + 1; + + String[] brightnessStrings = + latencyString.substring(startIndex).trim().split("\n"); + for (String str : brightnessStrings) { + String[] arr = str.split(" "); + final float timestampMs = Integer.parseInt(arr[0]) / 1000f; + final float value = Integer.parseInt(arr[1]); + entries.add(new Entry(timestampMs, value)); + } + return entries; + } + + static List smoothEntries(List entries, int windowSize) { + List smoothEntries = new ArrayList<>(); + for (int i = windowSize; i < entries.size() - windowSize; i++) { + final float time = entries.get(i).getX(); + float avg = 0; + for (int j = i - windowSize; j <= i + windowSize; j++) { + avg += entries.get(j).getY() / (2 * windowSize + 1); + } + smoothEntries.add(new Entry(time, avg)); + } + return smoothEntries; + } + + static double[] findShifts(List phoneEntries, List waltEntries) { + double[] phoneTimes = new double[phoneEntries.size()]; + double[] phoneValues = new double[phoneEntries.size()]; + double[] waltTimes = new double[waltEntries.size()]; + double[] waltValues = new double[waltEntries.size()]; + + for (int i = 0; i < phoneTimes.length; i++) { + phoneTimes[i] = phoneEntries.get(i).getX(); + phoneValues[i] = phoneEntries.get(i).getY(); + } + + for (int i = 0; i < waltTimes.length; i++) { + waltTimes[i] = waltEntries.get(i).getX(); + waltValues[i] = waltEntries.get(i).getY(); + } + + double[] shiftCorrelations = new double[401]; + for (int i = 0; i < shiftCorrelations.length; i++) { + double shift = i / 10.; + final double[] shiftedPhoneTimes = new double[phoneTimes.length]; + for (int j = 0; j < phoneTimes.length; j++) { + shiftedPhoneTimes[j] = phoneTimes[j] - shift; + } + final double[] interpolatedValues = interp(shiftedPhoneTimes, waltTimes, waltValues); + double sum = 0; + for (int j = 0; j < shiftedPhoneTimes.length; j++) { + // Calculate square dot product of phone and walt values + sum += Math.pow(phoneValues[j] * interpolatedValues[j], 2); + } + shiftCorrelations[i] = sum; + } + return shiftCorrelations; + } + + @Override + public View onCreateView(LayoutInflater inflater, ViewGroup container, + Bundle savedInstanceState) { + logger = SimpleLogger.getInstance(getContext()); + waltDevice = WaltDevice.getInstance(getContext()); + + // Inflate the layout for this fragment + final View view = inflater.inflate(R.layout.fragment_accelerometer, container, false); + logTextView = (TextView) view.findViewById(R.id.txt_log); + startButton = view.findViewById(R.id.button_start); + latencyChart = (ScatterChart) view.findViewById(R.id.latency_chart); + latencyChartLayout = view.findViewById(R.id.latency_chart_layout); + logTextView.setMovementMethod(new ScrollingMovementMethod()); + view.findViewById(R.id.button_close_chart).setOnClickListener(this); + sensorManager = (SensorManager) getContext().getSystemService(Context.SENSOR_SERVICE); + accelerometer = sensorManager.getDefaultSensor(Sensor.TYPE_ACCELEROMETER); + if (accelerometer == null) { + logger.log("ERROR! Accelerometer sensor not found"); + } + return view; + } + + @Override + public void onResume() { + super.onResume(); + logTextView.setText(logger.getLogText()); + logger.registerReceiver(logReceiver); + startButton.setOnClickListener(this); + sensorManager.registerListener( + AccelerometerFragment.this, accelerometer, SensorManager.SENSOR_DELAY_FASTEST); + } + + @Override + public void onPause() { + logger.unregisterReceiver(logReceiver); + sensorManager.unregisterListener(AccelerometerFragment.this, accelerometer); + super.onPause(); + } + + public void appendLogText(String msg) { + logTextView.append(msg + "\n"); + } + + void startMeasurement() { + logger.log("Starting accelerometer latency measurement"); + try { + accelerometerData = new StringBuilder(); + phoneAccelerometerData.clear(); + waltDevice.syncClock(); + waltDevice.startListener(); + realTimeOffsetMs = + SystemClock.elapsedRealtimeNanos() / 1e6 - waltDevice.clock.micros() / 1e3; + } catch (IOException e) { + logger.log("Error syncing clocks: " + e.getMessage()); + startButton.setEnabled(true); + return; + } + Toast.makeText(getContext(), "Start shaking the phone and WALT!", Toast.LENGTH_LONG).show(); + handler.postDelayed(startAccelerometer, 500); + } + + /** + * Handler for all the button clicks on this screen. + */ + @Override + public void onClick(View v) { + if (v.getId() == R.id.button_start) { + latencyChartLayout.setVisibility(View.GONE); + startButton.setEnabled(false); + startMeasurement(); + return; + } + + if (v.getId() == R.id.button_close_chart) { + latencyChartLayout.setVisibility(View.GONE); + } + } + + private void calculateAndDrawLatencyChart(final String latencyString) { + List phoneEntries = new ArrayList<>(); + List waltEntries = getEntriesFromString(latencyString); + List waltSmoothEntries = smoothEntries(waltEntries, 4); + + for (AccelerometerEvent e : phoneAccelerometerData) { + phoneEntries.add(new Entry(e.callbackTimeMs, e.value)); + } + + while (phoneEntries.get(0).getX() < waltSmoothEntries.get(0).getX()) { + // This event is earlier than any walt event, so discard it + phoneEntries.remove(0); + } + + while (phoneEntries.get(phoneEntries.size() - 1).getX() > + waltSmoothEntries.get(waltSmoothEntries.size() - 1).getX()) { + // This event is later than any walt event, so discard it + phoneEntries.remove(phoneEntries.size() - 1); + } + + // Adjust waltEntries so min and max is the same as phoneEntries + float phoneMean = mean(phoneEntries); + float phoneMax = max(phoneEntries); + float phoneMin = min(phoneEntries); + float waltMin = min(waltSmoothEntries); + float phoneRange = phoneMax - phoneMin; + float waltRange = max(waltSmoothEntries) - waltMin; + for (Entry e : waltSmoothEntries) { + e.setY((e.getY() - waltMin) * (phoneRange / waltRange) + phoneMin - phoneMean); + } + + // Adjust phoneEntries so mean=0 + for (Entry e : phoneEntries) { + e.setY(e.getY() - phoneMean); + } + + double[] shifts = findShifts(phoneEntries, waltSmoothEntries); + double bestShift = argmax(shifts) / 10d; + logger.log(String.format("Accelerometer latency: %.1fms", bestShift)); + + double[] deltasKernelToCallback = new double[phoneAccelerometerData.size()]; + for (int i = 0; i < deltasKernelToCallback.length; i++) { + deltasKernelToCallback[i] = phoneAccelerometerData.get(i).callbackTimeMs - + phoneAccelerometerData.get(i).kernelTimeMs; + } + + logger.log(String.format( + "Mean kernel-to-callback latency: %.1fms", mean(deltasKernelToCallback))); + + List phoneEntriesShifted = new ArrayList<>(); + for (Entry e : phoneEntries) { + phoneEntriesShifted.add(new Entry((float) (e.getX() - bestShift), e.getY())); + } + + drawLatencyChart(phoneEntriesShifted, waltSmoothEntries); + } + + private void drawLatencyChart(List phoneEntriesShifted, List waltEntries) { + final ScatterDataSet dataSetWalt = + new ScatterDataSet(waltEntries, "WALT Events"); + dataSetWalt.setColor(Color.BLUE); + dataSetWalt.setScatterShape(ScatterChart.ScatterShape.CIRCLE); + dataSetWalt.setScatterShapeSize(8f); + + final ScatterDataSet dataSetPhoneShifted = + new ScatterDataSet(phoneEntriesShifted, "Phone Events Shifted"); + dataSetPhoneShifted.setColor(Color.RED); + dataSetPhoneShifted.setScatterShapeSize(10f); + dataSetPhoneShifted.setScatterShape(ScatterChart.ScatterShape.X); + + final ScatterData scatterData = new ScatterData(dataSetWalt, dataSetPhoneShifted); + final Description desc = new Description(); + desc.setText(""); + desc.setTextSize(12f); + latencyChart.setDescription(desc); + latencyChart.setData(scatterData); + latencyChart.invalidate(); + latencyChartLayout.setVisibility(View.VISIBLE); + } + + @Override + public void onSensorChanged(SensorEvent event) { + if (isTestRunning) { + phoneAccelerometerData.add(new AccelerometerEvent(event)); + } + } + + @Override + public void onAccuracyChanged(Sensor sensor, int accuracy) { + } + + private class AccelerometerEvent { + float callbackTimeMs; + float kernelTimeMs; + float value; + + AccelerometerEvent(SensorEvent event) { + callbackTimeMs = waltDevice.clock.micros() / 1e3f; + kernelTimeMs = (float) (event.timestamp / 1e6f - realTimeOffsetMs); + value = event.values[2]; + } + } +} diff --git a/android/WALT/app/src/main/java/org/chromium/latency/walt/MainActivity.java b/android/WALT/app/src/main/java/org/chromium/latency/walt/MainActivity.java index ac1df47..e0d0d75 100644 --- a/android/WALT/app/src/main/java/org/chromium/latency/walt/MainActivity.java +++ b/android/WALT/app/src/main/java/org/chromium/latency/walt/MainActivity.java @@ -325,6 +325,11 @@ public class MainActivity extends AppCompatActivity { switchScreen(newFragment, "Drag Latency"); } + public void onClickAccelerometer(View view) { + AccelerometerFragment newFragment = new AccelerometerFragment(); + switchScreen(newFragment, "Accelerometer Latency"); + } + public void onClickOpenLog(View view) { LogFragment logFragment = new LogFragment(); // menu.findItem(R.id.action_help).setVisible(false); diff --git a/android/WALT/app/src/main/java/org/chromium/latency/walt/Utils.java b/android/WALT/app/src/main/java/org/chromium/latency/walt/Utils.java index 19c7488..97738ce 100644 --- a/android/WALT/app/src/main/java/org/chromium/latency/walt/Utils.java +++ b/android/WALT/app/src/main/java/org/chromium/latency/walt/Utils.java @@ -21,8 +21,11 @@ import android.content.SharedPreferences; import android.preference.PreferenceManager; import android.support.annotation.StringRes; +import com.github.mikephil.charting.data.Entry; + import java.util.ArrayList; import java.util.Collections; +import java.util.List; /** * Kitchen sink for small utility functions @@ -120,6 +123,11 @@ public class Utils { return sb.toString(); } + public static int argmax(double[] a) { + int imax = 0; + for (int i=1; i a[imax]) imax = i; + return imax; + } public static int argmin(double[] a) { int imin = 0; @@ -177,6 +185,30 @@ public class Utils { return preferences.getString(context.getString(keyId), defaultValue); } + static float min(List entries) { + float min = Float.MAX_VALUE; + for (Entry e : entries) { + min = Math.min(min, e.getY()); + } + return min; + } + + static float max(List entries) { + float max = Float.MIN_VALUE; + for (Entry e : entries) { + max = Math.max(max, e.getY()); + } + return max; + } + + static float mean(List entries) { + float mean = 0; + for (Entry e : entries) { + mean += e.getY()/entries.size(); + } + return mean; + } + public enum ListenerState { RUNNING, STARTING, diff --git a/android/WALT/app/src/main/java/org/chromium/latency/walt/WaltDevice.java b/android/WALT/app/src/main/java/org/chromium/latency/walt/WaltDevice.java index 8ec2cb4..566a890 100644 --- a/android/WALT/app/src/main/java/org/chromium/latency/walt/WaltDevice.java +++ b/android/WALT/app/src/main/java/org/chromium/latency/walt/WaltDevice.java @@ -31,7 +31,7 @@ public class WaltDevice implements WaltConnection.ConnectionStateListener { private static final int DEFAULT_DRIFT_LIMIT_US = 1500; private static final String TAG = "WaltDevice"; - public static final String PROTOCOL_VERSION = "5"; + public static final String PROTOCOL_VERSION = "6"; // Teensy side commands. Each command is a single char // Based on #defines section in walt.ino @@ -56,6 +56,7 @@ public class WaltDevice implements WaltConnection.ConnectionStateListener { static final char CMD_BEEP_STOP = 'S'; // Stop generating tone static final char CMD_MIDI = 'M'; // Start listening for a MIDI message static final char CMD_NOTE = 'N'; // Generate a MIDI NoteOn message + static final char CMD_ACCELEROMETER = 'O'; // Generate a MIDI NoteOn message private static final int BYTE_BUFFER_SIZE = 1024 * 4; private byte[] buffer = new byte[BYTE_BUFFER_SIZE]; diff --git a/android/WALT/app/src/main/res/layout/fragment_accelerometer.xml b/android/WALT/app/src/main/res/layout/fragment_accelerometer.xml new file mode 100644 index 0000000..664e246 --- /dev/null +++ b/android/WALT/app/src/main/res/layout/fragment_accelerometer.xml @@ -0,0 +1,78 @@ + + + + + + + + + + + + + + + + + + + + +