From 90fdbf7460fbd255194455331c6a38a7b8f41dd2 Mon Sep 17 00:00:00 2001 From: Emilian Peev Date: Mon, 16 Jan 2017 16:11:58 -0800 Subject: Add experimental depth stream support Configure one depth cloud stream in case the camera supports it. In order to avoid possible performance degrations the depth stream will only get enabled and start streaming when all additional YUV/RAW streams are disabled. Every N-th depth cloud buffer could be stored on the sd card if the storage flag is enabled. This is an experimental feature that could be used as reference on how to work with and access data from depth cloud streams. Bug: 33833999 Test: Manual Change-Id: I7bb84b038fd444a235910c46fae0e37d799b6fff --- src/com/android/devcamera/Api2Camera.java | 56 +++++++++++++++++++++++++- src/com/android/devcamera/CameraInfoCache.java | 8 ++++ src/com/android/devcamera/MediaSaver.java | 51 +++++++++++++++++++++-- 3 files changed, 110 insertions(+), 5 deletions(-) diff --git a/src/com/android/devcamera/Api2Camera.java b/src/com/android/devcamera/Api2Camera.java index 73e5c87..40b9be6 100644 --- a/src/com/android/devcamera/Api2Camera.java +++ b/src/com/android/devcamera/Api2Camera.java @@ -41,8 +41,11 @@ import android.os.SystemClock; import android.util.Log; import android.util.Size; import android.view.Surface; +import android.media.Image.Plane; import java.nio.ByteBuffer; +import java.nio.BufferUnderflowException; +import java.lang.IndexOutOfBoundsException; import java.util.ArrayList; import java.util.LinkedList; import java.util.List; @@ -120,6 +123,11 @@ public class Api2Camera implements CameraInterface, SurfaceTexture.OnFrameAvaila private int mYuv2ImageCounter; private ImageReader mRawImageReader; private int mRawImageCounter; + private boolean mIsDepthCloudSupported = false; + private ImageReader mDepthCloudImageReader; + private int mDepthCloudImageCounter = 0; + private static int STORE_NTH_DEPTH_CLOUD = 30; + private static boolean DEPTH_CLOUD_STORE_ENABLED = false; // Starting the preview requires each of these 3 to be true/non-null: volatile private Surface mPreviewSurface; @@ -175,6 +183,10 @@ public class Api2Camera implements CameraInterface, SurfaceTexture.OnFrameAvaila mReprocessingNoiseMode = CameraCharacteristics.NOISE_REDUCTION_MODE_HIGH_QUALITY; mReprocessingEdgeMode = CameraCharacteristics.EDGE_MODE_HIGH_QUALITY; } + + if (null != mCameraInfoCache.getDepthCloudSize()) { + mIsDepthCloudSupported = true; + } } // Ugh, why is this stuff so slow? @@ -201,6 +213,14 @@ public class Api2Camera implements CameraInterface, SurfaceTexture.OnFrameAvaila YUV1_IMAGEREADER_SIZE); mYuv1ImageReader.setOnImageAvailableListener(mYuv1ImageListener, mOpsHandler); + if (mIsDepthCloudSupported) { + mDepthCloudImageReader = ImageReader.newInstance( + mCameraInfoCache.getDepthCloudSize().getWidth(), + mCameraInfoCache.getDepthCloudSize().getHeight(), + ImageFormat.DEPTH_POINT_CLOUD, 2); + mDepthCloudImageReader.setOnImageAvailableListener(mDepthCloudImageListener, mOpsHandler); + } + if (SECOND_YUV_IMAGEREADER_STREAM) { // Create ImageReader to receive YUV image buffers. mYuv2ImageReader = ImageReader.newInstance( @@ -382,7 +402,7 @@ public class Api2Camera implements CameraInterface, SurfaceTexture.OnFrameAvaila CameraTimer.t_session_go = SystemClock.elapsedRealtime(); Log.v(TAG, "Configuring session.."); - List outputSurfaces = new ArrayList(3); + List outputSurfaces = new ArrayList(4); outputSurfaces.add(mPreviewSurface); Log.v(TAG, " .. added SurfaceView " + mCameraInfoCache.getPreviewSize().getWidth() + @@ -392,6 +412,11 @@ public class Api2Camera implements CameraInterface, SurfaceTexture.OnFrameAvaila Log.v(TAG, " .. added YUV ImageReader " + mCameraInfoCache.getYuvStream1Size().getWidth() + " x " + mCameraInfoCache.getYuvStream1Size().getHeight()); + if (mIsDepthCloudSupported) { + outputSurfaces.add(mDepthCloudImageReader.getSurface()); + Log.v(TAG, " .. added Depth cloud ImageReader"); + } + if (SECOND_YUV_IMAGEREADER_STREAM) { outputSurfaces.add(mYuv2ImageReader.getSurface()); Log.v(TAG, " .. added YUV ImageReader " + mCameraInfoCache.getYuvStream2Size().getWidth() + @@ -531,6 +556,10 @@ public class Api2Camera implements CameraInterface, SurfaceTexture.OnFrameAvaila b1.addTarget(mPreviewSurface); + if (mIsDepthCloudSupported && !mCaptureYuv1 && !mCaptureYuv2 && !mCaptureRaw) { + b1.addTarget(mDepthCloudImageReader.getSurface()); + } + if (mCaptureYuv2) { if (SECOND_SURFACE_TEXTURE_STREAM) { b1.addTarget(mSurfaceTextureSurface); @@ -601,6 +630,31 @@ public class Api2Camera implements CameraInterface, SurfaceTexture.OnFrameAvaila } }; + ImageReader.OnImageAvailableListener mDepthCloudImageListener = + new ImageReader.OnImageAvailableListener() { + @Override + public void onImageAvailable(ImageReader reader) + throws BufferUnderflowException, IndexOutOfBoundsException { + Image img = reader.acquireLatestImage(); + if (img == null) { + Log.e(TAG, "Null image returned Depth"); + return; + } + Plane[] planes = img.getPlanes(); + if (0 < planes.length) { + if (DEPTH_CLOUD_STORE_ENABLED) { + if ((mDepthCloudImageCounter % STORE_NTH_DEPTH_CLOUD) == 0) { + ByteBuffer b = planes[0].getBuffer(); + MediaSaver.saveDepth(mContext, b); + } + } + } else { + Log.e(TAG, "Depth buffer with empty planes!"); + } + img.close(); + mDepthCloudImageCounter++; + } + }; ImageReader.OnImageAvailableListener mJpegImageListener = new ImageReader.OnImageAvailableListener() { diff --git a/src/com/android/devcamera/CameraInfoCache.java b/src/com/android/devcamera/CameraInfoCache.java index 699fd97..7b97a4e 100644 --- a/src/com/android/devcamera/CameraInfoCache.java +++ b/src/com/android/devcamera/CameraInfoCache.java @@ -49,6 +49,7 @@ public class CameraInfoCache { private Integer mRawFormat; private int mBestFaceMode; private int mHardwareLevel; + private Size mDepthCloudSize = null; /** * Constructor. @@ -95,6 +96,10 @@ public class CameraInfoCache { lowestStall = stall; } } + if (formats[i] == ImageFormat.DEPTH_POINT_CLOUD) { + Size size = returnLargestSize(map.getOutputSizes(formats[i])); + mDepthCloudSize = size; + } } mActiveArea = mCameraCharacteristics.get(CameraCharacteristics.SENSOR_INFO_ACTIVE_ARRAY_SIZE); @@ -288,4 +293,7 @@ public class CameraInfoCache { return mRawSize; } + public Size getDepthCloudSize() { + return mDepthCloudSize; + } } diff --git a/src/com/android/devcamera/MediaSaver.java b/src/com/android/devcamera/MediaSaver.java index 4929e33..bf1ddba 100644 --- a/src/com/android/devcamera/MediaSaver.java +++ b/src/com/android/devcamera/MediaSaver.java @@ -27,6 +27,8 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; /** * This class has methods required to save a JPEG to disk as well as update the @@ -42,15 +44,56 @@ public class MediaSaver { private static final boolean UDPATE_MEDIA_STORE = true; - public static int getNextInt(Context context) { + public static int getNextInt(Context context, String id) { SharedPreferences prefs = context.getSharedPreferences(MY_PREFS_NAME, Context.MODE_PRIVATE); - int i = prefs.getInt("counter", 1); + int i = prefs.getInt(id, 1); SharedPreferences.Editor editor = prefs.edit(); - editor.putInt("counter", i+1); + editor.putInt(id, i+1); editor.commit(); return i; } + /** + * @param context Application context. + * @param depthCloudData Depth cloud byte buffer. + */ + public static String saveDepth(Context context, ByteBuffer depthCloudData) { + String filename = ""; + try { + File file; + int i = getNextInt(context, "depthCounter"); + filename = String.format("/sdcard/DCIM/Depth_%05d.img", i); + file = new File(filename); + if (!file.createNewFile()) { + throw new IOException(filename); + } + + long t0 = SystemClock.uptimeMillis(); + FileOutputStream fos = new FileOutputStream(file); + FileChannel channel = fos.getChannel(); + int bytesWritten = 0; + int byteCount = 0; + while (depthCloudData.hasRemaining()) { + byteCount = channel.write(depthCloudData); + if (0 == byteCount) { + throw new IOException(filename); + } else { + bytesWritten += byteCount; + } + } + channel.close(); + fos.flush(); + fos.close(); + long t1 = SystemClock.uptimeMillis(); + + Log.v(TAG, String.format("Wrote Depth %d bytes as %s in %.3f seconds", + bytesWritten, file, (t1 - t0) * 0.001)); + } catch (IOException e) { + Log.e(TAG, "Error creating new file: ", e); + } + return filename; + } + /** * @param context Application context. * @param jpegData JPEG byte stream. @@ -60,7 +103,7 @@ public class MediaSaver { try { File file; while (true) { - int i = getNextInt(context); + int i = getNextInt(context, "counter"); filename = String.format("/sdcard/DCIM/Camera/SNAP_%05d.JPG", i); file = new File(filename); if (file.createNewFile()) { -- cgit v1.2.3