SkJpegCodec: Gracefully handle malloc failure

Bug: 150353577
Test: Infeasible

Cherry-pick of 9eb78748448b853074c16d99da8bff42b6791782 in Skia.

If malloc fails, don't let SkJpegCodec attempt to decode into it.
Return kInternalError, which is what we return for OOM.

Change-Id: I6750e907c5e649d9e12ba75e84c8d9e91a66dd61
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/273809
Auto-Submit: Leon Scroggins <scroggo@google.com>
Reviewed-by: Brian Osman <brianosman@google.com>
Commit-Queue: Brian Osman <brianosman@google.com>

Merged-In: Iaec5e7d0f8603124acb8238bd4d12457f09e9090
Change-Id: I41527dc95c5565bc8985f0003260840e69555caa

3 files changed, 16 insertions, 7 deletions

diff --git a/src/codec/SkJpegCodec.cpp b/src/codec/SkJpegCodec.cpp
index 45bd35dd5d..f79c8cc310 100644
--- a/src/codec/SkJpegCodec.cpp
+++ b/src/codec/SkJpegCodec.cpp
@@ -553,7 +553,9 @@ SkCodec::Result SkJpegCodec::onGetPixels(const SkImageInfo& dstInfo,
         this->initializeSwizzler(dstInfo, options, true);
     }
 
-    this->allocateStorage(dstInfo);
+    if (!this->allocateStorage(dstInfo)) {
+        return kInternalError;
+    }
 
     int rows = this->readRows(dstInfo, dst, dstRowBytes, dstInfo.height(), options);
     if (rows < dstInfo.height()) {
@@ -564,7 +566,7 @@ SkCodec::Result SkJpegCodec::onGetPixels(const SkImageInfo& dstInfo,
     return kSuccess;
 }
 
-void SkJpegCodec::allocateStorage(const SkImageInfo& dstInfo) {
+bool SkJpegCodec::allocateStorage(const SkImageInfo& dstInfo) {
     int dstWidth = dstInfo.width();
 
     size_t swizzleBytes = 0;
@@ -582,11 +584,14 @@ void SkJpegCodec::allocateStorage(const SkImageInfo& dstInfo) {
 
     size_t totalBytes = swizzleBytes + xformBytes;
     if (totalBytes > 0) {
-        fStorage.reset(totalBytes);
+        if (!fStorage.reset(totalBytes)) {
+            return false;
+        }
         fSwizzleSrcRow = (swizzleBytes > 0) ? fStorage.get() : nullptr;
         fColorXformSrcRow = (xformBytes > 0) ?
                 SkTAddOffset<uint32_t>(fStorage.get(), swizzleBytes) : nullptr;
     }
+    return true;
 }
 
 void SkJpegCodec::initializeSwizzler(const SkImageInfo& dstInfo, const Options& options,
@@ -646,7 +651,9 @@ SkSampler* SkJpegCodec::getSampler(bool createIfNecessary) {
             fDecoderMgr->dinfo()->out_color_space, this->getEncodedInfo().profile(),
             this->colorXform());
     this->initializeSwizzler(this->dstInfo(), this->options(), needsCMYKToRGB);
-    this->allocateStorage(this->dstInfo());
+    if (!this->allocateStorage(this->dstInfo())) {
+        return nullptr;
+    }
     return fSwizzler.get();
 }
 
@@ -709,7 +716,9 @@ SkCodec::Result SkJpegCodec::onStartScanlineDecode(const SkImageInfo& dstInfo,
         this->initializeSwizzler(dstInfo, options, true);
     }
 
-    this->allocateStorage(dstInfo);
+    if (!this->allocateStorage(dstInfo)) {
+        return kInternalError;
+    }
 
     return kSuccess;
 }
diff --git a/src/codec/SkJpegCodec.h b/src/codec/SkJpegCodec.h
index 986d283b3a..609d928d75 100644
--- a/src/codec/SkJpegCodec.h
+++ b/src/codec/SkJpegCodec.h
@@ -106,7 +106,7 @@ private:
 
     void initializeSwizzler(const SkImageInfo& dstInfo, const Options& options,
                             bool needsCMYKToRGB);
-    void allocateStorage(const SkImageInfo& dstInfo);
+    bool SK_WARN_UNUSED_RESULT allocateStorage(const SkImageInfo& dstInfo);
     int readRows(const SkImageInfo& dstInfo, void* dst, size_t rowBytes, int count, const Options&);
 
     /*
diff --git a/src/codec/SkSampledCodec.cpp b/src/codec/SkSampledCodec.cpp
index c0c8cf02c4..b7ac0ebac1 100644
--- a/src/codec/SkSampledCodec.cpp
+++ b/src/codec/SkSampledCodec.cpp
@@ -273,7 +273,7 @@ SkCodec::Result SkSampledCodec::sampledDecode(const SkImageInfo& info, void* pix
 
     SkSampler* sampler = this->codec()->getSampler(true);
     if (!sampler) {
-        return SkCodec::kUnimplemented;
+        return SkCodec::kInternalError;
     }
 
     if (sampler->setSampleX(sampleX) != info.width()) {