diff options
author | Jeffrey Huang <jeffreyhuang@google.com> | 2024-04-26 18:28:03 +0000 |
---|---|---|
committer | Gerrit Code Review <noreply-gerritcodereview@google.com> | 2024-04-26 18:28:03 +0000 |
commit | ccb64f908e5cc556c3746e8f3b6d9bbaa4e81891 (patch) | |
tree | 902232ba0192e05812847e8f1ba7d57dafeb0894 | |
parent | ad018fc9207fdb48364c43672f4dd7bc2868e487 (diff) | |
parent | 10e016418d6a0779995224c03b700ac0faf6b8b9 (diff) | |
download | StatsD-main.tar.gz |
26 files changed, 377 insertions, 28 deletions
diff --git a/statsd/src/FieldValue.cpp b/statsd/src/FieldValue.cpp index 84fad927..92ed96d3 100644 --- a/statsd/src/FieldValue.cpp +++ b/statsd/src/FieldValue.cpp @@ -570,6 +570,14 @@ size_t getSize(const std::vector<FieldValue>& fieldValues) { return totalSize; } +size_t getFieldValuesSizeV2(const std::vector<FieldValue>& fieldValues) { + size_t totalSize = 0; + for (const FieldValue& fieldValue : fieldValues) { + totalSize += fieldValue.getSizeV2(); + } + return totalSize; +} + bool shouldKeepSample(const FieldValue& sampleFieldValue, int shardOffset, int shardCount) { int hashValue = 0; switch (sampleFieldValue.mValue.type) { diff --git a/statsd/src/FieldValue.h b/statsd/src/FieldValue.h index 66b54755..22fa89b1 100644 --- a/statsd/src/FieldValue.h +++ b/statsd/src/FieldValue.h @@ -448,6 +448,10 @@ struct FieldValue { return mField.getSize() + mValue.getSize(); } + size_t getSizeV2() const { + return mValue.getSize(); + } + Field mField; Value mValue; Annotations mAnnotations; @@ -482,6 +486,9 @@ bool subsetDimensions(const std::vector<Matcher>& dimension_a, // the size is computed at runtime using the actual contents stored in the FieldValue. size_t getSize(const std::vector<FieldValue>& fieldValues); +// Same as getSize but does not compute the size of Field. +size_t getFieldValuesSizeV2(const std::vector<FieldValue>& fieldValues); + bool shouldKeepSample(const FieldValue& sampleFieldValue, int shardOffset, int shardCount); } // namespace statsd diff --git a/statsd/src/HashableDimensionKey.cpp b/statsd/src/HashableDimensionKey.cpp index 837d9e90..03a42296 100644 --- a/statsd/src/HashableDimensionKey.cpp +++ b/statsd/src/HashableDimensionKey.cpp @@ -398,6 +398,21 @@ bool MetricDimensionKey::operator<(const MetricDimensionKey& that) const { return mStateValuesKey < that.getStateValuesKey(); } +size_t MetricDimensionKey::getSize(const bool usesNestedDimensions) const { + size_t dimensionKeySize = 0; + // Dimension/State values + if (usesNestedDimensions) { + // Assume nested dimension adds an additional atomTag + # of dimension fields + dimensionKeySize += sizeof(int32_t); + dimensionKeySize += sizeof(int32_t) * getDimensionKeyInWhat().getValues().size(); + } + dimensionKeySize += getFieldValuesSizeV2(getDimensionKeyInWhat().getValues()); + // Each state value has a atomId and group/value + dimensionKeySize += sizeof(int32_t) * getStateValuesKey().getValues().size(); + dimensionKeySize += getFieldValuesSizeV2(getStateValuesKey().getValues()); + return dimensionKeySize; +} + bool AtomDimensionKey::operator==(const AtomDimensionKey& that) const { return mAtomTag == that.getAtomTag() && mAtomFieldValues == that.getAtomFieldValues(); }; diff --git a/statsd/src/HashableDimensionKey.h b/statsd/src/HashableDimensionKey.h index 5753d98c..4792e8d9 100644 --- a/statsd/src/HashableDimensionKey.h +++ b/statsd/src/HashableDimensionKey.h @@ -135,6 +135,8 @@ public: bool operator<(const MetricDimensionKey& that) const; + size_t getSize(const bool usesNestedDimensions) const; + private: HashableDimensionKey mDimensionKeyInWhat; HashableDimensionKey mStateValuesKey; diff --git a/statsd/src/StatsLogProcessor.cpp b/statsd/src/StatsLogProcessor.cpp index 9fe282a3..95882d50 100644 --- a/statsd/src/StatsLogProcessor.cpp +++ b/statsd/src/StatsLogProcessor.cpp @@ -1114,8 +1114,11 @@ void StatsLogProcessor::flushIfNecessaryLocked(const ConfigKey& key, MetricsManager& metricsManager) { int64_t elapsedRealtimeNs = getElapsedRealtimeNs(); auto lastCheckTime = mLastByteSizeTimes.find(key); + int64_t minCheckPeriodNs = metricsManager.useV2SoftMemoryCalculation() + ? StatsdStats::kMinByteSizeV2CheckPeriodNs + : StatsdStats::kMinByteSizeCheckPeriodNs; if (lastCheckTime != mLastByteSizeTimes.end()) { - if (elapsedRealtimeNs - lastCheckTime->second < StatsdStats::kMinByteSizeCheckPeriodNs) { + if (elapsedRealtimeNs - lastCheckTime->second < minCheckPeriodNs) { return; } } diff --git a/statsd/src/guardrail/StatsdStats.cpp b/statsd/src/guardrail/StatsdStats.cpp index 8cd68dc3..16354b77 100644 --- a/statsd/src/guardrail/StatsdStats.cpp +++ b/statsd/src/guardrail/StatsdStats.cpp @@ -137,6 +137,7 @@ const int FIELD_ID_DB_DELETION_CONFIG_INVALID = 34; const int FIELD_ID_DB_DELETION_TOO_OLD = 35; const int FIELD_ID_DB_DELETION_CONFIG_REMOVED = 36; const int FIELD_ID_DB_DELETION_CONFIG_UPDATED = 37; +const int FIELD_ID_CONFIG_METADATA_PROVIDER_PROMOTION_FAILED = 38; const int FIELD_ID_INVALID_CONFIG_REASON_ENUM = 1; const int FIELD_ID_INVALID_CONFIG_REASON_METRIC_ID = 2; @@ -552,6 +553,16 @@ void StatsdStats::noteDbDeletionConfigUpdated(const ConfigKey& key) { it->second->db_deletion_config_updated++; } +void StatsdStats::noteConfigMetadataProviderPromotionFailed(const ConfigKey& key) { + lock_guard<std::mutex> lock(mLock); + auto it = mConfigStats.find(key); + if (it == mConfigStats.end()) { + ALOGE("Config key %s not found!", key.ToString().c_str()); + return; + } + it->second->config_metadata_provider_promote_failure++; +} + void StatsdStats::noteUidMapDropped(int deltas) { lock_guard<std::mutex> lock(mLock); mUidMapStats.dropped_changes += mUidMapStats.dropped_changes + deltas; @@ -1095,6 +1106,7 @@ void StatsdStats::resetInternalLocked() { config.second->db_deletion_too_old = 0; config.second->db_deletion_config_removed = 0; config.second->db_deletion_config_updated = 0; + config.second->config_metadata_provider_promote_failure = 0; } for (auto& pullStats : mPulledAtomStats) { pullStats.second.totalPull = 0; @@ -1212,6 +1224,10 @@ void StatsdStats::dumpStats(int out) const { configStats->db_deletion_too_old, configStats->db_deletion_config_removed, configStats->db_deletion_config_updated); } + if (configStats->config_metadata_provider_promote_failure > 0) { + dprintf(out, "ConfigMetadataProviderPromotionFailure=%d", + configStats->config_metadata_provider_promote_failure); + } dprintf(out, "\n"); if (!configStats->is_valid) { dprintf(out, "\tinvalid config reason: %s\n", @@ -1734,6 +1750,8 @@ void addConfigStatsToProto(const ConfigStats& configStats, ProtoOutputStream* pr configStats.db_deletion_config_removed, proto); writeNonZeroStatToStream(FIELD_TYPE_INT32 | FIELD_ID_DB_DELETION_CONFIG_UPDATED, configStats.db_deletion_config_updated, proto); + writeNonZeroStatToStream(FIELD_TYPE_INT32 | FIELD_ID_CONFIG_METADATA_PROVIDER_PROMOTION_FAILED, + configStats.config_metadata_provider_promote_failure, proto); for (int64_t latency : configStats.total_flush_latency_ns) { proto->write(FIELD_TYPE_INT64 | FIELD_ID_CONFIG_STATS_RESTRICTED_CONFIG_FLUSH_LATENCY | FIELD_COUNT_REPEATED, diff --git a/statsd/src/guardrail/StatsdStats.h b/statsd/src/guardrail/StatsdStats.h index 6f202603..03266ee3 100644 --- a/statsd/src/guardrail/StatsdStats.h +++ b/statsd/src/guardrail/StatsdStats.h @@ -91,6 +91,8 @@ struct ConfigStats { int32_t db_deletion_too_old = 0; int32_t db_deletion_config_removed = 0; int32_t db_deletion_config_updated = 0; + // Stores the number of ConfigMetadataProvider promotion failures + int32_t config_metadata_provider_promote_failure = 0; // Stores reasons for why config is valid or not std::optional<InvalidConfigReason> reason; @@ -234,7 +236,11 @@ public: static const int64_t kMinBroadcastPeriodNs = 60 * NS_PER_SEC; /* Min period between two checks of byte size per config key in nanoseconds. */ - static const int64_t kMinByteSizeCheckPeriodNs = 60 * NS_PER_SEC; + static const int64_t kMinByteSizeCheckPeriodNs = 1 * 60 * NS_PER_SEC; + + // Min period between two checks of byte size per config key in nanoseconds for V2 memory + // calculations. + static const int64_t kMinByteSizeV2CheckPeriodNs = 5 * 60 * NS_PER_SEC; /* Min period between two checks of restricted metrics TTLs. */ static const int64_t kMinTtlCheckPeriodNs = 60 * 60 * NS_PER_SEC; @@ -388,6 +394,11 @@ public: void noteDbDeletionConfigUpdated(const ConfigKey& key); /** + * Reports that the promotion for ConfigMetadataProvider failed. + */ + void noteConfigMetadataProviderPromotionFailed(const ConfigKey& key); + + /** * Report the size of output tuple of a condition. * * Note: only report when the condition has an output dimension, and the tuple @@ -1036,6 +1047,7 @@ private: FRIEND_TEST(StatsdStatsTest, TestAtomLoggedAndDroppedStats); FRIEND_TEST(StatsdStatsTest, TestAtomMetricsStats); FRIEND_TEST(StatsdStatsTest, TestAtomSkippedStats); + FRIEND_TEST(StatsdStatsTest, TestConfigMetadataProviderPromotionFailed); FRIEND_TEST(StatsdStatsTest, TestConfigRemove); FRIEND_TEST(StatsdStatsTest, TestHasHitDimensionGuardrail); FRIEND_TEST(StatsdStatsTest, TestInvalidConfigAdd); diff --git a/statsd/src/metrics/CountMetricProducer.cpp b/statsd/src/metrics/CountMetricProducer.cpp index fb5f4887..32be42e5 100644 --- a/statsd/src/metrics/CountMetricProducer.cpp +++ b/statsd/src/metrics/CountMetricProducer.cpp @@ -216,6 +216,7 @@ void CountMetricProducer::onSlicedConditionMayChangeLocked(bool overallCondition void CountMetricProducer::clearPastBucketsLocked(const int64_t dumpTimeNs) { mPastBuckets.clear(); + mTotalDataSize = 0; } void CountMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs, @@ -318,6 +319,7 @@ void CountMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs, if (erase_data) { mPastBuckets.clear(); mDimensionGuardrailHit = false; + mTotalDataSize = 0; } } @@ -325,6 +327,7 @@ void CountMetricProducer::dropDataLocked(const int64_t dropTimeNs) { flushIfNeededLocked(dropTimeNs); StatsdStats::getInstance().noteBucketDropped(mMetricId); mPastBuckets.clear(); + mTotalDataSize = 0; } void CountMetricProducer::onConditionChangedLocked(const bool conditionMet, @@ -459,7 +462,10 @@ void CountMetricProducer::flushCurrentBucketLocked(const int64_t eventTimeNs, if (countPassesThreshold(counter.second)) { info.mCount = counter.second; auto& bucketList = mPastBuckets[counter.first]; + const bool isFirstBucket = bucketList.empty(); bucketList.push_back(info); + mTotalDataSize += computeBucketSizeLocked(eventTimeNs < fullBucketEndTimeNs, + counter.first, isFirstBucket); VLOG("metric %lld, dump key value: %s -> %lld", (long long)mMetricId, counter.first.toString().c_str(), (long long)counter.second); } @@ -505,6 +511,11 @@ void CountMetricProducer::flushCurrentBucketLocked(const int64_t eventTimeNs, // greater than actual data size as it contains each dimension of // CountMetricData is duplicated. size_t CountMetricProducer::byteSizeLocked() const { + sp<ConfigMetadataProvider> configMetadataProvider = getConfigMetadataProvider(); + if (configMetadataProvider != nullptr && configMetadataProvider->useV2SoftMemoryCalculation()) { + return computeOverheadSizeLocked(!mPastBuckets.empty(), mDimensionGuardrailHit) + + mTotalDataSize; + } size_t totalSize = 0; for (const auto& pair : mPastBuckets) { totalSize += pair.second.size() * kBucketSize; @@ -512,6 +523,24 @@ size_t CountMetricProducer::byteSizeLocked() const { return totalSize; } +// Estimate for the size of a CountBucket. +size_t CountMetricProducer::computeBucketSizeLocked(const bool isFullBucket, + const MetricDimensionKey& dimKey, + const bool isFirstBucket) const { + size_t bucketSize = + MetricProducer::computeBucketSizeLocked(isFullBucket, dimKey, isFirstBucket); + + // Count Value + bucketSize += sizeof(int32_t); + + // ConditionTrueNanos + if (mConditionTrackerIndex >= 0 && mSlicedStateAtoms.empty() && !mConditionSliced) { + bucketSize += sizeof(int64_t); + } + + return bucketSize; +} + void CountMetricProducer::onActiveStateChangedLocked(const int64_t eventTimeNs, const bool isActive) { MetricProducer::onActiveStateChangedLocked(eventTimeNs, isActive); diff --git a/statsd/src/metrics/CountMetricProducer.h b/statsd/src/metrics/CountMetricProducer.h index 757dad2c..d34181c7 100644 --- a/statsd/src/metrics/CountMetricProducer.h +++ b/statsd/src/metrics/CountMetricProducer.h @@ -101,6 +101,9 @@ private: void onActiveStateChangedLocked(const int64_t eventTimeNs, const bool isActive) override; + size_t computeBucketSizeLocked(const bool isFullBucket, const MetricDimensionKey& dimKey, + const bool isFirstBucket) const override; + optional<InvalidConfigReason> onConfigUpdatedLocked( const StatsdConfig& config, int configIndex, int metricIndex, const std::vector<sp<AtomMatchingTracker>>& allAtomMatchingTrackers, diff --git a/statsd/src/metrics/DurationMetricProducer.cpp b/statsd/src/metrics/DurationMetricProducer.cpp index e18f83cf..b07f2e33 100644 --- a/statsd/src/metrics/DurationMetricProducer.cpp +++ b/statsd/src/metrics/DurationMetricProducer.cpp @@ -519,14 +519,15 @@ void DurationMetricProducer::onDumpReportLocked( protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_ID, (long long)mMetricId); protoOutput->write(FIELD_TYPE_BOOL | FIELD_ID_IS_ACTIVE, isActiveLocked()); - protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_ESTIMATED_MEMORY_BYTES, - (long long)byteSizeLocked()); if (mPastBuckets.empty()) { VLOG(" Duration metric, empty return"); return; } + protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_ESTIMATED_MEMORY_BYTES, + (long long)byteSizeLocked()); + if (StatsdStats::getInstance().hasHitDimensionGuardrail(mMetricId)) { protoOutput->write(FIELD_TYPE_BOOL | FIELD_ID_DIMENSION_GUARDRAIL_HIT, true); } @@ -846,8 +847,42 @@ void DurationMetricProducer::handleMatchedLogEventValuesLocked(const size_t matc eventTimeNs, values); } +// Estimate for the size of a DurationBucket. +size_t DurationMetricProducer::computeBucketSizeLocked(const bool isFullBucket, + const MetricDimensionKey& dimKey, + const bool isFirstBucket) const { + size_t bucketSize = + MetricProducer::computeBucketSizeLocked(isFullBucket, dimKey, isFirstBucket); + + // Duration Value + bucketSize += sizeof(int64_t); + + // ConditionTrueNanos + if (mConditionTrackerIndex >= 0 && mSlicedStateAtoms.empty() && !mConditionSliced) { + bucketSize += sizeof(int64_t); + } + + return bucketSize; +} + size_t DurationMetricProducer::byteSizeLocked() const { size_t totalSize = 0; + sp<ConfigMetadataProvider> configMetadataProvider = getConfigMetadataProvider(); + if (configMetadataProvider != nullptr && configMetadataProvider->useV2SoftMemoryCalculation()) { + bool hasHitDimensionGuardrail = + StatsdStats::getInstance().hasHitDimensionGuardrail(mMetricId); + totalSize += computeOverheadSizeLocked(!mPastBuckets.empty(), hasHitDimensionGuardrail); + for (const auto& pair : mPastBuckets) { + bool isFirstBucket = true; + for (const auto& bucket : pair.second) { + bool isFullBucket = bucket.mBucketEndNs - bucket.mBucketStartNs >= mBucketSizeNs; + totalSize += + computeBucketSizeLocked(isFullBucket, /*dimKey=*/pair.first, isFirstBucket); + isFirstBucket = false; + } + } + return totalSize; + } for (const auto& pair : mPastBuckets) { totalSize += pair.second.size() * kBucketSize; } diff --git a/statsd/src/metrics/DurationMetricProducer.h b/statsd/src/metrics/DurationMetricProducer.h index b7bc29ac..c5403ff8 100644 --- a/statsd/src/metrics/DurationMetricProducer.h +++ b/statsd/src/metrics/DurationMetricProducer.h @@ -139,6 +139,9 @@ private: void addAnomalyTrackerLocked(sp<AnomalyTracker>& anomalyTracker, const UpdateStatus& updateStatus, int64_t updateTimeNs); + size_t computeBucketSizeLocked(const bool isFullBucket, const MetricDimensionKey& dimKey, + const bool isFirstBucket) const override; + const DurationMetric_AggregationType mAggregationType; // Index of the SimpleAtomMatcher which defines the start. diff --git a/statsd/src/metrics/EventMetricProducer.cpp b/statsd/src/metrics/EventMetricProducer.cpp index 648e21c9..540e3d02 100644 --- a/statsd/src/metrics/EventMetricProducer.cpp +++ b/statsd/src/metrics/EventMetricProducer.cpp @@ -83,7 +83,6 @@ EventMetricProducer::EventMetricProducer( mConditionSliced = true; } - mTotalSize = 0; VLOG("metric %lld created. bucket size %lld start_time: %lld", (long long)mMetricId, (long long)mBucketSizeNs, (long long)mTimeBaseNs); } @@ -139,9 +138,9 @@ optional<InvalidConfigReason> EventMetricProducer::onConfigUpdatedLocked( void EventMetricProducer::dropDataLocked(const int64_t dropTimeNs) { mAggregatedAtoms.clear(); - mTotalSize = 0; mDataCorruptedDueToSocketLoss = false; mDataCorruptedDueToQueueOverflow = false; + mTotalDataSize = 0; StatsdStats::getInstance().noteBucketDropped(mMetricId); } @@ -168,9 +167,9 @@ std::unique_ptr<std::vector<uint8_t>> serializeProtoLocked(ProtoOutputStream& pr void EventMetricProducer::clearPastBucketsLocked(const int64_t dumpTimeNs) { mAggregatedAtoms.clear(); - mTotalSize = 0; mDataCorruptedDueToSocketLoss = false; mDataCorruptedDueToQueueOverflow = false; + mTotalDataSize = 0; } void EventMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs, @@ -181,12 +180,13 @@ void EventMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs, ProtoOutputStream* protoOutput) { protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_ID, (long long)mMetricId); protoOutput->write(FIELD_TYPE_BOOL | FIELD_ID_IS_ACTIVE, isActiveLocked()); - protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_ESTIMATED_MEMORY_BYTES, - (long long)byteSizeLocked()); // Data corrupted reason writeDataCorruptedReasons(*protoOutput, FIELD_ID_DATA_CORRUPTED_REASON, mDataCorruptedDueToQueueOverflow, mDataCorruptedDueToSocketLoss); - + if (!mAggregatedAtoms.empty()) { + protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_ESTIMATED_MEMORY_BYTES, + (long long)byteSizeLocked()); + } uint64_t protoToken = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_ID_EVENT_METRICS); for (const auto& [atomDimensionKey, elapsedTimestampsNs] : mAggregatedAtoms) { uint64_t wrapperToken = @@ -210,9 +210,9 @@ void EventMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs, protoOutput->end(protoToken); if (erase_data) { mAggregatedAtoms.clear(); - mTotalSize = 0; mDataCorruptedDueToSocketLoss = false; mDataCorruptedDueToQueueOverflow = false; + mTotalDataSize = 0; } } @@ -239,14 +239,24 @@ void EventMetricProducer::onMatchedLogEventInternalLocked( std::vector<int64_t>& aggregatedTimestampsNs = mAggregatedAtoms[key]; if (aggregatedTimestampsNs.empty()) { - mTotalSize += getSize(key.getAtomFieldValues().getValues()); + sp<ConfigMetadataProvider> provider = getConfigMetadataProvider(); + if (provider != nullptr && provider->useV2SoftMemoryCalculation()) { + mTotalDataSize += getFieldValuesSizeV2(key.getAtomFieldValues().getValues()); + } else { + mTotalDataSize += getSize(key.getAtomFieldValues().getValues()); + } } aggregatedTimestampsNs.push_back(elapsedTimeNs); - mTotalSize += sizeof(int64_t); // Add the size of the event timestamp + mTotalDataSize += sizeof(int64_t); // Add the size of the event timestamp } size_t EventMetricProducer::byteSizeLocked() const { - return mTotalSize; + sp<ConfigMetadataProvider> provider = getConfigMetadataProvider(); + if (provider != nullptr && provider->useV2SoftMemoryCalculation()) { + return mTotalDataSize + + computeOverheadSizeLocked(/*hasPastBuckets=*/false, /*dimensionGuardrailHit=*/false); + } + return mTotalDataSize; } } // namespace statsd diff --git a/statsd/src/metrics/EventMetricProducer.h b/statsd/src/metrics/EventMetricProducer.h index aed43583..d64552ba 100644 --- a/statsd/src/metrics/EventMetricProducer.h +++ b/statsd/src/metrics/EventMetricProducer.h @@ -51,9 +51,6 @@ public: return METRIC_TYPE_EVENT; } -protected: - size_t mTotalSize; - private: void onMatchedLogEventInternalLocked( const size_t matcherIndex, const MetricDimensionKey& eventKey, diff --git a/statsd/src/metrics/GaugeMetricProducer.cpp b/statsd/src/metrics/GaugeMetricProducer.cpp index f82f04e4..979b3444 100644 --- a/statsd/src/metrics/GaugeMetricProducer.cpp +++ b/statsd/src/metrics/GaugeMetricProducer.cpp @@ -244,6 +244,7 @@ void GaugeMetricProducer::clearPastBucketsLocked(const int64_t dumpTimeNs) { flushIfNeededLocked(dumpTimeNs); mPastBuckets.clear(); mSkippedBuckets.clear(); + mTotalDataSize = 0; } void GaugeMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs, @@ -261,13 +262,14 @@ void GaugeMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs, protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_ID, (long long)mMetricId); protoOutput->write(FIELD_TYPE_BOOL | FIELD_ID_IS_ACTIVE, isActiveLocked()); - protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_ESTIMATED_MEMORY_BYTES, - (long long)byteSizeLocked()); if (mPastBuckets.empty() && mSkippedBuckets.empty()) { return; } + protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_ESTIMATED_MEMORY_BYTES, + (long long)byteSizeLocked()); + if (mDimensionGuardrailHit) { protoOutput->write(FIELD_TYPE_BOOL | FIELD_ID_DIMENSION_GUARDRAIL_HIT, mDimensionGuardrailHit); @@ -373,6 +375,7 @@ void GaugeMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs, mPastBuckets.clear(); mSkippedBuckets.clear(); mDimensionGuardrailHit = false; + mTotalDataSize = 0; } } @@ -626,6 +629,7 @@ void GaugeMetricProducer::dropDataLocked(const int64_t dropTimeNs) { flushIfNeededLocked(dropTimeNs); StatsdStats::getInstance().noteBucketDropped(mMetricId); mPastBuckets.clear(); + mTotalDataSize = 0; } // When a new matched event comes in, we check if event falls into the current @@ -673,7 +677,11 @@ void GaugeMetricProducer::flushCurrentBucketLocked(const int64_t eventTimeNs, elapsedTimestampsNs.push_back(atom.mElapsedTimestampNs); } auto& bucketList = mPastBuckets[slice.first]; + const bool isFirstBucket = bucketList.empty(); bucketList.push_back(info); + mTotalDataSize += computeGaugeBucketSizeLocked(eventTimeNs >= fullBucketEndTimeNs, + /*dimKey=*/slice.first, isFirstBucket, + info.mAggregatedAtoms); VLOG("Gauge gauge metric %lld, dump key value: %s", (long long)mMetricId, slice.first.toString().c_str()); } @@ -685,6 +693,7 @@ void GaugeMetricProducer::flushCurrentBucketLocked(const int64_t eventTimeNs, buildDropEvent(eventTimeNs, BucketDropReason::BUCKET_TOO_SMALL)); } mSkippedBuckets.emplace_back(mCurrentSkippedBucket); + mTotalDataSize += computeSkippedBucketSizeLocked(mCurrentSkippedBucket); } // If we have anomaly trackers, we need to update the partial bucket values. @@ -708,7 +717,29 @@ void GaugeMetricProducer::flushCurrentBucketLocked(const int64_t eventTimeNs, mHasHitGuardrail = false; } +// Estimate for the size of a GaugeBucket. +size_t GaugeMetricProducer::computeGaugeBucketSizeLocked( + const bool isFullBucket, const MetricDimensionKey& dimKey, const bool isFirstBucket, + const std::unordered_map<AtomDimensionKey, std::vector<int64_t>>& aggregatedAtoms) const { + size_t bucketSize = + MetricProducer::computeBucketSizeLocked(isFullBucket, dimKey, isFirstBucket); + + // Gauge Atoms and timestamps + for (const auto& pair : aggregatedAtoms) { + bucketSize += getFieldValuesSizeV2(pair.first.getAtomFieldValues().getValues()); + bucketSize += sizeof(int64_t) * pair.second.size(); + } + + return bucketSize; +} + size_t GaugeMetricProducer::byteSizeLocked() const { + sp<ConfigMetadataProvider> configMetadataProvider = getConfigMetadataProvider(); + if (configMetadataProvider != nullptr && configMetadataProvider->useV2SoftMemoryCalculation()) { + return computeOverheadSizeLocked(!mPastBuckets.empty() || !mSkippedBuckets.empty(), + mDimensionGuardrailHit) + + mTotalDataSize; + } size_t totalSize = 0; for (const auto& pair : mPastBuckets) { for (const auto& bucket : pair.second) { diff --git a/statsd/src/metrics/GaugeMetricProducer.h b/statsd/src/metrics/GaugeMetricProducer.h index 18e0b43c..daff08c3 100644 --- a/statsd/src/metrics/GaugeMetricProducer.h +++ b/statsd/src/metrics/GaugeMetricProducer.h @@ -149,6 +149,11 @@ private: // Only call if mCondition == ConditionState::kTrue && metric is active. void pullAndMatchEventsLocked(const int64_t timestampNs); + size_t computeGaugeBucketSizeLocked( + const bool isFullBucket, const MetricDimensionKey& dimKey, const bool isFirstBucket, + const std::unordered_map<AtomDimensionKey, std::vector<int64_t>>& aggregatedAtoms) + const; + optional<InvalidConfigReason> onConfigUpdatedLocked( const StatsdConfig& config, int configIndex, int metricIndex, const std::vector<sp<AtomMatchingTracker>>& allAtomMatchingTrackers, diff --git a/statsd/src/metrics/KllMetricProducer.cpp b/statsd/src/metrics/KllMetricProducer.cpp index 26aafa41..cc59e3cc 100644 --- a/statsd/src/metrics/KllMetricProducer.cpp +++ b/statsd/src/metrics/KllMetricProducer.cpp @@ -153,7 +153,26 @@ PastBucket<unique_ptr<KllQuantile>> KllMetricProducer::buildPartialBucket( return bucket; } +// Estimate for the size of NumericValues. +size_t KllMetricProducer::getAggregatedValueSize(const std::unique_ptr<KllQuantile>& kll) const { + size_t valueSize = 0; + // Index + valueSize += sizeof(int32_t); + + // Value + valueSize += kll->SerializeToProto().ByteSizeLong(); + + return valueSize; +} + size_t KllMetricProducer::byteSizeLocked() const { + sp<ConfigMetadataProvider> configMetadataProvider = getConfigMetadataProvider(); + if (configMetadataProvider != nullptr && configMetadataProvider->useV2SoftMemoryCalculation()) { + bool dimensionGuardrailHit = StatsdStats::getInstance().hasHitDimensionGuardrail(mMetricId); + return computeOverheadSizeLocked(!mPastBuckets.empty() || !mSkippedBuckets.empty(), + dimensionGuardrailHit) + + mTotalDataSize; + } size_t totalSize = 0; for (const auto& [_, buckets] : mPastBuckets) { totalSize += buckets.size() * kBucketSize; diff --git a/statsd/src/metrics/KllMetricProducer.h b/statsd/src/metrics/KllMetricProducer.h index 16c1e2af..dd5dcbd8 100644 --- a/statsd/src/metrics/KllMetricProducer.h +++ b/statsd/src/metrics/KllMetricProducer.h @@ -103,6 +103,8 @@ private: const int sampleSize, ProtoOutputStream* const protoOutput) const override; + size_t getAggregatedValueSize(const std::unique_ptr<KllQuantile>& kll) const override; + bool aggregateFields(const int64_t eventTimeNs, const MetricDimensionKey& eventKey, const LogEvent& event, std::vector<Interval>& intervals, Empty& empty) override; diff --git a/statsd/src/metrics/MetricProducer.cpp b/statsd/src/metrics/MetricProducer.cpp index ad2c8ccc..ac80a209 100644 --- a/statsd/src/metrics/MetricProducer.cpp +++ b/statsd/src/metrics/MetricProducer.cpp @@ -391,6 +391,69 @@ bool MetricProducer::passesSampleCheckLocked(const vector<FieldValue>& values) c mShardCount); } +sp<ConfigMetadataProvider> MetricProducer::getConfigMetadataProvider() const { + sp<ConfigMetadataProvider> provider = mConfigMetadataProvider.promote(); + if (provider == nullptr) { + ALOGE("Could not promote ConfigMetadataProvider"); + StatsdStats::getInstance().noteConfigMetadataProviderPromotionFailed(mConfigKey); + } + return provider; +} + +size_t MetricProducer::computeBucketSizeLocked(const bool isFullBucket, + const MetricDimensionKey& dimKey, + const bool isFirstBucket) const { + size_t bucketSize = 0; + + // Bucket timestamps or bucket number + bucketSize += isFullBucket ? sizeof(int32_t) : 2 * sizeof(int64_t); + + // Each dimension / state key can have multiple buckets. Add the size only for the first bucket. + if (isFirstBucket) { + bucketSize += dimKey.getSize(mShouldUseNestedDimensions); + } + + return bucketSize; +} + +size_t MetricProducer::computeOverheadSizeLocked(const bool hasPastBuckets, + const bool dimensionGuardrailHit) const { + size_t overheadSize = 0; + + // MetricId + isActive + overheadSize += sizeof(int64_t) + sizeof(bool); + + if (hasPastBuckets) { + if (dimensionGuardrailHit) { + overheadSize += sizeof(int32_t); + } + + // estimated_memory_bytes + overheadSize += sizeof(int32_t); + // mTimeBase and mBucketSizeNs + overheadSize += 2 * sizeof(int64_t); + + if (!mShouldUseNestedDimensions) { + // Assume dimensions data adds an additional atomTag + # of dimension fields + overheadSize += sizeof(int32_t); + overheadSize += sizeof(int32_t) * mDimensionsInWhat.size(); + } + } + return overheadSize; +} + +size_t MetricProducer::computeSkippedBucketSizeLocked(const SkippedBucket& skippedBucket) const { + size_t skippedBucketSize = 0; + + // Bucket Start, Bucket End + skippedBucketSize += 2 * sizeof(int64_t); + + // DropType, Drop Time + skippedBucketSize += (sizeof(int32_t) + sizeof(int64_t)) * skippedBucket.dropEvents.size(); + + return skippedBucketSize; +} + } // namespace statsd } // namespace os } // namespace android diff --git a/statsd/src/metrics/MetricProducer.h b/statsd/src/metrics/MetricProducer.h index d134425e..cb97ae53 100644 --- a/statsd/src/metrics/MetricProducer.h +++ b/statsd/src/metrics/MetricProducer.h @@ -457,6 +457,15 @@ protected: void activateLocked(int activationTrackerIndex, int64_t elapsedTimestampNs); void cancelEventActivationLocked(int deactivationTrackerIndex); + // Computes the size of a newly added bucket to this metric, taking into account any new + // dimensions that are introduced if necessary. + virtual size_t computeBucketSizeLocked(const bool isFullBucket, + const MetricDimensionKey& dimKey, + const bool isFirstBucket) const; + size_t computeOverheadSizeLocked(const bool hasPastBuckets, + const bool dimensionGuardrailHit) const; + size_t computeSkippedBucketSizeLocked(const SkippedBucket& skippedBucket) const; + bool evaluateActiveStateLocked(int64_t elapsedTimestampNs); virtual void onActiveStateChangedLocked(const int64_t eventTimeNs, const bool isActive) { @@ -588,14 +597,14 @@ protected: int mShardCount; - inline wp<ConfigMetadataProvider> getConfigMetadataProvider() const { - return mConfigMetadataProvider; - } + sp<ConfigMetadataProvider> getConfigMetadataProvider() const; wp<ConfigMetadataProvider> mConfigMetadataProvider; bool mDataCorruptedDueToSocketLoss = false; bool mDataCorruptedDueToQueueOverflow = false; + size_t mTotalDataSize = 0; + FRIEND_TEST(CountMetricE2eTest, TestSlicedState); FRIEND_TEST(CountMetricE2eTest, TestSlicedStateWithMap); FRIEND_TEST(CountMetricE2eTest, TestMultipleSlicedStates); diff --git a/statsd/src/metrics/NumericValueMetricProducer.cpp b/statsd/src/metrics/NumericValueMetricProducer.cpp index 53a1901a..ed5032b3 100644 --- a/statsd/src/metrics/NumericValueMetricProducer.cpp +++ b/statsd/src/metrics/NumericValueMetricProducer.cpp @@ -630,7 +630,30 @@ void NumericValueMetricProducer::appendToFullBucket(const bool isFullBucketReach } } +// Estimate for the size of NumericValues. +size_t NumericValueMetricProducer::getAggregatedValueSize(const Value& value) const { + size_t valueSize = 0; + // Index + valueSize += sizeof(int32_t); + + // Value + valueSize += value.getSize(); + + // Sample Size + if (mIncludeSampleSize) { + valueSize += sizeof(int32_t); + } + return valueSize; +} + size_t NumericValueMetricProducer::byteSizeLocked() const { + sp<ConfigMetadataProvider> configMetadataProvider = getConfigMetadataProvider(); + if (configMetadataProvider != nullptr && configMetadataProvider->useV2SoftMemoryCalculation()) { + bool dimensionGuardrailHit = StatsdStats::getInstance().hasHitDimensionGuardrail(mMetricId); + return computeOverheadSizeLocked(!mPastBuckets.empty() || !mSkippedBuckets.empty(), + dimensionGuardrailHit) + + mTotalDataSize; + } size_t totalSize = 0; for (const auto& [_, buckets] : mPastBuckets) { totalSize += buckets.size() * kBucketSize; diff --git a/statsd/src/metrics/NumericValueMetricProducer.h b/statsd/src/metrics/NumericValueMetricProducer.h index 103f4045..8309aef5 100644 --- a/statsd/src/metrics/NumericValueMetricProducer.h +++ b/statsd/src/metrics/NumericValueMetricProducer.h @@ -150,6 +150,8 @@ private: return mAggregationTypes.size() == 1 ? mAggregationTypes[0] : mAggregationTypes[index]; } + size_t getAggregatedValueSize(const Value& value) const override; + const bool mUseAbsoluteValueOnReset; const std::vector<ValueMetric::AggregationType> mAggregationTypes; diff --git a/statsd/src/metrics/RestrictedEventMetricProducer.cpp b/statsd/src/metrics/RestrictedEventMetricProducer.cpp index 83454fb8..77682dd2 100644 --- a/statsd/src/metrics/RestrictedEventMetricProducer.cpp +++ b/statsd/src/metrics/RestrictedEventMetricProducer.cpp @@ -58,11 +58,11 @@ void RestrictedEventMetricProducer::onMatchedLogEventInternalLocked( StatsdStats::getInstance().noteRestrictedMetricCategoryChanged(mConfigKey, mMetricId); deleteMetricTable(); mLogEvents.clear(); - mTotalSize = 0; + mTotalDataSize = 0; } mRestrictedDataCategory = event.getRestrictionCategory(); mLogEvents.push_back(event); - mTotalSize += getSize(event.getValues()) + sizeof(event); + mTotalDataSize += getSize(event.getValues()) + sizeof(event); } void RestrictedEventMetricProducer::onDumpReportLocked( @@ -94,7 +94,7 @@ void RestrictedEventMetricProducer::clearPastBucketsLocked(const int64_t dumpTim void RestrictedEventMetricProducer::dropDataLocked(const int64_t dropTimeNs) { mLogEvents.clear(); - mTotalSize = 0; + mTotalDataSize = 0; StatsdStats::getInstance().noteBucketDropped(mMetricId); } @@ -130,7 +130,7 @@ void RestrictedEventMetricProducer::flushRestrictedData() { mConfigKey, mMetricId, getElapsedRealtimeNs() - flushStartNs); } mLogEvents.clear(); - mTotalSize = 0; + mTotalDataSize = 0; } bool RestrictedEventMetricProducer::writeMetricMetadataToProto( diff --git a/statsd/src/metrics/ValueMetricProducer.cpp b/statsd/src/metrics/ValueMetricProducer.cpp index f4620ddb..b1f0a331 100644 --- a/statsd/src/metrics/ValueMetricProducer.cpp +++ b/statsd/src/metrics/ValueMetricProducer.cpp @@ -226,6 +226,31 @@ ValueMetricProducer<AggregatedValue, DimExtras>::onConfigUpdatedLocked( } template <typename AggregatedValue, typename DimExtras> +size_t ValueMetricProducer<AggregatedValue, DimExtras>::computeValueBucketSizeLocked( + const bool isFullBucket, const MetricDimensionKey& dimKey, const bool isFirstBucket, + const PastBucket<AggregatedValue>& bucket) const { + size_t bucketSize = + MetricProducer::computeBucketSizeLocked(isFullBucket, dimKey, isFirstBucket); + + for (const auto& value : bucket.aggregates) { + bucketSize += getAggregatedValueSize(value); + } + + // ConditionTrueNanos + if (mConditionTrackerIndex >= 0 || !mSlicedStateAtoms.empty()) { + bucketSize += sizeof(int64_t); + } + + // ConditionCorrectionNanos + if (getDumpProtoFields().conditionCorrectionNsFieldId.has_value() && isPulled() && + mConditionCorrectionThresholdNs && + (abs(bucket.mConditionCorrectionNs) >= mConditionCorrectionThresholdNs)) { + bucketSize += sizeof(int64_t); + } + return bucketSize; +} + +template <typename AggregatedValue, typename DimExtras> void ValueMetricProducer<AggregatedValue, DimExtras>::onStateChanged( int64_t eventTimeNs, int32_t atomId, const HashableDimensionKey& primaryKey, const FieldValue& oldState, const FieldValue& newState) { @@ -291,6 +316,7 @@ void ValueMetricProducer<AggregatedValue, DimExtras>::clearPastBucketsLocked( const int64_t dumpTimeNs) { mPastBuckets.clear(); mSkippedBuckets.clear(); + mTotalDataSize = 0; } template <typename AggregatedValue, typename DimExtras> @@ -324,12 +350,13 @@ void ValueMetricProducer<AggregatedValue, DimExtras>::onDumpReportLocked( protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_ID, (long long)mMetricId); protoOutput->write(FIELD_TYPE_BOOL | FIELD_ID_IS_ACTIVE, isActiveLocked()); - protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_ESTIMATED_MEMORY_BYTES, - (long long)byteSizeLocked()); if (mPastBuckets.empty() && mSkippedBuckets.empty()) { return; } + protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_ESTIMATED_MEMORY_BYTES, + (long long)byteSizeLocked()); + if (StatsdStats::getInstance().hasHitDimensionGuardrail(mMetricId)) { protoOutput->write(FIELD_TYPE_BOOL | FIELD_ID_DIMENSION_GUARDRAIL_HIT, true); } @@ -447,6 +474,7 @@ void ValueMetricProducer<AggregatedValue, DimExtras>::onDumpReportLocked( if (eraseData) { mPastBuckets.clear(); mSkippedBuckets.clear(); + mTotalDataSize = 0; } } @@ -822,7 +850,10 @@ void ValueMetricProducer<AggregatedValue, DimExtras>::closeCurrentBucket( } auto& bucketList = mPastBuckets[metricDimensionKey]; + const bool isFirstBucket = bucketList.empty(); bucketList.push_back(std::move(bucket)); + mTotalDataSize += computeValueBucketSizeLocked( + eventTimeNs >= fullBucketEndTimeNs, metricDimensionKey, isFirstBucket, bucket); } if (!bucketHasData) { skipCurrentBucket(eventTimeNs, BucketDropReason::NO_DATA); @@ -833,6 +864,7 @@ void ValueMetricProducer<AggregatedValue, DimExtras>::closeCurrentBucket( mCurrentSkippedBucket.bucketStartTimeNs = mCurrentBucketStartTimeNs; mCurrentSkippedBucket.bucketEndTimeNs = bucketEndTimeNs; mSkippedBuckets.push_back(mCurrentSkippedBucket); + mTotalDataSize += computeSkippedBucketSizeLocked(mCurrentSkippedBucket); } // This means that the current bucket was not flushed before a forced bucket split. diff --git a/statsd/src/metrics/ValueMetricProducer.h b/statsd/src/metrics/ValueMetricProducer.h index bfde6a0f..78761ebe 100644 --- a/statsd/src/metrics/ValueMetricProducer.h +++ b/statsd/src/metrics/ValueMetricProducer.h @@ -232,6 +232,12 @@ protected: std::unordered_map<int, std::vector<int>>& deactivationAtomTrackerToMetricMap, std::vector<int>& metricsWithActivation) override; + size_t computeValueBucketSizeLocked(const bool isFullBucket, const MetricDimensionKey& dimKey, + const bool isFirstBucket, + const PastBucket<AggregatedValue>& bucket) const; + + virtual size_t getAggregatedValueSize(const AggregatedValue& value) const = 0; + virtual optional<int64_t> getConditionIdForMetric(const StatsdConfig& config, const int configIndex) const = 0; diff --git a/statsd/src/stats_log.proto b/statsd/src/stats_log.proto index 00763dea..93b0d8c8 100644 --- a/statsd/src/stats_log.proto +++ b/statsd/src/stats_log.proto @@ -505,6 +505,7 @@ message StatsdStatsReport { optional int32 db_deletion_too_old = 35; optional int32 db_deletion_config_removed = 36; optional int32 db_deletion_config_updated = 37; + optional int32 config_metadata_provider_promotion_failed = 38; } repeated ConfigStats config_stats = 3; diff --git a/statsd/tests/guardrail/StatsdStats_test.cpp b/statsd/tests/guardrail/StatsdStats_test.cpp index 86e80930..9232f5b0 100644 --- a/statsd/tests/guardrail/StatsdStats_test.cpp +++ b/statsd/tests/guardrail/StatsdStats_test.cpp @@ -88,6 +88,20 @@ TEST(StatsdStatsTest, TestValidConfigAdd) { EXPECT_FALSE(configReport.has_deletion_time_sec()); } +TEST(StatsdStatsTest, TestConfigMetadataProviderPromotionFailed) { + StatsdStats stats; + ConfigKey key(0, 12345); + stats.noteConfigReceived(key, /*metricsCount=*/0, /*conditionsCount=*/0, /*matchersCount=*/0, + /*alertCount=*/0, /*annotations=*/{}, nullopt /*valid config*/); + + stats.noteConfigMetadataProviderPromotionFailed(key); + + StatsdStatsReport report = getStatsdStatsReport(stats, /* reset stats */ false); + ASSERT_EQ(1, report.config_stats_size()); + const auto& configReport = report.config_stats(0); + EXPECT_EQ(1, configReport.config_metadata_provider_promotion_failed()); +} + TEST(StatsdStatsTest, TestInvalidConfigAdd) { StatsdStats stats; ConfigKey key(0, 12345); |