1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
|
/*
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "webrtc/common_video/libyuv/include/scaler.h"
#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
#include "webrtc/modules/video_processing/video_denoiser.h"
namespace webrtc {
VideoDenoiser::VideoDenoiser(bool runtime_cpu_detection)
: width_(0),
height_(0),
filter_(DenoiserFilter::Create(runtime_cpu_detection)) {}
void VideoDenoiser::TrailingReduction(int mb_rows,
int mb_cols,
const uint8_t* y_src,
int stride_y,
uint8_t* y_dst) {
for (int mb_row = 1; mb_row < mb_rows - 1; ++mb_row) {
for (int mb_col = 1; mb_col < mb_cols - 1; ++mb_col) {
int mb_index = mb_row * mb_cols + mb_col;
uint8_t* mb_dst = y_dst + (mb_row << 4) * stride_y + (mb_col << 4);
const uint8_t* mb_src = y_src + (mb_row << 4) * stride_y + (mb_col << 4);
// If the number of denoised neighbors is less than a threshold,
// do NOT denoise for the block. Set different threshold for skin MB.
// The change of denoising status will not propagate.
if (metrics_[mb_index].is_skin) {
// The threshold is high (more strict) for non-skin MB where the
// trailing usually happen.
if (metrics_[mb_index].denoise &&
metrics_[mb_index + 1].denoise + metrics_[mb_index - 1].denoise +
metrics_[mb_index + mb_cols].denoise +
metrics_[mb_index - mb_cols].denoise <=
2) {
metrics_[mb_index].denoise = 0;
filter_->CopyMem16x16(mb_src, stride_y, mb_dst, stride_y);
}
} else if (metrics_[mb_index].denoise &&
metrics_[mb_index + 1].denoise +
metrics_[mb_index - 1].denoise +
metrics_[mb_index + mb_cols + 1].denoise +
metrics_[mb_index + mb_cols - 1].denoise +
metrics_[mb_index - mb_cols + 1].denoise +
metrics_[mb_index - mb_cols - 1].denoise +
metrics_[mb_index + mb_cols].denoise +
metrics_[mb_index - mb_cols].denoise <=
7) {
filter_->CopyMem16x16(mb_src, stride_y, mb_dst, stride_y);
}
}
}
}
void VideoDenoiser::DenoiseFrame(const VideoFrame& frame,
VideoFrame* denoised_frame) {
int stride_y = frame.stride(kYPlane);
int stride_u = frame.stride(kUPlane);
int stride_v = frame.stride(kVPlane);
// If previous width and height are different from current frame's, then no
// denoising for the current frame.
if (width_ != frame.width() || height_ != frame.height()) {
width_ = frame.width();
height_ = frame.height();
denoised_frame->CreateFrame(frame.buffer(kYPlane), frame.buffer(kUPlane),
frame.buffer(kVPlane), width_, height_,
stride_y, stride_u, stride_v);
// Setting time parameters to the output frame.
denoised_frame->set_timestamp(frame.timestamp());
denoised_frame->set_render_time_ms(frame.render_time_ms());
return;
}
// For 16x16 block.
int mb_cols = width_ >> 4;
int mb_rows = height_ >> 4;
if (metrics_.get() == nullptr)
metrics_.reset(new DenoiseMetrics[mb_cols * mb_rows]());
// Denoise on Y plane.
uint8_t* y_dst = denoised_frame->buffer(kYPlane);
uint8_t* u_dst = denoised_frame->buffer(kUPlane);
uint8_t* v_dst = denoised_frame->buffer(kVPlane);
const uint8_t* y_src = frame.buffer(kYPlane);
const uint8_t* u_src = frame.buffer(kUPlane);
const uint8_t* v_src = frame.buffer(kVPlane);
// Temporary buffer to store denoising result.
uint8_t y_tmp[16 * 16] = {0};
for (int mb_row = 0; mb_row < mb_rows; ++mb_row) {
for (int mb_col = 0; mb_col < mb_cols; ++mb_col) {
const uint8_t* mb_src = y_src + (mb_row << 4) * stride_y + (mb_col << 4);
uint8_t* mb_dst = y_dst + (mb_row << 4) * stride_y + (mb_col << 4);
int mb_index = mb_row * mb_cols + mb_col;
// Denoise each MB at the very start and save the result to a temporary
// buffer.
if (filter_->MbDenoise(mb_dst, stride_y, y_tmp, 16, mb_src, stride_y, 0,
1) == FILTER_BLOCK) {
uint32_t thr_var = 0;
// Save var and sad to the buffer.
metrics_[mb_index].var = filter_->Variance16x8(
mb_dst, stride_y, y_tmp, 16, &metrics_[mb_index].sad);
// Get skin map.
metrics_[mb_index].is_skin = MbHasSkinColor(
y_src, u_src, v_src, stride_y, stride_u, stride_v, mb_row, mb_col);
// Variance threshold for skin/non-skin MB is different.
// Skin MB use a small threshold to reduce blockiness.
thr_var = metrics_[mb_index].is_skin ? 128 : 12 * 128;
if (metrics_[mb_index].var > thr_var) {
metrics_[mb_index].denoise = 0;
// Use the source MB.
filter_->CopyMem16x16(mb_src, stride_y, mb_dst, stride_y);
} else {
metrics_[mb_index].denoise = 1;
// Use the denoised MB.
filter_->CopyMem16x16(y_tmp, 16, mb_dst, stride_y);
}
} else {
metrics_[mb_index].denoise = 0;
filter_->CopyMem16x16(mb_src, stride_y, mb_dst, stride_y);
}
// Copy source U/V plane.
const uint8_t* mb_src_u =
u_src + (mb_row << 3) * stride_u + (mb_col << 3);
const uint8_t* mb_src_v =
v_src + (mb_row << 3) * stride_v + (mb_col << 3);
uint8_t* mb_dst_u = u_dst + (mb_row << 3) * stride_u + (mb_col << 3);
uint8_t* mb_dst_v = v_dst + (mb_row << 3) * stride_v + (mb_col << 3);
filter_->CopyMem8x8(mb_src_u, stride_u, mb_dst_u, stride_u);
filter_->CopyMem8x8(mb_src_v, stride_v, mb_dst_v, stride_v);
}
}
// Second round.
// This is to reduce the trailing artifact and blockiness by referring
// neighbors' denoising status.
TrailingReduction(mb_rows, mb_cols, y_src, stride_y, y_dst);
// Setting time parameters to the output frame.
denoised_frame->set_timestamp(frame.timestamp());
denoised_frame->set_render_time_ms(frame.render_time_ms());
return;
}
} // namespace webrtc
|
