1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
|
/*
* libjingle
* Copyright 2010 Google Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
* EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef TALK_MEDIA_BASE_VIDEOADAPTER_H_ // NOLINT
#define TALK_MEDIA_BASE_VIDEOADAPTER_H_
#include "talk/media/base/videocommon.h"
#include "webrtc/base/common.h" // For ASSERT
#include "webrtc/base/criticalsection.h"
#include "webrtc/base/scoped_ptr.h"
#include "webrtc/base/sigslot.h"
namespace cricket {
class VideoFrame;
// VideoAdapter adapts an input video frame to an output frame based on the
// specified input and output formats. The adaptation includes dropping frames
// to reduce frame rate and scaling frames. VideoAdapter is thread safe.
class VideoAdapter {
public:
VideoAdapter();
virtual ~VideoAdapter();
virtual void SetInputFormat(const VideoFormat& format);
void SetOutputFormat(const VideoFormat& format);
// Constrain output resolution to this many pixels overall
void SetOutputNumPixels(int num_pixels);
int GetOutputNumPixels() const;
const VideoFormat& input_format();
// Returns true if the adapter will always return zero size from
// AdaptFrameResolution.
bool drops_all_frames() const;
const VideoFormat& output_format();
// Return the adapted resolution given the input resolution. The returned
// resolution will be 0x0 if the frame should be dropped.
VideoFormat AdaptFrameResolution(int in_width, int in_height);
void set_scale_third(bool enable);
bool scale_third() const { return scale_third_; }
int adaptation_changes() const { return adaption_changes_; }
protected:
float FindClosestScale(int width, int height, int target_num_pixels);
float FindClosestViewScale(int width, int height, int target_num_pixels);
float FindLowerScale(int width, int height, int target_num_pixels);
private:
const float* GetViewScaleFactors() const;
float FindScale(const float* scale_factors,
const float upbias, int width, int height,
int target_num_pixels);
VideoFormat input_format_;
VideoFormat output_format_;
int output_num_pixels_;
bool scale_third_; // True if adapter allows scaling to 1/3 and 2/3.
int frames_in_; // Number of input frames.
int frames_out_; // Number of output frames.
int frames_scaled_; // Number of frames scaled.
int adaption_changes_; // Number of changes in scale factor.
size_t previous_width_; // Previous adapter output width.
size_t previous_height_; // Previous adapter output height.
int64_t interval_next_frame_;
// The critical section to protect the above variables.
rtc::CriticalSection critical_section_;
RTC_DISALLOW_COPY_AND_ASSIGN(VideoAdapter);
};
// CoordinatedVideoAdapter adapts the video input to the encoder by coordinating
// the format request from the server, the resolution request from the encoder,
// and the CPU load.
class CoordinatedVideoAdapter
: public VideoAdapter, public sigslot::has_slots<> {
public:
enum AdaptRequest { UPGRADE, KEEP, DOWNGRADE };
enum AdaptReasonEnum {
ADAPTREASON_NONE = 0,
ADAPTREASON_CPU = 1,
ADAPTREASON_BANDWIDTH = 2,
ADAPTREASON_VIEW = 4
};
typedef int AdaptReason;
CoordinatedVideoAdapter();
virtual ~CoordinatedVideoAdapter() {}
virtual void SetInputFormat(const VideoFormat& format);
// Enable or disable video adaptation due to the change of the CPU load.
void set_cpu_adaptation(bool enable) { cpu_adaptation_ = enable; }
bool cpu_adaptation() const { return cpu_adaptation_; }
// Enable or disable smoothing when doing CPU adaptation. When smoothing is
// enabled, system CPU load is tracked using an exponential weighted
// average.
void set_cpu_smoothing(bool enable);
bool cpu_smoothing() const { return cpu_smoothing_; }
// Enable or disable video adaptation due to the change of the GD
void set_gd_adaptation(bool enable) { gd_adaptation_ = enable; }
bool gd_adaptation() const { return gd_adaptation_; }
// Enable or disable video adaptation due to the change of the View
void set_view_adaptation(bool enable) { view_adaptation_ = enable; }
bool view_adaptation() const { return view_adaptation_; }
// Enable or disable video adaptation to fast switch View
void set_view_switch(bool enable) { view_switch_ = enable; }
bool view_switch() const { return view_switch_; }
CoordinatedVideoAdapter::AdaptReason adapt_reason() const {
return adapt_reason_;
}
// When the video is decreased, set the waiting time for CPU adaptation to
// decrease video again.
void set_cpu_load_min_samples(int cpu_load_min_samples);
int cpu_load_min_samples() const { return cpu_load_min_samples_; }
// CPU system load high threshold for reducing resolution. e.g. 0.85f
void set_high_system_threshold(float high_system_threshold);
float high_system_threshold() const { return high_system_threshold_; }
// CPU system load low threshold for increasing resolution. e.g. 0.70f
void set_low_system_threshold(float low_system_threshold);
float low_system_threshold() const { return low_system_threshold_; }
// CPU process load threshold for reducing resolution. e.g. 0.10f
void set_process_threshold(float process_threshold);
float process_threshold() const { return process_threshold_; }
// Handle the format request from the server via Jingle update message.
void OnOutputFormatRequest(const VideoFormat& format);
// Handle the resolution request from the encoder due to bandwidth changes.
void OnEncoderResolutionRequest(int width, int height, AdaptRequest request);
// Handle the resolution request for CPU overuse.
void OnCpuResolutionRequest(AdaptRequest request);
// Handle the CPU load provided by a CPU monitor.
void OnCpuLoadUpdated(int current_cpus, int max_cpus,
float process_load, float system_load);
sigslot::signal0<> SignalCpuAdaptationUnable;
private:
// Adapt to the minimum of the formats the server requests, the CPU wants, and
// the encoder wants. Returns true if resolution changed.
bool AdaptToMinimumFormat(int* new_width, int* new_height);
bool IsMinimumFormat(int pixels);
void StepPixelCount(CoordinatedVideoAdapter::AdaptRequest request,
int* num_pixels);
CoordinatedVideoAdapter::AdaptRequest FindCpuRequest(
int current_cpus, int max_cpus,
float process_load, float system_load);
bool cpu_adaptation_; // True if cpu adaptation is enabled.
bool cpu_smoothing_; // True if cpu smoothing is enabled (with adaptation).
bool gd_adaptation_; // True if gd adaptation is enabled.
bool view_adaptation_; // True if view adaptation is enabled.
bool view_switch_; // True if view switch is enabled.
int cpu_downgrade_count_;
int cpu_load_min_samples_;
int cpu_load_num_samples_;
// cpu system load thresholds relative to max cpus.
float high_system_threshold_;
float low_system_threshold_;
// cpu process load thresholds relative to current cpus.
float process_threshold_;
// Video formats that the server view requests, the CPU wants, and the encoder
// wants respectively. The adapted output format is the minimum of these.
int view_desired_num_pixels_;
int64_t view_desired_interval_;
int encoder_desired_num_pixels_;
int cpu_desired_num_pixels_;
CoordinatedVideoAdapter::AdaptReason adapt_reason_;
// The critical section to protect handling requests.
rtc::CriticalSection request_critical_section_;
// The weighted average of cpu load over time. It's always updated (if cpu
// adaptation is on), but only used if cpu_smoothing_ is set.
float system_load_average_;
RTC_DISALLOW_COPY_AND_ASSIGN(CoordinatedVideoAdapter);
};
} // namespace cricket
#endif // TALK_MEDIA_BASE_VIDEOADAPTER_H_ // NOLINT
|
