Line data Source code
1 : /*
2 : * Copyright (C) 2022 Sky UK
3 : *
4 : * This library is free software; you can redistribute it and/or
5 : * modify it under the terms of the GNU Lesser General Public
6 : * License as published by the Free Software Foundation;
7 : * version 2.1 of the License.
8 : *
9 : * This library is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 : * Lesser General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU Lesser General Public
15 : * License along with this library; if not, write to the Free Software
16 : * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 : */
18 : #include <mutex>
19 :
20 : #include <gst/audio/audio.h>
21 : #include <gst/gst.h>
22 :
23 : #include "GStreamerMSEUtils.h"
24 : #include "IMediaPipelineCapabilities.h"
25 : #include "PullModeAudioPlaybackDelegate.h"
26 : #include "PushModeAudioPlaybackDelegate.h"
27 : #include "RialtoGStreamerMSEAudioSink.h"
28 : #include "RialtoGStreamerMSEBaseSinkPrivate.h"
29 :
30 : using namespace firebolt::rialto::client;
31 :
32 : GST_DEBUG_CATEGORY_STATIC(RialtoMSEAudioSinkDebug);
33 : #define GST_CAT_DEFAULT RialtoMSEAudioSinkDebug
34 :
35 : #define rialto_mse_audio_sink_parent_class parent_class
36 4 : G_DEFINE_TYPE_WITH_CODE(RialtoMSEAudioSink, rialto_mse_audio_sink, RIALTO_TYPE_MSE_BASE_SINK,
37 : G_IMPLEMENT_INTERFACE(GST_TYPE_STREAM_VOLUME, NULL)
38 : GST_DEBUG_CATEGORY_INIT(RialtoMSEAudioSinkDebug, "rialtomseaudiosink", 0,
39 : "rialto mse audio sink"));
40 :
41 : enum
42 : {
43 : PROP_0,
44 : PROP_VOLUME,
45 : PROP_MUTE,
46 : PROP_GAP,
47 : PROP_LOW_LATENCY,
48 : PROP_SYNC,
49 : PROP_SYNC_OFF,
50 : PROP_STREAM_SYNC_MODE,
51 : PROP_AUDIO_FADE,
52 : PROP_FADE_VOLUME,
53 : PROP_LIMIT_BUFFERING_MS,
54 : PROP_USE_BUFFERING,
55 : PROP_ASYNC,
56 : PROP_WEBAUDIO,
57 : PROP_LAST
58 : };
59 :
60 : enum
61 : {
62 : SIGNAL_FIRST_AUDIO_FRAME_RECEIVED,
63 : SIGNAL_LAST
64 : };
65 :
66 : static guint g_signals[SIGNAL_LAST] = {0};
67 :
68 1 : void rialto_mse_audio_handle_rialto_server_sent_first_audio_frame_received(RialtoMSEAudioSink *sink)
69 : {
70 1 : GST_INFO_OBJECT(sink, "Sending first audio frame received signal");
71 1 : g_signal_emit(G_OBJECT(sink), g_signals[SIGNAL_FIRST_AUDIO_FRAME_RECEIVED], 0, 0, nullptr);
72 : }
73 :
74 786 : static GstStateChangeReturn rialto_mse_audio_sink_change_state(GstElement *element, GstStateChange transition)
75 : {
76 786 : RialtoMSEBaseSink *sink = RIALTO_MSE_BASE_SINK(element);
77 786 : if (GST_STATE_CHANGE_NULL_TO_READY == transition)
78 : {
79 239 : if (PlaybackMode::Pull == sink->priv->m_playbackMode)
80 : {
81 217 : GST_INFO_OBJECT(sink, "RialtoMSEAudioSink state change to READY. Initializing Pull Mode delegate");
82 217 : auto delegate = std::make_shared<PullModeAudioPlaybackDelegate>(element);
83 217 : delegate->createControlBackend();
84 217 : rialto_mse_base_sink_initialise_delegate(sink, delegate);
85 : }
86 : else // Push playback mode
87 : {
88 22 : GST_INFO_OBJECT(sink, "RialtoMSEAudioSink state change to READY. Initializing Push Mode delegate");
89 22 : rialto_mse_base_sink_initialise_delegate(sink, std::make_shared<PushModeAudioPlaybackDelegate>(element));
90 : }
91 : }
92 :
93 786 : GstStateChangeReturn result = GST_ELEMENT_CLASS(parent_class)->change_state(element, transition);
94 786 : if (G_UNLIKELY(result == GST_STATE_CHANGE_FAILURE))
95 : {
96 6 : GST_WARNING_OBJECT(sink, "State change failed");
97 6 : return result;
98 : }
99 :
100 780 : return result;
101 : }
102 :
103 30 : static void rialto_mse_audio_sink_get_property(GObject *object, guint propId, GValue *value, GParamSpec *pspec)
104 : {
105 30 : RialtoMSEBaseSink *sink = RIALTO_MSE_BASE_SINK(object);
106 30 : switch (propId)
107 : {
108 8 : case PROP_VOLUME:
109 : {
110 8 : g_value_set_double(value, kDefaultVolume);
111 8 : rialto_mse_base_sink_handle_get_property(sink, IPlaybackDelegate::Property::Volume, value);
112 8 : break;
113 : }
114 3 : case PROP_MUTE:
115 : {
116 3 : g_value_set_boolean(value, kDefaultMute);
117 3 : rialto_mse_base_sink_handle_get_property(sink, IPlaybackDelegate::Property::Mute, value);
118 3 : break;
119 : }
120 4 : case PROP_SYNC:
121 : {
122 4 : g_value_set_boolean(value, kDefaultSync);
123 4 : rialto_mse_base_sink_handle_get_property(sink, IPlaybackDelegate::Property::Sync, value);
124 4 : break;
125 : }
126 4 : case PROP_STREAM_SYNC_MODE:
127 : {
128 4 : g_value_set_int(value, kDefaultStreamSyncMode);
129 4 : rialto_mse_base_sink_handle_get_property(sink, IPlaybackDelegate::Property::StreamSyncMode, value);
130 4 : break;
131 : }
132 2 : case PROP_FADE_VOLUME:
133 : {
134 2 : g_value_set_uint(value, kDefaultFadeVolume);
135 2 : rialto_mse_base_sink_handle_get_property(sink, IPlaybackDelegate::Property::FadeVolume, value);
136 2 : break;
137 : }
138 3 : case PROP_LIMIT_BUFFERING_MS:
139 : {
140 3 : g_value_set_uint(value, kDefaultBufferingLimit);
141 3 : rialto_mse_base_sink_handle_get_property(sink, IPlaybackDelegate::Property::LimitBufferingMs, value);
142 3 : break;
143 : }
144 3 : case PROP_USE_BUFFERING:
145 : {
146 3 : g_value_set_boolean(value, kDefaultUseBuffering);
147 3 : rialto_mse_base_sink_handle_get_property(sink, IPlaybackDelegate::Property::UseBuffering, value);
148 3 : break;
149 : }
150 1 : case PROP_ASYNC:
151 : {
152 1 : g_value_set_boolean(value, TRUE);
153 1 : rialto_mse_base_sink_handle_get_property(sink, IPlaybackDelegate::Property::Async, value);
154 1 : break;
155 : }
156 1 : case PROP_WEBAUDIO:
157 : {
158 1 : g_value_set_boolean(value, (sink->priv->m_playbackMode == PlaybackMode::Push));
159 1 : break;
160 : }
161 1 : default:
162 : {
163 1 : G_OBJECT_WARN_INVALID_PROPERTY_ID(object, propId, pspec);
164 1 : break;
165 : }
166 : }
167 30 : }
168 :
169 68 : static void rialto_mse_audio_sink_set_property(GObject *object, guint propId, const GValue *value, GParamSpec *pspec)
170 : {
171 68 : RialtoMSEBaseSink *sink = RIALTO_MSE_BASE_SINK(object);
172 68 : switch (propId)
173 : {
174 8 : case PROP_VOLUME:
175 : {
176 8 : rialto_mse_base_sink_handle_set_property(sink, IPlaybackDelegate::Property::Volume, value);
177 8 : break;
178 : }
179 3 : case PROP_MUTE:
180 : {
181 3 : rialto_mse_base_sink_handle_set_property(sink, IPlaybackDelegate::Property::Mute, value);
182 3 : break;
183 : }
184 2 : case PROP_GAP:
185 : {
186 2 : rialto_mse_base_sink_handle_set_property(sink, IPlaybackDelegate::Property::Gap, value);
187 2 : break;
188 : }
189 5 : case PROP_LOW_LATENCY:
190 : {
191 5 : rialto_mse_base_sink_handle_set_property(sink, IPlaybackDelegate::Property::LowLatency, value);
192 5 : break;
193 : }
194 5 : case PROP_SYNC:
195 : {
196 5 : rialto_mse_base_sink_handle_set_property(sink, IPlaybackDelegate::Property::Sync, value);
197 5 : break;
198 : }
199 5 : case PROP_SYNC_OFF:
200 : {
201 5 : rialto_mse_base_sink_handle_set_property(sink, IPlaybackDelegate::Property::SyncOff, value);
202 5 : break;
203 : }
204 5 : case PROP_STREAM_SYNC_MODE:
205 : {
206 5 : rialto_mse_base_sink_handle_set_property(sink, IPlaybackDelegate::Property::StreamSyncMode, value);
207 5 : break;
208 : }
209 4 : case PROP_AUDIO_FADE:
210 : {
211 4 : rialto_mse_base_sink_handle_set_property(sink, IPlaybackDelegate::Property::AudioFade, value);
212 4 : break;
213 : }
214 3 : case PROP_LIMIT_BUFFERING_MS:
215 : {
216 3 : rialto_mse_base_sink_handle_set_property(sink, IPlaybackDelegate::Property::LimitBufferingMs, value);
217 3 : break;
218 : }
219 3 : case PROP_USE_BUFFERING:
220 : {
221 3 : rialto_mse_base_sink_handle_set_property(sink, IPlaybackDelegate::Property::UseBuffering, value);
222 3 : break;
223 : }
224 1 : case PROP_ASYNC:
225 : {
226 1 : rialto_mse_base_sink_handle_set_property(sink, IPlaybackDelegate::Property::Async, value);
227 1 : break;
228 : }
229 23 : case PROP_WEBAUDIO:
230 : {
231 23 : if (GST_STATE(sink) > GST_STATE_NULL)
232 : {
233 1 : GST_ERROR_OBJECT(object, "Playback mode set too late - sink is not in NULL state");
234 1 : break;
235 : }
236 22 : if (TRUE == g_value_get_boolean(value))
237 : {
238 22 : sink->priv->m_playbackMode = PlaybackMode::Push;
239 : }
240 : else
241 : {
242 0 : sink->priv->m_playbackMode = PlaybackMode::Pull;
243 : }
244 22 : break;
245 : }
246 1 : default:
247 : {
248 1 : G_OBJECT_WARN_INVALID_PROPERTY_ID(object, propId, pspec);
249 1 : break;
250 : }
251 : }
252 68 : }
253 :
254 249 : static void rialto_mse_audio_sink_init(RialtoMSEAudioSink *sink)
255 : {
256 249 : RialtoMSEBaseSinkPrivate *priv = sink->parent.priv;
257 :
258 249 : if (!rialto_mse_base_sink_initialise_sinkpad(RIALTO_MSE_BASE_SINK(sink)))
259 : {
260 0 : GST_ERROR_OBJECT(sink, "Failed to initialise AUDIO sink. Sink pad initialisation failed.");
261 0 : return;
262 : }
263 :
264 249 : gst_pad_set_chain_function(priv->m_sinkPad, rialto_mse_base_sink_chain);
265 249 : gst_pad_set_event_function(priv->m_sinkPad, rialto_mse_base_sink_event);
266 : }
267 :
268 1 : static void rialto_mse_audio_sink_class_init(RialtoMSEAudioSinkClass *klass)
269 : {
270 1 : GObjectClass *gobjectClass = G_OBJECT_CLASS(klass);
271 1 : GstElementClass *elementClass = GST_ELEMENT_CLASS(klass);
272 1 : gobjectClass->get_property = rialto_mse_audio_sink_get_property;
273 1 : gobjectClass->set_property = rialto_mse_audio_sink_set_property;
274 1 : elementClass->change_state = rialto_mse_audio_sink_change_state;
275 :
276 1 : g_object_class_install_property(gobjectClass, PROP_VOLUME,
277 : g_param_spec_double("volume", "Volume", "Volume of this stream", 0, 1.0,
278 : kDefaultVolume,
279 : GParamFlags(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
280 :
281 1 : g_object_class_install_property(gobjectClass, PROP_MUTE,
282 : g_param_spec_boolean("mute", "Mute", "Mute status of this stream", kDefaultMute,
283 : GParamFlags(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
284 :
285 1 : g_object_class_install_property(gobjectClass, PROP_GAP,
286 : g_param_spec_boxed("gap", "Gap", "Audio Gap", GST_TYPE_STRUCTURE,
287 : (GParamFlags)(G_PARAM_WRITABLE | G_PARAM_STATIC_STRINGS)));
288 :
289 1 : g_object_class_install_property(gobjectClass, PROP_USE_BUFFERING,
290 : g_param_spec_boolean("use-buffering",
291 : "Use buffering", "Emit GST_MESSAGE_BUFFERING based on low-/high-percent thresholds",
292 : kDefaultUseBuffering, G_PARAM_READWRITE));
293 1 : g_object_class_install_property(gobjectClass, PROP_ASYNC,
294 : g_param_spec_boolean("async", "Async", "Asynchronous mode", FALSE, G_PARAM_READWRITE));
295 1 : g_object_class_install_property(gobjectClass, PROP_WEBAUDIO,
296 : g_param_spec_boolean("web-audio",
297 : "Webaudio mode", "Enable webaudio mode. Property should be set before NULL->READY transition",
298 : FALSE, G_PARAM_READWRITE));
299 :
300 1 : g_signals[SIGNAL_FIRST_AUDIO_FRAME_RECEIVED] = g_signal_new("first-audio-frame-callback", G_TYPE_FROM_CLASS(klass),
301 : (GSignalFlags)(G_SIGNAL_RUN_LAST), 0, nullptr, nullptr,
302 : g_cclosure_marshal_VOID__UINT_POINTER, G_TYPE_NONE, 2,
303 : G_TYPE_UINT, G_TYPE_POINTER);
304 :
305 : std::unique_ptr<firebolt::rialto::IMediaPipelineCapabilities> mediaPlayerCapabilities =
306 1 : firebolt::rialto::IMediaPipelineCapabilitiesFactory::createFactory()->createMediaPipelineCapabilities();
307 1 : if (mediaPlayerCapabilities)
308 : {
309 : std::vector<std::string> supportedMimeTypes =
310 1 : mediaPlayerCapabilities->getSupportedMimeTypes(firebolt::rialto::MediaSourceType::AUDIO);
311 :
312 1 : rialto_mse_sink_setup_supported_caps(elementClass, supportedMimeTypes);
313 :
314 2 : const std::string kLowLatencyPropertyName{"low-latency"};
315 2 : const std::string kSyncPropertyName{"sync"};
316 2 : const std::string kSyncOffPropertyName{"sync-off"};
317 2 : const std::string kStreamSyncModePropertyName{"stream-sync-mode"};
318 2 : const std::string kAudioFadePropertyName{"audio-fade"};
319 2 : const std::string kFadeVolumePropertyName{"fade-volume"};
320 1 : const std::string kBufferingLimitPropertyName{"limit-buffering-ms"};
321 : const std::vector<std::string> kPropertyNamesToSearch{kLowLatencyPropertyName, kSyncPropertyName,
322 : kSyncOffPropertyName, kStreamSyncModePropertyName,
323 : kBufferingLimitPropertyName, kAudioFadePropertyName,
324 9 : kFadeVolumePropertyName};
325 : std::vector<std::string> supportedProperties{
326 1 : mediaPlayerCapabilities->getSupportedProperties(firebolt::rialto::MediaSourceType::AUDIO,
327 1 : kPropertyNamesToSearch)};
328 :
329 8 : for (auto it = supportedProperties.begin(); it != supportedProperties.end(); ++it)
330 : {
331 7 : if (kLowLatencyPropertyName == *it)
332 : {
333 1 : g_object_class_install_property(gobjectClass, PROP_LOW_LATENCY,
334 : g_param_spec_boolean(kLowLatencyPropertyName.c_str(),
335 : "low latency", "Turn on low latency mode, for use with gaming (no audio decoding, no a/v sync)",
336 : kDefaultLowLatency, GParamFlags(G_PARAM_WRITABLE)));
337 : }
338 6 : else if (kSyncPropertyName == *it)
339 : {
340 1 : g_object_class_install_property(gobjectClass, PROP_SYNC,
341 : g_param_spec_boolean(kSyncPropertyName.c_str(), "sync", "Clock sync",
342 : kDefaultSync, GParamFlags(G_PARAM_READWRITE)));
343 : }
344 5 : else if (kSyncOffPropertyName == *it)
345 : {
346 1 : g_object_class_install_property(gobjectClass, PROP_SYNC_OFF,
347 : g_param_spec_boolean(kSyncOffPropertyName.c_str(),
348 : "sync off", "Turn on free running audio. Must be set before pipeline is PLAYING state.",
349 : kDefaultSyncOff, GParamFlags(G_PARAM_WRITABLE)));
350 : }
351 4 : else if (kStreamSyncModePropertyName == *it)
352 : {
353 1 : g_object_class_install_property(gobjectClass, PROP_STREAM_SYNC_MODE,
354 : g_param_spec_int(kStreamSyncModePropertyName.c_str(),
355 : "stream sync mode", "1 - Frame to decode frame will immediately proceed next frame sync, 0 - Frame decoded with no frame sync",
356 : 0, G_MAXINT, kDefaultStreamSyncMode,
357 : GParamFlags(G_PARAM_READWRITE)));
358 : }
359 3 : else if (kAudioFadePropertyName == *it)
360 : {
361 1 : g_object_class_install_property(gobjectClass, PROP_AUDIO_FADE,
362 : g_param_spec_string(kAudioFadePropertyName.c_str(),
363 : "audio fade", "Start audio fade (vol[0-100],duration ms,easetype[(L)inear,Cubic(I)n,Cubic(O)ut])",
364 : kDefaultAudioFade, GParamFlags(G_PARAM_WRITABLE)));
365 : }
366 2 : else if (kFadeVolumePropertyName == *it)
367 : {
368 1 : g_object_class_install_property(gobjectClass, PROP_FADE_VOLUME,
369 : g_param_spec_uint(kFadeVolumePropertyName.c_str(), "fade volume",
370 : "Get current fade volume", 0, 100, kDefaultFadeVolume,
371 : G_PARAM_READABLE));
372 : }
373 1 : else if (kBufferingLimitPropertyName == *it)
374 : {
375 1 : constexpr uint32_t kMaxValue{20000};
376 1 : g_object_class_install_property(gobjectClass, PROP_LIMIT_BUFFERING_MS,
377 : g_param_spec_uint("limit-buffering-ms",
378 : "limit buffering ms", "Set millisecond threshold used if limit_buffering is set. Changing this value does not enable/disable limit_buffering",
379 : 0, kMaxValue, kDefaultBufferingLimit,
380 : G_PARAM_READWRITE));
381 : }
382 : else
383 : {
384 0 : GST_ERROR("Unexpected property %s returned from rialto", it->c_str());
385 : }
386 : }
387 1 : }
388 : else
389 : {
390 0 : GST_ERROR("Failed to get supported mime types for AUDIO");
391 : }
392 :
393 1 : gst_element_class_set_details_simple(elementClass, "Rialto Audio Sink", "Decoder/Audio/Sink/Audio",
394 : "Communicates with Rialto Server", "Sky");
395 2 : }
|
