Line data Source code
1 : /*
2 : * Copyright (C) 2022 Sky UK
3 : *
4 : * This library is free software; you can redistribute it and/or
5 : * modify it under the terms of the GNU Lesser General Public
6 : * License as published by the Free Software Foundation;
7 : * version 2.1 of the License.
8 : *
9 : * This library is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 : * Lesser General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU Lesser General Public
15 : * License along with this library; if not, write to the Free Software
16 : * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 : */
18 : #include <mutex>
19 :
20 : #include <gst/audio/audio.h>
21 : #include <gst/gst.h>
22 : #include <gst/pbutils/pbutils.h>
23 : #include <inttypes.h>
24 : #include <stdint.h>
25 :
26 : #include "Constants.h"
27 : #include "GStreamerEMEUtils.h"
28 : #include "GStreamerMSEUtils.h"
29 : #include "IMediaPipelineCapabilities.h"
30 : #include "RialtoGStreamerMSEAudioSink.h"
31 : #include "RialtoGStreamerMSEAudioSinkPrivate.h"
32 : #include "RialtoGStreamerMSEBaseSinkPrivate.h"
33 :
34 : using namespace firebolt::rialto::client;
35 :
36 : GST_DEBUG_CATEGORY_STATIC(RialtoMSEAudioSinkDebug);
37 : #define GST_CAT_DEFAULT RialtoMSEAudioSinkDebug
38 :
39 : #define rialto_mse_audio_sink_parent_class parent_class
40 773 : G_DEFINE_TYPE_WITH_CODE(RialtoMSEAudioSink, rialto_mse_audio_sink, RIALTO_TYPE_MSE_BASE_SINK,
41 : G_ADD_PRIVATE(RialtoMSEAudioSink) G_IMPLEMENT_INTERFACE(GST_TYPE_STREAM_VOLUME, NULL)
42 : GST_DEBUG_CATEGORY_INIT(RialtoMSEAudioSinkDebug, "rialtomseaudiosink", 0,
43 : "rialto mse audio sink"));
44 :
45 : enum
46 : {
47 : PROP_0,
48 : PROP_VOLUME,
49 : PROP_MUTE,
50 : PROP_GAP,
51 : PROP_LOW_LATENCY,
52 : PROP_SYNC,
53 : PROP_SYNC_OFF,
54 : PROP_STREAM_SYNC_MODE,
55 : PROP_AUDIO_FADE,
56 : PROP_FADE_VOLUME,
57 : PROP_LIMIT_BUFFERING_MS,
58 : PROP_USE_BUFFERING,
59 : PROP_ASYNC,
60 : PROP_LAST
61 : };
62 :
63 415 : static GstStateChangeReturn rialto_mse_audio_sink_change_state(GstElement *element, GstStateChange transition)
64 : {
65 415 : RialtoMSEAudioSink *sink = RIALTO_MSE_AUDIO_SINK(element);
66 415 : RialtoMSEBaseSinkPrivate *basePriv = sink->parent.priv;
67 415 : RialtoMSEAudioSinkPrivate *priv = sink->priv;
68 :
69 415 : switch (transition)
70 : {
71 100 : case GST_STATE_CHANGE_READY_TO_PAUSED:
72 : {
73 100 : if (!rialto_mse_base_sink_attach_to_media_client_and_set_streams_number(element))
74 : {
75 2 : return GST_STATE_CHANGE_FAILURE;
76 : }
77 :
78 98 : std::shared_ptr<GStreamerMSEMediaPlayerClient> client = basePriv->m_mediaPlayerManager.getMediaPlayerClient();
79 98 : if (!client)
80 : {
81 0 : GST_ERROR_OBJECT(sink, "MediaPlayerClient is nullptr");
82 0 : return GST_STATE_CHANGE_FAILURE;
83 : }
84 98 : if (priv->isVolumeQueued)
85 : {
86 1 : client->setVolume(priv->targetVolume, kDefaultVolumeDuration, kDefaultEaseType);
87 1 : priv->isVolumeQueued = false;
88 : }
89 98 : if (priv->isAudioFadeQueued)
90 : {
91 : AudioFadeConfig audioFadeConfig;
92 : {
93 1 : std::lock_guard<std::mutex> lock(priv->audioFadeConfigMutex);
94 1 : audioFadeConfig = priv->audioFadeConfig;
95 : }
96 1 : client->setVolume(audioFadeConfig.volume, audioFadeConfig.duration, audioFadeConfig.easeType);
97 1 : priv->isAudioFadeQueued = false;
98 : }
99 98 : break;
100 : }
101 315 : default:
102 315 : break;
103 : }
104 :
105 413 : GstStateChangeReturn result = GST_ELEMENT_CLASS(parent_class)->change_state(element, transition);
106 413 : if (G_UNLIKELY(result == GST_STATE_CHANGE_FAILURE))
107 : {
108 0 : GST_WARNING_OBJECT(sink, "State change failed");
109 0 : return result;
110 : }
111 :
112 413 : return result;
113 : }
114 :
115 : static std::unique_ptr<firebolt::rialto::IMediaPipeline::MediaSource>
116 96 : rialto_mse_audio_sink_create_media_source(RialtoMSEBaseSink *sink, GstCaps *caps)
117 : {
118 96 : GstStructure *structure = gst_caps_get_structure(caps, 0);
119 96 : const gchar *strct_name = gst_structure_get_name(structure);
120 :
121 96 : firebolt::rialto::AudioConfig audioConfig;
122 96 : firebolt::rialto::SegmentAlignment alignment = rialto_mse_base_sink_get_segment_alignment(sink, structure);
123 96 : std::shared_ptr<firebolt::rialto::CodecData> codecData = rialto_mse_base_sink_get_codec_data(sink, structure);
124 96 : firebolt::rialto::StreamFormat format = rialto_mse_base_sink_get_stream_format(sink, structure);
125 96 : std::string mimeType;
126 :
127 96 : if (strct_name)
128 : {
129 101 : if (g_str_has_prefix(strct_name, "audio/mpeg") || g_str_has_prefix(strct_name, "audio/x-eac3") ||
130 6 : g_str_has_prefix(strct_name, "audio/x-ac3"))
131 : {
132 90 : gint sample_rate = 0;
133 90 : gint number_of_channels = 0;
134 90 : gst_structure_get_int(structure, "rate", &sample_rate);
135 90 : gst_structure_get_int(structure, "channels", &number_of_channels);
136 :
137 180 : audioConfig = firebolt::rialto::AudioConfig{static_cast<uint32_t>(number_of_channels),
138 90 : static_cast<uint32_t>(sample_rate),
139 90 : {}};
140 :
141 90 : if (g_str_has_prefix(strct_name, "audio/mpeg"))
142 : {
143 88 : mimeType = "audio/mp4";
144 : }
145 : else
146 : {
147 2 : mimeType = "audio/x-eac3";
148 : }
149 : }
150 5 : else if (g_str_has_prefix(strct_name, "audio/x-opus"))
151 : {
152 2 : mimeType = "audio/x-opus";
153 2 : guint32 sample_rate = 48000;
154 : guint8 number_of_channels, streams, stereo_streams, channel_mapping_family;
155 : guint8 channel_mapping[256];
156 2 : guint16 pre_skip = 0;
157 2 : gint16 gain = 0;
158 2 : if (gst_codec_utils_opus_parse_caps(caps, &sample_rate, &number_of_channels, &channel_mapping_family,
159 : &streams, &stereo_streams, channel_mapping))
160 : {
161 : GstBuffer *id_header;
162 1 : id_header = gst_codec_utils_opus_create_header(sample_rate, number_of_channels, channel_mapping_family,
163 : streams, stereo_streams, channel_mapping, pre_skip, gain);
164 1 : std::vector<uint8_t> codec_specific_config;
165 : GstMapInfo lsMap;
166 1 : if (gst_buffer_map(id_header, &lsMap, GST_MAP_READ))
167 : {
168 1 : codec_specific_config.assign(lsMap.data, lsMap.data + lsMap.size);
169 1 : gst_buffer_unmap(id_header, &lsMap);
170 : }
171 : else
172 : {
173 0 : GST_ERROR_OBJECT(sink, "Failed to read opus header details from a GstBuffer!");
174 : }
175 1 : gst_buffer_unref(id_header);
176 :
177 1 : audioConfig = firebolt::rialto::AudioConfig{number_of_channels, sample_rate, codec_specific_config};
178 : }
179 : else
180 : {
181 1 : GST_ERROR("Failed to parse opus caps!");
182 1 : return nullptr;
183 : }
184 : }
185 3 : else if (g_str_has_prefix(strct_name, "audio/b-wav") || g_str_has_prefix(strct_name, "audio/x-raw"))
186 : {
187 2 : gint sample_rate = 0;
188 2 : gint number_of_channels = 0;
189 2 : std::optional<uint64_t> channelMask;
190 2 : gst_structure_get_int(structure, "rate", &sample_rate);
191 2 : gst_structure_get_int(structure, "channels", &number_of_channels);
192 : std::optional<firebolt::rialto::Layout> layout =
193 2 : rialto_mse_sink_convert_layout(gst_structure_get_string(structure, "layout"));
194 : std::optional<firebolt::rialto::Format> format =
195 2 : rialto_mse_sink_convert_format(gst_structure_get_string(structure, "format"));
196 2 : const GValue *channelMaskValue = gst_structure_get_value(structure, "channel-mask");
197 2 : if (channelMaskValue)
198 : {
199 2 : channelMask = gst_value_get_bitmask(channelMaskValue);
200 : }
201 :
202 2 : if (g_str_has_prefix(strct_name, "audio/b-wav"))
203 : {
204 1 : mimeType = "audio/b-wav";
205 : }
206 : else
207 : {
208 1 : mimeType = "audio/x-raw";
209 : }
210 :
211 4 : audioConfig = firebolt::rialto::AudioConfig{static_cast<uint32_t>(number_of_channels),
212 2 : static_cast<uint32_t>(sample_rate),
213 : {},
214 : format,
215 : layout,
216 2 : channelMask};
217 : }
218 1 : else if (g_str_has_prefix(strct_name, "audio/x-flac"))
219 : {
220 1 : mimeType = "audio/x-flac";
221 1 : gint sample_rate = 0;
222 1 : gint number_of_channels = 0;
223 1 : gst_structure_get_int(structure, "rate", &sample_rate);
224 1 : gst_structure_get_int(structure, "channels", &number_of_channels);
225 1 : std::vector<std::vector<uint8_t>> streamHeaderVec;
226 1 : const GValue *streamheader = gst_structure_get_value(structure, "streamheader");
227 1 : if (streamheader)
228 : {
229 2 : for (guint i = 0; i < gst_value_array_get_size(streamheader); ++i)
230 : {
231 1 : const GValue *headerValue = gst_value_array_get_value(streamheader, i);
232 1 : GstBuffer *headerBuffer = gst_value_get_buffer(headerValue);
233 1 : if (headerBuffer)
234 : {
235 1 : GstMappedBuffer mappedBuf(headerBuffer, GST_MAP_READ);
236 1 : if (mappedBuf)
237 : {
238 1 : streamHeaderVec.push_back(
239 3 : std::vector<std::uint8_t>(mappedBuf.data(), mappedBuf.data() + mappedBuf.size()));
240 : }
241 1 : }
242 : }
243 : }
244 1 : std::optional<bool> framed;
245 1 : gboolean framedValue{FALSE};
246 1 : if (gst_structure_get_boolean(structure, "framed", &framedValue))
247 : {
248 1 : framed = framedValue;
249 : }
250 :
251 2 : audioConfig = firebolt::rialto::AudioConfig{static_cast<uint32_t>(number_of_channels),
252 1 : static_cast<uint32_t>(sample_rate),
253 : {},
254 : std::nullopt,
255 : std::nullopt,
256 : std::nullopt,
257 : streamHeaderVec,
258 1 : framed};
259 : }
260 : else
261 : {
262 0 : GST_INFO_OBJECT(sink, "%s audio media source created", strct_name);
263 0 : mimeType = strct_name;
264 : }
265 :
266 188 : return std::make_unique<firebolt::rialto::IMediaPipeline::MediaSourceAudio>(mimeType, sink->priv->m_hasDrm,
267 : audioConfig, alignment, format,
268 94 : codecData);
269 : }
270 :
271 1 : GST_ERROR_OBJECT(sink, "Empty caps' structure name! Failed to set mime type for audio media source.");
272 1 : return nullptr;
273 96 : }
274 :
275 100 : static gboolean rialto_mse_audio_sink_event(GstPad *pad, GstObject *parent, GstEvent *event)
276 : {
277 100 : RialtoMSEBaseSink *sink = RIALTO_MSE_BASE_SINK(parent);
278 100 : RialtoMSEAudioSink *audioSink = RIALTO_MSE_AUDIO_SINK(parent);
279 100 : RialtoMSEBaseSinkPrivate *basePriv = sink->priv;
280 100 : switch (GST_EVENT_TYPE(event))
281 : {
282 94 : case GST_EVENT_CAPS:
283 : {
284 : GstCaps *caps;
285 94 : gst_event_parse_caps(event, &caps);
286 94 : if (basePriv->m_sourceAttached)
287 : {
288 1 : GST_INFO_OBJECT(sink, "Source already attached. Skip calling attachSource");
289 1 : break;
290 : }
291 :
292 93 : GST_INFO_OBJECT(sink, "Attaching AUDIO source with caps %" GST_PTR_FORMAT, caps);
293 :
294 : std::unique_ptr<firebolt::rialto::IMediaPipeline::MediaSource> asource =
295 93 : rialto_mse_audio_sink_create_media_source(sink, caps);
296 93 : if (asource)
297 : {
298 : std::shared_ptr<GStreamerMSEMediaPlayerClient> client =
299 92 : sink->priv->m_mediaPlayerManager.getMediaPlayerClient();
300 92 : if ((!client) || (!client->attachSource(asource, sink)))
301 : {
302 1 : GST_ERROR_OBJECT(sink, "Failed to attach AUDIO source");
303 : }
304 : else
305 : {
306 91 : basePriv->m_sourceAttached = true;
307 91 : RialtoMSEAudioSinkPrivate *priv = audioSink->priv;
308 :
309 91 : if (priv->isMuteQueued)
310 : {
311 1 : client->setMute(priv->mute, basePriv->m_sourceId);
312 1 : priv->isMuteQueued = false;
313 : }
314 91 : if (priv->isLowLatencyQueued)
315 : {
316 2 : if (!client->setLowLatency(priv->lowLatency))
317 : {
318 1 : GST_ERROR_OBJECT(audioSink, "Could not set queued low-latency");
319 : }
320 2 : priv->isLowLatencyQueued = false;
321 : }
322 91 : if (priv->isSyncQueued)
323 : {
324 2 : if (!client->setSync(priv->sync))
325 : {
326 1 : GST_ERROR_OBJECT(audioSink, "Could not set queued sync");
327 : }
328 2 : priv->isSyncQueued = false;
329 : }
330 91 : if (priv->isSyncOffQueued)
331 : {
332 2 : if (!client->setSyncOff(priv->syncOff))
333 : {
334 1 : GST_ERROR_OBJECT(audioSink, "Could not set queued sync-off");
335 : }
336 2 : priv->isSyncOffQueued = false;
337 : }
338 91 : if (priv->isStreamSyncModeQueued)
339 : {
340 2 : if (!client->setStreamSyncMode(basePriv->m_sourceId, audioSink->priv->streamSyncMode))
341 : {
342 1 : GST_ERROR_OBJECT(audioSink, "Could not set queued stream-sync-mode");
343 : }
344 2 : priv->isStreamSyncModeQueued = false;
345 : }
346 91 : if (priv->isBufferingLimitQueued)
347 : {
348 1 : client->setBufferingLimit(audioSink->priv->bufferingLimit);
349 1 : priv->isBufferingLimitQueued = false;
350 : }
351 91 : if (priv->isUseBufferingQueued)
352 : {
353 1 : client->setUseBuffering(audioSink->priv->useBuffering);
354 1 : priv->isUseBufferingQueued = false;
355 : }
356 :
357 : // check if READY -> PAUSED was requested before source was attached
358 91 : if (GST_STATE_NEXT(sink) == GST_STATE_PAUSED)
359 : {
360 91 : client->pause(sink->priv->m_sourceId);
361 : }
362 : }
363 92 : }
364 : else
365 : {
366 1 : GST_ERROR_OBJECT(sink, "Failed to create AUDIO source");
367 : }
368 93 : break;
369 : }
370 4 : case GST_EVENT_CUSTOM_DOWNSTREAM:
371 : case GST_EVENT_CUSTOM_DOWNSTREAM_OOB:
372 : {
373 4 : if (gst_event_has_name(event, "switch-source"))
374 : {
375 4 : GST_DEBUG_OBJECT(sink, "Switch source event received");
376 4 : const GstStructure *structure{gst_event_get_structure(event)};
377 4 : const GValue *value = gst_structure_get_value(structure, "caps");
378 4 : if (!value)
379 : {
380 1 : GST_ERROR_OBJECT(sink, "Caps not available in switch-source event");
381 2 : break;
382 : }
383 3 : const GstCaps *caps = gst_value_get_caps(value);
384 3 : GstCaps *mutableCaps = gst_caps_copy(caps);
385 : std::unique_ptr<firebolt::rialto::IMediaPipeline::MediaSource> asource =
386 3 : rialto_mse_audio_sink_create_media_source(sink, mutableCaps);
387 3 : gst_caps_unref(mutableCaps);
388 3 : if (!asource)
389 : {
390 1 : GST_ERROR_OBJECT(sink, "Not able to parse caps");
391 1 : break;
392 : }
393 : std::shared_ptr<GStreamerMSEMediaPlayerClient> client =
394 2 : sink->priv->m_mediaPlayerManager.getMediaPlayerClient();
395 2 : if ((!client) || (!client->switchSource(asource)))
396 : {
397 1 : GST_ERROR_OBJECT(sink, "Failed to switch AUDIO source");
398 : }
399 3 : }
400 2 : break;
401 : }
402 2 : default:
403 2 : break;
404 : }
405 :
406 100 : return rialto_mse_base_sink_event(pad, parent, event);
407 : }
408 :
409 26 : static void rialto_mse_audio_sink_get_property(GObject *object, guint propId, GValue *value, GParamSpec *pspec)
410 : {
411 26 : RialtoMSEAudioSink *sink = RIALTO_MSE_AUDIO_SINK(object);
412 26 : if (!sink)
413 : {
414 0 : GST_ERROR_OBJECT(object, "Sink not initalised");
415 11 : return;
416 : }
417 26 : RialtoMSEBaseSinkPrivate *basePriv = sink->parent.priv;
418 26 : RialtoMSEAudioSinkPrivate *priv = sink->priv;
419 26 : if (!basePriv || !priv)
420 : {
421 0 : GST_ERROR_OBJECT(object, "Private Sink not initalised");
422 0 : return;
423 : }
424 :
425 26 : std::shared_ptr<GStreamerMSEMediaPlayerClient> client = basePriv->m_mediaPlayerManager.getMediaPlayerClient();
426 :
427 26 : switch (propId)
428 : {
429 5 : case PROP_VOLUME:
430 : {
431 : double volume;
432 5 : if (client)
433 : {
434 3 : if (client->getVolume(volume))
435 2 : priv->targetVolume = volume;
436 : else
437 1 : volume = priv->targetVolume; // Use last known volume
438 : }
439 : else
440 : {
441 2 : volume = priv->targetVolume;
442 : }
443 5 : g_value_set_double(value, volume);
444 5 : break;
445 : }
446 3 : case PROP_MUTE:
447 : {
448 3 : if (!client)
449 : {
450 2 : g_value_set_boolean(value, priv->mute);
451 2 : return;
452 : }
453 1 : g_value_set_boolean(value, client->getMute(basePriv->m_sourceId));
454 1 : break;
455 : }
456 4 : case PROP_SYNC:
457 : {
458 4 : if (!client)
459 : {
460 2 : g_value_set_boolean(value, priv->sync);
461 2 : return;
462 : }
463 :
464 2 : bool sync{kDefaultSync};
465 2 : if (!client->getSync(sync))
466 : {
467 1 : GST_ERROR_OBJECT(sink, "Could not get sync");
468 : }
469 2 : g_value_set_boolean(value, sync);
470 2 : break;
471 : }
472 4 : case PROP_STREAM_SYNC_MODE:
473 : {
474 4 : if (!client)
475 : {
476 2 : g_value_set_int(value, priv->streamSyncMode);
477 2 : return;
478 : }
479 :
480 2 : int32_t streamSyncMode{kDefaultStreamSyncMode};
481 2 : if (!client->getStreamSyncMode(streamSyncMode))
482 : {
483 1 : GST_ERROR_OBJECT(sink, "Could not get stream-sync-mode");
484 : }
485 2 : g_value_set_int(value, streamSyncMode);
486 2 : break;
487 : }
488 2 : case PROP_FADE_VOLUME:
489 : {
490 : double volume;
491 2 : if (!client || !client->getVolume(volume))
492 : {
493 1 : g_value_set_uint(value, kDefaultFadeVolume);
494 1 : return;
495 : }
496 1 : g_value_set_uint(value, static_cast<uint32_t>(volume * 100.0));
497 1 : break;
498 : }
499 3 : case PROP_LIMIT_BUFFERING_MS:
500 : {
501 3 : if (!client)
502 : {
503 2 : g_value_set_uint(value, priv->bufferingLimit);
504 2 : return;
505 : }
506 1 : g_value_set_uint(value, client->getBufferingLimit());
507 1 : break;
508 : }
509 3 : case PROP_USE_BUFFERING:
510 : {
511 3 : if (!client)
512 : {
513 2 : g_value_set_boolean(value, priv->useBuffering);
514 2 : return;
515 : }
516 1 : g_value_set_boolean(value, client->getUseBuffering());
517 1 : break;
518 : }
519 1 : case PROP_ASYNC:
520 : {
521 : // Rialto MSE Audio sink is always async
522 1 : g_value_set_boolean(value, TRUE);
523 1 : break;
524 : }
525 1 : default:
526 : {
527 1 : G_OBJECT_WARN_INVALID_PROPERTY_ID(object, propId, pspec);
528 1 : break;
529 : }
530 : }
531 26 : }
532 :
533 3 : firebolt::rialto::EaseType convertCharToEaseType(char easeTypeChar)
534 : {
535 3 : switch (easeTypeChar)
536 : {
537 1 : case 'L':
538 1 : return firebolt::rialto::EaseType::EASE_LINEAR;
539 1 : case 'I':
540 1 : return firebolt::rialto::EaseType::EASE_IN_CUBIC;
541 1 : case 'O':
542 1 : return firebolt::rialto::EaseType::EASE_OUT_CUBIC;
543 0 : default:
544 0 : return firebolt::rialto::EaseType::EASE_LINEAR;
545 : }
546 : }
547 :
548 40 : static void rialto_mse_audio_sink_set_property(GObject *object, guint propId, const GValue *value, GParamSpec *pspec)
549 : {
550 40 : RialtoMSEAudioSink *sink = RIALTO_MSE_AUDIO_SINK(object);
551 40 : if (!sink)
552 : {
553 0 : GST_ERROR_OBJECT(object, "Sink not initalised");
554 22 : return;
555 : }
556 40 : RialtoMSEBaseSinkPrivate *basePriv = sink->parent.priv;
557 40 : RialtoMSEAudioSinkPrivate *priv = sink->priv;
558 40 : if (!basePriv || !priv)
559 : {
560 0 : GST_ERROR_OBJECT(object, "Private Sink not initalised");
561 0 : return;
562 : }
563 :
564 40 : std::shared_ptr<GStreamerMSEMediaPlayerClient> client = basePriv->m_mediaPlayerManager.getMediaPlayerClient();
565 :
566 40 : switch (propId)
567 : {
568 3 : case PROP_VOLUME:
569 : {
570 3 : priv->targetVolume = g_value_get_double(value);
571 3 : if (!client || !basePriv->m_sourceAttached)
572 : {
573 2 : GST_DEBUG_OBJECT(object, "Enqueue volume setting");
574 2 : priv->isVolumeQueued = true;
575 2 : return;
576 : }
577 1 : client->setVolume(priv->targetVolume, kDefaultVolumeDuration, kDefaultEaseType);
578 1 : break;
579 : }
580 3 : case PROP_MUTE:
581 : {
582 3 : priv->mute = g_value_get_boolean(value);
583 3 : if (!client || !basePriv->m_sourceAttached)
584 : {
585 2 : GST_DEBUG_OBJECT(object, "Enqueue mute setting");
586 2 : priv->isMuteQueued = true;
587 2 : return;
588 : }
589 1 : client->setMute(priv->mute, basePriv->m_sourceId);
590 1 : break;
591 : }
592 2 : case PROP_GAP:
593 : {
594 2 : gint64 position{0}, discontinuityGap{0};
595 2 : guint duration{0};
596 2 : gboolean audioAac{FALSE};
597 :
598 2 : GstStructure *gapData = GST_STRUCTURE_CAST(g_value_get_boxed(value));
599 2 : if (!gst_structure_get_int64(gapData, "position", &position))
600 : {
601 1 : GST_WARNING_OBJECT(object, "Set gap: position is missing!");
602 : }
603 2 : if (!gst_structure_get_uint(gapData, "duration", &duration))
604 : {
605 1 : GST_WARNING_OBJECT(object, "Set gap: duration is missing!");
606 : }
607 2 : if (!gst_structure_get_int64(gapData, "discontinuity-gap", &discontinuityGap))
608 : {
609 1 : GST_WARNING_OBJECT(object, "Set gap: discontinuity gap is missing!");
610 : }
611 2 : if (!gst_structure_get_boolean(gapData, "audio-aac", &audioAac))
612 : {
613 1 : GST_WARNING_OBJECT(object, "Set gap: audio aac is missing!");
614 : }
615 :
616 2 : GST_DEBUG_OBJECT(object, "Processing audio gap.");
617 2 : client->processAudioGap(position, duration, discontinuityGap, audioAac);
618 2 : break;
619 : }
620 5 : case PROP_LOW_LATENCY:
621 : {
622 5 : priv->lowLatency = g_value_get_boolean(value);
623 5 : if (!client)
624 : {
625 3 : GST_DEBUG_OBJECT(object, "Enqueue low latency setting");
626 3 : priv->isLowLatencyQueued = true;
627 3 : return;
628 : }
629 :
630 2 : if (!client->setLowLatency(priv->lowLatency))
631 : {
632 1 : GST_ERROR_OBJECT(sink, "Could not set low-latency");
633 : }
634 2 : break;
635 : }
636 5 : case PROP_SYNC:
637 : {
638 5 : priv->sync = g_value_get_boolean(value);
639 5 : if (!client)
640 : {
641 3 : GST_DEBUG_OBJECT(object, "Enqueue sync setting");
642 3 : priv->isSyncQueued = true;
643 3 : return;
644 : }
645 :
646 2 : if (!client->setSync(priv->sync))
647 : {
648 1 : GST_ERROR_OBJECT(sink, "Could not set sync");
649 : }
650 2 : break;
651 : }
652 5 : case PROP_SYNC_OFF:
653 : {
654 5 : priv->syncOff = g_value_get_boolean(value);
655 5 : if (!client)
656 : {
657 3 : GST_DEBUG_OBJECT(object, "Enqueue sync off setting");
658 3 : priv->isSyncOffQueued = true;
659 3 : return;
660 : }
661 :
662 2 : if (!client->setSyncOff(priv->syncOff))
663 : {
664 1 : GST_ERROR_OBJECT(sink, "Could not set sync-off");
665 : }
666 2 : break;
667 : }
668 5 : case PROP_STREAM_SYNC_MODE:
669 : {
670 5 : priv->streamSyncMode = g_value_get_int(value);
671 5 : if (!client || !basePriv->m_sourceAttached)
672 : {
673 3 : GST_DEBUG_OBJECT(object, "Enqueue stream sync mode setting");
674 3 : priv->isStreamSyncModeQueued = true;
675 3 : return;
676 : }
677 :
678 2 : if (!client->setStreamSyncMode(basePriv->m_sourceId, priv->streamSyncMode))
679 : {
680 1 : GST_ERROR_OBJECT(sink, "Could not set stream-sync-mode");
681 : }
682 2 : break;
683 : }
684 4 : case PROP_AUDIO_FADE:
685 : {
686 4 : const gchar *audioFadeStr = g_value_get_string(value);
687 :
688 4 : uint32_t fadeVolume = static_cast<uint32_t>(kDefaultVolume * 100);
689 4 : uint32_t duration = kDefaultVolumeDuration;
690 4 : char easeTypeChar = 'L';
691 :
692 4 : int parsedItems = sscanf(audioFadeStr, "%u,%u,%c", &fadeVolume, &duration, &easeTypeChar);
693 :
694 4 : if (parsedItems == 0)
695 : {
696 1 : GST_ERROR_OBJECT(object, "Failed to parse any values from audio fade string: %s.", audioFadeStr);
697 2 : return;
698 : }
699 3 : else if (parsedItems == 1 || parsedItems == 2)
700 : {
701 1 : GST_WARNING_OBJECT(object, "Partially parsed audio fade string: %s. Continuing with values: fadeVolume=%u, duration=%u, easeTypeChar=%c",
702 : audioFadeStr, fadeVolume, duration, easeTypeChar);
703 : }
704 :
705 3 : if (fadeVolume > 100)
706 : {
707 0 : GST_WARNING_OBJECT(object, "Fade volume is greater than 100. Setting it to 100.");
708 0 : fadeVolume = 100;
709 : }
710 3 : double volume = fadeVolume / 100.0;
711 :
712 3 : firebolt::rialto::EaseType easeType = convertCharToEaseType(easeTypeChar);
713 :
714 : {
715 3 : std::lock_guard<std::mutex> lock(priv->audioFadeConfigMutex);
716 3 : priv->audioFadeConfig.volume = volume;
717 3 : priv->audioFadeConfig.duration = duration;
718 3 : priv->audioFadeConfig.easeType = easeType;
719 : }
720 :
721 3 : if (!client)
722 : {
723 1 : GST_DEBUG_OBJECT(object, "Enqueue audio fade setting");
724 1 : priv->isAudioFadeQueued = true;
725 1 : return;
726 : }
727 :
728 2 : client->setVolume(volume, duration, easeType);
729 2 : break;
730 : }
731 3 : case PROP_LIMIT_BUFFERING_MS:
732 : {
733 3 : priv->bufferingLimit = g_value_get_uint(value);
734 3 : if (!client)
735 : {
736 2 : GST_DEBUG_OBJECT(object, "Enqueue buffering limit setting");
737 2 : priv->isBufferingLimitQueued = true;
738 2 : return;
739 : }
740 :
741 1 : client->setBufferingLimit(priv->bufferingLimit);
742 1 : break;
743 : }
744 3 : case PROP_USE_BUFFERING:
745 : {
746 3 : priv->useBuffering = g_value_get_boolean(value);
747 3 : if (!client)
748 : {
749 2 : GST_DEBUG_OBJECT(object, "Enqueue use buffering setting");
750 2 : priv->isUseBufferingQueued = true;
751 2 : return;
752 : }
753 :
754 1 : client->setUseBuffering(priv->useBuffering);
755 1 : break;
756 : }
757 1 : case PROP_ASYNC:
758 : {
759 1 : if (FALSE == g_value_get_boolean(value))
760 : {
761 1 : GST_WARNING_OBJECT(object, "Cannot set ASYNC to false - not supported");
762 : }
763 1 : break;
764 : }
765 1 : default:
766 : {
767 1 : G_OBJECT_WARN_INVALID_PROPERTY_ID(object, propId, pspec);
768 1 : break;
769 : }
770 : }
771 40 : }
772 :
773 1 : static void rialto_mse_audio_sink_qos_handle(GstElement *element, uint64_t processed, uint64_t dropped)
774 : {
775 1 : GstBus *bus = gst_element_get_bus(element);
776 : /* Hardcode isLive to FALSE and set invalid timestamps */
777 1 : GstMessage *message = gst_message_new_qos(GST_OBJECT(element), FALSE, GST_CLOCK_TIME_NONE, GST_CLOCK_TIME_NONE,
778 : GST_CLOCK_TIME_NONE, GST_CLOCK_TIME_NONE);
779 1 : gst_message_set_qos_stats(message, GST_FORMAT_DEFAULT, processed, dropped);
780 1 : gst_bus_post(bus, message);
781 1 : gst_object_unref(bus);
782 : }
783 :
784 189 : static void rialto_mse_audio_sink_init(RialtoMSEAudioSink *sink)
785 : {
786 189 : RialtoMSEBaseSinkPrivate *priv = sink->parent.priv;
787 :
788 189 : sink->priv = static_cast<RialtoMSEAudioSinkPrivate *>(rialto_mse_audio_sink_get_instance_private(sink));
789 189 : new (sink->priv) RialtoMSEAudioSinkPrivate();
790 :
791 189 : if (!rialto_mse_base_sink_initialise_sinkpad(RIALTO_MSE_BASE_SINK(sink)))
792 : {
793 0 : GST_ERROR_OBJECT(sink, "Failed to initialise AUDIO sink. Sink pad initialisation failed.");
794 0 : return;
795 : }
796 :
797 189 : priv->m_mediaSourceType = firebolt::rialto::MediaSourceType::AUDIO;
798 189 : gst_pad_set_chain_function(priv->m_sinkPad, rialto_mse_base_sink_chain);
799 189 : gst_pad_set_event_function(priv->m_sinkPad, rialto_mse_audio_sink_event);
800 :
801 378 : priv->m_callbacks.qosCallback = std::bind(rialto_mse_audio_sink_qos_handle, GST_ELEMENT_CAST(sink),
802 189 : std::placeholders::_1, std::placeholders::_2);
803 : }
804 :
805 1 : static void rialto_mse_audio_sink_class_init(RialtoMSEAudioSinkClass *klass)
806 : {
807 1 : GObjectClass *gobjectClass = G_OBJECT_CLASS(klass);
808 1 : GstElementClass *elementClass = GST_ELEMENT_CLASS(klass);
809 1 : gobjectClass->get_property = rialto_mse_audio_sink_get_property;
810 1 : gobjectClass->set_property = rialto_mse_audio_sink_set_property;
811 1 : elementClass->change_state = rialto_mse_audio_sink_change_state;
812 :
813 1 : g_object_class_install_property(gobjectClass, PROP_VOLUME,
814 : g_param_spec_double("volume", "Volume", "Volume of this stream", 0, 1.0,
815 : kDefaultVolume,
816 : GParamFlags(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
817 :
818 1 : g_object_class_install_property(gobjectClass, PROP_MUTE,
819 : g_param_spec_boolean("mute", "Mute", "Mute status of this stream", kDefaultMute,
820 : GParamFlags(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
821 :
822 1 : g_object_class_install_property(gobjectClass, PROP_GAP,
823 : g_param_spec_boxed("gap", "Gap", "Audio Gap", GST_TYPE_STRUCTURE,
824 : (GParamFlags)(G_PARAM_WRITABLE | G_PARAM_STATIC_STRINGS)));
825 :
826 1 : g_object_class_install_property(gobjectClass, PROP_USE_BUFFERING,
827 : g_param_spec_boolean("use-buffering",
828 : "Use buffering", "Emit GST_MESSAGE_BUFFERING based on low-/high-percent thresholds",
829 : kDefaultUseBuffering, G_PARAM_READWRITE));
830 1 : g_object_class_install_property(gobjectClass, PROP_ASYNC,
831 : g_param_spec_boolean("async", "Async", "Asynchronous mode", FALSE, G_PARAM_READWRITE));
832 :
833 : std::unique_ptr<firebolt::rialto::IMediaPipelineCapabilities> mediaPlayerCapabilities =
834 1 : firebolt::rialto::IMediaPipelineCapabilitiesFactory::createFactory()->createMediaPipelineCapabilities();
835 1 : if (mediaPlayerCapabilities)
836 : {
837 : std::vector<std::string> supportedMimeTypes =
838 1 : mediaPlayerCapabilities->getSupportedMimeTypes(firebolt::rialto::MediaSourceType::AUDIO);
839 :
840 1 : rialto_mse_sink_setup_supported_caps(elementClass, supportedMimeTypes);
841 :
842 2 : const std::string kLowLatencyPropertyName{"low-latency"};
843 2 : const std::string kSyncPropertyName{"sync"};
844 2 : const std::string kSyncOffPropertyName{"sync-off"};
845 2 : const std::string kStreamSyncModePropertyName{"stream-sync-mode"};
846 2 : const std::string kAudioFadePropertyName{"audio-fade"};
847 2 : const std::string kFadeVolumePropertyName{"fade-volume"};
848 1 : const std::string kBufferingLimitPropertyName{"limit-buffering-ms"};
849 : const std::vector<std::string> kPropertyNamesToSearch{kLowLatencyPropertyName, kSyncPropertyName,
850 : kSyncOffPropertyName, kStreamSyncModePropertyName,
851 : kBufferingLimitPropertyName, kAudioFadePropertyName,
852 9 : kFadeVolumePropertyName};
853 : std::vector<std::string> supportedProperties{
854 1 : mediaPlayerCapabilities->getSupportedProperties(firebolt::rialto::MediaSourceType::AUDIO,
855 1 : kPropertyNamesToSearch)};
856 :
857 8 : for (auto it = supportedProperties.begin(); it != supportedProperties.end(); ++it)
858 : {
859 7 : if (kLowLatencyPropertyName == *it)
860 : {
861 1 : g_object_class_install_property(gobjectClass, PROP_LOW_LATENCY,
862 : g_param_spec_boolean(kLowLatencyPropertyName.c_str(),
863 : "low latency", "Turn on low latency mode, for use with gaming (no audio decoding, no a/v sync)",
864 : kDefaultLowLatency, GParamFlags(G_PARAM_WRITABLE)));
865 : }
866 6 : else if (kSyncPropertyName == *it)
867 : {
868 1 : g_object_class_install_property(gobjectClass, PROP_SYNC,
869 : g_param_spec_boolean(kSyncPropertyName.c_str(), "sync", "Clock sync",
870 : kDefaultSync, GParamFlags(G_PARAM_READWRITE)));
871 : }
872 5 : else if (kSyncOffPropertyName == *it)
873 : {
874 1 : g_object_class_install_property(gobjectClass, PROP_SYNC_OFF,
875 : g_param_spec_boolean(kSyncOffPropertyName.c_str(),
876 : "sync off", "Turn on free running audio. Must be set before pipeline is PLAYING state.",
877 : kDefaultSyncOff, GParamFlags(G_PARAM_WRITABLE)));
878 : }
879 4 : else if (kStreamSyncModePropertyName == *it)
880 : {
881 1 : g_object_class_install_property(gobjectClass, PROP_STREAM_SYNC_MODE,
882 : g_param_spec_int(kStreamSyncModePropertyName.c_str(),
883 : "stream sync mode", "1 - Frame to decode frame will immediately proceed next frame sync, 0 - Frame decoded with no frame sync",
884 : 0, G_MAXINT, kDefaultStreamSyncMode,
885 : GParamFlags(G_PARAM_READWRITE)));
886 : }
887 3 : else if (kAudioFadePropertyName == *it)
888 : {
889 1 : g_object_class_install_property(gobjectClass, PROP_AUDIO_FADE,
890 : g_param_spec_string(kAudioFadePropertyName.c_str(),
891 : "audio fade", "Start audio fade (vol[0-100],duration ms,easetype[(L)inear,Cubic(I)n,Cubic(O)ut])",
892 : kDefaultAudioFade, GParamFlags(G_PARAM_WRITABLE)));
893 : }
894 2 : else if (kFadeVolumePropertyName == *it)
895 : {
896 1 : g_object_class_install_property(gobjectClass, PROP_FADE_VOLUME,
897 : g_param_spec_uint(kFadeVolumePropertyName.c_str(), "fade volume",
898 : "Get current fade volume", 0, 100, kDefaultFadeVolume,
899 : G_PARAM_READABLE));
900 : }
901 1 : else if (kBufferingLimitPropertyName == *it)
902 : {
903 1 : constexpr uint32_t kMaxValue{20000};
904 1 : g_object_class_install_property(gobjectClass, PROP_LIMIT_BUFFERING_MS,
905 : g_param_spec_uint("limit-buffering-ms",
906 : "limit buffering ms", "Set millisecond threshold used if limit_buffering is set. Changing this value does not enable/disable limit_buffering",
907 : 0, kMaxValue, kDefaultBufferingLimit,
908 : G_PARAM_READWRITE));
909 : }
910 : else
911 : {
912 0 : GST_ERROR("Unexpected property %s returned from rialto", it->c_str());
913 : }
914 : }
915 1 : }
916 : else
917 : {
918 0 : GST_ERROR("Failed to get supported mime types for AUDIO");
919 : }
920 :
921 1 : gst_element_class_set_details_simple(elementClass, "Rialto Audio Sink", "Decoder/Audio/Sink/Audio",
922 : "Communicates with Rialto Server", "Sky");
923 2 : }
|
