Line data Source code
1 : /*
2 : * If not stated otherwise in this file or this component's LICENSE file the
3 : * following copyright and licenses apply:
4 : *
5 : * Copyright 2022 Sky UK
6 : *
7 : * Licensed under the Apache License, Version 2.0 (the "License");
8 : * you may not use this file except in compliance with the License.
9 : * You may obtain a copy of the License at
10 : *
11 : * http://www.apache.org/licenses/LICENSE-2.0
12 : *
13 : * Unless required by applicable law or agreed to in writing, software
14 : * distributed under the License is distributed on an "AS IS" BASIS,
15 : * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 : * See the License for the specific language governing permissions and
17 : * limitations under the License.
18 : */
19 :
20 : #include "tasks/generic/SetupElement.h"
21 : #include "GenericPlayerContext.h"
22 : #include "IGlibWrapper.h"
23 : #include "IGstGenericPlayerPrivate.h"
24 : #include "IGstWrapper.h"
25 : #include "RialtoServerLogging.h"
26 : #include "Utils.h"
27 :
28 : namespace
29 : {
30 : /**
31 : * @brief Callback for audio underflow event from sink. Called by the Gstreamer thread.
32 : *
33 : * @param[in] object : the object that emitted the signal
34 : * @param[in] fifoDepth : the fifo depth (may be 0)
35 : * @param[in] queueDepth : the queue depth (may be NULL)
36 : * @param[in] self : The pointer to IGstGenericPlayerPrivate
37 : *
38 : * @retval true if the handling of the message is successful, false otherwise.
39 : */
40 1 : void audioUnderflowCallback(GstElement *object, guint fifoDepth, gpointer queueDepth, gpointer self)
41 : {
42 1 : firebolt::rialto::server::IGstGenericPlayerPrivate *player =
43 : static_cast<firebolt::rialto::server::IGstGenericPlayerPrivate *>(self);
44 1 : player->scheduleAudioUnderflow();
45 : }
46 :
47 : /**
48 : * @brief Callback for video underflow event from sink. Called by the Gstreamer thread.
49 : *
50 : * @param[in] object : the object that emitted the signal
51 : * @param[in] fifoDepth : the fifo depth (may be 0)
52 : * @param[in] queueDepth : the queue depth (may be NULL)
53 : * @param[in] self : The pointer to IGstGenericPlayerPrivate
54 : *
55 : * @retval true if the handling of the message is successful, false otherwise.
56 : */
57 1 : void videoUnderflowCallback(GstElement *object, guint fifoDepth, gpointer queueDepth, gpointer self)
58 : {
59 1 : firebolt::rialto::server::IGstGenericPlayerPrivate *player =
60 : static_cast<firebolt::rialto::server::IGstGenericPlayerPrivate *>(self);
61 1 : player->scheduleVideoUnderflow();
62 : }
63 :
64 : /**
65 : * @brief Callback for first video frame event from the emitting video element. Called by the Gstreamer thread.
66 : *
67 : * @param[in] object : the object that emitted the signal
68 : * @param[in] fifoDepth : the fifo depth (may be 0)
69 : * @param[in] queueDepth : the queue depth (may be NULL)
70 : * @param[in] self : The pointer to IGstGenericPlayerPrivate
71 : */
72 1 : void firstVideoFrameCallback(GstElement *object, guint fifoDepth, gpointer queueDepth, gpointer self)
73 : {
74 1 : firebolt::rialto::server::IGstGenericPlayerPrivate *player =
75 : static_cast<firebolt::rialto::server::IGstGenericPlayerPrivate *>(self);
76 1 : player->scheduleFirstVideoFrameReceived();
77 : }
78 :
79 : /**
80 : * @brief Callback for first audio frame event from the emitting audio element. Called by the Gstreamer thread.
81 : *
82 : * @param[in] object : the object that emitted the signal
83 : * @param[in] fifoDepth : the fifo depth (may be 0)
84 : * @param[in] queueDepth : the queue depth (may be NULL)
85 : * @param[in] self : The pointer to IGstGenericPlayerPrivate
86 : */
87 1 : void firstAudioFrameCallback(GstElement *object, guint fifoDepth, gpointer queueDepth, gpointer self)
88 : {
89 1 : firebolt::rialto::server::IGstGenericPlayerPrivate *player =
90 : static_cast<firebolt::rialto::server::IGstGenericPlayerPrivate *>(self);
91 1 : player->scheduleFirstAudioFrameReceived(firebolt::rialto::server::AudioFirstFrameAction::CLEAR_PROBE);
92 : }
93 :
94 : /**
95 : * @brief Fallback probe callback for first audio frame on sink pad.
96 : */
97 1 : GstPadProbeReturn firstAudioFrameProbeCallback(GstPad *pad, GstPadProbeInfo *info, gpointer self)
98 : {
99 1 : if (!(info->type & GST_PAD_PROBE_TYPE_BUFFER) || !GST_PAD_PROBE_INFO_BUFFER(info))
100 : {
101 0 : return GST_PAD_PROBE_OK;
102 : }
103 :
104 1 : firebolt::rialto::server::IGstGenericPlayerPrivate *player =
105 : static_cast<firebolt::rialto::server::IGstGenericPlayerPrivate *>(self);
106 1 : player->scheduleFirstAudioFrameReceived(firebolt::rialto::server::AudioFirstFrameAction::CLEAR_PROBE_STATE);
107 1 : return GST_PAD_PROBE_REMOVE;
108 : }
109 :
110 : /**
111 : * @brief Callback for a autovideosink when a child has been added to the sink.
112 : *
113 : * @param[in] obj : the parent element (autovideosink)
114 : * @param[in] object : the child element
115 : * @param[in] name : the name of the child element
116 : * @param[in] self : The pointer to IGstGenericPlayerPrivate
117 : */
118 1 : void autoVideoSinkChildAddedCallback(GstChildProxy *obj, GObject *object, gchar *name, gpointer self)
119 : {
120 1 : RIALTO_SERVER_LOG_DEBUG("AutoVideoSink added element %s", name);
121 1 : firebolt::rialto::server::IGstGenericPlayerPrivate *player =
122 : static_cast<firebolt::rialto::server::IGstGenericPlayerPrivate *>(self);
123 1 : player->addAutoVideoSinkChild(object);
124 : }
125 :
126 : /**
127 : * @brief Callback for a autoaudiosink when a child has been added to the sink.
128 : *
129 : * @param[in] obj : the parent element (autoaudiosink)
130 : * @param[in] object : the child element
131 : * @param[in] name : the name of the child element
132 : * @param[in] self : The pointer to IGstGenericPlayerPrivate
133 : */
134 1 : void autoAudioSinkChildAddedCallback(GstChildProxy *obj, GObject *object, gchar *name, gpointer self)
135 : {
136 1 : RIALTO_SERVER_LOG_DEBUG("AutoAudioSink added element %s", name);
137 1 : firebolt::rialto::server::IGstGenericPlayerPrivate *player =
138 : static_cast<firebolt::rialto::server::IGstGenericPlayerPrivate *>(self);
139 1 : player->addAutoAudioSinkChild(object);
140 : }
141 :
142 : /**
143 : * @brief Callback for a autovideosink when a child has been removed from the sink.
144 : *
145 : * @param[in] obj : the parent element (autovideosink)
146 : * @param[in] object : the child element
147 : * @param[in] name : the name of the child element
148 : * @param[in] self : The pointer to IGstGenericPlayerPrivate
149 : */
150 1 : void autoVideoSinkChildRemovedCallback(GstChildProxy *obj, GObject *object, gchar *name, gpointer self)
151 : {
152 1 : RIALTO_SERVER_LOG_DEBUG("AutoVideoSink removed element %s", name);
153 1 : firebolt::rialto::server::IGstGenericPlayerPrivate *player =
154 : static_cast<firebolt::rialto::server::IGstGenericPlayerPrivate *>(self);
155 1 : player->removeAutoVideoSinkChild(object);
156 : }
157 :
158 : /**
159 : * @brief Callback for a autoaudiosink when a child has been removed from the sink.
160 : *
161 : * @param[in] obj : the parent element (autoaudiosink)
162 : * @param[in] object : the child element
163 : * @param[in] name : the name of the child element
164 : * @param[in] self : The pointer to IGstGenericPlayerPrivate
165 : */
166 1 : void autoAudioSinkChildRemovedCallback(GstChildProxy *obj, GObject *object, gchar *name, gpointer self)
167 : {
168 1 : RIALTO_SERVER_LOG_DEBUG("AutoAudioSink removed element %s", name);
169 1 : firebolt::rialto::server::IGstGenericPlayerPrivate *player =
170 : static_cast<firebolt::rialto::server::IGstGenericPlayerPrivate *>(self);
171 1 : player->removeAutoAudioSinkChild(object);
172 : }
173 : } // namespace
174 :
175 : namespace firebolt::rialto::server::tasks::generic
176 : {
177 40 : SetupElement::SetupElement(GenericPlayerContext &context,
178 : const std::shared_ptr<firebolt::rialto::wrappers::IGstWrapper> &gstWrapper,
179 : const std::shared_ptr<firebolt::rialto::wrappers::IGlibWrapper> &glibWrapper,
180 40 : IGstGenericPlayerPrivate &player, GstElement *element)
181 40 : : m_context{context}, m_gstWrapper{gstWrapper}, m_glibWrapper{glibWrapper}, m_player{player}, m_element{element}
182 : {
183 40 : RIALTO_SERVER_LOG_DEBUG("Constructing SetupElement");
184 :
185 : // Property setting has to happen immediately
186 40 : if (isVideoParser(*m_gstWrapper, m_element))
187 : {
188 0 : RIALTO_SERVER_LOG_INFO("Setting syncmode-streaming to 1 on video parser (immediate)");
189 0 : bool streamSyncModePending{false};
190 0 : streamSyncModePending = m_context.pendingStreamSyncMode.find(MediaSourceType::VIDEO) !=
191 0 : m_context.pendingStreamSyncMode.end();
192 0 : if (streamSyncModePending)
193 : {
194 0 : m_player.setStreamSyncMode(MediaSourceType::VIDEO);
195 : }
196 : }
197 40 : else if (isAudioDecoder(*m_gstWrapper, m_element))
198 : {
199 7 : bool streamSyncModePending{false};
200 7 : bool bufferingLimitPending{false};
201 7 : streamSyncModePending = m_context.pendingStreamSyncMode.find(MediaSourceType::AUDIO) !=
202 7 : m_context.pendingStreamSyncMode.end();
203 7 : bufferingLimitPending = m_context.pendingBufferingLimit.has_value();
204 7 : if (streamSyncModePending)
205 : {
206 1 : m_player.setStreamSyncMode(MediaSourceType::AUDIO);
207 : }
208 7 : if (bufferingLimitPending)
209 : {
210 1 : m_player.setBufferingLimit();
211 : }
212 : }
213 40 : }
214 :
215 41 : SetupElement::~SetupElement()
216 : {
217 40 : RIALTO_SERVER_LOG_DEBUG("SetupElement finished");
218 41 : }
219 :
220 39 : void SetupElement::execute() const
221 : {
222 39 : RIALTO_SERVER_LOG_DEBUG("Executing SetupElement");
223 :
224 39 : const std::string kElementTypeName = m_glibWrapper->gTypeName(G_OBJECT_TYPE(m_element));
225 39 : if (kElementTypeName == "GstAutoVideoSink")
226 : {
227 : // Check and store child sink so we can set underlying properties
228 5 : m_glibWrapper->gSignalConnect(m_element, "child-added", G_CALLBACK(autoVideoSinkChildAddedCallback), &m_player);
229 5 : m_glibWrapper->gSignalConnect(m_element, "child-removed", G_CALLBACK(autoVideoSinkChildRemovedCallback),
230 5 : &m_player);
231 :
232 : // AutoVideoSink sets child before it is setup on the pipeline, so check for children here
233 5 : GstIterator *sinks = m_gstWrapper->gstBinIterateSinks(GST_BIN(m_element));
234 5 : if (sinks && sinks->size > 1)
235 : {
236 1 : RIALTO_SERVER_LOG_WARN("More than one child sink attached");
237 : }
238 :
239 5 : GValue elem = G_VALUE_INIT;
240 5 : if (m_gstWrapper->gstIteratorNext(sinks, &elem) == GST_ITERATOR_OK)
241 : {
242 2 : m_player.addAutoVideoSinkChild(G_OBJECT(m_glibWrapper->gValueGetObject(&elem)));
243 : }
244 5 : m_glibWrapper->gValueUnset(&elem);
245 :
246 5 : if (sinks)
247 5 : m_gstWrapper->gstIteratorFree(sinks);
248 : }
249 34 : else if (kElementTypeName == "GstAutoAudioSink")
250 : {
251 : // Check and store child sink so we can set underlying properties
252 5 : m_glibWrapper->gSignalConnect(m_element, "child-added", G_CALLBACK(autoAudioSinkChildAddedCallback), &m_player);
253 5 : m_glibWrapper->gSignalConnect(m_element, "child-removed", G_CALLBACK(autoAudioSinkChildRemovedCallback),
254 5 : &m_player);
255 :
256 : // AutoAudioSink sets child before it is setup on the pipeline, so check for children here
257 5 : GstIterator *sinks = m_gstWrapper->gstBinIterateSinks(GST_BIN(m_element));
258 5 : if (sinks && sinks->size > 1)
259 : {
260 1 : RIALTO_SERVER_LOG_WARN("More than one child sink attached");
261 : }
262 :
263 5 : GValue elem = G_VALUE_INIT;
264 5 : if (m_gstWrapper->gstIteratorNext(sinks, &elem) == GST_ITERATOR_OK)
265 : {
266 2 : m_player.addAutoAudioSinkChild(G_OBJECT(m_glibWrapper->gValueGetObject(&elem)));
267 : }
268 5 : m_glibWrapper->gValueUnset(&elem);
269 :
270 5 : if (sinks)
271 5 : m_gstWrapper->gstIteratorFree(sinks);
272 : }
273 :
274 39 : if (m_glibWrapper->gStrHasPrefix(GST_ELEMENT_NAME(m_element), "amlhalasink"))
275 : {
276 2 : if (m_context.streamInfo.find(MediaSourceType::VIDEO) != m_context.streamInfo.end())
277 : {
278 : // Wait for video so that the audio aligns at the starting point with timeout of 4000ms.
279 1 : m_glibWrapper->gObjectSet(m_element, "wait-video", TRUE, "a-wait-timeout", 4000, nullptr);
280 : }
281 :
282 : // Xrun occasionally pauses the underlying sink due to unstable playback, but the rest of the pipeline
283 : // remains in the playing state. This causes problems with the synchronization of gst element and rialto
284 : // ultimately hangs waiting for pipeline termination.
285 2 : m_glibWrapper->gObjectSet(m_element, "disable-xrun", TRUE, nullptr);
286 : }
287 37 : else if (m_glibWrapper->gStrHasPrefix(GST_ELEMENT_NAME(m_element), "brcmaudiosink"))
288 : {
289 1 : m_glibWrapper->gObjectSet(m_element, "async", TRUE, nullptr);
290 : }
291 36 : else if (m_glibWrapper->gStrHasPrefix(GST_ELEMENT_NAME(m_element), "rialtotexttracksink"))
292 : {
293 : // in cannot be set during construction, because playsink overwrites "sync" value of text-sink during setup
294 1 : m_glibWrapper->gObjectSet(m_element, "sync", FALSE, nullptr);
295 : }
296 :
297 39 : if (m_context.subtitleSink)
298 : {
299 3 : if (!m_context.isVideoHandleSet && isVideoDecoder(*m_gstWrapper, m_element))
300 : {
301 3 : if (m_glibWrapper->gStrHasPrefix(GST_ELEMENT_NAME(m_element), "westerossink"))
302 : {
303 2 : gpointer decoder{nullptr};
304 2 : m_glibWrapper->gObjectGet(m_element, "videodecoder", &decoder, nullptr);
305 2 : if (decoder)
306 : {
307 1 : m_glibWrapper->gObjectSet(m_context.subtitleSink, "video-decoder", decoder, nullptr);
308 1 : RIALTO_SERVER_LOG_INFO("Setting video decoder handle for subtitle sink: %p", decoder);
309 1 : m_context.isVideoHandleSet = true;
310 : }
311 : else
312 : {
313 1 : m_glibWrapper->gObjectSet(m_context.subtitleSink, "video-decoder", m_element, nullptr);
314 1 : RIALTO_SERVER_LOG_INFO("Setting video decoder handle for subtitle sink: %p", m_element);
315 1 : m_context.isVideoHandleSet = true;
316 : }
317 : }
318 1 : else if (m_glibWrapper->gStrHasPrefix(GST_ELEMENT_NAME(m_element), "omx"))
319 : {
320 1 : m_glibWrapper->gObjectSet(m_context.subtitleSink, "video-decoder", m_element, nullptr);
321 1 : RIALTO_SERVER_LOG_INFO("Setting video decoder handle for subtitle sink: %p", m_element);
322 1 : m_context.isVideoHandleSet = true;
323 : }
324 3 : if (m_context.pendingReportDecodeErrorsForVideo.has_value())
325 : {
326 0 : m_player.setReportDecodeErrors();
327 : }
328 : }
329 : }
330 :
331 39 : if (isDecoder(*m_gstWrapper, m_element) || isSink(*m_gstWrapper, m_element))
332 : {
333 33 : std::optional<std::string> underflowSignalName = getUnderflowSignalName(*m_glibWrapper, m_element);
334 33 : if (underflowSignalName)
335 : {
336 33 : if (isAudio(*m_gstWrapper, m_element))
337 : {
338 18 : RIALTO_SERVER_LOG_INFO("Connecting audio underflow callback for signal: %s",
339 : underflowSignalName.value().c_str());
340 36 : m_glibWrapper->gSignalConnect(m_element, underflowSignalName.value().c_str(),
341 18 : G_CALLBACK(audioUnderflowCallback), &m_player);
342 : }
343 15 : else if (isVideo(*m_gstWrapper, m_element))
344 : {
345 15 : RIALTO_SERVER_LOG_INFO("Connecting video underflow callback for signal: %s",
346 : underflowSignalName.value().c_str());
347 30 : m_glibWrapper->gSignalConnect(m_element, underflowSignalName.value().c_str(),
348 15 : G_CALLBACK(videoUnderflowCallback), &m_player);
349 : }
350 : }
351 :
352 33 : std::optional<std::string> firstFrameSignalName = getFirstFrameSignalName(*m_glibWrapper, m_element);
353 33 : if (firstFrameSignalName)
354 : {
355 2 : if (isVideo(*m_gstWrapper, m_element))
356 : {
357 1 : RIALTO_SERVER_LOG_INFO("Connecting first video frame callback for signal: %s",
358 : firstFrameSignalName.value().c_str());
359 2 : m_glibWrapper->gSignalConnect(m_element, firstFrameSignalName.value().c_str(),
360 1 : G_CALLBACK(firstVideoFrameCallback), &m_player);
361 : }
362 1 : else if (isAudio(*m_gstWrapper, m_element))
363 : {
364 1 : RIALTO_SERVER_LOG_INFO("Connecting first audio frame callback for signal: %s",
365 : firstFrameSignalName.value().c_str());
366 2 : m_glibWrapper->gSignalConnect(m_element, firstFrameSignalName.value().c_str(),
367 1 : G_CALLBACK(firstAudioFrameCallback), &m_player);
368 : }
369 : }
370 31 : else if (isAudioSink(*m_gstWrapper, m_element))
371 : {
372 11 : GstPad *sinkPad = m_gstWrapper->gstElementGetStaticPad(m_element, "sink");
373 11 : if (sinkPad)
374 : {
375 1 : gulong probeId = m_gstWrapper->gstPadAddProbe(sinkPad, GST_PAD_PROBE_TYPE_BUFFER,
376 1 : firstAudioFrameProbeCallback, &m_player, nullptr);
377 :
378 1 : if (probeId != 0)
379 : {
380 1 : RIALTO_SERVER_LOG_INFO("Installed first audio frame fallback probe on sink");
381 1 : m_player.setAudioFirstFrameFallbackProbe(sinkPad, probeId);
382 : }
383 : else
384 : {
385 0 : m_gstWrapper->gstObjectUnref(sinkPad);
386 : }
387 : }
388 : }
389 33 : }
390 :
391 39 : if (isVideoSink(*m_gstWrapper, m_element))
392 : {
393 13 : if (!m_context.videoSink)
394 : {
395 13 : m_gstWrapper->gstObjectRef(m_element);
396 13 : m_context.videoSink = m_element;
397 : }
398 13 : if (!m_context.pendingGeometry.empty())
399 : {
400 2 : m_player.setVideoSinkRectangle();
401 : }
402 11 : else if (!m_context.videoGeometrySetByApi.load() && !m_context.defaultVideoGeometry.empty())
403 : {
404 1 : m_context.pendingGeometry = m_context.defaultVideoGeometry;
405 1 : m_player.setVideoSinkRectangle();
406 : }
407 13 : if (m_context.pendingImmediateOutputForVideo.has_value())
408 : {
409 1 : m_player.setImmediateOutput();
410 : }
411 13 : if (m_context.pendingRenderFrame)
412 : {
413 1 : m_player.setRenderFrame();
414 : }
415 13 : if (m_context.pendingShowVideoWindow.has_value())
416 : {
417 1 : m_player.setShowVideoWindow();
418 : }
419 : }
420 26 : else if (isAudioDecoder(*m_gstWrapper, m_element))
421 : {
422 7 : if (m_context.pendingSyncOff.has_value())
423 : {
424 1 : m_player.setSyncOff();
425 : }
426 7 : bool streamSyncModePending{false};
427 7 : bool bufferingLimitPending{false};
428 7 : streamSyncModePending = m_context.pendingStreamSyncMode.find(MediaSourceType::AUDIO) !=
429 7 : m_context.pendingStreamSyncMode.end();
430 7 : bufferingLimitPending = m_context.pendingBufferingLimit.has_value();
431 7 : if (streamSyncModePending)
432 : {
433 0 : m_player.setStreamSyncMode(MediaSourceType::AUDIO);
434 : }
435 7 : if (bufferingLimitPending)
436 : {
437 0 : m_player.setBufferingLimit();
438 : }
439 :
440 8 : if (m_context.isLive &&
441 1 : m_glibWrapper->gObjectClassFindProperty(G_OBJECT_GET_CLASS(m_element), "enable-rate-correction"))
442 : {
443 1 : RIALTO_SERVER_LOG_INFO("Enabling rate correction for broadcom decoder.");
444 1 : m_glibWrapper->gObjectSet(m_element, "enable-rate-correction", TRUE, nullptr);
445 : }
446 : }
447 19 : else if (isAudioSink(*m_gstWrapper, m_element))
448 : {
449 11 : if (m_context.pendingLowLatency.has_value())
450 : {
451 1 : m_player.setLowLatency();
452 : }
453 11 : if (m_context.pendingSync.has_value())
454 : {
455 1 : m_player.setSync();
456 : }
457 : }
458 8 : else if (isVideoParser(*m_gstWrapper, m_element))
459 : {
460 0 : bool streamSyncModePending{false};
461 0 : streamSyncModePending = m_context.pendingStreamSyncMode.find(MediaSourceType::VIDEO) !=
462 0 : m_context.pendingStreamSyncMode.end();
463 0 : if (streamSyncModePending)
464 : {
465 0 : m_player.setStreamSyncMode(MediaSourceType::VIDEO);
466 : }
467 : }
468 :
469 39 : if (m_gstWrapper->gstIsBaseParse(m_element))
470 : {
471 1 : m_gstWrapper->gstBaseParseSetPtsInterpolation(GST_BASE_PARSE(m_element), FALSE);
472 : }
473 39 : m_gstWrapper->gstObjectUnref(m_element);
474 : }
475 : } // namespace firebolt::rialto::server::tasks::generic
|
