Line data Source code
1 : /*
2 : * Copyright (C) 2023 Sky UK
3 : *
4 : * This library is free software; you can redistribute it and/or
5 : * modify it under the terms of the GNU Lesser General Public
6 : * License as published by the Free Software Foundation;
7 : * version 2.1 of the License.
8 : *
9 : * This library is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 : * Lesser General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU Lesser General Public
15 : * License along with this library; if not, write to the Free Software
16 : * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 : */
18 :
19 : #include <gst/gst.h>
20 :
21 : #include "ControlBackend.h"
22 : #include "GStreamerWebAudioPlayerClient.h"
23 : #include "MessageQueue.h"
24 : #include "RialtoGStreamerWebAudioSink.h"
25 : #include "WebAudioClientBackend.h"
26 :
27 : using namespace firebolt::rialto::client;
28 :
29 : GST_DEBUG_CATEGORY_STATIC(RialtoWebAudioSinkDebug);
30 : #define GST_CAT_DEFAULT RialtoWebAudioSinkDebug
31 :
32 : #define rialto_web_audio_sink_parent_class parent_class
33 233 : G_DEFINE_TYPE_WITH_CODE(RialtoWebAudioSink, rialto_web_audio_sink, GST_TYPE_ELEMENT,
34 : G_ADD_PRIVATE(RialtoWebAudioSink)
35 : GST_DEBUG_CATEGORY_INIT(RialtoWebAudioSinkDebug, "rialtowebaudiosink", 0,
36 : "rialto web audio sink"));
37 : enum
38 : {
39 : PROP_0,
40 : PROP_TS_OFFSET,
41 : PROP_VOLUME,
42 : PROP_LAST
43 : };
44 :
45 16 : static void rialto_web_audio_async_start(RialtoWebAudioSink *sink)
46 : {
47 16 : sink->priv->m_isStateCommitNeeded = true;
48 16 : gst_element_post_message(GST_ELEMENT_CAST(sink), gst_message_new_async_start(GST_OBJECT(sink)));
49 : }
50 :
51 7 : static void rialto_web_audio_async_done(RialtoWebAudioSink *sink)
52 : {
53 7 : sink->priv->m_isStateCommitNeeded = false;
54 7 : gst_element_post_message(GST_ELEMENT_CAST(sink),
55 : gst_message_new_async_done(GST_OBJECT_CAST(sink), GST_CLOCK_TIME_NONE));
56 : }
57 :
58 2 : static void rialto_web_audio_sink_eos_handler(RialtoWebAudioSink *sink)
59 : {
60 2 : GstState currentState = GST_STATE(sink);
61 2 : if ((currentState != GST_STATE_PAUSED) && (currentState != GST_STATE_PLAYING))
62 : {
63 1 : GST_ERROR_OBJECT(sink, "Sink cannot post a EOS message in state '%s', posting an error instead",
64 : gst_element_state_get_name(currentState));
65 :
66 1 : const char *errMessage = "Web audio sink received EOS in non-playing state";
67 1 : GError *gError{g_error_new_literal(GST_STREAM_ERROR, 0, errMessage)};
68 1 : gst_element_post_message(GST_ELEMENT_CAST(sink),
69 : gst_message_new_error(GST_OBJECT_CAST(sink), gError, errMessage));
70 1 : g_error_free(gError);
71 : }
72 : else
73 : {
74 1 : gst_element_post_message(GST_ELEMENT_CAST(sink), gst_message_new_eos(GST_OBJECT_CAST(sink)));
75 : }
76 2 : }
77 :
78 1 : static void rialto_web_audio_sink_error_handler(RialtoWebAudioSink *sink, const char *message)
79 : {
80 1 : GError *gError{g_error_new_literal(GST_STREAM_ERROR, 0, message)};
81 1 : gst_element_post_message(GST_ELEMENT_CAST(sink), gst_message_new_error(GST_OBJECT_CAST(sink), gError, message));
82 1 : g_error_free(gError);
83 : }
84 :
85 7 : static void rialto_web_audio_sink_rialto_state_changed_handler(RialtoWebAudioSink *sink,
86 : firebolt::rialto::WebAudioPlayerState state)
87 : {
88 7 : GstState current = GST_STATE(sink);
89 7 : GstState next = GST_STATE_NEXT(sink);
90 7 : GstState pending = GST_STATE_PENDING(sink);
91 :
92 7 : GST_DEBUG_OBJECT(sink,
93 : "Received server's state change to %u. Sink's states are: current state: %s next state: %s "
94 : "pending state: %s, last return state %s",
95 : static_cast<uint32_t>(state), gst_element_state_get_name(current),
96 : gst_element_state_get_name(next), gst_element_state_get_name(pending),
97 : gst_element_state_change_return_get_name(GST_STATE_RETURN(sink)));
98 :
99 7 : RialtoWebAudioSinkPrivate *priv = sink->priv;
100 14 : if (priv->m_isStateCommitNeeded &&
101 7 : ((state == firebolt::rialto::WebAudioPlayerState::PAUSED && next == GST_STATE_PAUSED) ||
102 14 : (state == firebolt::rialto::WebAudioPlayerState::PLAYING && next == GST_STATE_PLAYING)))
103 : {
104 7 : GstState postNext = next == pending ? GST_STATE_VOID_PENDING : pending;
105 7 : GST_STATE(sink) = next;
106 7 : GST_STATE_NEXT(sink) = postNext;
107 7 : GST_STATE_PENDING(sink) = GST_STATE_VOID_PENDING;
108 7 : GST_STATE_RETURN(sink) = GST_STATE_CHANGE_SUCCESS;
109 :
110 7 : GST_INFO_OBJECT(sink, "Async state transition to state %s done", gst_element_state_get_name(next));
111 :
112 7 : gst_element_post_message(GST_ELEMENT_CAST(sink),
113 : gst_message_new_state_changed(GST_OBJECT_CAST(sink), current, next, pending));
114 7 : rialto_web_audio_async_done(sink);
115 : }
116 : }
117 :
118 1 : static void rialto_web_audio_sink_setup_supported_caps(GstElementClass *elementClass)
119 : {
120 1 : GstCaps *caps = gst_caps_from_string("audio/x-raw");
121 1 : GstPadTemplate *sinktempl = gst_pad_template_new("sink", GST_PAD_SINK, GST_PAD_ALWAYS, caps);
122 1 : gst_element_class_add_pad_template(elementClass, sinktempl);
123 1 : gst_caps_unref(caps);
124 : }
125 :
126 17 : static gboolean rialto_web_audio_sink_send_event(GstElement *element, GstEvent *event)
127 : {
128 17 : RialtoWebAudioSink *sink = RIALTO_WEB_AUDIO_SINK(element);
129 17 : switch (GST_EVENT_TYPE(event))
130 : {
131 1 : case GST_EVENT_CAPS:
132 : {
133 : GstCaps *caps;
134 1 : gst_event_parse_caps(event, &caps);
135 1 : GST_INFO_OBJECT(sink, "Attaching AUDIO source with caps %" GST_PTR_FORMAT, caps);
136 : }
137 17 : default:
138 17 : break;
139 : }
140 17 : GstElement *parent = GST_ELEMENT(&sink->parent);
141 17 : return GST_ELEMENT_CLASS(parent_class)->send_event(parent, event);
142 : }
143 :
144 99 : static GstStateChangeReturn rialto_web_audio_sink_change_state(GstElement *element, GstStateChange transition)
145 : {
146 99 : RialtoWebAudioSink *sink = RIALTO_WEB_AUDIO_SINK(element);
147 99 : RialtoWebAudioSinkPrivate *priv = sink->priv;
148 99 : const std::shared_ptr<GStreamerWebAudioPlayerClient> &kClient = priv->m_webAudioClient;
149 99 : GstPad *sinkPad = gst_element_get_static_pad(GST_ELEMENT_CAST(sink), "sink");
150 :
151 99 : GstState current_state = GST_STATE_TRANSITION_CURRENT(transition);
152 99 : GstState next_state = GST_STATE_TRANSITION_NEXT(transition);
153 99 : GST_INFO_OBJECT(sink, "State change: (%s) -> (%s)", gst_element_state_get_name(current_state),
154 : gst_element_state_get_name(next_state));
155 :
156 99 : GstStateChangeReturn result = GST_STATE_CHANGE_SUCCESS;
157 99 : switch (transition)
158 : {
159 21 : case GST_STATE_CHANGE_NULL_TO_READY:
160 : {
161 21 : GST_DEBUG("GST_STATE_CHANGE_NULL_TO_READY");
162 :
163 21 : if (!priv->m_rialtoControlClient->waitForRunning())
164 : {
165 1 : GST_ERROR_OBJECT(sink, "Rialto client cannot reach running state");
166 1 : result = GST_STATE_CHANGE_FAILURE;
167 : }
168 21 : break;
169 : }
170 19 : case GST_STATE_CHANGE_READY_TO_PAUSED:
171 : {
172 19 : GST_DEBUG("GST_STATE_CHANGE_READY_TO_PAUSED");
173 19 : break;
174 : }
175 9 : case GST_STATE_CHANGE_PAUSED_TO_PLAYING:
176 : {
177 9 : GST_DEBUG("GST_STATE_CHANGE_PAUSED_TO_PLAYING");
178 9 : if (!kClient->isOpen())
179 : {
180 2 : GST_INFO_OBJECT(sink, "Delay playing until the caps are recieved and the player is opened");
181 2 : priv->m_isPlayingDelayed = true;
182 2 : result = GST_STATE_CHANGE_ASYNC;
183 2 : rialto_web_audio_async_start(sink);
184 : }
185 : else
186 : {
187 7 : if (!kClient->play())
188 : {
189 1 : GST_ERROR_OBJECT(sink, "Failed to play web audio");
190 1 : result = GST_STATE_CHANGE_FAILURE;
191 : }
192 : else
193 : {
194 6 : result = GST_STATE_CHANGE_ASYNC;
195 6 : rialto_web_audio_async_start(sink);
196 : }
197 : }
198 9 : break;
199 : }
200 9 : case GST_STATE_CHANGE_PLAYING_TO_PAUSED:
201 : {
202 9 : GST_DEBUG("GST_STATE_CHANGE_PLAYING_TO_PAUSED");
203 9 : if (!kClient->pause())
204 : {
205 1 : GST_ERROR_OBJECT(sink, "Failed to pause web audio");
206 1 : result = GST_STATE_CHANGE_FAILURE;
207 : }
208 : else
209 : {
210 8 : result = GST_STATE_CHANGE_ASYNC;
211 8 : rialto_web_audio_async_start(sink);
212 : }
213 9 : break;
214 : }
215 19 : case GST_STATE_CHANGE_PAUSED_TO_READY:
216 : {
217 19 : GST_DEBUG("GST_STATE_CHANGE_PAUSED_TO_READY");
218 19 : if (!kClient->close())
219 : {
220 0 : GST_ERROR_OBJECT(sink, "Failed to close web audio");
221 0 : result = GST_STATE_CHANGE_FAILURE;
222 : }
223 19 : break;
224 : }
225 20 : case GST_STATE_CHANGE_READY_TO_NULL:
226 : {
227 20 : GST_DEBUG("GST_STATE_CHANGE_READY_TO_NULL");
228 :
229 20 : priv->m_rialtoControlClient->removeControlBackend();
230 : }
231 22 : default:
232 22 : break;
233 : }
234 :
235 99 : gst_object_unref(sinkPad);
236 :
237 99 : if (result == GST_STATE_CHANGE_SUCCESS)
238 : {
239 80 : GstStateChangeReturn stateChangeRet = GST_ELEMENT_CLASS(parent_class)->change_state(element, transition);
240 80 : if (G_UNLIKELY(stateChangeRet == GST_STATE_CHANGE_FAILURE))
241 : {
242 0 : GST_WARNING_OBJECT(sink, "State change failed");
243 0 : return stateChangeRet;
244 : }
245 : }
246 :
247 99 : return result;
248 : }
249 :
250 21 : static gboolean rialto_web_audio_sink_event(GstPad *pad, GstObject *parent, GstEvent *event)
251 : {
252 21 : RialtoWebAudioSink *sink = RIALTO_WEB_AUDIO_SINK(parent);
253 21 : RialtoWebAudioSinkPrivate *priv = sink->priv;
254 21 : bool result = false;
255 21 : switch (GST_EVENT_TYPE(event))
256 : {
257 1 : case GST_EVENT_EOS:
258 : {
259 1 : GST_DEBUG("GST_EVENT_EOS");
260 1 : result = priv->m_webAudioClient->setEos();
261 1 : gst_event_unref(event);
262 1 : break;
263 : }
264 19 : case GST_EVENT_CAPS:
265 : {
266 : GstCaps *caps;
267 19 : gst_event_parse_caps(event, &caps);
268 19 : GST_INFO_OBJECT(sink, "Opening WebAudio with caps %" GST_PTR_FORMAT, caps);
269 :
270 19 : if (!priv->m_webAudioClient->open(caps))
271 : {
272 1 : GST_ERROR_OBJECT(sink, "Failed to open web audio");
273 : }
274 : else
275 : {
276 18 : result = true;
277 :
278 18 : if (priv->isVolumeQueued)
279 : {
280 2 : if (!priv->m_webAudioClient->setVolume(priv->volume))
281 : {
282 1 : GST_ERROR_OBJECT(sink, "Failed to set volume");
283 1 : result = false;
284 : }
285 : else
286 : {
287 1 : priv->isVolumeQueued = false;
288 : }
289 : }
290 :
291 18 : if (priv->m_isPlayingDelayed)
292 : {
293 2 : if (!priv->m_webAudioClient->play())
294 : {
295 1 : GST_ERROR_OBJECT(sink, "Failed to play web audio");
296 1 : result = false;
297 : }
298 : else
299 : {
300 1 : priv->m_isPlayingDelayed = false;
301 : }
302 : }
303 : }
304 19 : gst_event_unref(event);
305 19 : break;
306 : }
307 1 : default:
308 1 : result = gst_pad_event_default(pad, parent, event);
309 1 : break;
310 : }
311 21 : return result;
312 : }
313 :
314 7 : static void rialto_web_audio_sink_get_property(GObject *object, guint propId, GValue *value, GParamSpec *pspec)
315 : {
316 7 : RialtoWebAudioSink *sink = RIALTO_WEB_AUDIO_SINK(object);
317 7 : RialtoWebAudioSinkPrivate *priv = sink->priv;
318 7 : const std::shared_ptr<GStreamerWebAudioPlayerClient> &kClient = priv->m_webAudioClient;
319 :
320 7 : switch (propId)
321 : {
322 1 : case PROP_TS_OFFSET:
323 : {
324 1 : GST_INFO_OBJECT(object, "ts-offset property not supported, RialtoWebAudioSink does not require the "
325 : "synchronisation of sources");
326 1 : break;
327 : }
328 :
329 5 : case PROP_VOLUME:
330 : {
331 : double volume;
332 5 : if (kClient && kClient->isOpen())
333 : {
334 3 : if (kClient->getVolume(volume))
335 2 : priv->volume = volume;
336 : else
337 1 : volume = priv->volume; // Use last known volume
338 : }
339 : else
340 : {
341 2 : volume = priv->volume;
342 : }
343 5 : g_value_set_double(value, volume);
344 5 : break;
345 : }
346 :
347 1 : default:
348 : {
349 1 : G_OBJECT_WARN_INVALID_PROPERTY_ID(object, propId, pspec);
350 1 : break;
351 : }
352 : }
353 7 : }
354 :
355 7 : static void rialto_web_audio_sink_set_property(GObject *object, guint propId, const GValue *value, GParamSpec *pspec)
356 : {
357 7 : RialtoWebAudioSink *sink = RIALTO_WEB_AUDIO_SINK(object);
358 7 : RialtoWebAudioSinkPrivate *priv = sink->priv;
359 7 : const std::shared_ptr<GStreamerWebAudioPlayerClient> &kClient = priv->m_webAudioClient;
360 :
361 7 : switch (propId)
362 : {
363 1 : case PROP_TS_OFFSET:
364 : {
365 1 : GST_INFO_OBJECT(object, "ts-offset property not supported, RialtoWebAudioSink does not require the "
366 : "synchronisation of sources");
367 1 : break;
368 : }
369 :
370 5 : case PROP_VOLUME:
371 : {
372 5 : priv->volume = g_value_get_double(value);
373 5 : if (!kClient || !kClient->isOpen())
374 : {
375 3 : GST_DEBUG_OBJECT(object, "Enqueue volume setting");
376 3 : priv->isVolumeQueued = true;
377 3 : return;
378 : }
379 2 : if (!kClient->setVolume(priv->volume))
380 : {
381 1 : GST_ERROR_OBJECT(object, "Failed to set volume");
382 : }
383 2 : break;
384 : }
385 :
386 1 : default:
387 : {
388 1 : G_OBJECT_WARN_INVALID_PROPERTY_ID(object, propId, pspec);
389 1 : break;
390 : }
391 : }
392 : }
393 :
394 1 : static GstFlowReturn rialto_web_audio_sink_chain(GstPad *pad, GstObject *parent, GstBuffer *buf)
395 : {
396 1 : RialtoWebAudioSink *sink = RIALTO_WEB_AUDIO_SINK(parent);
397 1 : bool res = sink->priv->m_webAudioClient->notifyNewSample(buf);
398 1 : if (res)
399 : {
400 1 : return GST_FLOW_OK;
401 : }
402 : else
403 : {
404 0 : GST_ERROR_OBJECT(sink, "Failed to push sample");
405 0 : return GST_FLOW_ERROR;
406 : }
407 : }
408 :
409 26 : static bool rialto_web_audio_sink_initialise_sinkpad(RialtoWebAudioSink *sink)
410 : {
411 : GstPadTemplate *pad_template =
412 26 : gst_element_class_get_pad_template(GST_ELEMENT_CLASS(G_OBJECT_GET_CLASS(sink)), "sink");
413 26 : if (!pad_template)
414 : {
415 0 : GST_ERROR_OBJECT(sink, "Could not find sink pad template");
416 0 : return false;
417 : }
418 :
419 26 : GstPad *sinkPad = gst_pad_new_from_template(pad_template, "sink");
420 26 : if (!sinkPad)
421 : {
422 0 : GST_ERROR_OBJECT(sink, "Could not create sinkpad");
423 0 : return false;
424 : }
425 :
426 26 : gst_element_add_pad(GST_ELEMENT_CAST(sink), sinkPad);
427 :
428 26 : gst_pad_set_event_function(sinkPad, rialto_web_audio_sink_event);
429 26 : gst_pad_set_chain_function(sinkPad, rialto_web_audio_sink_chain);
430 :
431 26 : return true;
432 : }
433 :
434 26 : static void rialto_web_audio_sink_init(RialtoWebAudioSink *sink)
435 : {
436 26 : GST_INFO_OBJECT(sink, "Init: %" GST_PTR_FORMAT, sink);
437 26 : sink->priv = static_cast<RialtoWebAudioSinkPrivate *>(rialto_web_audio_sink_get_instance_private(sink));
438 26 : new (sink->priv) RialtoWebAudioSinkPrivate();
439 :
440 26 : WebAudioSinkCallbacks callbacks;
441 26 : callbacks.eosCallback = std::bind(rialto_web_audio_sink_eos_handler, sink);
442 : callbacks.stateChangedCallback =
443 26 : std::bind(rialto_web_audio_sink_rialto_state_changed_handler, sink, std::placeholders::_1);
444 26 : callbacks.errorCallback = std::bind(rialto_web_audio_sink_error_handler, sink, std::placeholders::_1);
445 :
446 26 : sink->priv->m_rialtoControlClient = std::make_unique<firebolt::rialto::client::ControlBackend>();
447 26 : sink->priv->m_webAudioClient =
448 52 : std::make_shared<GStreamerWebAudioPlayerClient>(std::make_unique<firebolt::rialto::client::WebAudioClientBackend>(),
449 52 : std::make_unique<MessageQueue>(), callbacks,
450 78 : ITimerFactory::getFactory());
451 26 : GST_OBJECT_FLAG_SET(sink, GST_ELEMENT_FLAG_SINK);
452 26 : if (!rialto_web_audio_sink_initialise_sinkpad(sink))
453 : {
454 0 : GST_ERROR_OBJECT(sink, "Failed to initialise AUDIO sink. Sink pad initialisation failed.");
455 0 : return;
456 : }
457 26 : }
458 :
459 26 : static void rialto_web_audio_sink_finalize(GObject *object)
460 : {
461 26 : RialtoWebAudioSink *sink = RIALTO_WEB_AUDIO_SINK(object);
462 26 : RialtoWebAudioSinkPrivate *priv = sink->priv;
463 26 : sink->priv->m_webAudioClient = nullptr;
464 26 : GST_INFO_OBJECT(sink, "Finalize: %" GST_PTR_FORMAT " %" GST_PTR_FORMAT, sink, priv);
465 :
466 26 : priv->~RialtoWebAudioSinkPrivate();
467 :
468 26 : GST_CALL_PARENT(G_OBJECT_CLASS, finalize, (object));
469 : }
470 :
471 1 : static void rialto_web_audio_sink_class_init(RialtoWebAudioSinkClass *klass)
472 : {
473 1 : GObjectClass *gobjectClass = G_OBJECT_CLASS(klass);
474 1 : GstElementClass *elementClass = GST_ELEMENT_CLASS(klass);
475 :
476 1 : gst_element_class_set_metadata(elementClass, "Rialto Web Audio sink", "Generic", "A sink for Rialto Web Audio",
477 : "Sky");
478 :
479 1 : gobjectClass->finalize = rialto_web_audio_sink_finalize;
480 1 : gobjectClass->get_property = rialto_web_audio_sink_get_property;
481 1 : gobjectClass->set_property = rialto_web_audio_sink_set_property;
482 :
483 1 : elementClass->change_state = rialto_web_audio_sink_change_state;
484 1 : elementClass->send_event = rialto_web_audio_sink_send_event;
485 :
486 1 : g_object_class_install_property(gobjectClass, PROP_TS_OFFSET,
487 : g_param_spec_int64("ts-offset",
488 : "ts-offset", "Not supported, RialtoWebAudioSink does not require the synchronisation of sources",
489 : G_MININT64, G_MAXINT64, 0,
490 : GParamFlags(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
491 :
492 1 : g_object_class_install_property(gobjectClass, PROP_VOLUME,
493 : g_param_spec_double("volume", "Volume", "Volume of this stream", 0, 1.0,
494 : kDefaultVolume,
495 : GParamFlags(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
496 :
497 1 : rialto_web_audio_sink_setup_supported_caps(elementClass);
498 :
499 1 : gst_element_class_set_details_simple(elementClass, "Rialto Web Audio Sink", "Decoder/Audio/Sink/Audio",
500 : "Communicates with Rialto Server", "Sky");
501 : }
|
