Line data Source code
1 : /*
2 : * Copyright (C) 2024 Sky UK
3 : *
4 : * This library is free software; you can redistribute it and/or
5 : * modify it under the terms of the GNU Lesser General Public
6 : * License as published by the Free Software Foundation;
7 : * version 2.1 of the License.
8 : *
9 : * This library is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 : * Lesser General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU Lesser General Public
15 : * License along with this library; if not, write to the Free Software
16 : * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 : */
18 :
19 : #include <gst/gst.h>
20 : #include <inttypes.h>
21 : #include <stdint.h>
22 :
23 : #include "GStreamerEMEUtils.h"
24 : #include "GStreamerMSEUtils.h"
25 : #include "IMediaPipelineCapabilities.h"
26 : #include "RialtoGStreamerMSEBaseSinkPrivate.h"
27 : #include "RialtoGStreamerMSESubtitleSink.h"
28 :
29 : using namespace firebolt::rialto::client;
30 :
31 : GST_DEBUG_CATEGORY_STATIC(RialtoMSESubtitleSinkDebug);
32 : #define GST_CAT_DEFAULT RialtoMSESubtitleSinkDebug
33 :
34 : #define rialto_mse_subtitle_sink_parent_class parent_class
35 115 : G_DEFINE_TYPE_WITH_CODE(RialtoMSESubtitleSink, rialto_mse_subtitle_sink, RIALTO_TYPE_MSE_BASE_SINK,
36 : G_ADD_PRIVATE(RialtoMSESubtitleSink)
37 : GST_DEBUG_CATEGORY_INIT(RialtoMSESubtitleSinkDebug, "rialtomsesubtitlesink", 0,
38 : "rialto mse subtitle sink"));
39 :
40 : enum
41 : {
42 : PROP_0,
43 : PROP_MUTE,
44 : PROP_TEXT_TRACK_IDENTIFIER,
45 : PROP_WINDOW_ID,
46 : PROP_ASYNC,
47 : PROP_LAST
48 : };
49 :
50 48 : static GstStateChangeReturn rialto_mse_subtitle_sink_change_state(GstElement *element, GstStateChange transition)
51 : {
52 48 : RialtoMSESubtitleSink *sink = RIALTO_MSE_SUBTITLE_SINK(element);
53 :
54 48 : std::shared_ptr<GStreamerMSEMediaPlayerClient> client;
55 48 : switch (transition)
56 : {
57 11 : case GST_STATE_CHANGE_READY_TO_PAUSED:
58 : {
59 11 : if (!rialto_mse_base_sink_attach_to_media_client_and_set_streams_number(element))
60 : {
61 1 : return GST_STATE_CHANGE_FAILURE;
62 : }
63 :
64 10 : break;
65 : }
66 37 : default:
67 37 : break;
68 : }
69 :
70 47 : GstStateChangeReturn result = GST_ELEMENT_CLASS(parent_class)->change_state(element, transition);
71 47 : if (G_UNLIKELY(result == GST_STATE_CHANGE_FAILURE))
72 : {
73 0 : GST_WARNING_OBJECT(sink, "State change failed");
74 0 : return result;
75 : }
76 :
77 47 : return result;
78 48 : }
79 :
80 : static std::unique_ptr<firebolt::rialto::IMediaPipeline::MediaSource>
81 11 : rialto_mse_subtitle_sink_create_media_source(RialtoMSEBaseSink *sink, GstCaps *caps)
82 : {
83 11 : RialtoMSESubtitleSink *subSink = RIALTO_MSE_SUBTITLE_SINK(sink);
84 11 : GstStructure *structure = gst_caps_get_structure(caps, 0);
85 11 : const gchar *mimeName = gst_structure_get_name(structure);
86 :
87 11 : std::string mimeType;
88 11 : if (mimeName)
89 : {
90 11 : if (g_str_has_prefix(mimeName, "text/vtt") || g_str_has_prefix(mimeName, "application/x-subtitle-vtt"))
91 : {
92 1 : mimeType = "text/vtt";
93 : }
94 10 : else if (g_str_has_prefix(mimeName, "application/ttml+xml"))
95 : {
96 10 : mimeType = "text/ttml";
97 : }
98 : else
99 : {
100 0 : mimeType = mimeName;
101 : }
102 :
103 11 : GST_INFO_OBJECT(sink, "%s subtitle media source created", mimeType.c_str());
104 11 : return std::make_unique<firebolt::rialto::IMediaPipeline::MediaSourceSubtitle>(mimeType,
105 11 : subSink->priv->m_textTrackIdentifier);
106 : }
107 : else
108 : {
109 0 : GST_ERROR_OBJECT(sink,
110 : "Empty caps' structure name! Failed to set mime type when constructing subtitle media source");
111 : }
112 :
113 0 : return nullptr;
114 11 : }
115 :
116 16 : static gboolean rialto_mse_subtitle_sink_event(GstPad *pad, GstObject *parent, GstEvent *event)
117 : {
118 16 : RialtoMSEBaseSink *sink = RIALTO_MSE_BASE_SINK(parent);
119 16 : RialtoMSESubtitleSink *subtitleSink = RIALTO_MSE_SUBTITLE_SINK(parent);
120 16 : RialtoMSEBaseSinkPrivate *basePriv = sink->priv;
121 16 : switch (GST_EVENT_TYPE(event))
122 : {
123 12 : case GST_EVENT_CAPS:
124 : {
125 : GstCaps *caps;
126 12 : gst_event_parse_caps(event, &caps);
127 12 : if (basePriv->m_sourceAttached)
128 : {
129 1 : GST_INFO_OBJECT(sink, "Source already attached. Skip calling attachSource");
130 1 : break;
131 : }
132 :
133 11 : GST_INFO_OBJECT(sink, "Attaching SUBTITLE source with caps %" GST_PTR_FORMAT, caps);
134 :
135 : std::unique_ptr<firebolt::rialto::IMediaPipeline::MediaSource> subtitleSource =
136 11 : rialto_mse_subtitle_sink_create_media_source(sink, caps);
137 11 : if (subtitleSource)
138 : {
139 : std::shared_ptr<GStreamerMSEMediaPlayerClient> client =
140 11 : sink->priv->m_mediaPlayerManager.getMediaPlayerClient();
141 11 : if ((!client) || (!client->attachSource(subtitleSource, sink)))
142 : {
143 1 : GST_ERROR_OBJECT(sink, "Failed to attach SUBTITLE source");
144 : }
145 : else
146 : {
147 10 : basePriv->m_sourceAttached = true;
148 10 : if (subtitleSink->priv->m_isMuteQueued)
149 : {
150 1 : client->setMute(subtitleSink->priv->m_isMuted, basePriv->m_sourceId);
151 1 : subtitleSink->priv->m_isMuteQueued = false;
152 : }
153 :
154 : {
155 10 : std::unique_lock lock{subtitleSink->priv->m_mutex};
156 10 : if (subtitleSink->priv->m_isTextTrackIdentifierQueued)
157 : {
158 1 : client->setTextTrackIdentifier(subtitleSink->priv->m_textTrackIdentifier);
159 1 : subtitleSink->priv->m_isTextTrackIdentifierQueued = false;
160 : }
161 10 : }
162 :
163 : // check if READY -> PAUSED was requested before source was attached
164 10 : if (GST_STATE_NEXT(sink) == GST_STATE_PAUSED)
165 : {
166 10 : client->pause(sink->priv->m_sourceId);
167 : }
168 : }
169 11 : }
170 : else
171 : {
172 0 : GST_ERROR_OBJECT(sink, "Failed to create SUBTITLE source");
173 : }
174 :
175 11 : break;
176 : }
177 3 : case GST_EVENT_CUSTOM_DOWNSTREAM:
178 : case GST_EVENT_CUSTOM_DOWNSTREAM_OOB:
179 : {
180 3 : if (gst_event_has_name(event, "set-pts-offset"))
181 : {
182 3 : GST_DEBUG_OBJECT(sink, "Set pts offset event received");
183 3 : const GstStructure *structure{gst_event_get_structure(event)};
184 3 : guint64 ptsOffset{GST_CLOCK_TIME_NONE};
185 3 : if (gst_structure_get_uint64(structure, "pts-offset", &ptsOffset) == TRUE)
186 : {
187 2 : std::unique_lock lock{basePriv->m_sinkMutex};
188 2 : if (!basePriv->m_initialPositionSet)
189 : {
190 1 : GST_DEBUG_OBJECT(sink, "First segment not received yet. Queuing offset setting");
191 1 : basePriv->m_queuedOffset = static_cast<int64_t>(ptsOffset);
192 : }
193 : else
194 : {
195 : std::shared_ptr<GStreamerMSEMediaPlayerClient> client =
196 1 : basePriv->m_mediaPlayerManager.getMediaPlayerClient();
197 1 : if (client)
198 : {
199 1 : GST_DEBUG_OBJECT(sink, "Setting subtitle position to: %" GST_TIME_FORMAT,
200 : GST_TIME_ARGS(ptsOffset));
201 2 : client->setSourcePosition(basePriv->m_sourceId, ptsOffset, false,
202 1 : basePriv->m_lastSegment.applied_rate, sink->priv->m_lastSegment.stop);
203 : }
204 : }
205 2 : }
206 : else
207 : {
208 1 : GST_WARNING_OBJECT(sink, "Unable to set pts offset. Value not present");
209 : }
210 : }
211 3 : break;
212 : }
213 1 : default:
214 1 : break;
215 : }
216 :
217 16 : return rialto_mse_base_sink_event(pad, parent, event);
218 : }
219 :
220 7 : static void rialto_mse_subtitle_sink_get_property(GObject *object, guint propId, GValue *value, GParamSpec *pspec)
221 : {
222 7 : RialtoMSESubtitleSink *sink = RIALTO_MSE_SUBTITLE_SINK(object);
223 7 : if (!sink)
224 : {
225 0 : GST_ERROR_OBJECT(object, "Sink not initalised");
226 2 : return;
227 : }
228 7 : RialtoMSESubtitleSinkPrivate *priv = sink->priv;
229 7 : RialtoMSEBaseSinkPrivate *basePriv = sink->parent.priv;
230 7 : if (!priv || !basePriv)
231 : {
232 0 : GST_ERROR_OBJECT(object, "Private Sink not initalised");
233 0 : return;
234 : }
235 :
236 7 : std::shared_ptr<GStreamerMSEMediaPlayerClient> client = basePriv->m_mediaPlayerManager.getMediaPlayerClient();
237 :
238 7 : switch (propId)
239 : {
240 2 : case PROP_MUTE:
241 : {
242 2 : if (!client)
243 : {
244 1 : g_value_set_boolean(value, priv->m_isMuted);
245 1 : return;
246 : }
247 1 : g_value_set_boolean(value, client->getMute(basePriv->m_sourceId));
248 1 : break;
249 : }
250 2 : case PROP_TEXT_TRACK_IDENTIFIER:
251 : {
252 : {
253 2 : std::unique_lock lock{priv->m_mutex};
254 2 : if (!client)
255 : {
256 1 : g_value_set_string(value, priv->m_textTrackIdentifier.c_str());
257 1 : return;
258 : }
259 2 : }
260 1 : g_value_set_string(value, client->getTextTrackIdentifier().c_str());
261 :
262 1 : break;
263 : }
264 1 : case PROP_WINDOW_ID:
265 : {
266 1 : g_value_set_uint(value, priv->m_videoId);
267 1 : break;
268 : }
269 1 : case PROP_ASYNC:
270 : {
271 1 : g_value_set_boolean(value, basePriv->m_isAsync);
272 1 : break;
273 : }
274 1 : default:
275 1 : G_OBJECT_WARN_INVALID_PROPERTY_ID(object, propId, pspec);
276 1 : break;
277 : }
278 7 : }
279 :
280 10 : static void rialto_mse_subtitle_sink_set_property(GObject *object, guint propId, const GValue *value, GParamSpec *pspec)
281 : {
282 10 : RialtoMSESubtitleSink *sink = RIALTO_MSE_SUBTITLE_SINK(object);
283 10 : if (!sink)
284 : {
285 0 : GST_ERROR_OBJECT(object, "Sink not initalised");
286 2 : return;
287 : }
288 10 : RialtoMSESubtitleSinkPrivate *priv = sink->priv;
289 10 : RialtoMSEBaseSinkPrivate *basePriv = sink->parent.priv;
290 10 : if (!priv || !basePriv)
291 : {
292 0 : GST_ERROR_OBJECT(object, "Private sink not initalised");
293 0 : return;
294 : }
295 :
296 10 : std::shared_ptr<GStreamerMSEMediaPlayerClient> client = basePriv->m_mediaPlayerManager.getMediaPlayerClient();
297 :
298 10 : switch (propId)
299 : {
300 3 : case PROP_MUTE:
301 3 : priv->m_isMuted = g_value_get_boolean(value);
302 3 : if (!client || !basePriv->m_sourceAttached)
303 : {
304 2 : priv->m_isMuteQueued = true;
305 2 : return;
306 : }
307 :
308 1 : client->setMute(priv->m_isMuted, basePriv->m_sourceId);
309 :
310 1 : break;
311 4 : case PROP_TEXT_TRACK_IDENTIFIER:
312 : {
313 4 : const gchar *textTrackIdentifier = g_value_get_string(value);
314 4 : if (!textTrackIdentifier)
315 : {
316 1 : GST_WARNING_OBJECT(object, "TextTrackIdentifier string not valid");
317 1 : break;
318 : }
319 :
320 3 : std::unique_lock lock{priv->m_mutex};
321 3 : priv->m_textTrackIdentifier = std::string(textTrackIdentifier);
322 3 : if (!client || !basePriv->m_sourceAttached)
323 : {
324 2 : GST_DEBUG_OBJECT(object, "Rectangle setting enqueued");
325 2 : priv->m_isTextTrackIdentifierQueued = true;
326 : }
327 : else
328 : {
329 1 : client->setTextTrackIdentifier(priv->m_textTrackIdentifier);
330 : }
331 :
332 3 : break;
333 : }
334 1 : case PROP_WINDOW_ID:
335 : {
336 1 : priv->m_videoId = g_value_get_uint(value);
337 1 : break;
338 : }
339 1 : case PROP_ASYNC:
340 : {
341 1 : basePriv->m_isAsync = g_value_get_boolean(value);
342 1 : break;
343 : }
344 1 : default:
345 1 : G_OBJECT_WARN_INVALID_PROPERTY_ID(object, propId, pspec);
346 1 : break;
347 : }
348 10 : }
349 :
350 1 : static void rialto_mse_subtitle_sink_qos_handle(GstElement *element, uint64_t processed, uint64_t dropped)
351 : {
352 1 : GstBus *bus = gst_element_get_bus(element);
353 : /* Hardcode isLive to FALSE and set invalid timestamps */
354 1 : GstMessage *message = gst_message_new_qos(GST_OBJECT(element), FALSE, GST_CLOCK_TIME_NONE, GST_CLOCK_TIME_NONE,
355 : GST_CLOCK_TIME_NONE, GST_CLOCK_TIME_NONE);
356 :
357 1 : gst_message_set_qos_stats(message, GST_FORMAT_BUFFERS, processed, dropped);
358 1 : gst_bus_post(bus, message);
359 1 : gst_object_unref(bus);
360 : }
361 :
362 20 : static void rialto_mse_subtitle_sink_init(RialtoMSESubtitleSink *sink)
363 : {
364 20 : RialtoMSEBaseSinkPrivate *basePriv = sink->parent.priv;
365 :
366 20 : sink->priv = static_cast<RialtoMSESubtitleSinkPrivate *>(rialto_mse_subtitle_sink_get_instance_private(sink));
367 20 : new (sink->priv) RialtoMSESubtitleSinkPrivate();
368 :
369 20 : if (!rialto_mse_base_sink_initialise_sinkpad(RIALTO_MSE_BASE_SINK(sink)))
370 : {
371 0 : GST_ERROR_OBJECT(sink, "Failed to initialise SUBTITLE sink. Sink pad initialisation failed.");
372 0 : return;
373 : }
374 :
375 20 : basePriv->m_mediaSourceType = firebolt::rialto::MediaSourceType::SUBTITLE;
376 20 : basePriv->m_isAsync = false;
377 20 : gst_pad_set_chain_function(basePriv->m_sinkPad, rialto_mse_base_sink_chain);
378 20 : gst_pad_set_event_function(basePriv->m_sinkPad, rialto_mse_subtitle_sink_event);
379 :
380 40 : basePriv->m_callbacks.qosCallback = std::bind(rialto_mse_subtitle_sink_qos_handle, GST_ELEMENT_CAST(sink),
381 20 : std::placeholders::_1, std::placeholders::_2);
382 : }
383 :
384 1 : static void rialto_mse_subtitle_sink_class_init(RialtoMSESubtitleSinkClass *klass)
385 : {
386 1 : GObjectClass *gobjectClass = G_OBJECT_CLASS(klass);
387 1 : GstElementClass *elementClass = GST_ELEMENT_CLASS(klass);
388 1 : gobjectClass->get_property = rialto_mse_subtitle_sink_get_property;
389 1 : gobjectClass->set_property = rialto_mse_subtitle_sink_set_property;
390 1 : elementClass->change_state = rialto_mse_subtitle_sink_change_state;
391 :
392 1 : g_object_class_install_property(gobjectClass, PROP_MUTE,
393 : g_param_spec_boolean("mute", "Mute", "Mute subtitles", FALSE, G_PARAM_READWRITE));
394 :
395 1 : g_object_class_install_property(gobjectClass, PROP_TEXT_TRACK_IDENTIFIER,
396 : g_param_spec_string("text-track-identifier", "Text Track Identifier",
397 : "Identifier of text track. Valid input for service is "
398 : "\"CC[1-4]\", \"TEXT[1-4]\", \"SERVICE[1-64]\"",
399 : nullptr, GParamFlags(G_PARAM_READWRITE)));
400 :
401 1 : g_object_class_install_property(gobjectClass, PROP_WINDOW_ID,
402 : g_param_spec_uint("window-id", "Window ID", "Id of window (placeholder)", 0, 256, 0,
403 : GParamFlags(G_PARAM_READWRITE)));
404 :
405 1 : g_object_class_install_property(gobjectClass, PROP_ASYNC,
406 : g_param_spec_boolean("async", "Async", "Asynchronous mode", FALSE, G_PARAM_READWRITE));
407 :
408 : std::unique_ptr<firebolt::rialto::IMediaPipelineCapabilities> mediaPlayerCapabilities =
409 1 : firebolt::rialto::IMediaPipelineCapabilitiesFactory::createFactory()->createMediaPipelineCapabilities();
410 1 : if (mediaPlayerCapabilities)
411 : {
412 : std::vector<std::string> supportedMimeTypes =
413 1 : mediaPlayerCapabilities->getSupportedMimeTypes(firebolt::rialto::MediaSourceType::SUBTITLE);
414 :
415 1 : rialto_mse_sink_setup_supported_caps(elementClass, supportedMimeTypes);
416 : }
417 : else
418 : {
419 0 : GST_ERROR("Failed to get supported mime types for Subtitle");
420 : }
421 :
422 1 : gst_element_class_set_details_simple(elementClass, "Rialto Subtitle Sink", "Parser/Subtitle/Sink/Subtitle",
423 : "Communicates with Rialto Server", "Sky");
424 : }
|
