Line data Source code
1 : /*
2 : * Copyright (C) 2022 Sky UK
3 : *
4 : * This library is free software; you can redistribute it and/or
5 : * modify it under the terms of the GNU Lesser General Public
6 : * License as published by the Free Software Foundation;
7 : * version 2.1 of the License.
8 : *
9 : * This library is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 : * Lesser General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU Lesser General Public
15 : * License along with this library; if not, write to the Free Software
16 : * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 : */
18 :
19 : #pragma once
20 :
21 : #include <atomic>
22 : #include <condition_variable>
23 : #include <functional>
24 : #include <memory>
25 : #include <mutex>
26 : #include <thread>
27 : #include <unordered_map>
28 : #include <unordered_set>
29 : #include <vector>
30 :
31 : #include <gst/gst.h>
32 : #include <sys/syscall.h>
33 : #include <sys/types.h>
34 : #include <unistd.h>
35 :
36 : #include "BufferParser.h"
37 : #include "Constants.h"
38 : #include "FlushAndDataSynchronizer.h"
39 : #include "IMediaPipeline.h"
40 : #include "IMessageQueue.h"
41 : #include "IPullModePlaybackDelegate.h"
42 : #include "MediaCommon.h"
43 : #include "MediaPlayerClientBackendInterface.h"
44 : #include "RialtoGStreamerMSEBaseSink.h"
45 :
46 : #define DEFAULT_MAX_VIDEO_WIDTH 3840
47 : #define DEFAULT_MAX_VIDEO_HEIGHT 2160
48 :
49 : class GStreamerMSEMediaPlayerClient;
50 :
51 : enum class ClientState
52 : {
53 : IDLE,
54 : READY,
55 : AWAITING_PAUSED,
56 : PAUSED,
57 : AWAITING_PLAYING,
58 : PLAYING
59 : };
60 :
61 : class BufferPuller
62 : {
63 : public:
64 : BufferPuller(const std::shared_ptr<IMessageQueueFactory> &messageQueueFactory, GstElement *rialtoSink,
65 : const std::shared_ptr<BufferParser> &bufferParser,
66 : const std::shared_ptr<IPullModePlaybackDelegate> &delegate);
67 :
68 : void start();
69 : void stop();
70 : bool requestPullBuffer(int sourceId, size_t frameCount, unsigned int needDataRequestId,
71 : GStreamerMSEMediaPlayerClient *player);
72 :
73 : private:
74 : std::unique_ptr<IMessageQueue> m_queue;
75 : GstElement *m_rialtoSink;
76 : std::shared_ptr<BufferParser> m_bufferParser;
77 : std::shared_ptr<IPullModePlaybackDelegate> m_delegate;
78 : };
79 :
80 : class AttachedSource
81 : {
82 : friend class GStreamerMSEMediaPlayerClient;
83 :
84 : public:
85 211 : AttachedSource(RialtoMSEBaseSink *rialtoSink, std::shared_ptr<BufferPuller> puller,
86 : const std::shared_ptr<IPullModePlaybackDelegate> &delegate, firebolt::rialto::MediaSourceType type,
87 : ClientState state = ClientState::READY)
88 211 : : m_rialtoSink(rialtoSink), m_bufferPuller(puller), m_delegate{delegate}, m_type(type), m_state(state)
89 : {
90 : }
91 :
92 421 : firebolt::rialto::MediaSourceType getType() const { return m_type; }
93 1 : void setPosition(int64_t position) { m_position = position; }
94 :
95 : private:
96 : RialtoMSEBaseSink *m_rialtoSink;
97 : std::shared_ptr<BufferPuller> m_bufferPuller;
98 : std::shared_ptr<IPullModePlaybackDelegate> m_delegate;
99 : std::unordered_set<uint32_t> m_ongoingNeedDataRequests;
100 : firebolt::rialto::MediaSourceType m_type = firebolt::rialto::MediaSourceType::UNKNOWN;
101 : int64_t m_position = 0;
102 : bool m_isFlushing = false;
103 : ClientState m_state = ClientState::READY;
104 : };
105 :
106 : class HaveDataMessage : public Message
107 : {
108 : public:
109 : HaveDataMessage(firebolt::rialto::MediaSourceStatus status, int sourceId, unsigned int needDataRequestId,
110 : GStreamerMSEMediaPlayerClient *player);
111 : void handle() override;
112 :
113 : private:
114 : firebolt::rialto::MediaSourceStatus m_status;
115 : int m_sourceId;
116 : unsigned int m_needDataRequestId;
117 : GStreamerMSEMediaPlayerClient *m_player;
118 : };
119 :
120 : class PullBufferMessage : public Message
121 : {
122 : public:
123 : PullBufferMessage(int sourceId, size_t frameCount, unsigned int needDataRequestId, GstElement *rialtoSink,
124 : const std::shared_ptr<BufferParser> &bufferParser, IMessageQueue &pullerQueue,
125 : GStreamerMSEMediaPlayerClient *player, const std::shared_ptr<IPullModePlaybackDelegate> &delegate);
126 : void handle() override;
127 :
128 : private:
129 : int m_sourceId;
130 : size_t m_frameCount;
131 : unsigned int m_needDataRequestId;
132 : GstElement *m_rialtoSink;
133 : std::shared_ptr<BufferParser> m_bufferParser;
134 : IMessageQueue &m_pullerQueue;
135 : GStreamerMSEMediaPlayerClient *m_player;
136 : std::shared_ptr<IPullModePlaybackDelegate> m_delegate;
137 : };
138 :
139 : class NeedDataMessage : public Message
140 : {
141 : public:
142 : NeedDataMessage(int sourceId, size_t frameCount, unsigned int needDataRequestId,
143 : GStreamerMSEMediaPlayerClient *player);
144 : void handle() override;
145 :
146 : private:
147 : int m_sourceId;
148 : size_t m_frameCount;
149 : unsigned int m_needDataRequestId;
150 : GStreamerMSEMediaPlayerClient *m_player;
151 : };
152 :
153 : class PlaybackStateMessage : public Message
154 : {
155 : public:
156 : PlaybackStateMessage(firebolt::rialto::PlaybackState state, GStreamerMSEMediaPlayerClient *player);
157 : void handle() override;
158 :
159 : private:
160 : firebolt::rialto::PlaybackState m_state;
161 : GStreamerMSEMediaPlayerClient *m_player;
162 : };
163 :
164 : class QosMessage : public Message
165 : {
166 : public:
167 : QosMessage(int sourceId, firebolt::rialto::QosInfo qosInfo, GStreamerMSEMediaPlayerClient *player);
168 : void handle() override;
169 :
170 : private:
171 : int m_sourceId;
172 : firebolt::rialto::QosInfo m_qosInfo;
173 : GStreamerMSEMediaPlayerClient *m_player;
174 : };
175 :
176 : class BufferUnderflowMessage : public Message
177 : {
178 : public:
179 : BufferUnderflowMessage(int sourceId, GStreamerMSEMediaPlayerClient *player);
180 : void handle() override;
181 :
182 : private:
183 : int m_sourceId;
184 : GStreamerMSEMediaPlayerClient *m_player;
185 : };
186 :
187 : class FirstFrameReceivedMessage : public Message
188 : {
189 : public:
190 : FirstFrameReceivedMessage(int sourceId, GStreamerMSEMediaPlayerClient *player);
191 : void handle() override;
192 :
193 : private:
194 : int m_sourceId;
195 : GStreamerMSEMediaPlayerClient *m_player;
196 : };
197 :
198 : class PlaybackErrorMessage : public Message
199 : {
200 : public:
201 : PlaybackErrorMessage(int sourceId, firebolt::rialto::PlaybackError error, GStreamerMSEMediaPlayerClient *player);
202 : void handle() override;
203 :
204 : private:
205 : int m_sourceId;
206 : firebolt::rialto::PlaybackError m_error;
207 : GStreamerMSEMediaPlayerClient *m_player;
208 : };
209 :
210 : class SetPositionMessage : public Message
211 : {
212 : public:
213 : SetPositionMessage(int64_t newPosition, std::unordered_map<int32_t, AttachedSource> &attachedSources);
214 : void handle() override;
215 :
216 : private:
217 : int64_t m_newPosition;
218 : std::unordered_map<int32_t, AttachedSource> &m_attachedSources;
219 : };
220 :
221 : class SetDurationMessage : public Message
222 : {
223 : public:
224 : SetDurationMessage(int64_t newDuration, int64_t &targetDuration);
225 : void handle() override;
226 :
227 : private:
228 : int64_t m_newDuration;
229 : int64_t &m_targetDuration;
230 : };
231 :
232 : class SourceFlushedMessage : public Message
233 : {
234 : public:
235 : SourceFlushedMessage(int32_t sourceId, GStreamerMSEMediaPlayerClient *player);
236 : void handle() override;
237 :
238 : private:
239 : int32_t m_sourceId;
240 : GStreamerMSEMediaPlayerClient *m_player;
241 : };
242 :
243 : enum class StateChangeResult
244 : {
245 : SUCCESS_ASYNC,
246 : SUCCESS_SYNC,
247 : NOT_ATTACHED
248 : };
249 :
250 : class GStreamerMSEMediaPlayerClient : public firebolt::rialto::IMediaPipelineClient,
251 : public std::enable_shared_from_this<GStreamerMSEMediaPlayerClient>
252 : {
253 : friend class NeedDataMessage;
254 : friend class PullBufferMessage;
255 : friend class HaveDataMessage;
256 : friend class QosMessage;
257 :
258 : public:
259 : GStreamerMSEMediaPlayerClient(
260 : const std::shared_ptr<IMessageQueueFactory> &messageQueueFactory,
261 : const std::shared_ptr<firebolt::rialto::client::MediaPlayerClientBackendInterface> &MediaPlayerClientBackend,
262 : const uint32_t maxVideoWidth, const uint32_t maxVideoHeight, bool isLive);
263 : virtual ~GStreamerMSEMediaPlayerClient();
264 :
265 : void notifyDuration(int64_t duration) override;
266 : void notifyPosition(int64_t position) override;
267 : void notifyNativeSize(uint32_t width, uint32_t height, double aspect) override;
268 : void notifyNetworkState(firebolt::rialto::NetworkState state) override;
269 : void notifyPlaybackState(firebolt::rialto::PlaybackState state) override;
270 : void notifyVideoData(bool hasData) override;
271 : void notifyAudioData(bool hasData) override;
272 : void notifyNeedMediaData(int32_t sourceId, size_t frameCount, uint32_t needDataRequestId,
273 : const std::shared_ptr<firebolt::rialto::MediaPlayerShmInfo> &shmInfo) override;
274 : void notifyCancelNeedMediaData(int32_t sourceId) override;
275 : void notifyQos(int32_t sourceId, const firebolt::rialto::QosInfo &qosInfo) override;
276 : void notifyBufferUnderflow(int32_t sourceId) override;
277 : void notifyFirstFrameReceived(int32_t sourceId) override;
278 : void notifyPlaybackError(int32_t sourceId, firebolt::rialto::PlaybackError error) override;
279 : void notifySourceFlushed(int32_t sourceId) override;
280 : void notifyPlaybackInfo(const firebolt::rialto::PlaybackInfo &playbackInfo) override;
281 :
282 : int64_t getPosition(int32_t sourceId);
283 : bool getDuration(int64_t &duration);
284 : bool setImmediateOutput(int32_t sourceId, bool immediateOutput);
285 : bool getImmediateOutput(int32_t sourceId, bool &immediateOutput);
286 : bool getStats(int32_t sourceId, uint64_t &renderedFrames, uint64_t &droppedFrames);
287 :
288 : firebolt::rialto::AddSegmentStatus
289 : addSegment(unsigned int needDataRequestId,
290 : const std::unique_ptr<firebolt::rialto::IMediaPipeline::MediaSegment> &mediaSegment);
291 :
292 : bool createBackend();
293 : StateChangeResult play(int32_t sourceId);
294 : StateChangeResult pause(int32_t sourceId);
295 : void stop();
296 : void setPlaybackRate(double rate);
297 : void flush(int32_t sourceId, bool resetTime);
298 : void setSourcePosition(int32_t sourceId, int64_t position, bool resetTime, double appliedRate = 1.0,
299 : uint64_t stopPosition = GST_CLOCK_TIME_NONE);
300 : void setSubtitleOffset(int32_t sourceId, int64_t position);
301 : void processAudioGap(int64_t position, uint32_t duration, int64_t discontinuityGap, bool audioAac);
302 :
303 : bool attachSource(std::unique_ptr<firebolt::rialto::IMediaPipeline::MediaSource> &source,
304 : RialtoMSEBaseSink *rialtoSink, const std::shared_ptr<IPullModePlaybackDelegate> &delegate);
305 : void removeSource(int32_t sourceId);
306 : void handlePlaybackStateChange(firebolt::rialto::PlaybackState state);
307 : void handleSourceFlushed(int32_t sourceId);
308 : void sendAllSourcesAttachedIfPossible();
309 :
310 : void setVideoRectangle(const std::string &rectangleString);
311 : std::string getVideoRectangle();
312 :
313 : bool requestPullBuffer(int streamId, size_t frameCount, unsigned int needDataRequestId);
314 : bool handleQos(int sourceId, firebolt::rialto::QosInfo qosInfo);
315 : bool handleBufferUnderflow(int sourceId);
316 : bool handleFirstFrameReceived(int sourceId);
317 : bool handlePlaybackError(int sourceId, firebolt::rialto::PlaybackError error);
318 : void stopStreaming();
319 : void destroyClientBackend();
320 : bool renderFrame(int32_t sourceId);
321 : void setVolume(double targetVolume, uint32_t volumeDuration, firebolt::rialto::EaseType easeType);
322 : bool getVolume(double &volume);
323 : bool getCachedVolume(double &volume);
324 : void setMute(bool mute, int32_t sourceId);
325 : bool getMute(int sourceId);
326 : void setTextTrackIdentifier(const std::string &textTrackIdentifier);
327 : std::string getTextTrackIdentifier();
328 : bool setLowLatency(bool lowLatency);
329 : bool setSync(bool sync);
330 : bool getSync(bool &sync);
331 : bool setSyncOff(bool syncOff);
332 : bool setStreamSyncMode(int32_t sourceId, int32_t streamSyncMode);
333 : bool getStreamSyncMode(int32_t &streamSyncMode);
334 : ClientState getClientState();
335 : void handleStreamCollection(int32_t audioStreams, int32_t videoStreams, int32_t subtitleStreams);
336 : void setBufferingLimit(uint32_t limitBufferingMs);
337 : uint32_t getBufferingLimit();
338 : void setUseBuffering(bool useBuffering);
339 : bool getUseBuffering();
340 : bool switchSource(const std::unique_ptr<firebolt::rialto::IMediaPipeline::MediaSource> &source);
341 : IFlushAndDataSynchronizer &getFlushAndDataSynchronizer();
342 :
343 : private:
344 : bool areAllStreamsAttached();
345 : void sendAllSourcesAttachedIfPossibleInternal();
346 : bool checkIfAllAttachedSourcesInStates(const std::vector<ClientState> &states);
347 :
348 : std::unique_ptr<IMessageQueue> m_backendQueue;
349 : std::shared_ptr<IMessageQueueFactory> m_messageQueueFactory;
350 : std::shared_ptr<firebolt::rialto::client::MediaPlayerClientBackendInterface> m_clientBackend;
351 : int64_t m_position;
352 : int64_t m_duration;
353 : std::mutex m_playbackInfoMutex;
354 : std::unordered_map<int32_t, AttachedSource> m_attachedSources;
355 : bool m_wasAllSourcesAttachedSent = false;
356 : int32_t m_audioStreams;
357 : int32_t m_videoStreams;
358 : int32_t m_subtitleStreams;
359 : firebolt::rialto::PlaybackInfo m_playbackInfo{-1, 1.0};
360 : FlushAndDataSynchronizer m_flushAndDataSynchronizer;
361 : bool wasPlayingBeforeEos{false};
362 :
363 : struct Rectangle
364 : {
365 : unsigned int x, y, width, height;
366 : } m_videoRectangle;
367 :
368 : firebolt::rialto::PlaybackState m_serverPlaybackState = firebolt::rialto::PlaybackState::IDLE;
369 : ClientState m_clientState = ClientState::IDLE;
370 : // To check if the backend message queue and pulling of data to serve backend is stopped or not
371 : bool m_streamingStopped;
372 :
373 : const uint32_t m_maxWidth;
374 : const uint32_t m_maxHeight;
375 : const bool m_isLive;
376 : };
|
