Line data Source code
1 : /*
2 : * Copyright (C) 2022 Sky UK
3 : *
4 : * This library is free software; you can redistribute it and/or
5 : * modify it under the terms of the GNU Lesser General Public
6 : * License as published by the Free Software Foundation;
7 : * version 2.1 of the License.
8 : *
9 : * This library is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 : * Lesser General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU Lesser General Public
15 : * License along with this library; if not, write to the Free Software
16 : * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 : */
18 :
19 : #pragma once
20 :
21 : #include <atomic>
22 : #include <condition_variable>
23 : #include <functional>
24 : #include <memory>
25 : #include <mutex>
26 : #include <thread>
27 : #include <unordered_map>
28 : #include <unordered_set>
29 : #include <vector>
30 :
31 : #include <gst/gst.h>
32 : #include <sys/syscall.h>
33 : #include <sys/types.h>
34 : #include <unistd.h>
35 :
36 : #include "BufferParser.h"
37 : #include "Constants.h"
38 : #include "FlushAndDataSynchronizer.h"
39 : #include "IMediaPipeline.h"
40 : #include "IMessageQueue.h"
41 : #include "IPullModePlaybackDelegate.h"
42 : #include "MediaCommon.h"
43 : #include "MediaPlayerClientBackendInterface.h"
44 : #include "RialtoGStreamerMSEBaseSink.h"
45 :
46 : #define DEFAULT_MAX_VIDEO_WIDTH 3840
47 : #define DEFAULT_MAX_VIDEO_HEIGHT 2160
48 :
49 : class GStreamerMSEMediaPlayerClient;
50 :
51 : enum class ClientState
52 : {
53 : IDLE,
54 : READY,
55 : AWAITING_PAUSED,
56 : PAUSED,
57 : AWAITING_PLAYING,
58 : PLAYING
59 : };
60 :
61 : class BufferPuller
62 : {
63 : public:
64 : BufferPuller(const std::shared_ptr<IMessageQueueFactory> &messageQueueFactory, GstElement *rialtoSink,
65 : const std::shared_ptr<BufferParser> &bufferParser,
66 : const std::shared_ptr<IPullModePlaybackDelegate> &delegate);
67 :
68 : void start();
69 : void stop();
70 : bool requestPullBuffer(int sourceId, size_t frameCount, unsigned int needDataRequestId,
71 : GStreamerMSEMediaPlayerClient *player);
72 :
73 : private:
74 : std::unique_ptr<IMessageQueue> m_queue;
75 : GstElement *m_rialtoSink;
76 : std::shared_ptr<BufferParser> m_bufferParser;
77 : std::shared_ptr<IPullModePlaybackDelegate> m_delegate;
78 : };
79 :
80 : class AttachedSource
81 : {
82 : friend class GStreamerMSEMediaPlayerClient;
83 :
84 : public:
85 203 : AttachedSource(RialtoMSEBaseSink *rialtoSink, std::shared_ptr<BufferPuller> puller,
86 : const std::shared_ptr<IPullModePlaybackDelegate> &delegate, firebolt::rialto::MediaSourceType type,
87 : ClientState state = ClientState::READY)
88 203 : : m_rialtoSink(rialtoSink), m_bufferPuller(puller), m_delegate{delegate}, m_type(type), m_state(state)
89 : {
90 : }
91 :
92 406 : firebolt::rialto::MediaSourceType getType() const { return m_type; }
93 1 : void setPosition(int64_t position) { m_position = position; }
94 :
95 : private:
96 : RialtoMSEBaseSink *m_rialtoSink;
97 : std::shared_ptr<BufferPuller> m_bufferPuller;
98 : std::shared_ptr<IPullModePlaybackDelegate> m_delegate;
99 : std::unordered_set<uint32_t> m_ongoingNeedDataRequests;
100 : firebolt::rialto::MediaSourceType m_type = firebolt::rialto::MediaSourceType::UNKNOWN;
101 : int64_t m_position = 0;
102 : bool m_isFlushing = false;
103 : ClientState m_state = ClientState::READY;
104 : };
105 :
106 : class HaveDataMessage : public Message
107 : {
108 : public:
109 : HaveDataMessage(firebolt::rialto::MediaSourceStatus status, int sourceId, unsigned int needDataRequestId,
110 : GStreamerMSEMediaPlayerClient *player);
111 : void handle() override;
112 :
113 : private:
114 : firebolt::rialto::MediaSourceStatus m_status;
115 : int m_sourceId;
116 : unsigned int m_needDataRequestId;
117 : GStreamerMSEMediaPlayerClient *m_player;
118 : };
119 :
120 : class PullBufferMessage : public Message
121 : {
122 : public:
123 : PullBufferMessage(int sourceId, size_t frameCount, unsigned int needDataRequestId, GstElement *rialtoSink,
124 : const std::shared_ptr<BufferParser> &bufferParser, IMessageQueue &pullerQueue,
125 : GStreamerMSEMediaPlayerClient *player, const std::shared_ptr<IPullModePlaybackDelegate> &delegate);
126 : void handle() override;
127 :
128 : private:
129 : int m_sourceId;
130 : size_t m_frameCount;
131 : unsigned int m_needDataRequestId;
132 : GstElement *m_rialtoSink;
133 : std::shared_ptr<BufferParser> m_bufferParser;
134 : IMessageQueue &m_pullerQueue;
135 : GStreamerMSEMediaPlayerClient *m_player;
136 : std::shared_ptr<IPullModePlaybackDelegate> m_delegate;
137 : };
138 :
139 : class NeedDataMessage : public Message
140 : {
141 : public:
142 : NeedDataMessage(int sourceId, size_t frameCount, unsigned int needDataRequestId,
143 : GStreamerMSEMediaPlayerClient *player);
144 : void handle() override;
145 :
146 : private:
147 : int m_sourceId;
148 : size_t m_frameCount;
149 : unsigned int m_needDataRequestId;
150 : GStreamerMSEMediaPlayerClient *m_player;
151 : };
152 :
153 : class PlaybackStateMessage : public Message
154 : {
155 : public:
156 : PlaybackStateMessage(firebolt::rialto::PlaybackState state, GStreamerMSEMediaPlayerClient *player);
157 : void handle() override;
158 :
159 : private:
160 : firebolt::rialto::PlaybackState m_state;
161 : GStreamerMSEMediaPlayerClient *m_player;
162 : };
163 :
164 : class QosMessage : public Message
165 : {
166 : public:
167 : QosMessage(int sourceId, firebolt::rialto::QosInfo qosInfo, GStreamerMSEMediaPlayerClient *player);
168 : void handle() override;
169 :
170 : private:
171 : int m_sourceId;
172 : firebolt::rialto::QosInfo m_qosInfo;
173 : GStreamerMSEMediaPlayerClient *m_player;
174 : };
175 :
176 : class BufferUnderflowMessage : public Message
177 : {
178 : public:
179 : BufferUnderflowMessage(int sourceId, GStreamerMSEMediaPlayerClient *player);
180 : void handle() override;
181 :
182 : private:
183 : int m_sourceId;
184 : GStreamerMSEMediaPlayerClient *m_player;
185 : };
186 :
187 : class PlaybackErrorMessage : public Message
188 : {
189 : public:
190 : PlaybackErrorMessage(int sourceId, firebolt::rialto::PlaybackError error, GStreamerMSEMediaPlayerClient *player);
191 : void handle() override;
192 :
193 : private:
194 : int m_sourceId;
195 : firebolt::rialto::PlaybackError m_error;
196 : GStreamerMSEMediaPlayerClient *m_player;
197 : };
198 :
199 : class SetPositionMessage : public Message
200 : {
201 : public:
202 : SetPositionMessage(int64_t newPosition, std::unordered_map<int32_t, AttachedSource> &attachedSources);
203 : void handle() override;
204 :
205 : private:
206 : int64_t m_newPosition;
207 : std::unordered_map<int32_t, AttachedSource> &m_attachedSources;
208 : };
209 :
210 : class SetDurationMessage : public Message
211 : {
212 : public:
213 : SetDurationMessage(int64_t newDuration, int64_t &targetDuration);
214 : void handle() override;
215 :
216 : private:
217 : int64_t m_newDuration;
218 : int64_t &m_targetDuration;
219 : };
220 :
221 : class SourceFlushedMessage : public Message
222 : {
223 : public:
224 : SourceFlushedMessage(int32_t sourceId, GStreamerMSEMediaPlayerClient *player);
225 : void handle() override;
226 :
227 : private:
228 : int32_t m_sourceId;
229 : GStreamerMSEMediaPlayerClient *m_player;
230 : };
231 :
232 : enum class StateChangeResult
233 : {
234 : SUCCESS_ASYNC,
235 : SUCCESS_SYNC,
236 : NOT_ATTACHED
237 : };
238 :
239 : class GStreamerMSEMediaPlayerClient : public firebolt::rialto::IMediaPipelineClient,
240 : public std::enable_shared_from_this<GStreamerMSEMediaPlayerClient>
241 : {
242 : friend class NeedDataMessage;
243 : friend class PullBufferMessage;
244 : friend class HaveDataMessage;
245 : friend class QosMessage;
246 :
247 : public:
248 : GStreamerMSEMediaPlayerClient(
249 : const std::shared_ptr<IMessageQueueFactory> &messageQueueFactory,
250 : const std::shared_ptr<firebolt::rialto::client::MediaPlayerClientBackendInterface> &MediaPlayerClientBackend,
251 : const uint32_t maxVideoWidth, const uint32_t maxVideoHeight);
252 : virtual ~GStreamerMSEMediaPlayerClient();
253 :
254 : void notifyDuration(int64_t duration) override;
255 : void notifyPosition(int64_t position) override;
256 : void notifyNativeSize(uint32_t width, uint32_t height, double aspect) override;
257 : void notifyNetworkState(firebolt::rialto::NetworkState state) override;
258 : void notifyPlaybackState(firebolt::rialto::PlaybackState state) override;
259 : void notifyVideoData(bool hasData) override;
260 : void notifyAudioData(bool hasData) override;
261 : void notifyNeedMediaData(int32_t sourceId, size_t frameCount, uint32_t needDataRequestId,
262 : const std::shared_ptr<firebolt::rialto::MediaPlayerShmInfo> &shmInfo) override;
263 : void notifyCancelNeedMediaData(int32_t sourceId) override;
264 : void notifyQos(int32_t sourceId, const firebolt::rialto::QosInfo &qosInfo) override;
265 : void notifyBufferUnderflow(int32_t sourceId) override;
266 : void notifyPlaybackError(int32_t sourceId, firebolt::rialto::PlaybackError error) override;
267 : void notifySourceFlushed(int32_t sourceId) override;
268 : void notifyPlaybackInfo(const firebolt::rialto::PlaybackInfo &playbackInfo) override;
269 :
270 : int64_t getPosition(int32_t sourceId);
271 : bool setImmediateOutput(int32_t sourceId, bool immediateOutput);
272 : bool getImmediateOutput(int32_t sourceId, bool &immediateOutput);
273 : bool getStats(int32_t sourceId, uint64_t &renderedFrames, uint64_t &droppedFrames);
274 :
275 : firebolt::rialto::AddSegmentStatus
276 : addSegment(unsigned int needDataRequestId,
277 : const std::unique_ptr<firebolt::rialto::IMediaPipeline::MediaSegment> &mediaSegment);
278 :
279 : bool createBackend();
280 : StateChangeResult play(int32_t sourceId);
281 : StateChangeResult pause(int32_t sourceId);
282 : void stop();
283 : void setPlaybackRate(double rate);
284 : void flush(int32_t sourceId, bool resetTime);
285 : void setSourcePosition(int32_t sourceId, int64_t position, bool resetTime, double appliedRate = 1.0,
286 : uint64_t stopPosition = GST_CLOCK_TIME_NONE);
287 : void setSubtitleOffset(int32_t sourceId, int64_t position);
288 : void processAudioGap(int64_t position, uint32_t duration, int64_t discontinuityGap, bool audioAac);
289 :
290 : bool attachSource(std::unique_ptr<firebolt::rialto::IMediaPipeline::MediaSource> &source,
291 : RialtoMSEBaseSink *rialtoSink, const std::shared_ptr<IPullModePlaybackDelegate> &delegate);
292 : void removeSource(int32_t sourceId);
293 : void handlePlaybackStateChange(firebolt::rialto::PlaybackState state);
294 : void handleSourceFlushed(int32_t sourceId);
295 : void sendAllSourcesAttachedIfPossible();
296 :
297 : void setVideoRectangle(const std::string &rectangleString);
298 : std::string getVideoRectangle();
299 :
300 : bool requestPullBuffer(int streamId, size_t frameCount, unsigned int needDataRequestId);
301 : bool handleQos(int sourceId, firebolt::rialto::QosInfo qosInfo);
302 : bool handleBufferUnderflow(int sourceId);
303 : bool handlePlaybackError(int sourceId, firebolt::rialto::PlaybackError error);
304 : void stopStreaming();
305 : void destroyClientBackend();
306 : bool renderFrame(int32_t sourceId);
307 : void setVolume(double targetVolume, uint32_t volumeDuration, firebolt::rialto::EaseType easeType);
308 : bool getVolume(double &volume);
309 : void setMute(bool mute, int32_t sourceId);
310 : bool getMute(int sourceId);
311 : void setTextTrackIdentifier(const std::string &textTrackIdentifier);
312 : std::string getTextTrackIdentifier();
313 : bool setLowLatency(bool lowLatency);
314 : bool setSync(bool sync);
315 : bool getSync(bool &sync);
316 : bool setSyncOff(bool syncOff);
317 : bool setStreamSyncMode(int32_t sourceId, int32_t streamSyncMode);
318 : bool getStreamSyncMode(int32_t &streamSyncMode);
319 : ClientState getClientState();
320 : void handleStreamCollection(int32_t audioStreams, int32_t videoStreams, int32_t subtitleStreams);
321 : void setBufferingLimit(uint32_t limitBufferingMs);
322 : uint32_t getBufferingLimit();
323 : void setUseBuffering(bool useBuffering);
324 : bool getUseBuffering();
325 : bool switchSource(const std::unique_ptr<firebolt::rialto::IMediaPipeline::MediaSource> &source);
326 : IFlushAndDataSynchronizer &getFlushAndDataSynchronizer();
327 :
328 : private:
329 : bool areAllStreamsAttached();
330 : void sendAllSourcesAttachedIfPossibleInternal();
331 : bool checkIfAllAttachedSourcesInStates(const std::vector<ClientState> &states);
332 :
333 : std::unique_ptr<IMessageQueue> m_backendQueue;
334 : std::shared_ptr<IMessageQueueFactory> m_messageQueueFactory;
335 : std::shared_ptr<firebolt::rialto::client::MediaPlayerClientBackendInterface> m_clientBackend;
336 : int64_t m_position;
337 : int64_t m_duration;
338 : std::mutex m_playbackInfoMutex;
339 : std::unordered_map<int32_t, AttachedSource> m_attachedSources;
340 : bool m_wasAllSourcesAttachedSent = false;
341 : int32_t m_audioStreams;
342 : int32_t m_videoStreams;
343 : int32_t m_subtitleStreams;
344 : firebolt::rialto::PlaybackInfo m_playbackInfo{-1, 1.0};
345 : FlushAndDataSynchronizer m_flushAndDataSynchronizer;
346 : bool wasPlayingBeforeEos{false};
347 :
348 : struct Rectangle
349 : {
350 : unsigned int x, y, width, height;
351 : } m_videoRectangle;
352 :
353 : firebolt::rialto::PlaybackState m_serverPlaybackState = firebolt::rialto::PlaybackState::IDLE;
354 : ClientState m_clientState = ClientState::IDLE;
355 : // To check if the backend message queue and pulling of data to serve backend is stopped or not
356 : bool m_streamingStopped;
357 :
358 : const uint32_t m_maxWidth;
359 : const uint32_t m_maxHeight;
360 : };
|
