Line data Source code
1 : /*
2 : * Copyright (C) 2022 Sky UK
3 : *
4 : * This library is free software; you can redistribute it and/or
5 : * modify it under the terms of the GNU Lesser General Public
6 : * License as published by the Free Software Foundation;
7 : * version 2.1 of the License.
8 : *
9 : * This library is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 : * Lesser General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU Lesser General Public
15 : * License along with this library; if not, write to the Free Software
16 : * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 : */
18 :
19 : #pragma once
20 :
21 : #include <atomic>
22 : #include <condition_variable>
23 : #include <functional>
24 : #include <memory>
25 : #include <mutex>
26 : #include <thread>
27 : #include <unordered_map>
28 : #include <unordered_set>
29 : #include <vector>
30 :
31 : #include <gst/gst.h>
32 : #include <sys/syscall.h>
33 : #include <sys/types.h>
34 : #include <unistd.h>
35 :
36 : #include "BufferParser.h"
37 : #include "Constants.h"
38 : #include "IMediaPipeline.h"
39 : #include "IMessageQueue.h"
40 : #include "MediaCommon.h"
41 : #include "MediaPlayerClientBackendInterface.h"
42 : #include "RialtoGStreamerMSEBaseSink.h"
43 : #include "RialtoGStreamerMSEBaseSinkCallbacks.h"
44 :
45 : #define DEFAULT_MAX_VIDEO_WIDTH 3840
46 : #define DEFAULT_MAX_VIDEO_HEIGHT 2160
47 :
48 : class GStreamerMSEMediaPlayerClient;
49 :
50 : enum class ClientState
51 : {
52 : IDLE,
53 : READY,
54 : AWAITING_PAUSED,
55 : PAUSED,
56 : AWAITING_PLAYING,
57 : PLAYING
58 : };
59 :
60 : class BufferPuller
61 : {
62 : public:
63 : BufferPuller(const std::shared_ptr<IMessageQueueFactory> &messageQueueFactory, GstElement *rialtoSink,
64 : const std::shared_ptr<BufferParser> &bufferParser);
65 :
66 : void start();
67 : void stop();
68 : bool requestPullBuffer(int sourceId, size_t frameCount, unsigned int needDataRequestId,
69 : GStreamerMSEMediaPlayerClient *player);
70 :
71 : private:
72 : std::unique_ptr<IMessageQueue> m_queue;
73 : GstElement *m_rialtoSink;
74 : std::shared_ptr<BufferParser> m_bufferParser;
75 : };
76 :
77 : class AttachedSource
78 : {
79 : friend class GStreamerMSEMediaPlayerClient;
80 :
81 : public:
82 185 : AttachedSource(RialtoMSEBaseSink *rialtoSink, std::shared_ptr<BufferPuller> puller,
83 : firebolt::rialto::MediaSourceType type, ClientState state = ClientState::READY)
84 185 : : m_rialtoSink(rialtoSink), m_bufferPuller(puller), m_type(type), m_state(state)
85 : {
86 : }
87 :
88 370 : firebolt::rialto::MediaSourceType getType() const { return m_type; }
89 1 : void setPosition(int64_t position) { m_position = position; }
90 :
91 : private:
92 : RialtoMSEBaseSink *m_rialtoSink;
93 : std::shared_ptr<BufferPuller> m_bufferPuller;
94 : std::unordered_set<uint32_t> m_ongoingNeedDataRequests;
95 : firebolt::rialto::MediaSourceType m_type = firebolt::rialto::MediaSourceType::UNKNOWN;
96 : int64_t m_position = 0;
97 : bool m_isFlushing = false;
98 : ClientState m_state = ClientState::READY;
99 : };
100 :
101 : class HaveDataMessage : public Message
102 : {
103 : public:
104 : HaveDataMessage(firebolt::rialto::MediaSourceStatus status, int sourceId, unsigned int needDataRequestId,
105 : GStreamerMSEMediaPlayerClient *player);
106 : void handle() override;
107 :
108 : private:
109 : firebolt::rialto::MediaSourceStatus m_status;
110 : int m_sourceId;
111 : unsigned int m_needDataRequestId;
112 : GStreamerMSEMediaPlayerClient *m_player;
113 : };
114 :
115 : class PullBufferMessage : public Message
116 : {
117 : public:
118 : PullBufferMessage(int sourceId, size_t frameCount, unsigned int needDataRequestId, GstElement *rialtoSink,
119 : const std::shared_ptr<BufferParser> &bufferParser, IMessageQueue &pullerQueue,
120 : GStreamerMSEMediaPlayerClient *player);
121 : void handle() override;
122 :
123 : private:
124 : int m_sourceId;
125 : size_t m_frameCount;
126 : unsigned int m_needDataRequestId;
127 : GstElement *m_rialtoSink;
128 : std::shared_ptr<BufferParser> m_bufferParser;
129 : IMessageQueue &m_pullerQueue;
130 : GStreamerMSEMediaPlayerClient *m_player;
131 : };
132 :
133 : class NeedDataMessage : public Message
134 : {
135 : public:
136 : NeedDataMessage(int sourceId, size_t frameCount, unsigned int needDataRequestId,
137 : GStreamerMSEMediaPlayerClient *player);
138 : void handle() override;
139 :
140 : private:
141 : int m_sourceId;
142 : size_t m_frameCount;
143 : unsigned int m_needDataRequestId;
144 : GStreamerMSEMediaPlayerClient *m_player;
145 : };
146 :
147 : class PlaybackStateMessage : public Message
148 : {
149 : public:
150 : PlaybackStateMessage(firebolt::rialto::PlaybackState state, GStreamerMSEMediaPlayerClient *player);
151 : void handle() override;
152 :
153 : private:
154 : firebolt::rialto::PlaybackState m_state;
155 : GStreamerMSEMediaPlayerClient *m_player;
156 : };
157 :
158 : class QosMessage : public Message
159 : {
160 : public:
161 : QosMessage(int sourceId, firebolt::rialto::QosInfo qosInfo, GStreamerMSEMediaPlayerClient *player);
162 : void handle() override;
163 :
164 : private:
165 : int m_sourceId;
166 : firebolt::rialto::QosInfo m_qosInfo;
167 : GStreamerMSEMediaPlayerClient *m_player;
168 : };
169 :
170 : class BufferUnderflowMessage : public Message
171 : {
172 : public:
173 : BufferUnderflowMessage(int sourceId, GStreamerMSEMediaPlayerClient *player);
174 : void handle() override;
175 :
176 : private:
177 : int m_sourceId;
178 : GStreamerMSEMediaPlayerClient *m_player;
179 : };
180 :
181 : class PlaybackErrorMessage : public Message
182 : {
183 : public:
184 : PlaybackErrorMessage(int sourceId, firebolt::rialto::PlaybackError error, GStreamerMSEMediaPlayerClient *player);
185 : void handle() override;
186 :
187 : private:
188 : int m_sourceId;
189 : firebolt::rialto::PlaybackError m_error;
190 : GStreamerMSEMediaPlayerClient *m_player;
191 : };
192 :
193 : class SetPositionMessage : public Message
194 : {
195 : public:
196 : SetPositionMessage(int64_t newPosition, std::unordered_map<int32_t, AttachedSource> &attachedSources);
197 : void handle() override;
198 :
199 : private:
200 : int64_t m_newPosition;
201 : std::unordered_map<int32_t, AttachedSource> &m_attachedSources;
202 : };
203 :
204 : class SetDurationMessage : public Message
205 : {
206 : public:
207 : SetDurationMessage(int64_t newDuration, int64_t &targetDuration);
208 : void handle() override;
209 :
210 : private:
211 : int64_t m_newDuration;
212 : int64_t &m_targetDuration;
213 : };
214 :
215 : class SourceFlushedMessage : public Message
216 : {
217 : public:
218 : SourceFlushedMessage(int32_t sourceId, GStreamerMSEMediaPlayerClient *player);
219 : void handle() override;
220 :
221 : private:
222 : int32_t m_sourceId;
223 : GStreamerMSEMediaPlayerClient *m_player;
224 : };
225 :
226 : enum class StateChangeResult
227 : {
228 : SUCCESS_ASYNC,
229 : SUCCESS_SYNC,
230 : NOT_ATTACHED
231 : };
232 :
233 : class GStreamerMSEMediaPlayerClient : public firebolt::rialto::IMediaPipelineClient,
234 : public std::enable_shared_from_this<GStreamerMSEMediaPlayerClient>
235 : {
236 : friend class NeedDataMessage;
237 : friend class PullBufferMessage;
238 : friend class HaveDataMessage;
239 : friend class QosMessage;
240 :
241 : public:
242 : GStreamerMSEMediaPlayerClient(
243 : const std::shared_ptr<IMessageQueueFactory> &messageQueueFactory,
244 : const std::shared_ptr<firebolt::rialto::client::MediaPlayerClientBackendInterface> &MediaPlayerClientBackend,
245 : const uint32_t maxVideoWidth, const uint32_t maxVideoHeight);
246 : virtual ~GStreamerMSEMediaPlayerClient();
247 :
248 : void notifyDuration(int64_t duration) override;
249 : void notifyPosition(int64_t position) override;
250 : void notifyNativeSize(uint32_t width, uint32_t height, double aspect) override;
251 : void notifyNetworkState(firebolt::rialto::NetworkState state) override;
252 : void notifyPlaybackState(firebolt::rialto::PlaybackState state) override;
253 : void notifyVideoData(bool hasData) override;
254 : void notifyAudioData(bool hasData) override;
255 : void notifyNeedMediaData(int32_t sourceId, size_t frameCount, uint32_t needDataRequestId,
256 : const std::shared_ptr<firebolt::rialto::MediaPlayerShmInfo> &shmInfo) override;
257 : void notifyCancelNeedMediaData(int32_t sourceId) override;
258 : void notifyQos(int32_t sourceId, const firebolt::rialto::QosInfo &qosInfo) override;
259 : void notifyBufferUnderflow(int32_t sourceId) override;
260 : void notifyPlaybackError(int32_t sourceId, firebolt::rialto::PlaybackError error) override;
261 : void notifySourceFlushed(int32_t sourceId) override;
262 :
263 : void getPositionDo(int64_t *position, int32_t sourceId);
264 : int64_t getPosition(int32_t sourceId);
265 : bool setImmediateOutput(int32_t sourceId, bool immediateOutput);
266 : bool getImmediateOutput(int32_t sourceId, bool &immediateOutput);
267 : bool getStats(int32_t sourceId, uint64_t &renderedFrames, uint64_t &droppedFrames);
268 :
269 : firebolt::rialto::AddSegmentStatus
270 : addSegment(unsigned int needDataRequestId,
271 : const std::unique_ptr<firebolt::rialto::IMediaPipeline::MediaSegment> &mediaSegment);
272 :
273 : bool createBackend();
274 : StateChangeResult play(int32_t sourceId);
275 : StateChangeResult pause(int32_t sourceId);
276 : void stop();
277 : void setPlaybackRate(double rate);
278 : void flush(int32_t sourceId, bool resetTime);
279 : void setSourcePosition(int32_t sourceId, int64_t position, bool resetTime, double appliedRate = 1.0,
280 : uint64_t stopPosition = GST_CLOCK_TIME_NONE);
281 : void processAudioGap(int64_t position, uint32_t duration, int64_t discontinuityGap, bool audioAac);
282 :
283 : bool attachSource(std::unique_ptr<firebolt::rialto::IMediaPipeline::MediaSource> &source,
284 : RialtoMSEBaseSink *rialtoSink);
285 : void removeSource(int32_t sourceId);
286 : void handlePlaybackStateChange(firebolt::rialto::PlaybackState state);
287 : void handleSourceFlushed(int32_t sourceId);
288 : void sendAllSourcesAttachedIfPossible();
289 :
290 : void setVideoRectangle(const std::string &rectangleString);
291 : std::string getVideoRectangle();
292 :
293 : bool requestPullBuffer(int streamId, size_t frameCount, unsigned int needDataRequestId);
294 : bool handleQos(int sourceId, firebolt::rialto::QosInfo qosInfo);
295 : bool handleBufferUnderflow(int sourceId);
296 : bool handlePlaybackError(int sourceId, firebolt::rialto::PlaybackError error);
297 : void stopStreaming();
298 : void destroyClientBackend();
299 : bool renderFrame(RialtoMSEBaseSink *sink);
300 : void setVolume(double targetVolume, uint32_t volumeDuration, firebolt::rialto::EaseType easeType);
301 : bool getVolume(double &volume);
302 : void setMute(bool mute, int32_t sourceId);
303 : bool getMute(int sourceId);
304 : void setTextTrackIdentifier(const std::string &textTrackIdentifier);
305 : std::string getTextTrackIdentifier();
306 : bool setLowLatency(bool lowLatency);
307 : bool setSync(bool sync);
308 : bool getSync(bool &sync);
309 : bool setSyncOff(bool syncOff);
310 : bool setStreamSyncMode(int32_t sourceId, int32_t streamSyncMode);
311 : bool getStreamSyncMode(int32_t &streamSyncMode);
312 : ClientState getClientState();
313 : void handleStreamCollection(int32_t audioStreams, int32_t videoStreams, int32_t subtitleStreams);
314 : void setBufferingLimit(uint32_t limitBufferingMs);
315 : uint32_t getBufferingLimit();
316 : void setUseBuffering(bool useBuffering);
317 : bool getUseBuffering();
318 : bool switchSource(const std::unique_ptr<firebolt::rialto::IMediaPipeline::MediaSource> &source);
319 :
320 : private:
321 : bool areAllStreamsAttached();
322 : void sendAllSourcesAttachedIfPossibleInternal();
323 : bool checkIfAllAttachedSourcesInStates(const std::vector<ClientState> &states);
324 :
325 : std::unique_ptr<IMessageQueue> m_backendQueue;
326 : std::shared_ptr<IMessageQueueFactory> m_messageQueueFactory;
327 : std::shared_ptr<firebolt::rialto::client::MediaPlayerClientBackendInterface> m_clientBackend;
328 : int64_t m_position;
329 : int64_t m_duration;
330 : std::mutex m_playerMutex;
331 : std::unordered_map<int32_t, AttachedSource> m_attachedSources;
332 : bool m_wasAllSourcesAttachedSent = false;
333 : int32_t m_audioStreams;
334 : int32_t m_videoStreams;
335 : int32_t m_subtitleStreams;
336 :
337 : struct Rectangle
338 : {
339 : unsigned int x, y, width, height;
340 : } m_videoRectangle;
341 :
342 : firebolt::rialto::PlaybackState m_serverPlaybackState = firebolt::rialto::PlaybackState::IDLE;
343 : ClientState m_clientState = ClientState::IDLE;
344 : // To check if the backend message queue and pulling of data to serve backend is stopped or not
345 : bool m_streamingStopped;
346 :
347 : const uint32_t m_maxWidth;
348 : const uint32_t m_maxHeight;
349 : };
|
