IMediaPipeline.h

Go to the documentation of this file.

/*
 * If not stated otherwise in this file or this component's LICENSE file the
 * following copyright and licenses apply:
 *
 * Copyright 2022 Sky UK
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
 
#ifndef FIREBOLT_RIALTO_I_MEDIA_PIPELINE_H_
#define FIREBOLT_RIALTO_I_MEDIA_PIPELINE_H_
 
#include <stdint.h>
 
#include <algorithm>
#include <memory>
#include <string>
#include <vector>
 
#include "IMediaPipelineClient.h"
#include "MediaCommon.h"
 
namespace firebolt::rialto
{
class IMediaPipeline;
 
class IMediaPipelineFactory
{
public:
    IMediaPipelineFactory() = default;
    virtual ~IMediaPipelineFactory() = default;
 
    static std::shared_ptr<IMediaPipelineFactory> createFactory();
 
    virtual std::unique_ptr<IMediaPipeline> createMediaPipeline(std::weak_ptr<IMediaPipelineClient> client,
                                                                const VideoRequirements &videoRequirements) const = 0;
};
 
class IMediaPipeline
{
public:
    IMediaPipeline() = default;
    virtual ~IMediaPipeline() = default;
 
    IMediaPipeline(const IMediaPipeline &) = delete;
    IMediaPipeline &operator=(const IMediaPipeline &) = delete;
    IMediaPipeline(IMediaPipeline &&) = delete;
    IMediaPipeline &operator=(IMediaPipeline &&) = delete;
 
    class MediaSource
    {
    public:
        virtual ~MediaSource() {}
 
        virtual std::unique_ptr<MediaSource> copy() const = 0;
 
        virtual MediaSourceType getType() const { return MediaSourceType::UNKNOWN; }
 
        std::string getMimeType() const { return m_mimeType; }
 
        bool getHasDrm() const { return m_hasDrm; }
 
        SourceConfigType getConfigType() const { return m_configType; }
 
        int32_t getId() const { return m_id; }
 
        void setId(int32_t id) { m_id = id; }
 
    protected:
        explicit MediaSource(SourceConfigType configType = SourceConfigType::UNKNOWN,
                             const std::string &mimeType = std::string(), bool hasDrm = true)
            : m_id(0), m_configType(configType), m_mimeType(mimeType), m_hasDrm(hasDrm)
        {
        }
        int32_t m_id;
 
        SourceConfigType m_configType;
 
        std::string m_mimeType;
 
        bool m_hasDrm;
    };
 
    class MediaSourceAV : public MediaSource
    {
    public:
        ~MediaSourceAV() {}
        std::unique_ptr<MediaSource> copy() const { return std::make_unique<MediaSourceAV>(*this); }
 
        SegmentAlignment getSegmentAlignment() const { return m_alignment; }
 
        const std::shared_ptr<CodecData> &getCodecData() const { return m_codecData; }
 
        StreamFormat getStreamFormat() const { return m_streamFormat; }
 
    protected:
        explicit MediaSourceAV(SourceConfigType configType = SourceConfigType::UNKNOWN,
                               const std::string &mimeType = std::string(), bool hasDrm = true,
                               SegmentAlignment alignment = SegmentAlignment::UNDEFINED,
                               StreamFormat streamFormat = StreamFormat::UNDEFINED,
                               const std::shared_ptr<CodecData> &codecData = nullptr)
            : MediaSource(configType, mimeType, hasDrm), m_alignment(alignment), m_streamFormat(streamFormat),
              m_codecData(codecData)
        {
        }
        SegmentAlignment m_alignment;
 
        StreamFormat m_streamFormat;
 
        std::shared_ptr<CodecData> m_codecData;
    };
 
    class MediaSourceAudio : public MediaSourceAV
    {
    public:
        MediaSourceAudio(const std::string &mimeType, bool hasDrm = true, const AudioConfig &audioConfig = AudioConfig(),
                         SegmentAlignment alignment = SegmentAlignment::UNDEFINED,
                         StreamFormat streamFormat = StreamFormat::UNDEFINED,
                         const std::shared_ptr<CodecData> &codecData = nullptr)
            : MediaSourceAV(SourceConfigType::AUDIO, mimeType, hasDrm, alignment, streamFormat, codecData),
              m_audioConfig(audioConfig)
        {
        }
 
        ~MediaSourceAudio() {}
 
        MediaSourceType getType() const override { return MediaSourceType::AUDIO; }
        std::unique_ptr<MediaSource> copy() const override { return std::make_unique<MediaSourceAudio>(*this); }
 
        const AudioConfig &getAudioConfig() const { return m_audioConfig; }
 
    protected:
        AudioConfig m_audioConfig;
    };
 
    class MediaSourceVideo : public MediaSourceAV
    {
    public:
        MediaSourceVideo(const std::string &mimeType, bool hasDrm = true,
                         int32_t width = firebolt::rialto::kUndefinedSize,
                         int32_t height = firebolt::rialto::kUndefinedSize,
                         SegmentAlignment alignment = SegmentAlignment::UNDEFINED,
                         StreamFormat streamFormat = StreamFormat::UNDEFINED,
                         const std::shared_ptr<CodecData> &codecData = nullptr)
            : MediaSourceAV(SourceConfigType::VIDEO, mimeType, hasDrm, alignment, streamFormat, codecData),
              m_width(width), m_height(height)
        {
        }
        ~MediaSourceVideo() {}
 
        MediaSourceType getType() const override { return MediaSourceType::VIDEO; }
        std::unique_ptr<MediaSource> copy() const { return std::make_unique<MediaSourceVideo>(*this); }
 
        int32_t getWidth() const { return m_width; }
 
        int32_t getHeight() const { return m_height; }
 
    protected:
        MediaSourceVideo(SourceConfigType sourceConfigType, const std::string &mimeType, bool hasDrm = true,
                         int32_t width = firebolt::rialto::kUndefinedSize,
                         int32_t height = firebolt::rialto::kUndefinedSize,
                         SegmentAlignment alignment = SegmentAlignment::UNDEFINED,
                         StreamFormat streamFormat = StreamFormat::UNDEFINED,
                         const std::shared_ptr<CodecData> &codecData = nullptr)
            : MediaSourceAV(sourceConfigType, mimeType, hasDrm, alignment, streamFormat, codecData), m_width(width),
              m_height(height)
        {
        }
 
    private:
        int m_width;
 
        int m_height;
    };
 
    class MediaSourceVideoDolbyVision : public MediaSourceVideo
    {
    public:
        MediaSourceVideoDolbyVision(const std::string &mimeType, int32_t dolbyVisionProfile, bool hasDrm = true,
                                    int32_t width = firebolt::rialto::kUndefinedSize,
                                    int32_t height = firebolt::rialto::kUndefinedSize,
                                    SegmentAlignment alignment = SegmentAlignment::UNDEFINED,
                                    StreamFormat streamFormat = StreamFormat::UNDEFINED,
                                    const std::shared_ptr<CodecData> &codecData = nullptr)
            : MediaSourceVideo(SourceConfigType::VIDEO_DOLBY_VISION, mimeType, hasDrm, width, height, alignment,
                               streamFormat, codecData),
              m_dolbyVisionProfile(dolbyVisionProfile)
        {
        }
        ~MediaSourceVideoDolbyVision() {}
        std::unique_ptr<MediaSource> copy() const { return std::make_unique<MediaSourceVideoDolbyVision>(*this); }
 
        uint32_t getDolbyVisionProfile() const { return m_dolbyVisionProfile; }
 
    protected:
        uint32_t m_dolbyVisionProfile;
    };
 
    class MediaSourceSubtitle : public MediaSource
    {
    public:
        MediaSourceSubtitle(const std::string &mimeType, const std::string &textTrackIdentifier)
            : MediaSource(SourceConfigType::SUBTITLE, mimeType, false), m_textTrackIdentifier(textTrackIdentifier)
        {
        }
 
        ~MediaSourceSubtitle() {}
 
        MediaSourceType getType() const override { return MediaSourceType::SUBTITLE; }
        std::unique_ptr<MediaSource> copy() const override { return std::make_unique<MediaSourceSubtitle>(*this); }
 
        const std::string &getTextTrackIdentifier() const { return m_textTrackIdentifier; }
 
    protected:
        std::string m_textTrackIdentifier;
    };
 
    class MediaSegment
    {
    public:
        MediaSegment(int32_t sourceId = 0, MediaSourceType type = MediaSourceType::UNKNOWN, int64_t timeStamp = 0,
                     int64_t duration = 0)
            : m_sourceId(sourceId), m_type(type), m_data(nullptr), m_dataLength(0u), m_timeStamp(timeStamp),
              m_duration(duration), m_encrypted(false), m_mediaKeySessionId(0), m_initWithLast15(0),
              m_alignment(SegmentAlignment::UNDEFINED), m_cipherMode(CipherMode::UNKNOWN), m_crypt(0), m_skip(0),
              m_encryptionPatternSet(false)
        {
        }
 
        virtual ~MediaSegment() {}
 
        virtual std::unique_ptr<MediaSegment> copy() const { return std::make_unique<MediaSegment>(*this); }
 
        int32_t getId() const { return m_sourceId; }
 
        MediaSourceType getType() const { return m_type; }
 
        const uint8_t *getData() const { return m_data; }
 
        uint32_t getDataLength() const { return m_dataLength; }
 
        int64_t getTimeStamp() const { return m_timeStamp; }
 
        void setTimeStamp(int64_t timeStamp) { m_timeStamp = timeStamp; }
 
        int64_t getDuration() const { return m_duration; }
 
        const std::vector<uint8_t> &getExtraData() const { return m_extraData; }
 
        bool isEncrypted() const { return m_encrypted; }
 
        const int32_t getMediaKeySessionId() const { return m_mediaKeySessionId; }
 
        const std::vector<uint8_t> &getKeyId() const { return m_keyId; }
 
        const std::vector<uint8_t> &getInitVector() const { return m_initVector; }
 
        const std::vector<SubSamplePair> &getSubSamples() const { return m_subSamples; }
 
        const uint32_t getInitWithLast15() const { return m_initWithLast15; }
 
        const SegmentAlignment getSegmentAlignment() const { return m_alignment; }
 
        const std::shared_ptr<CodecData> &getCodecData() const { return m_codecData; }
 
        const CipherMode &getCipherMode() const { return m_cipherMode; }
 
        const bool getEncryptionPattern(uint32_t &crypt, uint32_t &skip) const
        {
            crypt = m_crypt;
            skip = m_skip;
            return m_encryptionPatternSet;
        }
 
    protected:
        int32_t m_sourceId;
 
        MediaSourceType m_type;
 
        const uint8_t *m_data;
 
        uint32_t m_dataLength;
 
        int64_t m_timeStamp;
 
        int64_t m_duration;
 
        std::shared_ptr<CodecData> m_codecData;
 
        std::vector<uint8_t> m_extraData;
 
        bool m_encrypted;
 
        int32_t m_mediaKeySessionId;
 
        std::vector<uint8_t> m_keyId;
 
        std::vector<uint8_t> m_initVector;
 
        std::vector<SubSamplePair> m_subSamples;
 
        uint32_t m_initWithLast15;
 
        SegmentAlignment m_alignment;
 
        CipherMode m_cipherMode;
 
        uint32_t m_crypt;
 
        uint32_t m_skip;
 
        bool m_encryptionPatternSet;
 
    public:
        bool setData(uint32_t dataLength, const uint8_t *data)
        {
            m_dataLength = dataLength;
            m_data = data;
            return true;
        }
 
        bool setExtraData(const std::vector<uint8_t> &extraData)
        {
            m_extraData = extraData;
            return true;
        }
 
        void setSegmentAlignment(const SegmentAlignment &alignment) { m_alignment = alignment; }
 
        void setCodecData(const std::shared_ptr<CodecData> &codecData) { m_codecData = codecData; }
 
        void setEncrypted(bool encrypted) { m_encrypted = encrypted; }
 
        void setMediaKeySessionId(int32_t mksId) { m_mediaKeySessionId = mksId; }
 
        void setKeyId(const std::vector<uint8_t> &keyId) { m_keyId = keyId; }
 
        void setInitVector(const std::vector<uint8_t> &initVector) { m_initVector = initVector; }
 
        void addSubSample(size_t numClearBytes, size_t numEncryptedBytes)
        {
            m_subSamples.emplace_back(SubSamplePair{numClearBytes, numEncryptedBytes});
        }
 
        void setInitWithLast15(uint32_t initWithLast15) { m_initWithLast15 = initWithLast15; }
 
        void setCipherMode(CipherMode cipherMode) { m_cipherMode = cipherMode; }
 
        void setEncryptionPattern(uint32_t crypt, uint32_t skip)
        {
            m_crypt = crypt;
            m_skip = skip;
            m_encryptionPatternSet = true;
        }
 
        void copy(const MediaSegment &other);
    };
 
    class MediaSegmentAudio : public MediaSegment
    {
    public:
        MediaSegmentAudio(int32_t sourceId = 0, int64_t timeStamp = 0, int64_t duration = 0, int32_t sampleRate = 0,
                          int32_t numberOfChannels = 0, uint64_t clippingStart = 0, uint64_t clippingEnd = 0)
            : MediaSegment(sourceId, MediaSourceType::AUDIO, timeStamp, duration), m_sampleRate(sampleRate),
              m_numberOfChannels(numberOfChannels), m_clippingStart(clippingStart), m_clippingEnd(clippingEnd)
        {
        }
 
        MediaSegmentAudio(const MediaSegmentAudio &other) : MediaSegment(other)
        {
            m_sampleRate = other.m_sampleRate;
            m_numberOfChannels = other.m_numberOfChannels;
        }
 
        std::unique_ptr<MediaSegment> copy() const override { return std::make_unique<MediaSegmentAudio>(*this); }
 
        int32_t getSampleRate() const { return m_sampleRate; }
 
        int32_t getNumberOfChannels() const { return m_numberOfChannels; }
 
        uint64_t getClippingStart() const { return m_clippingStart; }
 
        uint64_t getClippingEnd() const { return m_clippingEnd; }
 
        MediaSegmentAudio &operator=(const MediaSegmentAudio &other)
        {
            copy(other);
            return *this;
        }
 
    protected:
        void copy(const MediaSegmentAudio &other);
 
        int32_t m_sampleRate;
 
        int32_t m_numberOfChannels;
 
        uint64_t m_clippingStart;
 
        uint64_t m_clippingEnd;
    };
 
    class MediaSegmentVideo : public MediaSegment
    {
    public:
        MediaSegmentVideo(int32_t sourceId = 0, int64_t timeStamp = 0, int64_t duration = 0,
                          int32_t width = firebolt::rialto::kUndefinedSize,
                          int32_t height = firebolt::rialto::kUndefinedSize,
                          firebolt::rialto::Fraction frameRate = {firebolt::rialto::kUndefinedSize,
                                                                  firebolt::rialto::kUndefinedSize})
            : MediaSegment(sourceId, MediaSourceType::VIDEO, timeStamp, duration), m_width(width), m_height(height),
              m_frameRate(frameRate)
        {
        }
 
        MediaSegmentVideo(const MediaSegmentVideo &other) : MediaSegment(other)
        {
            m_width = other.m_width;
            m_height = other.m_height;
            m_frameRate = other.m_frameRate;
        }
 
        std::unique_ptr<MediaSegment> copy() const override { return std::make_unique<MediaSegmentVideo>(*this); }
 
        int32_t getWidth() const { return m_width; }
 
        int32_t getHeight() const { return m_height; }
 
        firebolt::rialto::Fraction getFrameRate() const { return m_frameRate; }
 
        MediaSegmentVideo &operator=(const MediaSegmentVideo &other)
        {
            copy(other);
            return *this;
        }
 
    protected:
        void copy(const MediaSegmentVideo &other);
 
        int32_t m_width;
 
        int32_t m_height;
 
        firebolt::rialto::Fraction m_frameRate;
    };
 
    typedef std::vector<std::unique_ptr<MediaSegment>> MediaSegmentVector;
 
    virtual std::weak_ptr<IMediaPipelineClient> getClient() = 0;
 
    virtual bool load(MediaType type, const std::string &mimeType, const std::string &url) = 0;
 
    virtual bool attachSource(const std::unique_ptr<MediaSource> &source) = 0;
 
    virtual bool removeSource(int32_t id) = 0;
 
    virtual bool allSourcesAttached() = 0;
 
    virtual bool play() = 0;
 
    virtual bool pause() = 0;
 
    virtual bool stop() = 0;
 
    virtual bool setPlaybackRate(double rate) = 0;
 
    virtual bool setPosition(int64_t position) = 0;
 
    virtual bool getPosition(int64_t &position) = 0;
 
    virtual bool getStats(int32_t sourceId, uint64_t &renderedFrames, uint64_t &droppedFrames) = 0;
 
    virtual bool setImmediateOutput(int32_t sourceId, bool immediateOutput) = 0;
 
    virtual bool getImmediateOutput(int32_t sourceId, bool &immediateOutput) = 0;
 
    virtual bool setVideoWindow(uint32_t x, uint32_t y, uint32_t width, uint32_t height) = 0;
 
    virtual bool haveData(MediaSourceStatus status, uint32_t needDataRequestId) = 0;
 
    virtual AddSegmentStatus addSegment(uint32_t needDataRequestId, const std::unique_ptr<MediaSegment> &mediaSegment) = 0;
 
    virtual bool renderFrame() = 0;
 
    virtual bool setVolume(double targetVolume, uint32_t volumeDuration = 0,
                           EaseType easeType = EaseType::EASE_LINEAR) = 0;
 
    virtual bool getVolume(double &currentVolume) = 0;
 
    virtual bool setMute(int32_t sourceId, bool mute) = 0;
 
    virtual bool getMute(int32_t sourceId, bool &mute) = 0;
 
    virtual bool setTextTrackIdentifier(const std::string &textTrackIdentifier) = 0;
 
    virtual bool getTextTrackIdentifier(std::string &textTrackIdentifier) = 0;
 
    virtual bool setLowLatency(bool lowLatency) = 0;
 
    virtual bool setSync(bool sync) = 0;
 
    virtual bool getSync(bool &sync) = 0;
 
    virtual bool setSyncOff(bool syncOff) = 0;
 
    virtual bool setStreamSyncMode(int32_t sourceId, int32_t streamSyncMode) = 0;
 
    virtual bool getStreamSyncMode(int32_t &streamSyncMode) = 0;
 
    virtual bool flush(int32_t sourceId, bool resetTime) = 0;
 
    virtual bool setSourcePosition(int32_t sourceId, int64_t position, bool resetTime = false, double appliedRate = 1.0,
                                   uint64_t stopPosition = kUndefinedPosition) = 0;
 
    virtual bool processAudioGap(int64_t position, uint32_t duration, int64_t discontinuityGap, bool audioAac) = 0;
 
    virtual bool setBufferingLimit(uint32_t limitBufferingMs) = 0;
 
    virtual bool getBufferingLimit(uint32_t &limitBufferingMs) = 0;
 
    virtual bool setUseBuffering(bool useBuffering) = 0;
 
    virtual bool getUseBuffering(bool &useBuffering) = 0;
 
    virtual bool switchSource(const std::unique_ptr<MediaSource> &source) = 0;
};
 
}; // namespace firebolt::rialto
 
#endif // FIREBOLT_RIALTO_I_MEDIA_PIPELINE_H_