The main object which starts / stops / manages the containers. More...

#include <DobbyManager.h>

Public Types
typedef std::function< void(int32_t cd, const ContainerId &id)>	ContainerStartedFunc

typedef std::function< void(int32_t cd, const ContainerId &id, int32_t status)>	ContainerStoppedFunc

typedef std::function< void(int32_t cd, const ContainerId &id)>	ContainerHibernatedFunc

Public Member Functions
	DobbyManager (const std::shared_ptr< IDobbyEnv > &env, const std::shared_ptr< IDobbyUtils > &utils, const std::shared_ptr< IDobbyIPCUtils > &ipcUtils, const std::shared_ptr< const IDobbySettings > &settings, const ContainerStartedFunc &containerStartedCb, const ContainerStoppedFunc &containerStoppedCb, const ContainerHibernatedFunc &containerHibernatedCb, const ContainerHibernatedFunc &containerAwokenCb)

int32_t	startContainerFromBundle (const ContainerId &id, const std::string &bundlePath, const std::list< int > &files, const std::string &command, const std::string &displaySocket, const std::vector< std::string > &envVars)
	Where the magic begins ... attempts to create a container from an OCI bundle*. More...

bool	stopContainer (int32_t cd, bool withPrejudice)
	Stops a running container. More...

bool	pauseContainer (int32_t cd)
	Freezes a running container. More...

bool	resumeContainer (int32_t cd)
	Thaws a frozen container. More...

bool	hibernateContainer (int32_t cd, const std::string &options)
	Dumps a running container's processes. More...

bool	wakeupContainer (int32_t cd)
	Wakeup a checkpointed container from existing dump. More...

bool	addMount (int32_t cd, const std::string &source, const std::string &destination, const std::vector< std::string > &mountFlags, const std::string &mountData)
	adds a mount to a running container More...

bool	removeMount (int32_t cd, const std::string &source)
	removes a mount from a running container More...

bool	execInContainer (int32_t cd, const std::string &options, const std::string &command)
	Executes a command in a running container. More...

std::list< std::pair< int32_t, ContainerId > >	listContainers () const
	Returns a list of all the containers. More...

int32_t	stateOfContainer (int32_t cd) const
	Returns the state of a given container. More...

std::string	statsOfContainer (int32_t cd) const
	Gets the stats for the container. More...

std::string	ociConfigOfContainer (int32_t cd) const
	Debugging method to allow you to retrieve the OCI config.json spec used to create the container. More...

Private Member Functions
void	setupSystem ()
	Configures the linux system for enabling features needed for runc. More...

void	setupWorkspace (const std::shared_ptr< IDobbyEnv > &env)
	Configures the workspace directory. More...

void	cleanupContainers ()
	Gets a list of running containers and tries to kill and delete them. More...

bool	cleanupContainer (const DobbyRunC::ContainerListItem &container)
	Cleans up a container that is in an unknown state - used at Dobby startup to ensure the box is in a clean state with no leftover containers. More...

void	cleanupContainersShutdown ()
	Gracefully stops and cleans up any running containers. Will emit the container stop event when a container stops. More...

void	handleContainerTerminate (const ContainerId &id, const std::unique_ptr< DobbyContainer > &container, const int status)
	Perform all the necessary cleanup and run plugins required when a container has terminated. More...

void	onChildExit ()
	Called when we detect a child process has terminated. More...

std::shared_ptr< IDobbyRdkLoggingPlugin >	GetContainerLogger (const std::unique_ptr< DobbyContainer > &container)
	Get the instance of the logging plugin for the current container (if one is loaded)

bool	createAndStart (const ContainerId &id, const std::unique_ptr< DobbyContainer > &container, const std::list< int > &files)
	Create and start a container. Set up and capture logs from all container hooks if an RDK logging plugin is loaded. More...

bool	customiseConfig (const std::shared_ptr< DobbyConfig > &config, const std::string &command, const std::string &displaySocket, const std::vector< std::string > &envVars)
	Updates the container config with custom options provided by the start command. More...

bool	createAndStartContainer (const ContainerId &id, const std::unique_ptr< DobbyContainer > &container, const std::list< int > &files)
	Creates and attempts to start the container. More...

bool	restartContainer (const ContainerId &id, const std::unique_ptr< DobbyContainer > &container)
	Attempts to restart the container. More...

bool	onPostInstallationHook (const std::unique_ptr< DobbyContainer > &container)
	Called at the post-installation stage of container startup. More...

bool	onPreCreationHook (const std::unique_ptr< DobbyContainer > &container)
	Called at the pre-create stage of container startup. More...

bool	onPostHaltHook (const std::unique_ptr< DobbyContainer > &container)
	Called at the post-halt stage of container startup. More...

void	startRuncMonitorThread ()
	Starts a thread that monitors for SIGCHILD signals. More...

void	stopRuncMonitorThread ()
	Stops the monitor thread and cleans up it's resources.

void	runcMonitorThread ()
	Thread function that monitors for any SIGCHILD signals and if detected loops through the running containers to see if it was the runc process that spawned it (i.e. a containered init process)

bool	invalidContainerCleanupTask ()
	Task that will try and cleanup invalid/unknown state containers periodically - if the container can be killed then kill it and release the id back to the pool so it can be restarted.

bool	shouldEnableSTrace (const std::shared_ptr< DobbyConfig > &config) const

Private Attributes
ContainerStartedFunc	mContainerStartedCb

ContainerStoppedFunc	mContainerStoppedCb

ContainerHibernatedFunc	mContainerHibernatedCb

ContainerHibernatedFunc	mContainerAwokenCb

std::mutex	mLock

std::map< ContainerId, std::unique_ptr< DobbyContainer > >	mContainers

std::multimap< ContainerId, pid_t >	mContainerExecPids

const std::shared_ptr< IDobbyEnv >	mEnvironment

const std::shared_ptr< IDobbyUtils >	mUtilities

const std::shared_ptr< IDobbyIPCUtils >	mIPCUtilities

const std::shared_ptr< const IDobbySettings >	mSettings

std::unique_ptr< DobbyLogger >	mLogger

std::unique_ptr< DobbyRunC >	mRunc

sem_t	mRuncMonitorThreadStartedSem

std::thread	mRuncMonitorThread

std::atomic< bool >	mRuncMonitorTerminate

int	mCleanupTaskTimerId

Detailed Description

The main object which starts / stops / manages the containers.

This is where most of the work is done for creating and monitoring containers.

https://groups.google.com/a/opencontainers.org/forum/m/#!topic/dev/Y7p6YW8zr4s

Member Function Documentation

◆ addMount()

bool DobbyManager::addMount	(	int32_t	cd,
		const std::string &	source,
		const std::string &	destination,
		const std::vector< std::string > &	mountFlags,
		const std::string &	mountData
	)

adds a mount to a running container

Parameters

[in]	cd	The descriptor of the container to checkpoint.
[in]	source	The source path of the mount
[in]	destination	The destination path of the mount
[in]	mountFlags	The mount flags is vector of strings containing the mount flags e.g. "rbind, ro"
[in]	mountData	a string containing the mountdata

Returns: true if a container was successfully restored.

◆ cleanupContainer()

bool DobbyManager::cleanupContainer ( const DobbyRunC::ContainerListItem & container )

private

Cleans up a container that is in an unknown state - used at Dobby startup to ensure the box is in a clean state with no leftover containers.

Parameters

[in] container Container info returned by crun list command

◆ cleanupContainers()

void DobbyManager::cleanupContainers ( )

private

Gets a list of running containers and tries to kill and delete them.

Will run the postHalt hook for the container where possible (some hooks might fail as the container bundle cannot be guaranteed to exist at this time)

Designed as a crash-recovery mechanism as we should clean up all our containers if the daemon shut down gracefully

◆ cleanupContainersShutdown()

void DobbyManager::cleanupContainersShutdown ( )

private

Gracefully stops and cleans up any running containers. Will emit the container stop event when a container stops.

Designed to be called when the daemon is going down (e.g. SIGTERM)

◆ createAndStart()

bool DobbyManager::createAndStart	(	const ContainerId &	id,
		const std::unique_ptr< DobbyContainer > &	container,
		const std::list< int > &	files
	)

private

Create and start a container. Set up and capture logs from all container hooks if an RDK logging plugin is loaded.

If container->customConfigFilePath is set, the container will use that config.json file instead of the onein the bundle

Parameters

[in]	id	id of the container
[in]	container	The object that wraps up the container details.
[in]	files	List of fds to preserve inside the container

◆ createAndStartContainer()

bool DobbyManager::createAndStartContainer	(	const ContainerId &	id,
		const std::unique_ptr< DobbyContainer > &	container,
		const std::list< int > &	files
	)

private

Creates and attempts to start the container.

Parameters

[in]	id	The id string for the container.
[in]	container	The object that wraps up the container details.
[in]	startState	The start state object of the files.
[in]	command	The custom command to run instead of the args in the config file

Returns: true on success, false on failure.

◆ customiseConfig()

bool DobbyManager::customiseConfig	(	const std::shared_ptr< DobbyConfig > &	config,
		const std::string &	command,
		const std::string &	displaySocket,
		const std::vector< std::string > &	envVars
	)

private

Updates the container config with custom options provided by the start command.

Parameters

[in]	command	The custom command to run instead of the args in the config file
[in]	displaySocket	Path to a westeros socket to mount into the container
[in]	envVars	Custom env vars to add to the container

Returns: true if modifications were made, false if no changes made

◆ execInContainer()

bool DobbyManager::execInContainer	(	int32_t	cd,
		const std::string &	options,
		const std::string &	command
	)

Executes a command in a running container.

Parameters

[in]	cd	The descriptor of the container to execute the command in.
[in]	command	Command to be executed.
[in]	options	Options to execute the command with.

Returns: true if a container with a matching descriptor was found and the command was run

◆ handleContainerTerminate()

void DobbyManager::handleContainerTerminate	(	const ContainerId &	id,
		const std::unique_ptr< DobbyContainer > &	container,
		const int	status
	)

private

Perform all the necessary cleanup and run plugins required when a container has terminated.

Will also delete the container so the ID can be re-used

Parameters

[in]	id	ID of the container that has terminated
[in]	container	Information about the container that has terminated (rootfs, config etc)
[in]	status	Exit status of the container runtime

◆ hibernateContainer()

bool DobbyManager::hibernateContainer	(	int32_t	cd,
		const std::string &	options
	)

Dumps a running container's processes.

Parameters

[in]	cd	The descriptor of the container to checkpoint.
[in]	options	Additional options

Returns: true if a container with a matching descriptor was found and its processes were dumped.

◆ listContainers()

std::list< std::pair< int32_t, ContainerId > > DobbyManager::listContainers ( ) const

Returns a list of all the containers.

The returned list contains the id of all the containers we know about in their various states. Just because a container id is in the list it doesn't necessarily mean it's actually running, it could be in either the starting or stopping phase.

See also: DobbyManager::stateOfContainer for a way to retrieve the status of the container.

Returns: a list of all the containers.

◆ ociConfigOfContainer()

std::string DobbyManager::ociConfigOfContainer ( int32_t cd ) const

Debugging method to allow you to retrieve the OCI config.json spec used to create the container.

Parameters

[in] cd The descriptor of the container to get the config.json of.

Returns: the config.json string.

◆ onChildExit()

void DobbyManager::onChildExit ( )

private

Called when we detect a child process has terminated.

The child process will typically be a runc instance for a container. To check we need to iterate over the pids we know about and check if they have terminated.

This is how we determine when a container dies, we look for the child process death signal.

◆ onPostHaltHook()

bool DobbyManager::onPostHaltHook ( const std::unique_ptr< DobbyContainer > & container )

private

Called at the post-halt stage of container startup.

Uses the map of rdkPlugin names/data in the container config to determine which plugins to execute. RDK plugins are loaded from disk in the ctor of DobbyContainer if any rdk plugins are required

Parameters

[in] container Container wrapper object for the container to execute the plugins on

Returns: true if all required postInstallation hooks were successful

◆ onPostInstallationHook()

bool DobbyManager::onPostInstallationHook ( const std::unique_ptr< DobbyContainer > & container )

private

Called at the post-installation stage of container startup.

Uses the map of rdkPlugin names/data in the container config to determine which plugins to execute. RDK plugins are loaded from disk in the constructor of DobbyContainer if any rdk plugins are required

Parameters

[in] container Container wrapper object for the container to execute the plugins on

Returns: true if all required postInstallation hooks were successful

◆ onPreCreationHook()

bool DobbyManager::onPreCreationHook ( const std::unique_ptr< DobbyContainer > & container )

private

Called at the pre-create stage of container startup.

Uses the map of rdkPlugin names/data in the container config to determine which plugins to execute. RDK plugins are loaded from disk in the constructor of DobbyContainer if any rdk plugins are required

Parameters

[in] container Container wrapper object for the container to execute the plugins on

Returns: true if all required preCreate hooks were successful

◆ pauseContainer()

bool DobbyManager::pauseContainer ( int32_t cd )

Freezes a running container.

Currently we have no use case for pause/resume containers so the method hasn't been implemented, however when testing manually I've discovered it actually works quite well.

If wanting to have a play you can run the following on the command line

runc --root /var/run/runc pause <id>

Parameters

[in] cd The descriptor of the container to pause.

Returns: true if a container with a matching descriptor was found and it was frozen.

◆ removeMount()

bool DobbyManager::removeMount	(	int32_t	cd,
		const std::string &	source
	)

removes a mount from a running container

Parameters

[in]	cd	The descriptor of the container to checkpoint.
[in]	source

Returns: true if a container was successfully restored.

◆ restartContainer()

bool DobbyManager::restartContainer	(	const ContainerId &	id,
		const std::unique_ptr< DobbyContainer > &	container
	)

private

Attempts to restart the container.

Called internally when we've detected a container shutdown and the config has indicated we should try and auto-restart the container.

From the runc tool's POV this is start of a new container.

Note when restarting, we don't regenerate the rootfs or config.json files for the bundle and the postConstruction hook is not called. However the preStart, postStart and postStop hooks will be called.

Any file descriptors that were passed into the container when it was first launched will be passed in again. The DobbyContainer object will have stored them when the container was created the first time.

Warning: this function is called from the thread that monitors the child pids, not the main thread, however it is called with mLock held.

Parameters

[in]	id	The id of the container to re-start.
[in]	container	The container object storing the container details.

Returns: true if runc executed the request, however this doesn't necessarily mean the container is up and running successfully.

◆ resumeContainer()

bool DobbyManager::resumeContainer ( int32_t cd )

Thaws a frozen container.

Parameters

[in] cd The descriptor of the container to resume.

Returns: true if a container with a matching descriptor was found and it was resumed.

◆ setupSystem()

void DobbyManager::setupSystem ( )

private

Configures the linux system for enabling features needed for runc.

This method is equivalent to performing the following on the cmdline

ulimit -c unlimited

echo "1" > /proc/sys/net/ipv4/ip_forward

See the comments in the code for why each step is needed

◆ setupWorkspace()

void DobbyManager::setupWorkspace ( const std::shared_ptr< IDobbyEnv > & env )

private

Configures the workspace directory.

The supplied path will be created if it doesn't exist. It should be on writable mount point with an adequate amount of space available.

The workspace is where bundle directories are created for each container and also where various temporary files are created. However the runc tool doesn't use it and log files are written to /var/log/...

TODO: show tree of workspace layout

Parameters

[in] path The absolute path to the workspace

◆ startContainerFromBundle()

int32_t DobbyManager::startContainerFromBundle	(	const ContainerId &	id,
		const std::string &	bundlePath,
		const std::list< int > &	files,
		const std::string &	command,
		const std::string &	displaySocket,
		const std::vector< std::string > &	envVars
	)

Where the magic begins ... attempts to create a container from an OCI bundle*.

Parameters

[in]	id	The id string for the container
[in]	bundlePath	The absolute path to the OCI bundle*
[in]	files	A list of file descriptors to pass into the container, can be empty.
[in]	command	The custom command to run instead of the args in the config file (optional)

Returns: a container descriptor, which is just a unique number that identifies the container.

◆ startRuncMonitorThread()

void DobbyManager::startRuncMonitorThread ( )

private

Starts a thread that monitors for SIGCHILD signals.

This thread is used to determine when one of our spawned child runc processes has died.

Failure to create the thread should be considered fatal.

Returns: true on success, false on failure.

◆ stateOfContainer()

int32_t DobbyManager::stateOfContainer ( int32_t cd ) const

Returns the state of a given container.

Parameters

[in] cd The descriptor of the container to get the state of.

Returns: one of the possible state values.

◆ statsOfContainer()

std::string DobbyManager::statsOfContainer ( int32_t cd ) const

Gets the stats for the container.

This is primarily a debugging method, used to get statistics on the container and roughly correlates to the 'runc events –stats <id>' call.

The reply is a json formatted string containing some info, it's form may change over time.

{
    "id": "blah",
    "state": "running",
    "timestamp": 348134887768,
    "pids": [ 1234, 1245 ],
    "cpu": {
        "usage": {
            "total":734236982,
            "percpu":[348134887,386102095]
        }
    },
    "memory":{
        "user": {
            "limit":41943040,
            "usage":356352,
            "max":524288,
            "failcnt":0
        }
    }
    "gpu":{
        "memory": {
            "limit":41943040,
            "usage":356352,
            "max":524288,
            "failcnt":0
        }
    }
    ...
}

Parameters

[in] cd The container descriptor

Returns: Json formatted string with the info for the container, on failure an empty string.

◆ stopContainer()

bool DobbyManager::stopContainer	(	int32_t	cd,
		bool	withPrejudice
	)

Stops a running container.

If withPrejudice is not specified (the default) then we send the init process within the container a SIGTERM.

If the withPrejudice is true then we use the SIGKILL signal.

This call is asynchronous, i.e. it is a request to stop rather than a blocking call that ensures the container is stopped before returning.

The mContainerStoppedCb callback will be called when the container has actually been torn down.

Parameters

[in]	cd	The descriptor of the container to stop.
[in]	withPrejudice	If true the container process is killed with SIGKILL, otherwise SIGTERM is used.

Returns: true if a container with a matching id was found and a signal sent successfully to it.

◆ wakeupContainer()

bool DobbyManager::wakeupContainer ( int32_t cd )

Wakeup a checkpointed container from existing dump.

Parameters

[in] cd The descriptor of the container to checkpoint.

Returns: true if a container was successfully restored.

The documentation for this class was generated from the following files:

daemon/lib/source/include/DobbyManager.h
daemon/lib/source/DobbyManager.cpp

Public Types

Public Member Functions

Private Member Functions

Private Attributes

Detailed Description

Member Function Documentation

◆ addMount()

◆ cleanupContainer()

◆ cleanupContainers()

◆ cleanupContainersShutdown()

◆ createAndStart()

◆ createAndStartContainer()

◆ customiseConfig()

◆ execInContainer()

◆ handleContainerTerminate()

◆ hibernateContainer()

◆ listContainers()

◆ ociConfigOfContainer()

◆ onChildExit()

◆ onPostHaltHook()

◆ onPostInstallationHook()

◆ onPreCreationHook()

◆ pauseContainer()

◆ removeMount()

◆ restartContainer()

◆ resumeContainer()

◆ setupSystem()

◆ setupWorkspace()

◆ startContainerFromBundle()

◆ startRuncMonitorThread()

◆ stateOfContainer()

◆ statsOfContainer()

◆ stopContainer()

◆ wakeupContainer()