org.apache.hadoop.yarn.server.nodemanager.recovery

Class NMStateStoreService

java.lang.Object

org.apache.hadoop.service.AbstractService

org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService

All Implemented Interfaces:

Closeable, AutoCloseable, org.apache.hadoop.service.Service

Direct Known Subclasses:

NMLeveldbStateStoreService, NMNullStateStoreService

@InterfaceAudience.Private
 @InterfaceStability.Unstable
public abstract class NMStateStoreService
extends org.apache.hadoop.service.AbstractService

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	NMStateStoreService.LocalResourceTrackerState
static class	NMStateStoreService.RecoveredApplicationsState
static class	NMStateStoreService.RecoveredContainerState
static class	NMStateStoreService.RecoveredContainerStatus
static class	NMStateStoreService.RecoveredContainerTokensState
static class	NMStateStoreService.RecoveredDeletionServiceState
static class	NMStateStoreService.RecoveredLocalizationState
static class	NMStateStoreService.RecoveredLogDeleterState
static class	NMStateStoreService.RecoveredNMTokensState
static class	NMStateStoreService.RecoveredUserResources

Nested classes/interfaces inherited from interface org.apache.hadoop.service.Service

org.apache.hadoop.service.Service.STATE

Constructor Summary

Constructors

Constructor and Description
NMStateStoreService(String name)

Method Summary

Modifier and Type	Method and Description
boolean	canRecover()
protected abstract void	closeStorage()
abstract void	finishResourceLocalization(String user, org.apache.hadoop.yarn.api.records.ApplicationId appId, YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto proto) Record the completion of a resource localization
protected abstract void	initStorage(org.apache.hadoop.conf.Configuration conf)
boolean	isNewlyCreated()
abstract NMStateStoreService.RecoveredApplicationsState	loadApplicationsState() Load the state of applications
abstract List<NMStateStoreService.RecoveredContainerState>	loadContainersState() Load the state of containers
abstract NMStateStoreService.RecoveredContainerTokensState	loadContainerTokensState() Load the state of container tokens
abstract NMStateStoreService.RecoveredDeletionServiceState	loadDeletionServiceState() Load the state of the deletion service
abstract NMStateStoreService.RecoveredLocalizationState	loadLocalizationState() Load the state of localized resources
abstract NMStateStoreService.RecoveredLogDeleterState	loadLogDeleterState() Load the state of log deleters
abstract NMStateStoreService.RecoveredNMTokensState	loadNMTokensState() Load the state of NM tokens
abstract void	removeApplication(org.apache.hadoop.yarn.api.records.ApplicationId appId) Remove records corresponding to an application
abstract void	removeContainer(org.apache.hadoop.yarn.api.records.ContainerId containerId) Remove records corresponding to a container
abstract void	removeContainerToken(org.apache.hadoop.yarn.api.records.ContainerId containerId) Remove records for a container token
abstract void	removeDeletionTask(int taskId) Remove records corresponding to a deletion task
abstract void	removeLocalizedResource(String user, org.apache.hadoop.yarn.api.records.ApplicationId appId, org.apache.hadoop.fs.Path localPath) Remove records related to a resource localization
abstract void	removeLogDeleter(org.apache.hadoop.yarn.api.records.ApplicationId appId) Remove the state of a log deleter
abstract void	removeNMTokenApplicationMasterKey(org.apache.hadoop.yarn.api.records.ApplicationAttemptId attempt) Remove a master key corresponding to an application
void	serviceInit(org.apache.hadoop.conf.Configuration conf) Initialize the state storage
void	serviceStart() Start the state storage for use
void	serviceStop() Shutdown the state storage.
abstract void	startResourceLocalization(String user, org.apache.hadoop.yarn.api.records.ApplicationId appId, org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto proto, org.apache.hadoop.fs.Path localPath) Record the start of localization for a resource
protected abstract void	startStorage()
abstract void	storeApplication(org.apache.hadoop.yarn.api.records.ApplicationId appId, YarnServerNodemanagerRecoveryProtos.ContainerManagerApplicationProto p) Record the start of an application
abstract void	storeContainer(org.apache.hadoop.yarn.api.records.ContainerId containerId, org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest startRequest) Record a container start request
abstract void	storeContainerCompleted(org.apache.hadoop.yarn.api.records.ContainerId containerId, int exitCode) Record that a container has completed
abstract void	storeContainerDiagnostics(org.apache.hadoop.yarn.api.records.ContainerId containerId, StringBuilder diagnostics) Record diagnostics for a container
abstract void	storeContainerKilled(org.apache.hadoop.yarn.api.records.ContainerId containerId) Record a request to kill a container
abstract void	storeContainerLaunched(org.apache.hadoop.yarn.api.records.ContainerId containerId) Record that a container has been launched
abstract void	storeContainerToken(org.apache.hadoop.yarn.api.records.ContainerId containerId, Long expirationTime) Record the expiration time for a container token
abstract void	storeContainerTokenCurrentMasterKey(org.apache.hadoop.yarn.server.api.records.MasterKey key) Record the current container token master key
abstract void	storeContainerTokenPreviousMasterKey(org.apache.hadoop.yarn.server.api.records.MasterKey key) Record the previous container token master key
abstract void	storeDeletionTask(int taskId, YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto taskProto) Record a deletion task
abstract void	storeLogDeleter(org.apache.hadoop.yarn.api.records.ApplicationId appId, YarnServerNodemanagerRecoveryProtos.LogDeleterProto proto) Store the state of a log deleter
abstract void	storeNMTokenApplicationMasterKey(org.apache.hadoop.yarn.api.records.ApplicationAttemptId attempt, org.apache.hadoop.yarn.server.api.records.MasterKey key) Record a master key corresponding to an application
abstract void	storeNMTokenCurrentMasterKey(org.apache.hadoop.yarn.server.api.records.MasterKey key) Record the current NM token master key
abstract void	storeNMTokenPreviousMasterKey(org.apache.hadoop.yarn.server.api.records.MasterKey key) Record the previous NM token master key

Methods inherited from class org.apache.hadoop.service.AbstractService

close, getBlockers, getConfig, getFailureCause, getFailureState, getLifecycleHistory, getName, getServiceState, getStartTime, init, isInState, noteFailure, putBlocker, registerGlobalListener, registerServiceListener, removeBlocker, setConfig, start, stop, toString, unregisterGlobalListener, unregisterServiceListener, waitForServiceToStop

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait

Constructor Detail

NMStateStoreService

public NMStateStoreService(String name)

Method Detail

serviceInit

public void serviceInit(org.apache.hadoop.conf.Configuration conf)
                 throws IOException

Initialize the state storage

Overrides:

serviceInit in class org.apache.hadoop.service.AbstractService

Throws:

IOException

serviceStart

public void serviceStart()
                  throws IOException

Start the state storage for use

Overrides:

serviceStart in class org.apache.hadoop.service.AbstractService

Throws:

IOException

serviceStop

public void serviceStop()
                 throws IOException

Shutdown the state storage.

Overrides:

serviceStop in class org.apache.hadoop.service.AbstractService

Throws:

IOException

canRecover

public boolean canRecover()

isNewlyCreated

public boolean isNewlyCreated()

loadApplicationsState

public abstract NMStateStoreService.RecoveredApplicationsState loadApplicationsState()
                                                                              throws IOException

Load the state of applications

Returns:

recovered state for applications

Throws:

IOException

storeApplication

public abstract void storeApplication(org.apache.hadoop.yarn.api.records.ApplicationId appId,
                                      YarnServerNodemanagerRecoveryProtos.ContainerManagerApplicationProto p)
                               throws IOException

Record the start of an application

Parameters:

appId - the application ID

p - state to store for the application

Throws:

IOException

removeApplication

public abstract void removeApplication(org.apache.hadoop.yarn.api.records.ApplicationId appId)
                                throws IOException

Remove records corresponding to an application

Parameters:

appId - the application ID

Throws:

IOException

loadContainersState

public abstract List<NMStateStoreService.RecoveredContainerState> loadContainersState()
                                                                               throws IOException

Load the state of containers

Returns:

recovered state for containers

Throws:

IOException

storeContainer

public abstract void storeContainer(org.apache.hadoop.yarn.api.records.ContainerId containerId,
                                    org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest startRequest)
                             throws IOException

Record a container start request

Parameters:

containerId - the container ID

startRequest - the container start request

Throws:

IOException

storeContainerLaunched

public abstract void storeContainerLaunched(org.apache.hadoop.yarn.api.records.ContainerId containerId)
                                     throws IOException

Record that a container has been launched

Parameters:

containerId - the container ID

Throws:

IOException

storeContainerCompleted

public abstract void storeContainerCompleted(org.apache.hadoop.yarn.api.records.ContainerId containerId,
                                             int exitCode)
                                      throws IOException

Record that a container has completed

Parameters:

containerId - the container ID

exitCode - the exit code from the container

Throws:

IOException

storeContainerKilled

public abstract void storeContainerKilled(org.apache.hadoop.yarn.api.records.ContainerId containerId)
                                   throws IOException

Record a request to kill a container

Parameters:

containerId - the container ID

Throws:

IOException

storeContainerDiagnostics

public abstract void storeContainerDiagnostics(org.apache.hadoop.yarn.api.records.ContainerId containerId,
                                               StringBuilder diagnostics)
                                        throws IOException

Record diagnostics for a container

Parameters:

containerId - the container ID

diagnostics - the container diagnostics

Throws:

IOException

removeContainer

public abstract void removeContainer(org.apache.hadoop.yarn.api.records.ContainerId containerId)
                              throws IOException

Remove records corresponding to a container

Parameters:

containerId - the container ID

Throws:

IOException

loadLocalizationState

public abstract NMStateStoreService.RecoveredLocalizationState loadLocalizationState()
                                                                              throws IOException

Load the state of localized resources

Returns:

recovered localized resource state

Throws:

IOException

startResourceLocalization

public abstract void startResourceLocalization(String user,
                                               org.apache.hadoop.yarn.api.records.ApplicationId appId,
                                               org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto proto,
                                               org.apache.hadoop.fs.Path localPath)
                                        throws IOException

Record the start of localization for a resource

Parameters:

user - the username or null if the resource is public

appId - the application ID if the resource is app-specific or null

proto - the resource request

localPath - local filesystem path where the resource will be stored

Throws:

IOException

finishResourceLocalization

public abstract void finishResourceLocalization(String user,
                                                org.apache.hadoop.yarn.api.records.ApplicationId appId,
                                                YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto proto)
                                         throws IOException

Record the completion of a resource localization

Parameters:

user - the username or null if the resource is public

appId - the application ID if the resource is app-specific or null

proto - the serialized localized resource

Throws:

IOException

removeLocalizedResource

public abstract void removeLocalizedResource(String user,
                                             org.apache.hadoop.yarn.api.records.ApplicationId appId,
                                             org.apache.hadoop.fs.Path localPath)
                                      throws IOException

Remove records related to a resource localization

Parameters:

user - the username or null if the resource is public

appId - the application ID if the resource is app-specific or null

localPath - local filesystem path where the resource will be stored

Throws:

IOException

loadDeletionServiceState

public abstract NMStateStoreService.RecoveredDeletionServiceState loadDeletionServiceState()
                                                                                    throws IOException

Load the state of the deletion service

Returns:

recovered deletion service state

Throws:

IOException

storeDeletionTask

public abstract void storeDeletionTask(int taskId,
                                       YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto taskProto)
                                throws IOException

Record a deletion task

Parameters:

taskId - the deletion task ID

taskProto - the deletion task protobuf

Throws:

IOException

removeDeletionTask

public abstract void removeDeletionTask(int taskId)
                                 throws IOException

Remove records corresponding to a deletion task

Parameters:

taskId - the deletion task ID

Throws:

IOException

loadNMTokensState

public abstract NMStateStoreService.RecoveredNMTokensState loadNMTokensState()
                                                                      throws IOException

Load the state of NM tokens

Returns:

recovered state of NM tokens

Throws:

IOException

storeNMTokenCurrentMasterKey

public abstract void storeNMTokenCurrentMasterKey(org.apache.hadoop.yarn.server.api.records.MasterKey key)
                                           throws IOException

Record the current NM token master key

Parameters:

key - the master key

Throws:

IOException

storeNMTokenPreviousMasterKey

public abstract void storeNMTokenPreviousMasterKey(org.apache.hadoop.yarn.server.api.records.MasterKey key)
                                            throws IOException

Record the previous NM token master key

Parameters:

key - the previous master key

Throws:

IOException

storeNMTokenApplicationMasterKey

public abstract void storeNMTokenApplicationMasterKey(org.apache.hadoop.yarn.api.records.ApplicationAttemptId attempt,
                                                      org.apache.hadoop.yarn.server.api.records.MasterKey key)
                                               throws IOException

Record a master key corresponding to an application

Parameters:

attempt - the application attempt ID

key - the master key

Throws:

IOException

removeNMTokenApplicationMasterKey

public abstract void removeNMTokenApplicationMasterKey(org.apache.hadoop.yarn.api.records.ApplicationAttemptId attempt)
                                                throws IOException

Remove a master key corresponding to an application

Parameters:

attempt - the application attempt ID

Throws:

IOException

loadContainerTokensState

public abstract NMStateStoreService.RecoveredContainerTokensState loadContainerTokensState()
                                                                                    throws IOException

Load the state of container tokens

Returns:

recovered state of container tokens

Throws:

IOException

storeContainerTokenCurrentMasterKey

public abstract void storeContainerTokenCurrentMasterKey(org.apache.hadoop.yarn.server.api.records.MasterKey key)
                                                  throws IOException

Record the current container token master key

Parameters:

key - the master key

Throws:

IOException

storeContainerTokenPreviousMasterKey

public abstract void storeContainerTokenPreviousMasterKey(org.apache.hadoop.yarn.server.api.records.MasterKey key)
                                                   throws IOException

Record the previous container token master key

Parameters:

key - the previous master key

Throws:

IOException

storeContainerToken

public abstract void storeContainerToken(org.apache.hadoop.yarn.api.records.ContainerId containerId,
                                         Long expirationTime)
                                  throws IOException

Record the expiration time for a container token

Parameters:

containerId - the container ID

expirationTime - the container token expiration time

Throws:

IOException

removeContainerToken

public abstract void removeContainerToken(org.apache.hadoop.yarn.api.records.ContainerId containerId)
                                   throws IOException

Remove records for a container token

Parameters:

containerId - the container ID

Throws:

IOException

loadLogDeleterState

public abstract NMStateStoreService.RecoveredLogDeleterState loadLogDeleterState()
                                                                          throws IOException

Load the state of log deleters

Returns:

recovered log deleter state

Throws:

IOException

storeLogDeleter

public abstract void storeLogDeleter(org.apache.hadoop.yarn.api.records.ApplicationId appId,
                                     YarnServerNodemanagerRecoveryProtos.LogDeleterProto proto)
                              throws IOException

Store the state of a log deleter

Parameters:

appId - the application ID for the log deleter

proto - the serialized state of the log deleter

Throws:

IOException

removeLogDeleter

public abstract void removeLogDeleter(org.apache.hadoop.yarn.api.records.ApplicationId appId)
                               throws IOException

Remove the state of a log deleter

Parameters:

appId - the application ID for the log deleter

Throws:

IOException

initStorage

protected abstract void initStorage(org.apache.hadoop.conf.Configuration conf)
                             throws IOException

Throws:

IOException

startStorage

protected abstract void startStorage()
                              throws IOException

Throws:

IOException

closeStorage

protected abstract void closeStorage()
                              throws IOException

Throws:

IOException