This project has retired. For details please refer to its Attic page.
Source code
001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *   http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing,
013 * software distributed under the License is distributed on an
014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015 * KIND, either express or implied.  See the License for the
016 * specific language governing permissions and limitations
017 * under the License.
018 */
019package org.apache.reef.client;
020
021import org.apache.reef.annotations.Provided;
022import org.apache.reef.annotations.audience.ClientSide;
023import org.apache.reef.annotations.audience.Public;
024import org.apache.reef.driver.context.ActiveContext;
025import org.apache.reef.driver.context.ClosedContext;
026import org.apache.reef.driver.context.ContextMessage;
027import org.apache.reef.driver.context.FailedContext;
028import org.apache.reef.driver.evaluator.AllocatedEvaluator;
029import org.apache.reef.driver.evaluator.CompletedEvaluator;
030import org.apache.reef.driver.evaluator.FailedEvaluator;
031import org.apache.reef.driver.parameters.*;
032import org.apache.reef.driver.task.*;
033import org.apache.reef.runtime.common.DriverRestartCompleted;
034import org.apache.reef.runtime.common.driver.DriverRuntimeConfiguration;
035import org.apache.reef.tang.formats.*;
036import org.apache.reef.wake.EventHandler;
037import org.apache.reef.wake.time.Clock;
038import org.apache.reef.wake.time.event.StartTime;
039import org.apache.reef.wake.time.event.StopTime;
040
041/**
042 * A ConfigurationModule for Drivers.
043 */
044@ClientSide
045@Public
046@Provided
047public final class DriverConfiguration extends ConfigurationModuleBuilder {
048
049  /**
050   * Identifies the driver and therefore the JOB. Expect to see this e.g. on YARN's dashboard.
051   */
052  public static final OptionalParameter<String> DRIVER_IDENTIFIER = new OptionalParameter<>();
053
054  /**
055   * The amount of memory to be allocated for the Driver. This is the size of the AM container in YARN.
056   */
057  public static final OptionalParameter<Integer> DRIVER_MEMORY = new OptionalParameter<>();
058
059  /**
060   * Files to be made available on the Driver and all Evaluators.
061   */
062  public static final OptionalParameter<String> GLOBAL_FILES = new OptionalParameter<>();
063
064  /**
065   * Libraries to be made available on the Driver and all Evaluators.
066   */
067  public static final OptionalParameter<String> GLOBAL_LIBRARIES = new OptionalParameter<>();
068
069  /**
070   * Files to be made available on the Driver only.
071   */
072  public static final OptionalParameter<String> LOCAL_FILES = new OptionalParameter<>();
073
074  /**
075   * Libraries to be made available on the Driver only.
076   */
077  public static final OptionalParameter<String> LOCAL_LIBRARIES = new OptionalParameter<>();
078
079  /**
080   * Job submission directory to be used by driver. This is the folder on the DFS used to stage the files
081   * for the Driver and subsequently for the Evaluators. It will be created if it doesn't exist yet.
082   * If this is set by the user, user must make sure its uniqueness across different jobs.
083   */
084  public static final OptionalParameter<String> DRIVER_JOB_SUBMISSION_DIRECTORY = new OptionalParameter<>();
085
086  /**
087   * The event handler invoked right after the driver boots up.
088   */
089  public static final RequiredImpl<EventHandler<StartTime>> ON_DRIVER_STARTED = new RequiredImpl<>();
090
091  /**
092   * This event is fired in place of the ON_DRIVER_STARTED when the Driver is in fact restarted after failure.
093   */
094  public static final OptionalImpl<EventHandler<StartTime>> ON_DRIVER_RESTARTED = new OptionalImpl<>();
095
096  /**
097   * The event handler invoked right before the driver shuts down. Defaults to ignore.
098   */
099  public static final OptionalImpl<EventHandler<StopTime>> ON_DRIVER_STOP = new OptionalImpl<>();
100
101  // ***** EVALUATOR HANDLER BINDINGS:
102
103  /**
104   * Event handler for allocated evaluators. Defaults to returning the evaluator if not bound.
105   */
106  public static final OptionalImpl<EventHandler<AllocatedEvaluator>> ON_EVALUATOR_ALLOCATED = new OptionalImpl<>();
107
108  /**
109   * Event handler for completed evaluators. Defaults to logging if not bound.
110   */
111  public static final OptionalImpl<EventHandler<CompletedEvaluator>> ON_EVALUATOR_COMPLETED = new OptionalImpl<>();
112
113  /**
114   * Event handler for failed evaluators. Defaults to job failure if not bound.
115   */
116  public static final OptionalImpl<EventHandler<FailedEvaluator>> ON_EVALUATOR_FAILED = new OptionalImpl<>();
117
118  // ***** TASK HANDLER BINDINGS:
119
120  /**
121   * Event handler for task messages. Defaults to logging if not bound.
122   */
123  public static final OptionalImpl<EventHandler<TaskMessage>> ON_TASK_MESSAGE = new OptionalImpl<>();
124
125  /**
126   * Event handler for completed tasks. Defaults to closing the context the task ran on if not bound.
127   */
128  public static final OptionalImpl<EventHandler<CompletedTask>> ON_TASK_COMPLETED = new OptionalImpl<>();
129
130  /**
131   * Event handler for failed tasks. Defaults to job failure if not bound.
132   */
133  public static final OptionalImpl<EventHandler<FailedTask>> ON_TASK_FAILED = new OptionalImpl<>();
134
135  /**
136   * Event handler for running tasks. Defaults to logging if not bound.
137   */
138  public static final OptionalImpl<EventHandler<RunningTask>> ON_TASK_RUNNING = new OptionalImpl<>();
139
140  /**
141   * Event handler for running tasks in previous evaluator, when driver restarted. Defaults to crash if not bound.
142   */
143  public static final OptionalImpl<EventHandler<RunningTask>> ON_DRIVER_RESTART_TASK_RUNNING = new OptionalImpl<>();
144
145  /**
146   * Event handler for suspended tasks. Defaults to job failure if not bound. Rationale: many jobs don't support
147   * task suspension. Hence, this parameter should be optional. The only sane default is to crash the job, then.
148   */
149  public static final OptionalImpl<EventHandler<SuspendedTask>> ON_TASK_SUSPENDED = new OptionalImpl<>();
150
151  // ***** CLIENT HANDLER BINDINGS:
152
153  /**
154   * Event handler for client messages. Defaults to logging if not bound.
155   */
156  public static final OptionalImpl<EventHandler<byte[]>> ON_CLIENT_MESSAGE = new OptionalImpl<>();
157
158  /**
159   * Event handler for close messages sent by the client. Defaults to job failure if not bound.
160   */
161  public static final OptionalImpl<EventHandler<Void>> ON_CLIENT_CLOSED = new OptionalImpl<>();
162
163  /**
164   * Event handler for close messages sent by the client. Defaults to job failure if not bound.
165   */
166  public static final OptionalImpl<EventHandler<byte[]>> ON_CLIENT_CLOSED_MESSAGE = new OptionalImpl<>();
167
168  // ***** CONTEXT HANDLER BINDINGS:
169
170  /**
171   * Event handler for active context. Defaults to closing the context if not bound.
172   */
173  public static final OptionalImpl<EventHandler<ActiveContext>> ON_CONTEXT_ACTIVE = new OptionalImpl<>();
174
175  /**
176   * Event handler for active context when driver restart. Defaults to closing the context if not bound.
177   */
178  public static final OptionalImpl<EventHandler<ActiveContext>> ON_DRIVER_RESTART_CONTEXT_ACTIVE = new OptionalImpl<>();
179
180  /**
181   * Event handler for closed context. Defaults to logging if not bound.
182   */
183  public static final OptionalImpl<EventHandler<ClosedContext>> ON_CONTEXT_CLOSED = new OptionalImpl<>();
184
185  /**
186   * Event handler for closed context. Defaults to job failure if not bound.
187   */
188  public static final OptionalImpl<EventHandler<FailedContext>> ON_CONTEXT_FAILED = new OptionalImpl<>();
189
190  /**
191   * Event handler for context messages. Defaults to logging if not bound.
192   */
193  public static final OptionalImpl<EventHandler<ContextMessage>> ON_CONTEXT_MESSAGE = new OptionalImpl<>();
194
195  /**
196   * "Number of threads allocated per evaluator to dispatch events from this Evaluator.
197   */
198  public static final OptionalParameter<Integer> EVALUATOR_DISPATCHER_THREADS = new OptionalParameter<>();
199
200  /**
201   * Event handler for the event of driver restart completion, default to logging if not bound.
202   */
203  public static final OptionalImpl<EventHandler<DriverRestartCompleted>> ON_DRIVER_RESTART_COMPLETED = new OptionalImpl<>();
204
205  /**
206   * ConfigurationModule to fill out to get a legal Driver Configuration.
207   */
208  public static final ConfigurationModule CONF = new DriverConfiguration().merge(DriverRuntimeConfiguration.CONF)
209
210      .bindNamedParameter(DriverIdentifier.class, DRIVER_IDENTIFIER)
211      .bindNamedParameter(DriverMemory.class, DRIVER_MEMORY)
212      .bindNamedParameter(DriverJobSubmissionDirectory.class, DRIVER_JOB_SUBMISSION_DIRECTORY)
213      .bindSetEntry(JobGlobalFiles.class, GLOBAL_FILES)
214      .bindSetEntry(JobGlobalLibraries.class, GLOBAL_LIBRARIES)
215      .bindSetEntry(DriverLocalFiles.class, LOCAL_FILES)
216      .bindSetEntry(DriverLocalLibraries.class, LOCAL_LIBRARIES)
217
218          // Driver start/stop handlers
219      .bindSetEntry(DriverStartHandler.class, ON_DRIVER_STARTED)
220      .bindNamedParameter(DriverRestartHandler.class, ON_DRIVER_RESTARTED)
221      .bindSetEntry(Clock.StartHandler.class, org.apache.reef.runtime.common.driver.DriverStartHandler.class)
222      .bindSetEntry(Clock.StopHandler.class, ON_DRIVER_STOP)
223
224          // Evaluator handlers
225      .bindSetEntry(EvaluatorAllocatedHandlers.class, ON_EVALUATOR_ALLOCATED)
226      .bindSetEntry(EvaluatorCompletedHandlers.class, ON_EVALUATOR_COMPLETED)
227      .bindSetEntry(EvaluatorFailedHandlers.class, ON_EVALUATOR_FAILED)
228
229          // Task handlers
230      .bindSetEntry(TaskRunningHandlers.class, ON_TASK_RUNNING)
231      .bindSetEntry(DriverRestartTaskRunningHandlers.class, ON_DRIVER_RESTART_TASK_RUNNING)
232      .bindSetEntry(TaskFailedHandlers.class, ON_TASK_FAILED)
233      .bindSetEntry(TaskMessageHandlers.class, ON_TASK_MESSAGE)
234      .bindSetEntry(TaskCompletedHandlers.class, ON_TASK_COMPLETED)
235      .bindSetEntry(TaskSuspendedHandlers.class, ON_TASK_SUSPENDED)
236
237          // Context handlers
238      .bindSetEntry(ContextActiveHandlers.class, ON_CONTEXT_ACTIVE)
239      .bindSetEntry(DriverRestartContextActiveHandlers.class, ON_DRIVER_RESTART_CONTEXT_ACTIVE)
240      .bindSetEntry(ContextClosedHandlers.class, ON_CONTEXT_CLOSED)
241      .bindSetEntry(ContextMessageHandlers.class, ON_CONTEXT_MESSAGE)
242      .bindSetEntry(ContextFailedHandlers.class, ON_CONTEXT_FAILED)
243
244          // Client handlers
245      .bindSetEntry(ClientMessageHandlers.class, ON_CLIENT_MESSAGE)
246      .bindSetEntry(ClientCloseHandlers.class, ON_CLIENT_CLOSED)
247      .bindSetEntry(ClientCloseWithMessageHandlers.class, ON_CLIENT_CLOSED_MESSAGE)
248
249          // Various parameters
250      .bindNamedParameter(EvaluatorDispatcherThreads.class, EVALUATOR_DISPATCHER_THREADS)
251      .bindSetEntry(DriverRestartCompletedHandlers.class, ON_DRIVER_RESTART_COMPLETED)
252      .build();
253}