Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *   http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing,
013 * software distributed under the License is distributed on an
014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015 * KIND, either express or implied.  See the License for the
016 * specific language governing permissions and limitations
017 * under the License.
018 */
019package org.apache.reef.client;
020
021import org.apache.reef.annotations.Provided;
022import org.apache.reef.annotations.audience.ClientSide;
023import org.apache.reef.annotations.audience.Public;
024import org.apache.reef.driver.ProgressProvider;
025import org.apache.reef.driver.context.ActiveContext;
026import org.apache.reef.driver.context.ClosedContext;
027import org.apache.reef.driver.context.ContextMessage;
028import org.apache.reef.driver.context.FailedContext;
029import org.apache.reef.driver.evaluator.AllocatedEvaluator;
030import org.apache.reef.driver.evaluator.CompletedEvaluator;
031import org.apache.reef.driver.evaluator.FailedEvaluator;
032import org.apache.reef.driver.parameters.*;
033import org.apache.reef.driver.task.*;
034import org.apache.reef.runtime.common.driver.DriverRuntimeConfiguration;
035import org.apache.reef.tang.formats.*;
036import org.apache.reef.wake.EventHandler;
037import org.apache.reef.wake.time.Clock;
038import org.apache.reef.wake.time.event.StartTime;
039import org.apache.reef.wake.time.event.StopTime;
040
041/**
042 * A ConfigurationModule for Drivers.
043 */
044@ClientSide
045@Public
046@Provided
047public final class DriverConfiguration extends ConfigurationModuleBuilder {
048
049  /**
050   * Identifies the driver and therefore the JOB. Expect to see this e.g. on YARN's dashboard.
051   */
052  public static final OptionalParameter<String> DRIVER_IDENTIFIER = new OptionalParameter<>();
053
054  /**
055   * The amount of memory to be allocated for the Driver. This is the size of the AM container in YARN.
056   */
057  public static final OptionalParameter<Integer> DRIVER_MEMORY = new OptionalParameter<>();
058
059  /**
060   * Files to be made available on the Driver and all Evaluators.
061   */
062  public static final OptionalParameter<String> GLOBAL_FILES = new OptionalParameter<>();
063
064  /**
065   * Libraries to be made available on the Driver and all Evaluators.
066   */
067  public static final OptionalParameter<String> GLOBAL_LIBRARIES = new OptionalParameter<>();
068
069  /**
070   * Files to be made available on the Driver only.
071   */
072  public static final OptionalParameter<String> LOCAL_FILES = new OptionalParameter<>();
073
074  /**
075   * Libraries to be made available on the Driver only.
076   */
077  public static final OptionalParameter<String> LOCAL_LIBRARIES = new OptionalParameter<>();
078
079  /**
080   * Job submission directory to be used by driver. This is the folder on the DFS used to stage the files
081   * for the Driver and subsequently for the Evaluators. It will be created if it doesn't exist yet.
082   * If this is set by the user, user must make sure its uniqueness across different jobs.
083   */
084  public static final OptionalParameter<String> DRIVER_JOB_SUBMISSION_DIRECTORY = new OptionalParameter<>();
085
086  /**
087   * The event handler invoked right after the driver boots up.
088   */
089  public static final RequiredImpl<EventHandler<StartTime>> ON_DRIVER_STARTED = new RequiredImpl<>();
090
091  /**
092   * The event handler invoked right before the driver shuts down. Defaults to ignore.
093   */
094  public static final OptionalImpl<EventHandler<StopTime>> ON_DRIVER_STOP = new OptionalImpl<>();
095
096  // ***** EVALUATOR HANDLER BINDINGS:
097
098  /**
099   * Event handler for allocated evaluators. Defaults to returning the evaluator if not bound.
100   */
101  public static final OptionalImpl<EventHandler<AllocatedEvaluator>> ON_EVALUATOR_ALLOCATED = new OptionalImpl<>();
102
103  /**
104   * Event handler for completed evaluators. Defaults to logging if not bound.
105   */
106  public static final OptionalImpl<EventHandler<CompletedEvaluator>> ON_EVALUATOR_COMPLETED = new OptionalImpl<>();
107
108  /**
109   * Event handler for failed evaluators. Defaults to job failure if not bound.
110   */
111  public static final OptionalImpl<EventHandler<FailedEvaluator>> ON_EVALUATOR_FAILED = new OptionalImpl<>();
112
113  // ***** TASK HANDLER BINDINGS:
114
115  /**
116   * Event handler for task messages. Defaults to logging if not bound.
117   */
118  public static final OptionalImpl<EventHandler<TaskMessage>> ON_TASK_MESSAGE = new OptionalImpl<>();
119
120  /**
121   * Event handler for completed tasks. Defaults to closing the context the task ran on if not bound.
122   */
123  public static final OptionalImpl<EventHandler<CompletedTask>> ON_TASK_COMPLETED = new OptionalImpl<>();
124
125  /**
126   * Event handler for failed tasks. Defaults to job failure if not bound.
127   */
128  public static final OptionalImpl<EventHandler<FailedTask>> ON_TASK_FAILED = new OptionalImpl<>();
129
130  /**
131   * Event handler for running tasks. Defaults to logging if not bound.
132   */
133  public static final OptionalImpl<EventHandler<RunningTask>> ON_TASK_RUNNING = new OptionalImpl<>();
134
135  /**
136   * Event handler for suspended tasks. Defaults to job failure if not bound. Rationale: many jobs don't support
137   * task suspension. Hence, this parameter should be optional. The only sane default is to crash the job, then.
138   */
139  public static final OptionalImpl<EventHandler<SuspendedTask>> ON_TASK_SUSPENDED = new OptionalImpl<>();
140
141  // ***** CLIENT HANDLER BINDINGS:
142
143  /**
144   * Event handler for client messages. Defaults to logging if not bound.
145   */
146  public static final OptionalImpl<EventHandler<byte[]>> ON_CLIENT_MESSAGE = new OptionalImpl<>();
147
148  /**
149   * Event handler for close messages sent by the client. Defaults to job failure if not bound.
150   */
151  public static final OptionalImpl<EventHandler<Void>> ON_CLIENT_CLOSED = new OptionalImpl<>();
152
153  /**
154   * Event handler for close messages sent by the client. Defaults to job failure if not bound.
155   */
156  public static final OptionalImpl<EventHandler<byte[]>> ON_CLIENT_CLOSED_MESSAGE = new OptionalImpl<>();
157
158  // ***** CONTEXT HANDLER BINDINGS:
159
160  /**
161   * Event handler for active context. Defaults to closing the context if not bound.
162   */
163  public static final OptionalImpl<EventHandler<ActiveContext>> ON_CONTEXT_ACTIVE = new OptionalImpl<>();
164
165  /**
166   * Event handler for closed context. Defaults to logging if not bound.
167   */
168  public static final OptionalImpl<EventHandler<ClosedContext>> ON_CONTEXT_CLOSED = new OptionalImpl<>();
169
170  /**
171   * Event handler for closed context. Defaults to job failure if not bound.
172   */
173  public static final OptionalImpl<EventHandler<FailedContext>> ON_CONTEXT_FAILED = new OptionalImpl<>();
174
175  /**
176   * Event handler for context messages. Defaults to logging if not bound.
177   */
178  public static final OptionalImpl<EventHandler<ContextMessage>> ON_CONTEXT_MESSAGE = new OptionalImpl<>();
179
180  /**
181   * Progress provider. See {@link ProgressProvider}.
182   */
183  public static final OptionalImpl<ProgressProvider> PROGRESS_PROVIDER = new OptionalImpl<>();
184
185  /**
186   * Number of threads allocated per evaluator to dispatch events from this Evaluator.
187   */
188  public static final OptionalParameter<Integer> EVALUATOR_DISPATCHER_THREADS = new OptionalParameter<>();
189
190  /**
191   * The number of submissions that the resource manager will attempt to submit the application. Defaults to 1.
192   */
193  public static final OptionalParameter<Integer> MAX_APPLICATION_SUBMISSIONS = new OptionalParameter<>();
194
195  /**
196   * ConfigurationModule to fill out to get a legal Driver Configuration.
197   */
198  public static final ConfigurationModule CONF = new DriverConfiguration().merge(DriverRuntimeConfiguration.CONF)
199      .bindNamedParameter(DriverIdentifier.class, DRIVER_IDENTIFIER)
200      .bindNamedParameter(DriverMemory.class, DRIVER_MEMORY)
201      .bindNamedParameter(DriverJobSubmissionDirectory.class, DRIVER_JOB_SUBMISSION_DIRECTORY)
202      .bindNamedParameter(MaxApplicationSubmissions.class, MAX_APPLICATION_SUBMISSIONS)
203      .bindSetEntry(JobGlobalFiles.class, GLOBAL_FILES)
204      .bindSetEntry(JobGlobalLibraries.class, GLOBAL_LIBRARIES)
205      .bindSetEntry(DriverLocalFiles.class, LOCAL_FILES)
206      .bindSetEntry(DriverLocalLibraries.class, LOCAL_LIBRARIES)
207
208          // Driver start/stop handlers
209      .bindSetEntry(DriverStartHandler.class, ON_DRIVER_STARTED)
210      .bindSetEntry(Clock.StartHandler.class, org.apache.reef.runtime.common.driver.DriverStartHandler.class)
211      .bindSetEntry(Clock.StopHandler.class, ON_DRIVER_STOP)
212
213          // Evaluator handlers
214      .bindSetEntry(EvaluatorAllocatedHandlers.class, ON_EVALUATOR_ALLOCATED)
215      .bindSetEntry(EvaluatorCompletedHandlers.class, ON_EVALUATOR_COMPLETED)
216      .bindSetEntry(EvaluatorFailedHandlers.class, ON_EVALUATOR_FAILED)
217
218          // Task handlers
219      .bindSetEntry(TaskRunningHandlers.class, ON_TASK_RUNNING)
220      .bindSetEntry(TaskFailedHandlers.class, ON_TASK_FAILED)
221      .bindSetEntry(TaskMessageHandlers.class, ON_TASK_MESSAGE)
222      .bindSetEntry(TaskCompletedHandlers.class, ON_TASK_COMPLETED)
223      .bindSetEntry(TaskSuspendedHandlers.class, ON_TASK_SUSPENDED)
224
225          // Context handlers
226      .bindSetEntry(ContextActiveHandlers.class, ON_CONTEXT_ACTIVE)
227      .bindSetEntry(ContextClosedHandlers.class, ON_CONTEXT_CLOSED)
228      .bindSetEntry(ContextMessageHandlers.class, ON_CONTEXT_MESSAGE)
229      .bindSetEntry(ContextFailedHandlers.class, ON_CONTEXT_FAILED)
230
231          // Client handlers
232      .bindSetEntry(ClientMessageHandlers.class, ON_CLIENT_MESSAGE)
233      .bindSetEntry(ClientCloseHandlers.class, ON_CLIENT_CLOSED)
234      .bindSetEntry(ClientCloseWithMessageHandlers.class, ON_CLIENT_CLOSED_MESSAGE)
235
236          // Various parameters
237      .bindNamedParameter(EvaluatorDispatcherThreads.class, EVALUATOR_DISPATCHER_THREADS)
238      .bindImplementation(ProgressProvider.class, PROGRESS_PROVIDER)
239      .build();
240}