This project has retired. For details please refer to its Attic page.
Source code
001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *   http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing,
013 * software distributed under the License is distributed on an
014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015 * KIND, either express or implied.  See the License for the
016 * specific language governing permissions and limitations
017 * under the License.
018 */
019package org.apache.reef.javabridge.generic;
020
021import org.apache.reef.client.ClientConfiguration;
022import org.apache.reef.runtime.local.client.LocalRuntimeConfiguration;
023import org.apache.reef.runtime.yarn.client.YarnClientConfiguration;
024import org.apache.reef.tang.Configuration;
025import org.apache.reef.tang.Injector;
026import org.apache.reef.tang.JavaConfigurationBuilder;
027import org.apache.reef.tang.Tang;
028import org.apache.reef.tang.annotations.Name;
029import org.apache.reef.tang.annotations.NamedParameter;
030import org.apache.reef.tang.exceptions.BindException;
031import org.apache.reef.tang.exceptions.InjectionException;
032import org.apache.reef.tang.formats.CommandLine;
033import org.apache.reef.util.logging.LoggingScope;
034import org.apache.reef.util.logging.LoggingScopeFactory;
035
036import java.io.File;
037import java.io.IOException;
038import java.util.Arrays;
039import java.util.Date;
040import java.util.logging.Level;
041import java.util.logging.Logger;
042
043/**
044 * Clr Bridge example - main class.
045 */
046public final class Launch {
047
048  /**
049   * The upper limit on the number of Evaluators that the local resourcemanager will hand out concurrently.
050   */
051  private static final int MAX_NUMBER_OF_EVALUATORS = 10;
052  /**
053   * Standard Java logger.
054   */
055  private static final Logger LOG = Logger.getLogger(Launch.class.getName());
056
057  /**
058   * This class should not be instantiated.
059   */
060  private Launch() {
061    throw new RuntimeException("Do not instantiate this class!");
062  }
063
064  /**
065   * Parse the command line arguments.
066   *
067   * @param args command line arguments, as passed to main()
068   * @return Configuration object.
069   * @throws org.apache.reef.tang.exceptions.BindException configuration error.
070   * @throws java.io.IOException                           error reading the configuration.
071   */
072  private static Configuration parseCommandLine(final String[] args)
073      throws BindException, IOException {
074    final JavaConfigurationBuilder confBuilder = Tang.Factory.getTang().newConfigurationBuilder();
075    final CommandLine cl = new CommandLine(confBuilder);
076    cl.registerShortNameOfClass(Local.class);
077    cl.registerShortNameOfClass(NumRuns.class);
078    cl.registerShortNameOfClass(WaitTimeForDriver.class);
079    cl.registerShortNameOfClass(DriverMemoryInMb.class);
080    cl.registerShortNameOfClass(DriverIdentifier.class);
081    cl.registerShortNameOfClass(DriverJobSubmissionDirectory.class);
082    cl.registerShortNameOfClass(Submit.class);
083    cl.processCommandLine(args);
084    return confBuilder.build();
085  }
086
087  private static Configuration cloneCommandLineConfiguration(final Configuration commandLineConf)
088      throws InjectionException, BindException {
089    final Injector injector = Tang.Factory.getTang().newInjector(commandLineConf);
090    final JavaConfigurationBuilder cb = Tang.Factory.getTang().newConfigurationBuilder();
091    cb.bindNamedParameter(NumRuns.class, String.valueOf(injector.getNamedInstance(NumRuns.class)));
092    return cb.build();
093  }
094
095  /**
096   * Parse command line arguments and create TANG configuration ready to be submitted to REEF.
097   *
098   * @param args Command line arguments, as passed into main().
099   * @return (immutable) TANG Configuration object.
100   * @throws org.apache.reef.tang.exceptions.BindException      if configuration commandLineInjector fails.
101   * @throws org.apache.reef.tang.exceptions.InjectionException if configuration commandLineInjector fails.
102   * @throws java.io.IOException                                error reading the configuration.
103   */
104  private static Configuration getClientConfiguration(final String[] args)
105      throws BindException, InjectionException, IOException {
106
107    try (final LoggingScope ls = LoggingScopeFactory.getNewLoggingScope(Level.INFO, "Launch::getClientConfiguration")) {
108      final Configuration commandLineConf = parseCommandLine(args);
109
110      final Configuration clientConfiguration = ClientConfiguration.CONF
111          .set(ClientConfiguration.ON_JOB_COMPLETED, JobClient.CompletedJobHandler.class)
112          .set(ClientConfiguration.ON_JOB_FAILED, JobClient.FailedJobHandler.class)
113          .set(ClientConfiguration.ON_RUNTIME_ERROR, JobClient.RuntimeErrorHandler.class)
114          .build();
115
116      final Injector commandLineInjector = Tang.Factory.getTang().newInjector(commandLineConf);
117      final boolean isLocal = commandLineInjector.getNamedInstance(Local.class);
118      final Configuration runtimeConfiguration;
119      if (isLocal) {
120        LOG.log(Level.INFO, "Running on the local runtime");
121        runtimeConfiguration = LocalRuntimeConfiguration.CONF
122            .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, MAX_NUMBER_OF_EVALUATORS)
123            .build();
124      } else {
125        LOG.log(Level.INFO, "Running on YARN");
126        runtimeConfiguration = YarnClientConfiguration.CONF.build();
127      }
128
129      return Tang.Factory.getTang()
130          .newConfigurationBuilder(runtimeConfiguration, clientConfiguration,
131              cloneCommandLineConfiguration(commandLineConf))
132          .build();
133    }
134  }
135
136  /**
137   * Main method that starts the CLR Bridge from Java.
138   *
139   * @param args command line parameters.
140   */
141  public static void main(final String[] args) {
142    LOG.log(Level.INFO, "Entering Launch at :::" + new Date());
143    try {
144      if (args == null || args.length == 0) {
145        throw new IllegalArgumentException("No arguments provided, at least a clrFolder should be supplied.");
146      }
147      final File dotNetFolder = new File(args[0]).getAbsoluteFile();
148      final String[] removedArgs = Arrays.copyOfRange(args, 1, args.length);
149
150      final Configuration config = getClientConfiguration(removedArgs);
151      final Injector commandLineInjector = Tang.Factory.getTang().newInjector(parseCommandLine(removedArgs));
152      final int waitTime = commandLineInjector.getNamedInstance(WaitTimeForDriver.class);
153      final int driverMemory = commandLineInjector.getNamedInstance(DriverMemoryInMb.class);
154      final boolean isLocal = commandLineInjector.getNamedInstance(Local.class);
155      final String driverIdentifier = commandLineInjector.getNamedInstance(DriverIdentifier.class);
156      final String jobSubmissionDirectory = commandLineInjector.getNamedInstance(DriverJobSubmissionDirectory.class);
157      final boolean submit = commandLineInjector.getNamedInstance(Submit.class);
158      final Injector injector = Tang.Factory.getTang().newInjector(config);
159      final JobClient client = injector.getInstance(JobClient.class);
160      client.setDriverInfo(driverIdentifier, driverMemory, jobSubmissionDirectory);
161
162      if (submit) {
163        client.submit(dotNetFolder, true, isLocal, null);
164        client.waitForCompletion(waitTime);
165      } else {
166        client.submit(dotNetFolder, false, isLocal, config);
167        client.waitForCompletion(0);
168      }
169
170
171      LOG.info("Done!");
172    } catch (final BindException | InjectionException | IOException ex) {
173      LOG.log(Level.SEVERE, "Job configuration error", ex);
174    }
175  }
176
177  /**
178   * Command line parameter: number of experiments to run.
179   */
180  @NamedParameter(doc = "Number of times to run the command",
181      short_name = "num_runs", default_value = "1")
182  public static final class NumRuns implements Name<Integer> {
183  }
184
185  /**
186   * Command line parameter = true to run locally, or false to run on YARN.
187   */
188  @NamedParameter(doc = "Whether or not to run on the local runtime",
189      short_name = "local", default_value = "true")
190  public static final class Local implements Name<Boolean> {
191  }
192
193  /**
194   * Command line parameter, number of seconds  to wait till driver finishes ,
195   * = -1 : waits forever
196   * = 0: exit immediately without wait for driver.
197   */
198  @NamedParameter(doc = "Whether or not to wait for driver to finish",
199      short_name = "wait_time", default_value = "-1")
200  public static final class WaitTimeForDriver implements Name<Integer> {
201  }
202
203  /**
204   * Command line parameter, driver memory, in MB.
205   */
206  @NamedParameter(doc = "memory allocated to driver JVM",
207      short_name = "driver_memory", default_value = "512")
208  public static final class DriverMemoryInMb implements Name<Integer> {
209  }
210
211  /**
212   * Command line parameter, driver identifier.
213   */
214  @NamedParameter(doc = "driver identifier for clr bridge",
215      short_name = "driver_id", default_value = "ReefClrBridge")
216  public static final class DriverIdentifier implements Name<String> {
217  }
218
219  /**
220   * Command line parameter = true to submit the job with driver config, or false to write config to current directory.
221   */
222  @NamedParameter(doc = "Whether or not to submit the reef job after driver config is constructed",
223      short_name = "submit", default_value = "true")
224  public static final class Submit implements Name<Boolean> {
225  }
226
227  /**
228   * Command line parameter, job submission directory, if set, user should guarantee its uniqueness.
229   */
230  @NamedParameter(doc = "driver job submission directory",
231      short_name = "submission_directory", default_value = "empty")
232  public static final class DriverJobSubmissionDirectory implements Name<String> {
233  }
234}