This project has retired. For details please refer to its Attic page.
Source code
001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *   http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing,
013 * software distributed under the License is distributed on an
014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015 * KIND, either express or implied.  See the License for the
016 * specific language governing permissions and limitations
017 * under the License.
018 */
019package org.apache.reef.runtime.multi.client;
020
021import org.apache.commons.lang.Validate;
022import org.apache.reef.annotations.Unstable;
023import org.apache.reef.runtime.local.client.parameters.MaxNumberOfEvaluators;
024import org.apache.reef.runtime.multi.client.parameters.DefaultRuntimeName;
025import org.apache.reef.runtime.multi.client.parameters.RuntimeNames;
026import org.apache.reef.runtime.yarn.client.ExtensibleYarnClientConfiguration;
027import org.apache.reef.runtime.yarn.driver.RuntimeIdentifier;
028import org.apache.reef.tang.Configuration;
029import org.apache.reef.tang.Configurations;
030import org.apache.reef.tang.JavaConfigurationBuilder;
031import org.apache.reef.tang.Tang;
032import org.apache.reef.util.Optional;
033
034import java.util.*;
035
036/**
037 * A builder for Multi Runtime Configuration.
038 */
039@Unstable
040public final class MultiRuntimeConfigurationBuilder {
041  private static final Set<String> SUPPORTED_RUNTIMES = new HashSet<>(Arrays.asList(
042          org.apache.reef.runtime.yarn.driver.RuntimeIdentifier.RUNTIME_NAME,
043          org.apache.reef.runtime.local.driver.RuntimeIdentifier.RUNTIME_NAME));
044  private static final Set<String> SUPPORTED_SUBMISSION_RUNTIMES = new HashSet<>(Arrays.asList(
045          org.apache.reef.runtime.yarn.driver.RuntimeIdentifier.RUNTIME_NAME));
046
047  private final HashMap<Class, Object> namedParameters = new HashMap<>();
048
049  private Set<String> runtimeNames = new HashSet<>();
050  private Optional<String> defaultRuntime = Optional.empty();
051  private String submissionRuntime;
052
053  private void addNamedParameter(final Class namedParameter,
054                                 final Object namedParameterValue) {
055    Validate.notNull(namedParameterValue);
056
057    this.namedParameters.put(namedParameter, namedParameterValue);
058  }
059
060  /**
061   * Adds runtime name to the builder.
062   * @param runtimeName The name to add
063   * @return The builder instance
064   */
065  public MultiRuntimeConfigurationBuilder addRuntime(final String runtimeName) {
066    Validate.isTrue(SUPPORTED_RUNTIMES.contains(runtimeName), "unsupported runtime " + runtimeName);
067
068    this.runtimeNames.add(runtimeName);
069    return this;
070  }
071
072  /**
073   * Sets default runtime. Default runtime is used when no runtime was specified for evaluator
074   * @param runtimeName the default runtime name
075   * @return The builder instance
076   */
077  public MultiRuntimeConfigurationBuilder setDefaultRuntime(final String runtimeName) {
078    Validate.isTrue(SUPPORTED_RUNTIMES.contains(runtimeName), "Unsupported runtime " + runtimeName);
079    Validate.isTrue(!this.defaultRuntime.isPresent(), "Default runtime was already added");
080
081    this.defaultRuntime = Optional.of(runtimeName);
082    return this;
083  }
084
085  /**
086   * Sets the submission runtime. Submission runtime is used for launching the job driver.
087   * @param runtimeName the submission runtime name
088   * @return The builder instance
089   */
090  public MultiRuntimeConfigurationBuilder setSubmissionRuntime(final String runtimeName) {
091    Validate.isTrue(
092            SUPPORTED_SUBMISSION_RUNTIMES.contains(runtimeName), "Unsupported submission runtime " +
093            runtimeName);
094    Validate.isTrue(this.submissionRuntime == null, "Submission runtime was already added");
095
096    this.submissionRuntime = runtimeName;
097    return this;
098  }
099
100  /**
101   * Sets the max number of local evaluators for local runtime. This parameter is ignored when local runtime is not used
102   * @param maxLocalEvaluators The max evaluators number
103   * @return The builder instance
104   */
105  public MultiRuntimeConfigurationBuilder setMaxEvaluatorsNumberForLocalRuntime(final int maxLocalEvaluators) {
106    Validate.isTrue(maxLocalEvaluators > 0, "Max evaluators number should be greater then 0");
107
108    addNamedParameter(MaxNumberOfEvaluators.class, maxLocalEvaluators);
109    return this;
110  }
111
112  /**
113   * Builds the configuration.
114   * @return The built configuration
115   */
116  public Configuration build() {
117    Validate.notNull(this.submissionRuntime, "Default Runtime was not defined");
118
119    if(!this.defaultRuntime.isPresent() || this.runtimeNames.size() == 1){
120      this.defaultRuntime = Optional.of(this.runtimeNames.toArray(new String[0])[0]);
121    }
122
123    Validate.isTrue(this.defaultRuntime.isPresent(),
124            "Default runtime was not defined, and multiple runtimes were specified");
125
126    if(!this.runtimeNames.contains(this.defaultRuntime.get())){
127      this.runtimeNames.add(this.defaultRuntime.get());
128    }
129
130    JavaConfigurationBuilder conf = Tang.Factory.getTang().newConfigurationBuilder();
131
132    for(Map.Entry<Class, Object> entry: this.namedParameters.entrySet()){
133      conf = conf.bindNamedParameter(entry.getKey(), entry.getValue().toString());
134    }
135
136    conf = conf.bindNamedParameter(DefaultRuntimeName.class, this.defaultRuntime.get());
137
138    for(final String runtimeName : this.runtimeNames){
139      conf = conf.bindSetEntry(RuntimeNames.class, runtimeName);
140    }
141
142    if(!this.submissionRuntime.equalsIgnoreCase(RuntimeIdentifier.RUNTIME_NAME)){
143      throw new RuntimeException("Unsupported submission runtime " + this.submissionRuntime);
144    }
145
146    conf = conf.bindImplementation(MultiRuntimeMainConfigurationGenerator.class,
147            YarnMultiRuntimeMainConfigurationGeneratorImpl.class);
148
149    // Currently only local runtime is supported as a secondary runtime
150    return Configurations.merge(conf.build(),
151            ExtensibleYarnClientConfiguration.CONF
152                    .set(ExtensibleYarnClientConfiguration.DRIVER_CONFIGURATION_PROVIDER,
153                            MultiRuntimeDriverConfigurationProvider.class).build());
154  }
155}