001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, 013 * software distributed under the License is distributed on an 014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 015 * KIND, either express or implied. See the License for the 016 * specific language governing permissions and limitations 017 * under the License. 018 */ 019package org.apache.reef.runtime.multi.client; 020 021import org.apache.commons.lang.Validate; 022import org.apache.reef.annotations.Unstable; 023import org.apache.reef.runtime.local.client.parameters.MaxNumberOfEvaluators; 024import org.apache.reef.runtime.multi.client.parameters.DefaultRuntimeName; 025import org.apache.reef.runtime.multi.client.parameters.RuntimeNames; 026import org.apache.reef.runtime.yarn.client.ExtensibleYarnClientConfiguration; 027import org.apache.reef.runtime.yarn.driver.RuntimeIdentifier; 028import org.apache.reef.tang.Configuration; 029import org.apache.reef.tang.Configurations; 030import org.apache.reef.tang.JavaConfigurationBuilder; 031import org.apache.reef.tang.Tang; 032import org.apache.reef.util.Optional; 033 034import java.util.*; 035 036/** 037 * A builder for Multi Runtime Configuration. 038 */ 039@Unstable 040public final class MultiRuntimeConfigurationBuilder { 041 private static final Set<String> SUPPORTED_RUNTIMES = new HashSet<>(Arrays.asList( 042 org.apache.reef.runtime.yarn.driver.RuntimeIdentifier.RUNTIME_NAME, 043 org.apache.reef.runtime.local.driver.RuntimeIdentifier.RUNTIME_NAME)); 044 private static final Set<String> SUPPORTED_SUBMISSION_RUNTIMES = new HashSet<>(Arrays.asList( 045 org.apache.reef.runtime.yarn.driver.RuntimeIdentifier.RUNTIME_NAME)); 046 047 private final HashMap<Class, Object> namedParameters = new HashMap<>(); 048 049 private Set<String> runtimeNames = new HashSet<>(); 050 private Optional<String> defaultRuntime = Optional.empty(); 051 private String submissionRuntime; 052 053 private void addNamedParameter(final Class namedParameter, 054 final Object namedParameterValue) { 055 Validate.notNull(namedParameterValue); 056 057 this.namedParameters.put(namedParameter, namedParameterValue); 058 } 059 060 /** 061 * Adds runtime name to the builder. 062 * @param runtimeName The name to add 063 * @return The builder instance 064 */ 065 public MultiRuntimeConfigurationBuilder addRuntime(final String runtimeName) { 066 Validate.isTrue(SUPPORTED_RUNTIMES.contains(runtimeName), "unsupported runtime " + runtimeName); 067 068 this.runtimeNames.add(runtimeName); 069 return this; 070 } 071 072 /** 073 * Sets default runtime. Default runtime is used when no runtime was specified for evaluator 074 * @param runtimeName the default runtime name 075 * @return The builder instance 076 */ 077 public MultiRuntimeConfigurationBuilder setDefaultRuntime(final String runtimeName) { 078 Validate.isTrue(SUPPORTED_RUNTIMES.contains(runtimeName), "Unsupported runtime " + runtimeName); 079 Validate.isTrue(!this.defaultRuntime.isPresent(), "Default runtime was already added"); 080 081 this.defaultRuntime = Optional.of(runtimeName); 082 return this; 083 } 084 085 /** 086 * Sets the submission runtime. Submission runtime is used for launching the job driver. 087 * @param runtimeName the submission runtime name 088 * @return The builder instance 089 */ 090 public MultiRuntimeConfigurationBuilder setSubmissionRuntime(final String runtimeName) { 091 Validate.isTrue( 092 SUPPORTED_SUBMISSION_RUNTIMES.contains(runtimeName), "Unsupported submission runtime " + 093 runtimeName); 094 Validate.isTrue(this.submissionRuntime == null, "Submission runtime was already added"); 095 096 this.submissionRuntime = runtimeName; 097 return this; 098 } 099 100 /** 101 * Sets the max number of local evaluators for local runtime. This parameter is ignored when local runtime is not used 102 * @param maxLocalEvaluators The max evaluators number 103 * @return The builder instance 104 */ 105 public MultiRuntimeConfigurationBuilder setMaxEvaluatorsNumberForLocalRuntime(final int maxLocalEvaluators) { 106 Validate.isTrue(maxLocalEvaluators > 0, "Max evaluators number should be greater then 0"); 107 108 addNamedParameter(MaxNumberOfEvaluators.class, maxLocalEvaluators); 109 return this; 110 } 111 112 /** 113 * Builds the configuration. 114 * @return The built configuration 115 */ 116 public Configuration build() { 117 Validate.notNull(this.submissionRuntime, "Default Runtime was not defined"); 118 119 if(!this.defaultRuntime.isPresent() || this.runtimeNames.size() == 1){ 120 this.defaultRuntime = Optional.of(this.runtimeNames.toArray(new String[0])[0]); 121 } 122 123 Validate.isTrue(this.defaultRuntime.isPresent(), 124 "Default runtime was not defined, and multiple runtimes were specified"); 125 126 if(!this.runtimeNames.contains(this.defaultRuntime.get())){ 127 this.runtimeNames.add(this.defaultRuntime.get()); 128 } 129 130 JavaConfigurationBuilder conf = Tang.Factory.getTang().newConfigurationBuilder(); 131 132 for(Map.Entry<Class, Object> entry: this.namedParameters.entrySet()){ 133 conf = conf.bindNamedParameter(entry.getKey(), entry.getValue().toString()); 134 } 135 136 conf = conf.bindNamedParameter(DefaultRuntimeName.class, this.defaultRuntime.get()); 137 138 for(final String runtimeName : this.runtimeNames){ 139 conf = conf.bindSetEntry(RuntimeNames.class, runtimeName); 140 } 141 142 if(!this.submissionRuntime.equalsIgnoreCase(RuntimeIdentifier.RUNTIME_NAME)){ 143 throw new RuntimeException("Unsupported submission runtime " + this.submissionRuntime); 144 } 145 146 conf = conf.bindImplementation(MultiRuntimeMainConfigurationGenerator.class, 147 YarnMultiRuntimeMainConfigurationGeneratorImpl.class); 148 149 // Currently only local runtime is supported as a secondary runtime 150 return Configurations.merge(conf.build(), 151 ExtensibleYarnClientConfiguration.CONF 152 .set(ExtensibleYarnClientConfiguration.DRIVER_CONFIGURATION_PROVIDER, 153 MultiRuntimeDriverConfigurationProvider.class).build()); 154 } 155}