Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *   http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing,
013 * software distributed under the License is distributed on an
014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015 * KIND, either express or implied.  See the License for the
016 * specific language governing permissions and limitations
017 * under the License.
018 */
019package org.apache.reef.runtime.yarn;
020
021import net.jcip.annotations.Immutable;
022import org.apache.commons.lang.StringUtils;
023import org.apache.hadoop.yarn.conf.YarnConfiguration;
024import org.apache.reef.runtime.common.files.RuntimeClasspathProvider;
025import org.apache.reef.util.OSUtils;
026import org.apache.reef.util.Optional;
027
028import javax.inject.Inject;
029import java.util.List;
030import java.util.Map;
031import java.util.logging.Level;
032import java.util.logging.Logger;
033
034/**
035 * Access to the classpath according to the REEF file system standard.
036 */
037@Immutable
038public final class YarnClasspathProvider implements RuntimeClasspathProvider {
039  private static final Logger LOG = Logger.getLogger(YarnClasspathProvider.class.getName());
040  private static final Level CLASSPATH_LOG_LEVEL = Level.FINE;
041
042  private static final String YARN_TOO_OLD_MESSAGE =
043      "The version of YARN you are using is too old to support classpath assembly. Reverting to legacy method.";
044  private static final String HADOOP_CONF_DIR = OSUtils.formatVariable("HADOOP_CONF_DIR");
045  private static final String HADOOP_HOME = OSUtils.formatVariable("HADOOP_HOME");
046  private static final String HADOOP_COMMON_HOME = OSUtils.formatVariable("HADOOP_COMMON_HOME");
047  private static final String HADOOP_YARN_HOME = OSUtils.formatVariable("HADOOP_YARN_HOME");
048  private static final String HADOOP_HDFS_HOME = OSUtils.formatVariable("HADOOP_HDFS_HOME");
049  private static final String HADOOP_MAPRED_HOME = OSUtils.formatVariable("HADOOP_MAPRED_HOME");
050
051  // Used when we can't get a classpath from YARN
052  private static final String[] LEGACY_CLASSPATH_LIST = new String[]{
053      HADOOP_HOME + "/*",
054      HADOOP_HOME + "/lib/*",
055      HADOOP_COMMON_HOME + "/*",
056      HADOOP_COMMON_HOME + "/lib/*",
057      HADOOP_YARN_HOME + "/*",
058      HADOOP_YARN_HOME + "/lib/*",
059      HADOOP_HDFS_HOME + "/*",
060      HADOOP_HDFS_HOME + "/lib/*",
061      HADOOP_MAPRED_HOME + "/*",
062      HADOOP_MAPRED_HOME + "/lib/*",
063      HADOOP_COMMON_HOME + "/etc/hadoop/client/*",
064      HADOOP_HOME + "/etc/hadoop",
065      HADOOP_HOME + "/share/hadoop/common/*",
066      HADOOP_HOME + "/share/hadoop/common/lib/*",
067      HADOOP_HOME + "/share/hadoop/yarn/*",
068      HADOOP_HOME + "/share/hadoop/yarn/lib/*",
069      HADOOP_HOME + "/share/hadoop/hdfs/*",
070      HADOOP_HOME + "/share/hadoop/hdfs/lib/*",
071      HADOOP_HOME + "/share/hadoop/mapreduce/*",
072      HADOOP_HOME + "/share/hadoop/mapreduce/lib/*"
073  };
074  private final List<String> classPathPrefix;
075  private final List<String> classPathSuffix;
076
077  @Inject
078  YarnClasspathProvider(final YarnConfiguration yarnConfiguration) {
079    logEnvVariable();
080    boolean needsLegacyClasspath = false;
081    // will be set to true below whenever we encounter issues with the YARN Configuration
082    final ClassPathBuilder builder = new ClassPathBuilder();
083
084
085    try {
086      // Add the classpath actually configured on this cluster
087      final Optional<String[]> yarnClassPath =
088          getTrimmedStrings(yarnConfiguration, YarnConfiguration.YARN_APPLICATION_CLASSPATH);
089      final String[] yarnDefaultClassPath = YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH;
090
091      if (!yarnClassPath.isPresent()) {
092        LOG.log(Level.SEVERE,
093            "YarnConfiguration.YARN_APPLICATION_CLASSPATH is empty. This indicates a broken cluster configuration.");
094        needsLegacyClasspath = true;
095      } else {
096        if (LOG.isLoggable(CLASSPATH_LOG_LEVEL)) {
097          LOG.log(CLASSPATH_LOG_LEVEL,
098              "YarnConfiguration.YARN_APPLICATION_CLASSPATH is [" + StringUtils.join(yarnClassPath.get(), '|') + "]");
099          LOG.log(CLASSPATH_LOG_LEVEL,
100              "YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH is [" +
101                  StringUtils.join(yarnDefaultClassPath, '|') + "]");
102        }
103        builder.addAll(yarnClassPath.get());
104        builder.addAll(yarnDefaultClassPath);
105      }
106    } catch (final NoSuchFieldError e) {
107      // This means that one of the static fields above aren't actually in YarnConfiguration.
108      // The reason for that is most likely that we encounter a really old version of YARN.
109      needsLegacyClasspath = true;
110      LOG.log(Level.SEVERE, YARN_TOO_OLD_MESSAGE);
111    }
112
113    if (needsLegacyClasspath) {
114      builder.addToPrefix(HADOOP_CONF_DIR);
115      builder.addAllToSuffix(LEGACY_CLASSPATH_LIST);
116    }
117
118    this.classPathPrefix = builder.getPrefixAsImmutableList();
119    this.classPathSuffix = builder.getSuffixAsImmutableList();
120    this.logClasspath();
121  }
122
123  /**
124   * Fetches the string[] under the given key, if it exists and contains entries (.length >0).
125   *
126   * @param configuration
127   * @param key
128   * @return
129   */
130  private static Optional<String[]> getTrimmedStrings(final YarnConfiguration configuration, final String key) {
131    final String[] result = configuration.getTrimmedStrings(key);
132    if (null == result || result.length == 0) {
133      return Optional.empty();
134    } else {
135      return Optional.of(result);
136    }
137  }
138
139  @Override
140  public List<String> getDriverClasspathPrefix() {
141    return this.classPathPrefix;
142  }
143
144  @Override
145  public List<String> getDriverClasspathSuffix() {
146    return this.classPathSuffix;
147  }
148
149  @Override
150  public List<String> getEvaluatorClasspathPrefix() {
151    return this.classPathPrefix;
152  }
153
154  @Override
155  public List<String> getEvaluatorClasspathSuffix() {
156    return this.classPathSuffix;
157  }
158
159  private static void logEnvVariable() {
160    if (LOG.isLoggable(CLASSPATH_LOG_LEVEL)) {
161      for (final Map.Entry<String, String> entry : System.getenv().entrySet()) {
162        LOG.log(CLASSPATH_LOG_LEVEL, "Environment variable: Key: {0}, Value: {1}.",
163            new Object[]{entry.getKey(), entry.getValue()});
164      }
165    }
166  }
167
168  private void logClasspath() {
169    if (LOG.isLoggable(CLASSPATH_LOG_LEVEL)) {
170      final StringBuilder message = new StringBuilder("Classpath:\n\t");
171      message.append(StringUtils.join(classPathPrefix, "\n\t"));
172      message.append("\n--------------------------------\n\t");
173      message.append(StringUtils.join(classPathSuffix, "\n\t"));
174      LOG.log(CLASSPATH_LOG_LEVEL, message.toString());
175    }
176  }
177}