Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *   http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing,
013 * software distributed under the License is distributed on an
014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015 * KIND, either express or implied.  See the License for the
016 * specific language governing permissions and limitations
017 * under the License.
018 */
019package org.apache.reef.runtime.yarn;
020
021import net.jcip.annotations.Immutable;
022import org.apache.commons.lang.StringUtils;
023import org.apache.hadoop.yarn.conf.YarnConfiguration;
024import org.apache.reef.runtime.common.files.RuntimeClasspathProvider;
025import org.apache.reef.util.OSUtils;
026import org.apache.reef.util.Optional;
027
028import javax.inject.Inject;
029import java.util.List;
030import java.util.logging.Level;
031import java.util.logging.Logger;
032
033/**
034 * Access to the classpath according to the REEF file system standard.
035 */
036@Immutable
037public final class YarnClasspathProvider implements RuntimeClasspathProvider {
038  private static final Logger LOG = Logger.getLogger(YarnClasspathProvider.class.getName());
039  private static final Level CLASSPATH_LOG_LEVEL = Level.FINE;
040
041  private static final String YARN_TOO_OLD_MESSAGE =
042      "The version of YARN you are using is too old to support classpath assembly. Reverting to legacy method.";
043  private static final String HADOOP_CONF_DIR = OSUtils.formatVariable("HADOOP_CONF_DIR");
044  private static final String HADOOP_HOME = OSUtils.formatVariable("HADOOP_HOME");
045  private static final String HADOOP_COMMON_HOME = OSUtils.formatVariable("HADOOP_COMMON_HOME");
046  private static final String HADOOP_YARN_HOME = OSUtils.formatVariable("HADOOP_YARN_HOME");
047  private static final String HADOOP_HDFS_HOME = OSUtils.formatVariable("HADOOP_HDFS_HOME");
048  private static final String HADOOP_MAPRED_HOME = OSUtils.formatVariable("HADOOP_MAPRED_HOME");
049
050  // Used when we can't get a classpath from YARN
051  private static final String[] LEGACY_CLASSPATH_LIST = new String[]{
052      HADOOP_HOME + "/*",
053      HADOOP_HOME + "/lib/*",
054      HADOOP_COMMON_HOME + "/*",
055      HADOOP_COMMON_HOME + "/lib/*",
056      HADOOP_YARN_HOME + "/*",
057      HADOOP_YARN_HOME + "/lib/*",
058      HADOOP_HDFS_HOME + "/*",
059      HADOOP_HDFS_HOME + "/lib/*",
060      HADOOP_MAPRED_HOME + "/*",
061      HADOOP_MAPRED_HOME + "/lib/*",
062      HADOOP_HOME + "/etc/hadoop",
063      HADOOP_HOME + "/share/hadoop/common/*",
064      HADOOP_HOME + "/share/hadoop/common/lib/*",
065      HADOOP_HOME + "/share/hadoop/yarn/*",
066      HADOOP_HOME + "/share/hadoop/yarn/lib/*",
067      HADOOP_HOME + "/share/hadoop/hdfs/*",
068      HADOOP_HOME + "/share/hadoop/hdfs/lib/*",
069      HADOOP_HOME + "/share/hadoop/mapreduce/*",
070      HADOOP_HOME + "/share/hadoop/mapreduce/lib/*"
071  };
072  private final List<String> classPathPrefix;
073  private final List<String> classPathSuffix;
074
075  @Inject
076  YarnClasspathProvider(final YarnConfiguration yarnConfiguration) {
077    boolean needsLegacyClasspath = false;
078    // will be set to true below whenever we encounter issues with the YARN Configuration
079    final ClassPathBuilder builder = new ClassPathBuilder();
080
081
082    try {
083      // Add the classpath actually configured on this cluster
084      final Optional<String[]> yarnClassPath =
085          getTrimmedStrings(yarnConfiguration, YarnConfiguration.YARN_APPLICATION_CLASSPATH);
086      final String[] yarnDefaultClassPath = YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH;
087
088      if (!yarnClassPath.isPresent()) {
089        LOG.log(Level.SEVERE,
090            "YarnConfiguration.YARN_APPLICATION_CLASSPATH is empty. This indicates a broken cluster configuration.");
091        needsLegacyClasspath = true;
092      } else {
093        if (LOG.isLoggable(CLASSPATH_LOG_LEVEL)) {
094          LOG.log(CLASSPATH_LOG_LEVEL,
095              "YarnConfiguration.YARN_APPLICATION_CLASSPATH is [" + StringUtils.join(yarnClassPath.get(), '|') + "]");
096          LOG.log(CLASSPATH_LOG_LEVEL,
097              "YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH is [" +
098                  StringUtils.join(yarnDefaultClassPath, '|') + "]");
099        }
100        builder.addAll(yarnClassPath.get());
101        builder.addAll(yarnDefaultClassPath);
102      }
103    } catch (final NoSuchFieldError e) {
104      // This means that one of the static fields above aren't actually in YarnConfiguration.
105      // The reason for that is most likely that we encounter a really old version of YARN.
106      needsLegacyClasspath = true;
107      LOG.log(Level.SEVERE, YARN_TOO_OLD_MESSAGE);
108    }
109
110    if (needsLegacyClasspath) {
111      builder.addToPrefix(HADOOP_CONF_DIR);
112      builder.addAllToSuffix(LEGACY_CLASSPATH_LIST);
113    }
114
115    this.classPathPrefix = builder.getPrefixAsImmutableList();
116    this.classPathSuffix = builder.getSuffixAsImmutableList();
117    this.logClasspath();
118  }
119
120  /**
121   * Fetches the string[] under the given key, if it exists and contains entries (.length >0).
122   *
123   * @param configuration
124   * @param key
125   * @return
126   */
127  private static Optional<String[]> getTrimmedStrings(final YarnConfiguration configuration, final String key) {
128    final String[] result = configuration.getTrimmedStrings(key);
129    if (null == result || result.length == 0) {
130      return Optional.empty();
131    } else {
132      return Optional.of(result);
133    }
134  }
135
136  @Override
137  public List<String> getDriverClasspathPrefix() {
138    return this.classPathPrefix;
139  }
140
141  @Override
142  public List<String> getDriverClasspathSuffix() {
143    return this.classPathSuffix;
144  }
145
146  @Override
147  public List<String> getEvaluatorClasspathPrefix() {
148    return this.classPathPrefix;
149  }
150
151  @Override
152  public List<String> getEvaluatorClasspathSuffix() {
153    return this.classPathSuffix;
154  }
155
156
157  private void logClasspath() {
158    if (LOG.isLoggable(CLASSPATH_LOG_LEVEL)) {
159      final StringBuilder message = new StringBuilder("Classpath:\n\t");
160      message.append(StringUtils.join(classPathPrefix, "\n\t"));
161      message.append("\n--------------------------------\n\t");
162      message.append(StringUtils.join(classPathSuffix, "\n\t"));
163      LOG.log(CLASSPATH_LOG_LEVEL, message.toString());
164    }
165  }
166}