Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 * <p>
010 * http://www.apache.org/licenses/LICENSE-2.0
011 * <p>
012 * Unless required by applicable law or agreed to in writing,
013 * software distributed under the License is distributed on an
014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015 * KIND, either express or implied.  See the License for the
016 * specific language governing permissions and limitations
017 * under the License.
018 */
019package org.apache.reef.runtime.yarn.client.uploader;
020
021import org.apache.hadoop.fs.FileContext;
022import org.apache.hadoop.fs.FileStatus;
023import org.apache.hadoop.fs.FileSystem;
024import org.apache.hadoop.fs.Path;
025import org.apache.hadoop.yarn.api.records.LocalResource;
026import org.apache.hadoop.yarn.api.records.LocalResourceType;
027import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
028import org.apache.hadoop.yarn.util.ConverterUtils;
029import org.apache.hadoop.yarn.util.Records;
030
031import java.io.File;
032import java.io.FileNotFoundException;
033import java.io.IOException;
034import java.util.logging.Level;
035import java.util.logging.Logger;
036
037/**
038 * Helper class that represents a folder on the destination filesystem.
039 */
040public final class JobFolder {
041
042  private static final Logger LOG = Logger.getLogger(JobFolder.class.getName());
043  private final FileSystem fileSystem;
044  private final Path path;
045
046  /**
047   * This constructor is only called by JobUploader.
048   *
049   * @param fileSystem
050   * @param path
051   * @throws IOException when the given path can't be created on the given FileSystem
052   */
053  JobFolder(final FileSystem fileSystem, final Path path) throws IOException {
054    this.fileSystem = fileSystem;
055    this.path = path;
056    this.fileSystem.mkdirs(this.path);
057  }
058
059  /**
060   * Upload given file to the DFS.
061   * @param localFile File on local FS to upload to the (remote) DFS.
062   * @return the Path representing the file on the DFS.
063   * @throws IOException if unable to upload the file or local file cannot be read.
064   */
065  public Path upload(final File localFile) throws IOException {
066
067    if (!localFile.exists()) {
068      throw new FileNotFoundException(localFile.getAbsolutePath());
069    }
070
071    final Path source = new Path(localFile.getAbsolutePath());
072    final Path destination = new Path(this.path, localFile.getName());
073
074    try {
075      this.fileSystem.copyFromLocalFile(source, destination);
076    } catch (final IOException ex) {
077      LOG.log(Level.SEVERE, "Unable to upload " + source + " to " + destination, ex);
078      throw ex;
079    }
080
081    LOG.log(Level.FINE, "Uploaded {0} to {1}", new Object[] {source, destination});
082
083    return destination;
084  }
085
086  /**
087   * Shortcut to first upload the file and then form a LocalResource for the YARN submission.
088   * @param localFile File on local FS to upload to the (remote) DFS.
089   * @return an instance of a LocalResource on (remote) DFS.
090   * @throws IOException if unable to upload the file or local file cannot be read.
091   */
092  public LocalResource uploadAsLocalResource(final File localFile, final LocalResourceType type) throws IOException {
093    final Path remoteFile = upload(localFile);
094    return getLocalResourceForPath(remoteFile, type);
095  }
096
097  /**
098   * Creates a LocalResource instance for the JAR file referenced by the given Path.
099   */
100  public LocalResource getLocalResourceForPath(final Path jarPath, final LocalResourceType type) throws IOException {
101
102    final FileStatus status = FileContext.getFileContext(fileSystem.getUri()).getFileStatus(jarPath);
103
104    final LocalResource localResource = Records.newRecord(LocalResource.class);
105    localResource.setType(type);
106    localResource.setVisibility(LocalResourceVisibility.APPLICATION);
107    localResource.setResource(ConverterUtils.getYarnUrlFromPath(status.getPath()));
108    localResource.setTimestamp(status.getModificationTime());
109    localResource.setSize(status.getLen());
110
111    return localResource;
112  }
113
114  /**
115   * @return the Path on the DFS represented by this object.
116   */
117  public Path getPath() {
118    return this.path;
119  }
120}