Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 * <p>
010 * http://www.apache.org/licenses/LICENSE-2.0
011 * <p>
012 * Unless required by applicable law or agreed to in writing,
013 * software distributed under the License is distributed on an
014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015 * KIND, either express or implied.  See the License for the
016 * specific language governing permissions and limitations
017 * under the License.
018 */
019package org.apache.reef.runtime.yarn.client.uploader;
020
021import org.apache.hadoop.fs.FileContext;
022import org.apache.hadoop.fs.FileStatus;
023import org.apache.hadoop.fs.FileSystem;
024import org.apache.hadoop.fs.Path;
025import org.apache.hadoop.yarn.api.records.LocalResource;
026import org.apache.hadoop.yarn.api.records.LocalResourceType;
027import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
028import org.apache.hadoop.yarn.util.ConverterUtils;
029import org.apache.hadoop.yarn.util.Records;
030
031import java.io.File;
032import java.io.FileNotFoundException;
033import java.io.IOException;
034import java.util.logging.Level;
035import java.util.logging.Logger;
036
037/**
038 * Helper class that represents a folder on the destination filesystem.
039 */
040public final class JobFolder {
041
042  private static final Logger LOG = Logger.getLogger(JobFolder.class.getName());
043  private final FileSystem fileSystem;
044  private final Path path;
045
046  /**
047   * This constructor is only called by JobUploader.
048   *
049   * @param fileSystem
050   * @param path
051   * @throws IOException when the given path can't be created on the given FileSystem
052   */
053  JobFolder(final FileSystem fileSystem, final Path path) throws IOException {
054    this.fileSystem = fileSystem;
055    this.path = path;
056    this.fileSystem.mkdirs(this.path);
057  }
058
059  /**
060   * Uploads the given file to the DFS.
061   *
062   * @param localFile
063   * @return the Path representing the file on the DFS.
064   * @throws IOException
065   */
066  public Path upload(final File localFile) throws IOException {
067    if (!localFile.exists()) {
068      throw new FileNotFoundException(localFile.getAbsolutePath());
069    }
070
071    final Path source = new Path(localFile.getAbsolutePath());
072    final Path destination = new Path(this.path, localFile.getName());
073    try {
074      this.fileSystem.copyFromLocalFile(source, destination);
075    } catch (final IOException e) {
076      LOG.log(Level.SEVERE, "Unable to upload {0} to {1}", new Object[]{source, destination});
077      throw e;
078    }
079    LOG.log(Level.FINE, "Uploaded {0} to {1}", new Object[]{source, destination});
080
081    return destination;
082  }
083
084  /**
085   * Shortcut to first upload the file and then form a LocalResource for the YARN submission.
086   *
087   * @param localFile
088   * @return
089   * @throws IOException
090   */
091  public LocalResource uploadAsLocalResource(final File localFile, final LocalResourceType type) throws IOException {
092    final Path p = upload(localFile);
093    return getLocalResourceForPath(p, type);
094  }
095
096  /**
097   * Creates a LocalResource instance for the JAR file referenced by the given Path.
098   */
099  public LocalResource getLocalResourceForPath(final Path jarPath, final LocalResourceType type) throws IOException {
100    final LocalResource localResource = Records.newRecord(LocalResource.class);
101    final FileStatus status = FileContext.getFileContext(fileSystem.getUri()).getFileStatus(jarPath);
102    localResource.setType(type);
103    localResource.setVisibility(LocalResourceVisibility.APPLICATION);
104    localResource.setResource(ConverterUtils.getYarnUrlFromPath(status.getPath()));
105    localResource.setTimestamp(status.getModificationTime());
106    localResource.setSize(status.getLen());
107    return localResource;
108  }
109
110  /**
111   * @return the Path on the DFS represented by this object.
112   */
113  public Path getPath() {
114    return this.path;
115  }
116}