Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 * <p>
010 * http://www.apache.org/licenses/LICENSE-2.0
011 * <p>
012 * Unless required by applicable law or agreed to in writing,
013 * software distributed under the License is distributed on an
014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015 * KIND, either express or implied.  See the License for the
016 * specific language governing permissions and limitations
017 * under the License.
018 */
019package org.apache.reef.runtime.yarn.client.uploader;
020
021import org.apache.hadoop.fs.FileSystem;
022import org.apache.hadoop.fs.Path;
023import org.apache.hadoop.yarn.conf.YarnConfiguration;
024import org.apache.reef.runtime.yarn.driver.JobSubmissionDirectoryProvider;
025import javax.inject.Inject;
026import java.io.IOException;
027import java.util.logging.Level;
028import java.util.logging.Logger;
029
030/**
031 * Helper class to upload the driver files to HDFS.
032 */
033public final class JobUploader {
034
035  private static final Logger LOG = Logger.getLogger(JobUploader.class.getName());
036
037  private final FileSystem fileSystem;
038  private final JobSubmissionDirectoryProvider jobSubmissionDirectoryProvider;
039
040  @Inject
041  JobUploader(final YarnConfiguration yarnConfiguration,
042              final JobSubmissionDirectoryProvider jobSubmissionDirectoryProvider) throws IOException {
043    this.jobSubmissionDirectoryProvider = jobSubmissionDirectoryProvider;
044    this.fileSystem = FileSystem.get(yarnConfiguration);
045  }
046
047  /**
048   * Creates the Job folder on the DFS.
049   *
050   * @param applicationId
051   * @return a reference to the JobFolder that can be used to upload files to it.
052   * @throws IOException
053   */
054  public JobFolder createJobFolderWithApplicationId(final String applicationId) throws IOException {
055    final Path jobFolderPath = jobSubmissionDirectoryProvider.getJobSubmissionDirectoryPath(applicationId);
056    final String finalJobFolderPath = jobFolderPath.toString();
057    LOG.log(Level.FINE, "Final job submission Directory: " + finalJobFolderPath);
058    return createJobFolder(finalJobFolderPath);
059  }
060
061
062  /**
063   * Convenience override for int ids.
064   *
065   * @param finalJobFolderPath
066   * @return
067   * @throws IOException
068   */
069  public JobFolder createJobFolder(final String finalJobFolderPath) throws IOException {
070    LOG.log(Level.FINE, "Final job submission Directory: " + finalJobFolderPath);
071    return new JobFolder(this.fileSystem, new Path(finalJobFolderPath));
072  }
073
074  /**
075   * Convenience override for int ids.
076   *
077   * @param applicationId
078   * @return
079   * @throws IOException
080   */
081  public JobFolder createJobFolder(final int applicationId) throws IOException {
082    return this.createJobFolderWithApplicationId(Integer.toString(applicationId));
083  }
084}