Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *   http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing,
013 * software distributed under the License is distributed on an
014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015 * KIND, either express or implied.  See the License for the
016 * specific language governing permissions and limitations
017 * under the License.
018 */
019package org.apache.reef.bridge.client;
020
021import org.apache.commons.lang.Validate;
022import org.apache.hadoop.yarn.api.records.LocalResource;
023import org.apache.hadoop.yarn.api.records.LocalResourceType;
024import org.apache.hadoop.yarn.api.records.URL;
025import org.apache.reef.runtime.common.files.RuntimeClasspathProvider;
026import org.apache.reef.runtime.yarn.YarnClasspathProvider;
027import org.apache.reef.runtime.yarn.client.uploader.JobUploader;
028import org.apache.reef.runtime.yarn.util.YarnConfigurationConstructor;
029import org.apache.reef.tang.Configuration;
030import org.apache.reef.tang.Tang;
031import org.apache.reef.tang.exceptions.InjectionException;
032
033import java.io.*;
034import java.util.logging.Level;
035import java.util.logging.Logger;
036
037/**
038 * Helper class that uploads job resource to HDFS.
039 */
040public final class JobResourceUploader {
041  private static final Logger LOG = Logger.getLogger(JobResourceUploader.class.getName());
042
043  private JobResourceUploader(){}
044
045  /**
046   * This class is invoked from Org.Apache.REEF.Client.Yarn.LegacyJobResourceUploader in .NET code.
047   * Arguments:
048   * [0] : Local path for file.
049   * [1] : Type for file.
050   * [2] : Path of job submission directory
051   * [3] : File path for output with details of uploaded resource
052   */
053  public static void main(final String[] args) throws InjectionException, IOException {
054    Validate.isTrue(args.length == 4, "Job resource uploader requires 4 args");
055    final File localFile = new File(args[0]);
056    Validate.isTrue(localFile.exists(), "Local file does not exist " + localFile.getAbsolutePath());
057    final String fileType = args[1];
058    final String jobSubmissionDirectory = args[2];
059    final String localOutputPath = args[3];
060
061    LOG.log(Level.INFO, "Received args: LocalPath " + localFile.getAbsolutePath() + " Submission directory " +
062        jobSubmissionDirectory + " LocalOutputPath " + localOutputPath);
063    final Configuration configuration = Tang.Factory.getTang().newConfigurationBuilder()
064        .bindImplementation(RuntimeClasspathProvider.class, YarnClasspathProvider.class)
065        .bindConstructor(org.apache.hadoop.yarn.conf.YarnConfiguration.class, YarnConfigurationConstructor.class)
066        .build();
067
068    final JobUploader jobUploader = Tang.Factory.getTang()
069        .newInjector(configuration)
070        .getInstance(JobUploader.class);
071    final LocalResource localResource = jobUploader.createJobFolder(jobSubmissionDirectory)
072        .uploadAsLocalResource(localFile, LocalResourceType.valueOf(fileType));
073
074    // Output: <UploadedPath>;<LastModificationUnixTimestamp>;<ResourceSize>
075    final URL resource = localResource.getResource();
076    final String outputString = String.format("%s://%s:%d%s;%d;%d", resource.getScheme(), resource.getHost(),
077        resource.getPort(), resource.getFile(), localResource.getTimestamp(), localResource.getSize());
078    LOG.log(Level.INFO, "Writing output: " + outputString);
079    try (Writer writer = new BufferedWriter(new OutputStreamWriter(
080      new FileOutputStream(localOutputPath), "utf-8"))) {
081      writer.write(outputString);
082    }
083
084    LOG.log(Level.FINER, "Done writing output file");
085  }
086}