Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *   http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing,
013 * software distributed under the License is distributed on an
014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015 * KIND, either express or implied.  See the License for the
016 * specific language governing permissions and limitations
017 * under the License.
018 */
019package org.apache.reef.io.checkpoint.fs;
020
021import org.apache.hadoop.conf.Configuration;
022import org.apache.hadoop.fs.FileSystem;
023import org.apache.reef.annotations.audience.DriverSide;
024import org.apache.reef.annotations.audience.Public;
025import org.apache.reef.io.checkpoint.CheckpointID;
026import org.apache.reef.io.checkpoint.CheckpointNamingService;
027import org.apache.reef.io.checkpoint.CheckpointService;
028import org.apache.reef.io.checkpoint.RandomNameCNS;
029import org.apache.reef.tang.ExternalConstructor;
030import org.apache.reef.tang.annotations.Name;
031import org.apache.reef.tang.annotations.NamedParameter;
032import org.apache.reef.tang.annotations.Parameter;
033import org.apache.reef.tang.formats.ConfigurationModule;
034import org.apache.reef.tang.formats.ConfigurationModuleBuilder;
035import org.apache.reef.tang.formats.OptionalParameter;
036import org.apache.reef.tang.formats.RequiredParameter;
037
038import javax.inject.Inject;
039import java.io.IOException;
040
041/**
042 * ConfigurationModule for the FSCheckPointService.
043 * This can be used to create Evaluator-side configurations of the checkpointing service.
044 */
045@DriverSide
046@Public
047public class FSCheckPointServiceConfiguration extends ConfigurationModuleBuilder {
048
049  /**
050   * Use local file system if true; otherwise, use HDFS.
051   */
052  public static final RequiredParameter<Boolean> IS_LOCAL = new RequiredParameter<>();
053
054  /**
055   * Path to be used to store the checkpoints on file system.
056   */
057  public static final RequiredParameter<String> PATH = new RequiredParameter<>();
058
059  /**
060   * Replication factor to be used for the checkpoints.
061   */
062  public static final OptionalParameter<Short> REPLICATION_FACTOR = new OptionalParameter<>();
063
064  /**
065   * Prefix for checkpoint files (optional).
066   */
067  public static final OptionalParameter<String> PREFIX = new OptionalParameter<>();
068  public static final ConfigurationModule CONF = new FSCheckPointServiceConfiguration()
069      .bindImplementation(CheckpointService.class, FSCheckpointService.class) // Use the HDFS based checkpoints
070      .bindImplementation(CheckpointNamingService.class, RandomNameCNS.class) // Use Random Names for the checkpoints
071      .bindImplementation(CheckpointID.class, FSCheckpointID.class)
072      .bindConstructor(FileSystem.class, FileSystemConstructor.class)
073      .bindNamedParameter(FileSystemConstructor.IsLocal.class, IS_LOCAL)
074      .bindNamedParameter(FSCheckpointService.PATH.class, PATH)
075      .bindNamedParameter(FSCheckpointService.ReplicationFactor.class, REPLICATION_FACTOR)
076      .bindNamedParameter(RandomNameCNS.PREFIX.class, PREFIX)
077      .build();
078
079  /**
080   * Constructor for Hadoop FileSystem instances.
081   * This assumes that Hadoop Configuration is in the CLASSPATH.
082   */
083  public static class FileSystemConstructor implements ExternalConstructor<FileSystem> {
084
085    /**
086     * If false, use default values for Hadoop configuration; otherwise, load from config file.
087     * Set to false when REEF is running in local mode.
088     */
089    private final boolean loadConfig;
090
091    @Inject
092    public FileSystemConstructor(@Parameter(IsLocal.class) final boolean isLocal) {
093      this.loadConfig = !isLocal;
094    }
095
096    @Override
097    public FileSystem newInstance() {
098      try {
099        return FileSystem.get(new Configuration(this.loadConfig));
100      } catch (final IOException ex) {
101        throw new RuntimeException("Unable to create a FileSystem instance." +
102            " Probably Hadoop configuration is not in the CLASSPATH", ex);
103      }
104    }
105
106    @NamedParameter(doc = "Use local file system if true; otherwise, use HDFS.")
107    static class IsLocal implements Name<Boolean> {
108    }
109  }
110}