Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *   http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing,
013 * software distributed under the License is distributed on an
014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015 * KIND, either express or implied.  See the License for the
016 * specific language governing permissions and limitations
017 * under the License.
018 */
019package org.apache.reef.io.checkpoint.fs;
020
021import org.apache.hadoop.conf.Configuration;
022import org.apache.hadoop.fs.FileSystem;
023import org.apache.reef.annotations.audience.DriverSide;
024import org.apache.reef.annotations.audience.Public;
025import org.apache.reef.io.checkpoint.CheckpointID;
026import org.apache.reef.io.checkpoint.CheckpointNamingService;
027import org.apache.reef.io.checkpoint.CheckpointService;
028import org.apache.reef.io.checkpoint.RandomNameCNS;
029import org.apache.reef.tang.ExternalConstructor;
030import org.apache.reef.tang.annotations.Name;
031import org.apache.reef.tang.annotations.NamedParameter;
032import org.apache.reef.tang.annotations.Parameter;
033import org.apache.reef.tang.formats.ConfigurationModule;
034import org.apache.reef.tang.formats.ConfigurationModuleBuilder;
035import org.apache.reef.tang.formats.OptionalParameter;
036import org.apache.reef.tang.formats.RequiredParameter;
037
038import javax.inject.Inject;
039import java.io.IOException;
040
041/**
042 * ConfigurationModule for the FSCheckpointService.
043 * This can be used to create Evaluator-side configurations of the checkpointing service.
044 */
045@DriverSide
046@Public
047public class FSCheckPointServiceConfiguration extends ConfigurationModuleBuilder {
048
049  /**
050   * Use local file system if true; otherwise, use HDFS.
051   */
052  public static final RequiredParameter<Boolean> IS_LOCAL = new RequiredParameter<>();
053
054  /**
055   * Path to be used to store the checkpoints on file system.
056   */
057  public static final RequiredParameter<String> PATH = new RequiredParameter<>();
058
059  /**
060   * Replication factor to be used for the checkpoints.
061   */
062  public static final OptionalParameter<Short> REPLICATION_FACTOR = new OptionalParameter<>();
063
064  /**
065   * Prefix for checkpoint files (optional).
066   */
067  public static final OptionalParameter<String> PREFIX = new OptionalParameter<>();
068
069
070  public static final ConfigurationModule CONF = new FSCheckPointServiceConfiguration()
071
072      .bindImplementation(CheckpointService.class, FSCheckpointService.class) // Use the HDFS based checkpoints
073      .bindImplementation(CheckpointNamingService.class, RandomNameCNS.class) // Use Random Names for the checkpoints
074      .bindImplementation(CheckpointID.class, FSCheckpointID.class)
075      .bindConstructor(FileSystem.class, FileSystemConstructor.class)
076
077      .bindNamedParameter(FileSystemConstructor.IsLocal.class, IS_LOCAL)
078      .bindNamedParameter(FSCheckpointService.PATH.class, PATH)
079      .bindNamedParameter(FSCheckpointService.ReplicationFactor.class, REPLICATION_FACTOR)
080      .bindNamedParameter(RandomNameCNS.PREFIX.class, PREFIX)
081      .build();
082
083  /**
084   * Constructor for Hadoop FileSystem instances.
085   * This assumes that Hadoop Configuration is in the CLASSPATH.
086   */
087  public static class FileSystemConstructor implements ExternalConstructor<FileSystem> {
088
089    /**
090     * If false, use default values for Hadoop configuration; otherwise, load from config file.
091     * Set to false when REEF is running in local mode.
092     */
093    private final boolean loadConfig;
094
095    @Inject
096    public FileSystemConstructor(@Parameter(IsLocal.class) final boolean isLocal) {
097      this.loadConfig = !isLocal;
098    }
099
100    @Override
101    public FileSystem newInstance() {
102      try {
103        return FileSystem.get(new Configuration(this.loadConfig));
104      } catch (final IOException ex) {
105        throw new RuntimeException("Unable to create a FileSystem instance." +
106            " Probably Hadoop configuration is not in the CLASSPATH", ex);
107      }
108    }
109
110    @NamedParameter(doc = "Use local file system if true; otherwise, use HDFS.")
111    static class IsLocal implements Name<Boolean> {
112    }
113  }
114}