cephClient

#!/usr/bin/env ruby
# Copyright (c) 2015-17 Anirvan BASU, INRIA Rennes - Bretagne Atlantique
#
# Licensed under the CeCCIL-B license (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.cecill.info/licences/Licence_CeCILL-B_V1-en.html
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License. 

require 'cute'
require 'logger'
require 'cute/taktuk'
require 'net/sftp'
require 'erb'
require 'socket'
require 'trollop'
require 'json'
require "net/http"
require "uri"
require "fileutils"
require_relative "helpers/abstractions"


g5k = Cute::G5K::API.new()
user = g5k.g5k_user

# Get the script dir
scriptDir = File.expand_path(File.dirname(__FILE__))
# Make the temporary files directory (if not created already)
tempDir = scriptDir + "/.generated"
FileUtils.mkpath(tempDir)
FileUtils.mkpath(tempDir + "/config")

currentConfigFile = tempDir + "/config/defaults.yml" # config file to be used.
# Read all options from CLI or config file
opts = readOptions(scriptDir, currentConfigFile, "cephClient")

# Adapt some CLI arguments into global variables. Later change to class attributes.
opts[:'pool-name'] = "#{user}_" + opts[:'pool-name'] # Name of pool to create on clusters.
opts[:'rbd-name'] = "#{user}_" + opts[:'rbd-name'] # Name of pool to create on clusters.

# Prepare logFile for logging all actions
logger = logCreate(tempDir, "cephClient")


# Get the job for Ceph cluster if it exists
jobCephCluster = getJob(g5k, opts[:jobid], opts[:'job-name'], opts[:site])

# Abort script if no deployed Ceph cluster
if jobCephCluster.nil?
   logger.error "No deployed Ceph cluster found. First deploy Ceph cluster, then run script."
   abort()
end # if jobCephCluster.nil?

# Get the monitor node
monitor = jobCephCluster["assigned_nodes"][0]


# Recover job details of clients
jobCephClient = getJob(g5k, 0, opts[:'job-client'], opts[:'client-site'])
clients = [] # Array of client nodes

# If client-list specified in CLI argument, get list of clients & fill array
unless opts[:'file'].empty?
   clients = File.open(opts[:'file'], 'r'){ |file| 
      file.readlines.collect{ |line| line.chomp }
   } # File.open
else # when no client-list is specified. Do deployment or start from scratch.

   # Finally, if job does not yet exist reserve nodes
   jobCephClient = g5k.reserve(:name => opts[:'job-client'], :nodes => opts[:'num-client'], :site => opts[:'client-site'], :cluster => opts[:'client-cluster'], :walltime => opts[:walltime], :type => :deploy) if jobCephClient.nil?

   # Assign roles to each node
   clients = jobCephClient["assigned_nodes"]

   # if not yet deployed, then deploy it
   if jobCephClient["deploy"].nil?
      logger.info "Deploying #{opts[:'env-client']} on client node(s): #{clients}"
      depCephClient = g5k.deploy(jobCephClient, :nodes => clients, :env => opts[:'env-client']) 
      g5k.wait_for_deploy(jobCephClient)
   end # if jobCephClient["deploy"].nil?

end # unless opts[:'file'].empty?

# At this point Ceph client job was created / fetched
logger.info "Deploying Ceph clients on nodes: #{clients}" 


# Additionally create a directory for saving details of clients deployed
jobID = jobCephClient["uid"]
clientStateDir = tempDir + "/#{opts[:'client-site']}/#{jobID}"
FileUtils.mkpath(clientStateDir)

# Prepare clients-list-file locally
clientsFile = File.open("#{clientStateDir}/clients-list", "w") do |file|
   clients.each do |client|
      file.puts("#{client}")
   end
end

# Abort script if only Linux deployment flag was set
abort("Linux #{opts[:'env-client']} deployed on clients: #{clients}. \n Clients list file in: #{clientStateDir}/clients-list. \n Rerun script with option -f <nodes list file> to configure Ceph clients.") if opts[:'only-deploy']


# Remind where is the deployed Ceph monitor
logger.info "Deployed Ceph cluster details:"
logger.info "   monitor on: #{monitor}"


#1 Preflight Checklist
logger.info "Doing pre-flight checklist..."

# Add Ceph & Extras to each Ceph node ('firefly' is the most complete)
argDebian = opts[:'env-client'].slice(0,6)

# Specify explicit dependencies for Ceph packages 
# See Bug #832714: Ceph from Jessie-backports
aptgetPurgeCmd = ""
aptgetInstallCmd = ""
if argDebian.include? "jessie"
   aptgetPurgeCmd = "apt-get -y autoremove ceph ceph ceph-common libcephfs1 librados2 librbd1 python-ceph && apt-get -y purge ceph ceph-common libcephfs1 librados2 librbd1 python-ceph"
   aptgetInstallCmd = "apt-get -y --force-yes install ceph=0.80.10-2~bpo8+1 chrony ceph-common=0.80.10-2~bpo8+1 python-ceph=0.80.10-2~bpo8+1 librbd1=0.80.10-2~bpo8+1 libcephfs1=0.80.10-2~bpo8+1 librados2=0.80.10-2~bpo8+1"
else 
   aptgetPurgeCmd = "apt-get -y autoremove ceph ceph-common && apt-get -y purge ceph ceph-common"
   aptgetInstallCmd = "apt-get -y update && apt-get -y --force-yes install ceph"
end # if argDebian.include? "jessie"

Cute::TakTuk.start(clients, :user => "root") do |tak|
     tak.exec!("echo deb http://apt.grid5000.fr/ceph5k/#{argDebian}/#{opts[:release]} / | tee /etc/apt/sources.list.d/ceph.list")
     tak.exec!(aptgetPurgeCmd) # Purge previous ceph packages & dependencies
     tak.exec!(aptgetInstallCmd) # Install ceph packages & dependencies
     tak.loop()
end


# Prepare config file locally with list of nodes for clients
configFile = File.open("#{tempDir}/config/config", "w") do |file|
   clients.each do |client|
      file.puts("Host #{client}")
      file.puts("   Hostname #{client}")
      file.puts("   User root")
      file.puts("   StrictHostKeyChecking no")
   end # clients.each do |client|

end

# Get ssh_config file from master/monitor
Net::SFTP.start(monitor, 'root') do |sftp|
  sftp.download!("/etc/ssh/ssh_config", "#{tempDir}/config/ssh_config")
end

# Copy first updated config for Ceph clients on monitor/master node
ssh_key =  'id_rsa'
Cute::TakTuk.start([monitor], :user => "root") do |tak|
     result = tak.put("#{tempDir}/config/config", "/root/.ssh/config") # copy the config file to master/monitor
     tak.loop()
end

# Push ssh_config file & ssh public key to all nodes
Cute::TakTuk.start(clients, :user => "root") do |tak|
     tak.put("#{tempDir}/config/ssh_config", "/etc/ssh/ssh_config")
     tak.put("/home/#{user}/.ssh/#{ssh_key}.pub", "/root/.ssh/#{ssh_key}.pub")
     tak.exec!("cat /root/.ssh/#{ssh_key}.pub >> /root/.ssh/authorized_keys")
     tak.loop()
end

# Preflight checklist completed.
logger.info "Pre-flight checklist completed." + "\n"


# Install & administer clients to Ceph deployed cluster.
logger.info "Adding following clients to deployed Ceph cluster:"
clients.each do |client|
     clientShort = client.split(".").first
     Cute::TakTuk.start([monitor], :user => "root") do |tak|
          tak.exec!("ceph-deploy install --release #{opts[:release]} #{clientShort}")
          result = tak.exec!("ceph-deploy --overwrite-conf admin #{clientShort}")
          logger.info "Added client: #{client}" if result[monitor][:status] == 0
          tak.loop()
     end
end # clients.each do


# Create Ceph pools on deployed cluster.
logger.info "Creating Ceph pools on deployed cluster ..."
# Create Ceph pools & RBDs
Cute::TakTuk.start(clients, :user => "root") do |tak|
     tak.exec!("modprobe rbd")
     tak.exec!("rados mkpool #{opts[:'pool-name']}")
     tak.exec!("rbd create --image-format 2 #{opts[:'rbd-name']} --pool #{opts[:'pool-name']} --size #{opts[:'rbd-size']}")
     tak.loop()
end

# Created Pools & RBDs for Ceph deployed cluster.
logger.info "Created Ceph pool on deployed cluster as follows :" + "\n"
logger.info "Pool name: #{opts[:'pool-name']} , RBD Name: #{opts[:'rbd-name']} , RBD Size: #{opts[:'rbd-size']} " + "\n"


# Map RBDs and create File Systems.
logger.info "Mapping RBD in deployed Ceph clusters ..."
Cute::TakTuk.start(clients, :user => "root") do |tak|
     # Map RBD & create FS on deployed cluster
     tak.exec!("rbd map #{opts[:'rbd-name']} --pool #{opts[:'pool-name']}")
     tak.exec!("mkfs.#{opts[:'file-system']} -m0 /dev/rbd/#{opts[:'pool-name']}/#{opts[:'rbd-name']}")
     tak.loop()
end
# Mapped RBDs & created FS for clients on Ceph deployed cluster.
logger.info "Mapped RBDs #{opts[:'rbd-name']} for clients on deployed Ceph." + "\n"


# Mount RBDs on Ceph client(s).
logger.info "Mounting RBDs in deployed Ceph cluster on client(s) ..."
clients.each do |client|
   Cute::TakTuk.start([client], :user => "root") do |tak|

        # mount RBD from deployed cluster
        tak.exec!("umount /dev/rbd/#{opts[:'pool-name']}/#{opts[:'rbd-name']} /mnt/#{opts[:'mnt-depl']}")
        tak.exec!("rmdir /mnt/#{opts[:'mnt-depl']}")
        tak.exec!("mkdir /mnt/#{opts[:'mnt-depl']}")
        result = tak.exec!("mount /dev/rbd/#{opts[:'pool-name']}/#{opts[:'rbd-name']} /mnt/#{opts[:'mnt-depl']}")
        logger.info "Mounted RBD as File System on client: #{client}" if result[client][:status] == 0

        tak.loop()
   end
end # clients.each do