cephManaged

#!/usr/bin/env ruby
# Copyright (c) 2015-17 Anirvan BASU, INRIA Rennes - Bretagne Atlantique
#
# Licensed under the CeCCIL-B license (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.cecill.info/licences/Licence_CeCILL-B_V1-en.html
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License. 

require 'cute'
require 'logger'
require 'cute/taktuk'
require 'net/sftp'
require 'erb'
require 'socket'
require 'trollop'
require 'json'
require 'net/http'
require 'uri'
require 'fileutils'
require_relative "helpers/abstractions"


g5k = Cute::G5K::API.new()
user = g5k.g5k_user

# Get the script dir
scriptDir = File.expand_path(File.dirname(__FILE__))
# Make the temporary files directory (if not created already)
tempDir = scriptDir + "/.generated"
FileUtils.mkpath(tempDir)
FileUtils.mkpath(tempDir + "/config")

# Additionally create a directory for managed Ceph config files
prodDir = tempDir + "/config/prod"
FileUtils.mkpath(prodDir)

currentConfigFile = tempDir + "/config/defaults.yml" # config file to be used.
# Read all options from CLI or config file
opts = readOptions(scriptDir, currentConfigFile, "cephManaged")

# Adapt some CLI arguments into global variables. Later change to class attributes.
opts[:'pool-name'] = "#{user}_" + opts[:'pool-name'] # Name of pool to find on cluster.
opts[:'rbd-name'] = "#{user}_" + opts[:'rbd-name'] # Name of RBD to create on cluster.


# Prepare logFile for logging all actions
logger = logCreate(tempDir, "cephManaged")


# Recover job details of clients
jobCephClient = getJob(g5k, 0, opts[:'job-client'], opts[:'client-site'])
clients = [] # Array of client nodes

# If client-list specified in CLI argument, get list of clients & fill array
unless opts[:'file'].empty?
   clients = File.open(opts[:'file'], 'r'){ |file| 
      file.readlines.collect{ |line| line.chomp }
   } # File.open
else # when no client-list is specified. Do deployment or start from scratch.

   # Finally, if job does not yet exist reserve nodes
   jobCephClient = g5k.reserve(:name => opts[:'job-client'], :nodes => opts[:'num-client'], :site => opts[:'client-site'], :cluster => opts[:'client-cluster'], :walltime => opts[:walltime], :type => :deploy) if jobCephClient.nil?

   # Assign roles to each node
   clients = jobCephClient["assigned_nodes"]

   # if not yet deployed, then deploy it
   if jobCephClient["deploy"].nil?
      logger.info "Deploying #{opts[:'env-client']} on client node(s): #{clients}"
      depCephClient = g5k.deploy(jobCephClient, :nodes => clients, :env => opts[:'env-client']) 
      g5k.wait_for_deploy(jobCephClient)
   end # if jobCephClient["deploy"].nil?

end # unless opts[:'file'].empty?

# At this point Ceph client job was created / fetched
logger.info "Deploying Ceph clients on nodes: #{clients}" 


# Additionally create a directory for saving details of clients deployed
jobID = jobCephClient["uid"]
clientStateDir = tempDir + "/#{opts[:'client-site']}/#{jobID}"
FileUtils.mkpath(clientStateDir)

# Prepare clients-list-file locally
clientsFile = File.open("#{clientStateDir}/clients-list", "w") do |file|
   clients.each do |client|
      file.puts("#{client}")
   end
end

# Get the client(s) for the managed Ceph cluster
clients = jobCephClient["assigned_nodes"]

# if not yet deployed, then deploy it
if jobCephClient["deploy"].nil?
   logger.info "Deploying #{opts[:'env-client']} on client node(s): #{clients}"
   depCephClient = g5k.deploy(jobCephClient, :nodes => clients, :env => opts[:'env-client']) 
   g5k.wait_for_deploy(jobCephClient)
end # if jobCephClient["deploy"].nil?

# Remind where is the Ceph client
logger.info "Managed Ceph client(s) on: #{clients}"


#1 Preflight Checklist
logger.info "Doing pre-flight checklist..."

# Add Ceph & Extras to each Ceph node ('firefly' is the most complete)
argDebian = opts[:'env-client'].slice(0,6)

# Specify explicit dependencies for Ceph packages 
# See Bug #832714: Ceph from Jessie-backports
aptgetPurgeCmd = ""
aptgetInstallCmd = ""
if argDebian.include? "jessie"
   aptgetPurgeCmd = "apt-get -y autoremove ceph ceph ceph-common libcephfs1 librados2 librbd1 python-ceph && apt-get -y purge ceph ceph-common libcephfs1 librados2 librbd1 python-ceph"
   aptgetInstallCmd = "apt-get -y --force-yes install ceph=0.80.10-2~bpo8+1 chrony ceph-common=0.80.10-2~bpo8+1 python-ceph=0.80.10-2~bpo8+1 librbd1=0.80.10-2~bpo8+1 libcephfs1=0.80.10-2~bpo8+1 librados2=0.80.10-2~bpo8+1"
else 
   aptgetPurgeCmd = "apt-get -y autoremove ceph ceph-common && apt-get -y purge ceph ceph-common"
   aptgetInstallCmd = "apt-get -y update && apt-get -y --force-yes install ceph"
end # if argDebian.include? "jessie"

Cute::TakTuk.start(clients, :user => "root") do |tak|
#     result = tak.exec!("apt-get autoremove -y ceph && apt-get autoremove -y ceph-deploy")
#     tak.exec!("echo deb #{ceph_extras}  | sudo tee /etc/apt/sources.list.d/ceph-extras.list")
     tak.exec!("echo deb http://apt.grid5000.fr/ceph5k/#{argDebian}/#{opts[:release]} / | tee /etc/apt/sources.list.d/ceph.list")
     tak.exec!(aptgetPurgeCmd) # Purge previous ceph packages & dependencies
     tak.exec!(aptgetInstallCmd) # Install ceph packages & dependencies
     tak.loop()
end

# Preflight checklist completed.
logger.info "Pre-flight checklist completed."


# Abort script if no managed Ceph cluster specified
if opts[:'managed-cluster'].nil?
   logger.info "Script exited - no managed Ceph cluster specified. Use option --managed-cluster to specify 'rennes' or 'nantes'"
   abort() 
end



# Prepare ceph.conf file for managed Ceph cluster
FileUtils.mkpath(prodDir + "/#{opts[:'managed-cluster']}")
configFile = File.open(prodDir + "/#{opts[:'managed-cluster']}/ceph.conf", "w") do |file|
   file.puts("[global]")
   file.puts("  mon initial members = ceph0")
   if opts[:'managed-cluster'] == "rennes"
      file.puts("  mon host = 172.16.111.30")
   else # nantes cluster
      file.puts("  mon host = 172.16.207.10")
   end # if managedCluster
end

# Copy ceph.conf file to client state directory
FileUtils.mkpath(clientStateDir + "/prod/#{opts[:'managed-cluster']}")
FileUtils.cp("#{prodDir}/#{opts[:'managed-cluster']}/ceph.conf", "#{clientStateDir}/prod/#{opts[:'managed-cluster']}/ceph.conf")
FileUtils.cp("/tmp/#{opts[:'managed-cluster']}/ceph.client.#{user}.keyring", "#{clientStateDir}/prod/#{opts[:'managed-cluster']}/ceph.client.#{user}.keyring")

# Then put ceph.conf file to the client
Cute::TakTuk.start(clients, :user => "root") do |tak|
     tak.exec!("mkdir -p /etc/ceph/#{opts[:'managed-cluster']} && touch /etc/ceph/#{opts[:'managed-cluster']}/ceph.client.#{user}.keyring")
     tak.exec!("mkdir -p prod/#{opts[:'managed-cluster']} && touch prod/#{opts[:'managed-cluster']}/ceph.conf")
     tak.put("#{prodDir}/#{opts[:'managed-cluster']}/ceph.conf", "/root/prod/#{opts[:'managed-cluster']}/ceph.conf")
     tak.put("/tmp/#{opts[:'managed-cluster']}/ceph.client.#{user}.keyring", "/etc/ceph/#{opts[:'managed-cluster']}/ceph.client.#{user}.keyring")
     tak.loop()
end

# Created & pushed config file for Managed Ceph cluster.
logger.info "Created & pushed config file for managed Ceph cluster to client(s)."


# Creating Ceph pools on managed cluster.
logger.info "Configuring Ceph pool & RBD on managed cluster ..."
poolsList = []
userPool = ""
userPoolMatch = ""
userRBD = ""
prodCluster = false
abortFlag = false
# Check Ceph pools & RBD on Managed cluster, using first client.
client = clients[0]
Cute::TakTuk.start([client], :user => "root") do |tak|
     tak.exec!("modprobe rbd")
     # Create RBD on managed cluster
     result = tak.exec!("rados -c /root/prod/#{opts[:'managed-cluster']}/ceph.conf --id #{user} -k /etc/ceph/#{opts[:'managed-cluster']}/ceph.client.#{user}.keyring lspools")
     poolsList = result[client][:output].split("\n") # Get list of pools in an array

     poolsList.each do |pool|  # logic: it will take the alphabetic-last pool from user
        userPool = pool if pool == opts[:'pool-name']
     end # poolsList.each do

     if userPool.empty?   # There is no pool created on managed Ceph
        logger.error "No Ceph pool found with name: #{opts[:'pool-name']}"
        logger.error "Specify correct pool name in config file or using option --client-pool-name"
        logger.error "Or verify / create the Ceph pool from the Ceph managed cluster frontend"
        logger.error "Use this link to create pool: https://api.grid5000.fr/sid/storage/ceph/ui/"
        logger.error "Then rerun this script."
        abortFlag = true
     end # userPool.empty?
     tak.loop()
end

# Abort script if no pool in managed Ceph
if abortFlag
   logger.error "Script exited - no pool for user in Managed Ceph clusters"
   abort() 
end

# At this point pool identified on Managed Ceph, now identify/create RBDs

# Get list of RBD images in specified Ceph "pool", as an array
poolRBDList = [] 
firstClient = clients[0]
Cute::TakTuk.start([firstClient], :user => "root") do |tak|
     result = tak.exec!("rbd -c /root/prod/#{opts[:'managed-cluster']}/ceph.conf --id #{user} -k /etc/ceph/#{opts[:'managed-cluster']}/ceph.client.#{user}.keyring --pool #{userPool} ls")
     poolRBDList = result[firstClient][:output].split("\n") unless result[firstClient][:output].nil?
     tak.loop()
end

unless poolRBDList.count == 0 || poolRBDList.count == opts[:'num-client']
   logger.error "Script exited - number mismatch between list of RBDs in pool #{userPool} and number of clients: #{opts[:'num-client']}"
   abort() 
end # unless poolRBDList.count == 0


# Get list of client RBD images if specified in file
clientRBDList = []
unless opts[:'rbd-list-file'].nil?
   clientRBDList = begin
     YAML.load(File.open(opts[:'rbd-list-file']))
   rescue ArgumentError => e
     logger.info "Could not parse YAML: #{e.message}"
   end

   unless clientRBDList & poolRBDList == clientRBDList || poolRBDList.count == 0
      logger.error "Script exited - mismatch between supplied list of RBDs and RBDs found in pool #{userPool}"
      abort() 
   end # unless clientRBDList & poolRBDList == clientRBDList 

   unless clientRBDList.count == opts[:'num-client']
      logger.error "Script exited - Count mismatch between supplied list of RBDs and number of clients: #{opts[:'num-client']}"
      abort() 
   end # unless clientRBDList.count == opts[:'num-client']

end # unless opts[:'rbd-list-file'].nil?

# At this point we have the RBDs to map/create either in clientRBDList or poolRBDList

createRBD = false
rbdList = []
case
   when clientRBDList.any? && poolRBDList.any?
     rbdList = poolRBDList
     logger.info "Mapping & mounting RBDs from list: #{rbdList} \nto client(s): #{clients}"

   when clientRBDList.any? && poolRBDList.empty?
     createRBD = true
     rbdList = clientRBDList
     logger.info "Creating, mapping & mounting RBDs from list: #{rbdList} \nto client(s): #{clients}"

   when clientRBDList.empty? && poolRBDList.any?
     rbdList = poolRBDList
     logger.info "Mapping & mounting RBDs from list: #{rbdList} \nto client(s): #{clients}"

   when clientRBDList.empty? && poolRBDList.empty?
     clients.each_with_index do |cephClient, index|
        rbdList[index] = "#{opts[:'rbd-name']}_#{index}"
     end # clients.each_with_index do
     createRBD = true
     logger.info "Creating, mapping & mounting RBDs from list: #{rbdList} \nto client(s): #{clients}"

   else # rbdList.count <= opts[:'num-client']
      logger.error "Script exited - possible mismatch between RBDs in pool and clients. Please try with a different pool and/or RBD list."
      abort() 
end

clients.each_with_index do |cephClient, index| # loop over each client to create RBD
   Cute::TakTuk.start([cephClient], :user => "root") do |tak|

        # Create RBDs if pool is empty
        if createRBD
           result = tak.exec!("rbd -c /root/prod/#{opts[:'managed-cluster']}/ceph.conf --id #{user} --pool #{userPool} -k /etc/ceph/#{opts[:'managed-cluster']}/ceph.client.#{user}.keyring create #{rbdList[index]} --size #{opts[:'rbd-size']}")
           logger.info "RBD #{rbdList[index]} created on Ceph managed cluster for client #{cephClient}." if result[cephClient][:status] == 0
        end # if createRBD


        # Map RBD on managed cluster
        result = tak.exec!("rbd -c /root/prod/#{opts[:'managed-cluster']}/ceph.conf --id #{user} --pool #{userPool} -k /etc/ceph/#{opts[:'managed-cluster']}/ceph.client.#{user}.keyring map #{rbdList[index]}")
        logger.info "RBD #{rbdList[index]} mapped on Ceph managed cluster for client #{cephClient}." if result[cephClient][:status] == 0

        # Create an FS only if RBD is newly created
        if createRBD 
           result = tak.exec!("mkfs.#{opts[:'file-system']} /dev/rbd/#{userPool}/#{rbdList[index]}")
           logger.info "RBD #{rbdList[index]} formatted with File system #{opts[:'file-system']} on client #{cephClient}." if result[cephClient][:status] == 0
        end # if createRBD

        # mount RBD from managed cluster
        tak.exec!("umount /dev/rbd/#{userPool}/#{rbdList[index]} /mnt/#{opts[:'mnt-prod']}")
        tak.exec!("rmdir /mnt/#{opts[:'mnt-prod']}")
        tak.exec!("mkdir /mnt/#{opts[:'mnt-prod']}")
        result = tak.exec!("mount /dev/rbd/#{userPool}/#{rbdList[index]} /mnt/#{opts[:'mnt-prod']}")
        logger.info "Mounted RBD #{rbdList[index]} on managed Ceph client #{cephClient}." if result[cephClient][:status] == 0

     tak.loop()

   end # Cute::TakTuk.start

end # clients.each do |client, index|

# Map RBD and create File Systems.
logger.info "Mapping RBD in managed Ceph clusters ..."
Cute::TakTuk.start(clients, :user => "root") do |tak|
     unless opts[:'no-deployed']
        # Map RBD & create FS on deployed Ceph cluster
        results = tak.exec!("rbd map #{opts[:'rbd-name']} --pool #{opts[:'pool-name']}")
        tak.exec!("mkfs.#{opts[:'file-system']} /dev/rbd/#{opts[:'pool-name']}/#{opts[:'rbd-name']}")

        results.each do |client, result|
           logger.info "Mapped RBD #{myRBDName} on deployed Ceph to client #{client}." if result[:status] == 0
        end # results.each do |client, result|

     end # unless opts[:'no-deployed']

  myRBDName = ""
  # Map RBD & create FS on managed cluster
  results = tak.exec!("rbd -c /root/prod/#{opts[:'managed-cluster']}/ceph.conf --id #{user} --pool #{userPool} map #{opts[:'rbd-name']} -k /etc/ceph/#{opts[:'managed-cluster']}/ceph.client.#{user}.keyring")
  if userRBD.empty? # Do it only the first time when the RBD is created.
     tak.exec!("mkfs.#{opts[:'file-system']} /dev/rbd/#{userPool}/#{opts[:'rbd-name']}")
     myRBDName = opts[:'rbd-name']
  else              # This case is when RBD is already created earlier.
     myRBDName = userRBD
  end # if userRBD.empty?

  tak.loop()
  results.each do |client, result|
     logger.info "Mapped RBD #{myRBDName} on managed Ceph to client #{client}." if result[:status] == 0
  end # results.each do |client, result|
end


# Mount RBDs as File Systems.
logger.info "Mounting RBD as File Systems in managed Ceph cluster ..."
Cute::TakTuk.start(clients, :user => "root") do |tak|

     result = nil
     # mount RBD from managed cluster
     if userRBD.empty? # Do it only the first time when the RBD is created.
        tak.exec!("umount /dev/rbd/#{userPool}/#{opts[:'rbd-name']} /mnt/#{opts[:'mnt-prod']}")
        tak.exec!("rmdir /mnt/#{opts[:'mnt-prod']}")
        tak.exec!("mkdir /mnt/#{opts[:'mnt-prod']}")
        results = tak.exec!("mount /dev/rbd/#{userPool}/#{opts[:'rbd-name']} /mnt/#{opts[:'mnt-prod']}")
     else              # This case is when RBD is already created earlier.
        tak.exec!("umount /dev/rbd/#{userPool}/#{userRBD} /mnt/#{opts[:'mnt-prod']}")
        tak.exec!("rmdir /mnt/#{opts[:'mnt-prod']}")
        tak.exec!("mkdir /mnt/#{opts[:'mnt-prod']}")
        results = tak.exec!("mount /dev/rbd/#{userPool}/#{userRBD} /mnt/#{opts[:'mnt-prod']}")
     end # if userRBD.empty?
     results.each do |client, result|
        logger.info "Mounted RBD on managed Ceph to client #{client}." if result[:status] == 0
     end # results.each do |client, result|

     tak.loop()
end