cephDeploy

#!/usr/bin/env ruby
# Copyright (c) 2015-17 Anirvan BASU, INRIA Rennes - Bretagne Atlantique
#
# Licensed under the CeCCIL-B license (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.cecill.info/licences/Licence_CeCILL-B_V1-en.html
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License. 

require 'cute'
require 'logger'
require 'cute/taktuk'
require 'net/sftp'
require 'erb'
require 'socket'
require 'trollop'
require 'json'
require "net/http"
require "uri"
require "fileutils"
require_relative "helpers/abstractions"


g5k = Cute::G5K::API.new()
user = g5k.g5k_user

# Get the script dir
scriptDir = File.expand_path(File.dirname(__FILE__))
# Make the temporary files directory (if not created already)
tempDir = scriptDir + "/.generated"
FileUtils.mkpath(tempDir)
FileUtils.mkpath(tempDir + "/config")

currentConfigFile = tempDir + "/config/defaults.yml" # config file to be used.
# Read all options from CLI or config file
opts = readOptions(scriptDir, currentConfigFile, "cephDeploy")

# Prepare logFile for logging all actions
logger = logCreate(tempDir, "cephDeploy")

# Show parameters for creating Ceph cluster
logger.info "Deploying Ceph cluster with the following parameters:"
logger.info "Grid 5000 site: #{opts[:site]}"
logger.info "Grid 5000 cluster: #{opts[:cluster]}"
logger.info "Ceph Release: #{opts[:release]}"
logger.info "Grid'5000 deployment: #{opts[:env]}"
logger.info "Job name (for nodes reservation): #{opts[:'job-name']}"
logger.info "Ceph cluster name: #{opts[:'cluster-name']}"
logger.info "Total nodes in Ceph cluster: #{opts[:'num-nodes']}"
logger.info "Deployment time: #{opts[:walltime]}\n"
logger.info "Option for multiple OSDs per node: #{opts[:'multi-osd']}\n" + "\n"

# Get the job for Ceph cluster if it exists
jobCephCluster = getJob(g5k, opts[:jobid], opts[:'job-name'], opts[:site])

# Finally create job if not yet created
jobCephCluster = g5k.reserve(:name => opts[:'job-name'], :nodes => opts[:'num-nodes'], :site => opts[:site], :cluster => opts[:cluster], :walltime => opts[:walltime], :type => :deploy) if jobCephCluster.nil?


# Assign roles to each node
nodes = jobCephCluster["assigned_nodes"]
monitor = nodes[0] # Currently single monitor. Later make multiple monitors.
osdNodes = nodes - [monitor]
radosGW = monitor # as of now the machine is the same for monitor & rados GW
monAllNodes = [monitor] # List of all monitors. As of now, only single monitor.

# Then, deploy the nodes with respective environments
depCeph = g5k.deploy(jobCephCluster, :nodes => nodes, :env => opts[:env])
g5k.wait_for_deploy(jobCephCluster)


# At this point job was created or fetched
logger.info "Ceph deployment job details recovered." + "\n"

# At this point job was created / fetched
logger.info "Deploying Ceph cluster #{opts[:'cluster-name']} as follows:"
logger.info "Cluster on nodes: #{nodes}" 
logger.info "Monitor node on: #{monAllNodes}"
logger.info "OSDs on: #{osdNodes}"

# Additionally create a directory for saving details of deployment
jobID = jobCephCluster["uid"]
depStateDir = tempDir + "/#{opts[:site]}/#{jobID}"
FileUtils.mkpath(depStateDir)
FileUtils.cp(currentConfigFile, depStateDir + "/defaults.yml")

# Copy deployed-list file at frontend
depListFile = File.open("#{depStateDir}/deployed-list", "w") do |file|
   nodes.each do |node|
      file.puts("#{node}")
   end
end



#1 Preflight Checklist
logger.info "Doing pre-flight checklist..."

# Add Ceph & Extras to each Ceph node ('firefly' is the most complete)
argDebian = opts[:env].slice(0,6)

# Specify explicit dependencies for Ceph packages 
# See Bug #832714: Ceph from Jessie-backports
aptgetPurgeCmd = ""
aptgetInstallCmd = ""
if argDebian.include? "jessie"
   aptgetPurgeCmd = "apt-get -y autoremove ceph ceph ceph-common libcephfs1 librados2 librbd1 python-ceph && apt-get -y purge ceph ceph-common libcephfs1 librados2 librbd1 python-ceph"
   aptgetInstallCmd = "apt-get -y --force-yes install ceph=0.80.10-2~bpo8+1 chrony ceph-common=0.80.10-2~bpo8+1 python-ceph=0.80.10-2~bpo8+1 librbd1=0.80.10-2~bpo8+1 libcephfs1=0.80.10-2~bpo8+1 librados2=0.80.10-2~bpo8+1"
else 
   aptgetPurgeCmd = "apt-get -y autoremove ceph ceph-common && apt-get -y purge ceph ceph-common"
   aptgetInstallCmd = "apt-get -y update && apt-get -y --force-yes install ceph"
end # if argDebian.include? "jessie"

Cute::TakTuk.start(nodes, :user => "root") do |tak|
     tak.exec!("echo deb http://apt.grid5000.fr/ceph5k/#{argDebian}/#{opts[:release]} / | tee /etc/apt/sources.list.d/ceph.list")
     tak.exec!(aptgetPurgeCmd) # Purge previous ceph packages & dependencies
     tak.exec!(aptgetInstallCmd) # Install ceph packages & dependencies
     tak.loop()
end


# Prepare .ssh/config file locally
configFile = File.open("#{tempDir}/config/config", "w") do |file|
   nodes.each do |node|
      file.puts("Host #{node}")
      file.puts("   Hostname #{node}")
      file.puts("   User root")
      file.puts("   StrictHostKeyChecking no")
   end
end

# Get ssh_config file from master/monitor
Net::SFTP.start(monitor, 'root') do |sftp|
  sftp.download!("/etc/ssh/ssh_config", "#{tempDir}/config/ssh_config")
end

# In ssh_config file (local) add a line to avoid StrictHostKeyChecking
configFile = File.open("#{tempDir}/config/ssh_config", "a") do |file|
   file.puts("    StrictHostKeyChecking no") # append only once
end

# Copy ssh keys & config for Ceph on monitor/master node
ssh_key =  'id_rsa'
Cute::TakTuk.start([monitor], :user => "root") do |tak|
     tak.put("/home/#{user}/.ssh/#{ssh_key}", "/root/.ssh/#{ssh_key}") # copy ssh private key to master/monitor
     tak.put("/home/#{user}/.ssh/#{ssh_key}.pub", "/root/.ssh/#{ssh_key}.pub") # copy ssh public key to master/monitor
     tak.put("#{tempDir}/config/config", "/root/.ssh/config") # copy the config file to master/monitor
     tak.loop()
end

# Push ssh_config file & ssh public key to all nodes
Cute::TakTuk.start(nodes, :user => "root") do |tak|
     tak.put("#{tempDir}/config/ssh_config", "/etc/ssh/ssh_config")
     tak.put("/home/#{user}/.ssh/#{ssh_key}.pub", "/root/.ssh/#{ssh_key}.pub")
     tak.exec!("cat /root/.ssh/#{ssh_key}.pub >> /root/.ssh/authorized_keys")
     tak.loop()
end

# Preflight checklist completed.
logger.info "Pre-flight checklist completed."


# Purging any previous Ceph installations.
logger.info "Purging any previous Ceph installations..."

nodesShort = nodes.map do |node|  # array of short names of nodes
     node.split(".").first
end
nodesList = nodesShort.join(' ') # text list of short names separated by spaces

# Cleanup: Purge previous Ceph installations if any & clear config
Cute::TakTuk.start([monitor], :user => "root") do |tak|
     tak.exec!("ceph-deploy purge #{nodesList}")
     tak.exec!("ceph-deploy forgetkeys")
     tak.exec!("rm -f ceph.conf")
     tak.loop()
end

# Purged previous Ceph installations.
logger.info "Purged previous Ceph installations." + "\n"


# Creating & installing Ceph cluster.
logger.info "Creating & installing Ceph cluster..."

# Create the ceph cluster on the monitor node
monitorShort = monitor.split(".").first
Cute::TakTuk.start([monitor], :user => "root") do |tak|
     tak.exec!("ceph-deploy new #{monitorShort}")
     tak.loop()
end


# Get initial config file from ceph master/monitor
Net::SFTP.start(monitor, 'root') do |sftp|
  sftp.download!("ceph.conf", "#{tempDir}/config/ceph.conf")
end

# Copy the Ceph config file to the deployment directory
FileUtils.cp("#{tempDir}/config/ceph.conf", "#{depStateDir}/ceph.conf")

# Read the following important parameters from file & keep in memory
fsid = ""
confFile = File.open("#{tempDir}/config/ceph.conf", "r") do |file|
   file.each_line do |line|
      if line.include? "fsid"
         fsid = line.split(" = ").last.split("\n").first
      end
   end
end

# Update certain lines in ceph.conf file
# Prepare the list of short names of all monitors as a text string
monAllNodesShort = monAllNodes.map do |node|  # array of short names of monitors
     node.split(".").first
end
monAllNodesList = monAllNodesShort.join(', ') # text list of short names separated by comma

# Prepare the list of IP addresses of all monitors as a text string
monAllNodesIP = monAllNodes.map do |node|  # array of IP addresses of monitors
     Socket.getaddrinfo(node, "http", nil, :STREAM)[0][2]
end
monAllNodesIPList = monAllNodesIP.join(', ') # text list of IP address separated by comma

# Read template file ceph.conf.erb
template = ERB.new File.new("./ceph5k/ceph.conf.erb").read, nil, "%"
# Fill up variables
mon_initial_members = monAllNodesList
mon_host = monAllNodesIPList
public_network = monAllNodesIP[0]
radosgw_host = monAllNodes[0]
# Write result to config file ceph.conf
cephFileText = template.result(binding)
File.open("#{tempDir}/config/ceph.conf", 'w+') do |file|
   file.write(cephFileText)
end


# Then put ceph.conf file to all nodes
Cute::TakTuk.start(nodes, :user => "root") do |tak|
     tak.exec!("rm ceph.conf")
     tak.exec!("mkdir /etc/ceph; touch /etc/ceph/ceph.conf")
     tak.put("#{tempDir}/config/ceph.conf", "ceph.conf")
     tak.put("#{tempDir}/config/ceph.conf", "/etc/ceph/ceph.conf")
     tak.loop()
end


# Install ceph on all nodes of cluster
nodes.each do |node|
     nodeShort = node.split(".").first
     Cute::TakTuk.start([node], :user => "root") do |tak|
          tak.exec!("ceph-deploy install --release #{opts[:release]} #{nodeShort}")
          tak.loop()
     end
end

# Ceph installation on all nodes completed.
logger.info "Ceph cluster installation completed."


# Adding & preparing monitor.
logger.info "Adding monitor #{monitor} to cluster..."

# Add initial monitor/master
Cute::TakTuk.start([monitor], :user => "root") do |tak|
     tak.exec!("ceph-deploy --overwrite-conf mon create-initial")
     tak.loop()
end


# Push config file and admin keys from master/monitor/admin node to all ceph nodes
otherNodesShort = nodesShort - [monitorShort]
otherNodesList = otherNodesShort.join(' ') # short names of all nodes other than master
Cute::TakTuk.start([monitor], :user => "root") do |tak|
     tak.exec!("ceph-deploy --overwrite-conf admin #{monitorShort} #{otherNodesList}")
     tak.loop()
end

# Ensure correct permissions for ceph.client.admin.keyring
Cute::TakTuk.start(nodes, :user => "root") do |tak|
     tak.exec!("chmod +r /etc/ceph/ceph.client.admin.keyring")
     tak.loop()
end

# Monitor added and prepared.
logger.info "Monitor added to Ceph cluster."


# Prepare & Activate OSDs.
logger.info "Preparing & activating OSDs..."

osdIndex = 0 # Iteration counter for OSDs created.
# mkdir, Prepare & Activate each OSD
if opts[:'multi-osd'] # Option for activating multiple OSDs per node

   osdNodes.each do |node|    # loop over all OSD nodes
     nodeShort = node.split(".").first       # the shortname of the node
     g5kCluster = nodeShort.split("-").first # the G5K cluster of the node
     storageDevices = []

     # Prepare URI & get node resources.
     nodeURI = "https://api.grid5000.fr/stable/sites/#{opts[:site]}/clusters/#{g5kCluster}/nodes/#{nodeShort}"
     uri = URI(nodeURI)
     http = Net::HTTP.new(uri.host, uri.port)
     request = Net::HTTP::Get.new(uri.request_uri)
     http.use_ssl = true
     http.verify_mode = OpenSSL::SSL::VERIFY_NONE
     response = http.request(request)
     result = response.body if response.is_a?(Net::HTTPSuccess)

     # Understand node resources
     parsedResult = JSON.parse(result)
     storageDevices = parsedResult["storage_devices"] # Get list of storage devices

     storageDevices.each do |storageDev, index| # loop over each physical disc
        device = storageDev["device"]
        nodeShort = node.split(".").first

        if device == "sda" # deploy OSD only on partition /dev/sda5
           Cute::TakTuk.start([node], :user => "root") do |tak|
               tak.exec!("umount /tmp")
               tak.exec!("mkdir -p /osd.#{osdIndex}")
               tak.exec!("mount /dev/#{device}5 /osd.#{osdIndex}")
               tak.loop()
           end
           Cute::TakTuk.start([monitor], :user => "root") do |tak|
               tak.exec!("ceph-deploy osd prepare #{nodeShort}:/osd.#{osdIndex}")
               tak.exec!("ceph-deploy osd activate #{nodeShort}:/osd.#{osdIndex}")
               tak.loop()
           end
           logger.info "Prepared & activated OSD.#{osdIndex} on: #{nodeShort}:/dev/#{device}5."

        else  # case of /dev/sdb, /dev/sdc, required to zap disc before deploy 
           # Remove all partitions, create primary partition (1), install FS
           Cute::TakTuk.start([node], :user => "root") do |tak|
               tak.exec!("parted -s /dev/#{device} mklabel msdos")
               tak.exec!("parted -s /dev/#{device} --align optimal mkpart primary 0 100%")
               tak.exec!("mkfs -t #{opts[:'file-system']} /dev/#{device}1")
               tak.loop()
           end # end of TakTuk loop for OSD node

           # Mount the partition on /osd# 
           Cute::TakTuk.start([node], :user => "root") do |tak|
               tak.exec!("rm -rf /osd.#{osdIndex}")
               tak.exec!("mkdir /osd.#{osdIndex}")
               tak.exec!("mount /dev/#{device}1 /osd.#{osdIndex}")
               tak.loop()
           end

           # Prepare & Activate the OSD 
           Cute::TakTuk.start([monitor], :user => "root") do |tak|
               tak.exec!("ceph-deploy osd prepare #{nodeShort}:/osd.#{osdIndex}")
               tak.exec!("ceph-deploy osd activate #{nodeShort}:/osd.#{osdIndex}")
               tak.loop()
           end # end of TakTuk loop for monitor
           logger.info "Prepared & activated OSD.#{osdIndex} on: #{nodeShort}:/dev/#{device}1."

        end # end of if-else device == "sda"
        osdIndex += 1

     end # loop over storage devices

   end # loop over all OSD nodes

else # Option for single OSD per node

   osdNodes.each do |node|
        nodeShort = node.split(".").first

        Cute::TakTuk.start([node], :user => "root") do |tak|
          tak.exec!("rm -rf /osd.#{osdIndex}")
          tak.exec!("umount /tmp")
          tak.exec!("mkdir /osd.#{osdIndex}")
          tak.exec!("mount /dev/sda5 /osd.#{osdIndex}")
          tak.loop()
        end # end of TakTuk loop for OSD node

        Cute::TakTuk.start([monitor], :user => "root") do |tak|
          tak.exec!("ceph-deploy osd prepare #{nodeShort}:/osd.#{osdIndex}")
          tak.exec!("ceph-deploy osd activate #{nodeShort}:/osd.#{osdIndex}")
          tak.loop()
        end # end of TakTuk loop for monitor

   end # loop over all OSD nodes

end # if-else for single/multi-OSD per node


# Distribute config & keyrings for cluster.
logger.info "Distributing config and keyrings for cluster..."

# Pull config and keyrings for cluster
Cute::TakTuk.start([monitor], :user => "root") do |tak|
     tak.exec!("ceph-deploy config pull #{monitorShort}")
     tak.exec!("ceph-deploy gatherkeys #{monitorShort}")
     tak.loop()
end


# Push config file and admin keys from master/monitor/admin node to all ceph nodes
otherNodesShort = nodesShort - [monitorShort]
otherNodesList = otherNodesShort.join(' ') # short names of all nodes other than master
Cute::TakTuk.start([monitor], :user => "root") do |tak|
     tak.exec!("ceph-deploy admin #{monitorShort} #{otherNodesList}")
     tak.loop()
end


# Ensure correct permissions for ceph.client.admin.keyring
Cute::TakTuk.start(nodes, :user => "root") do |tak|
     tak.exec!("chmod +r /etc/ceph/ceph.client.admin.keyring")
     tak.loop()
end

# Config & keyrings distributed.
logger.info "Ceph configuration & keyrings distributed throughout cluster."


# Finally check if Ceph Cluster was correctly deployed - result should be "active+clean"
Cute::TakTuk.start([monitor], :user => "root") do |tak|
     result = tak.exec!("ceph status")
     resultText = result[monitor][:output]
     logger.info "Ceph cluster up and running. In state 'active+clean'" if resultText.include? "active+clean"
     tak.loop()
end