5f34e76b9b
Options such as: osd mount options xfs = -i size=2048 -n size=16k were parsed as: key: 'osd mount options xfs = -i size=2048 -n size' value: '16k' This is due to the greedy .* in the beginning. Making it non-greedy fixes this. Fixes #851
343 lines
7.7 KiB
Perl
343 lines
7.7 KiB
Perl
package PVE::CephTools;
|
|
|
|
use strict;
|
|
use warnings;
|
|
use File::Basename;
|
|
use File::Path;
|
|
use POSIX qw (LONG_MAX);
|
|
use Cwd qw(abs_path);
|
|
use IO::Dir;
|
|
|
|
use PVE::Tools qw(extract_param run_command file_get_contents file_read_firstline dir_glob_regex dir_glob_foreach);
|
|
|
|
my $ccname = 'ceph'; # ceph cluster name
|
|
my $ceph_cfgdir = "/etc/ceph";
|
|
my $pve_ceph_cfgpath = "/etc/pve/$ccname.conf";
|
|
my $ceph_cfgpath = "$ceph_cfgdir/$ccname.conf";
|
|
|
|
my $pve_mon_key_path = "/etc/pve/priv/$ccname.mon.keyring";
|
|
my $pve_ckeyring_path = "/etc/pve/priv/$ccname.client.admin.keyring";
|
|
my $ceph_bootstrap_osd_keyring = "/var/lib/ceph/bootstrap-osd/$ccname.keyring";
|
|
my $ceph_bootstrap_mds_keyring = "/var/lib/ceph/bootstrap-mds/$ccname.keyring";
|
|
|
|
my $ceph_bin = "/usr/bin/ceph";
|
|
|
|
my $config_hash = {
|
|
ccname => $ccname,
|
|
pve_ceph_cfgpath => $pve_ceph_cfgpath,
|
|
pve_mon_key_path => $pve_mon_key_path,
|
|
pve_ckeyring_path => $pve_ckeyring_path,
|
|
ceph_bootstrap_osd_keyring => $ceph_bootstrap_osd_keyring,
|
|
ceph_bootstrap_mds_keyring => $ceph_bootstrap_mds_keyring,
|
|
long_rados_timeout => 60,
|
|
};
|
|
|
|
sub get_config {
|
|
my $key = shift;
|
|
|
|
my $value = $config_hash->{$key};
|
|
|
|
die "no such ceph config '$key'" if !$value;
|
|
|
|
return $value;
|
|
}
|
|
|
|
sub verify_blockdev_path {
|
|
my ($rel_path) = @_;
|
|
|
|
die "missing path" if !$rel_path;
|
|
my $path = abs_path($rel_path);
|
|
die "failed to get absolute path to $rel_path" if !$path;
|
|
|
|
die "got unusual device path '$path'\n" if $path !~ m|^/dev/(.*)$|;
|
|
|
|
$path = "/dev/$1"; # untaint
|
|
|
|
die "no such block device '$path'\n"
|
|
if ! -b $path;
|
|
|
|
return $path;
|
|
};
|
|
|
|
sub purge_all_ceph_files {
|
|
# fixme: this is very dangerous - should we really support this function?
|
|
|
|
unlink $ceph_cfgpath;
|
|
|
|
unlink $pve_ceph_cfgpath;
|
|
unlink $pve_ckeyring_path;
|
|
unlink $pve_mon_key_path;
|
|
|
|
unlink $ceph_bootstrap_osd_keyring;
|
|
unlink $ceph_bootstrap_mds_keyring;
|
|
|
|
system("rm -rf /var/lib/ceph/mon/ceph-*");
|
|
|
|
# remove osd?
|
|
}
|
|
|
|
sub check_ceph_installed {
|
|
my ($noerr) = @_;
|
|
|
|
if (! -x $ceph_bin) {
|
|
die "ceph binaries not installed\n" if !$noerr;
|
|
return undef;
|
|
}
|
|
|
|
return 1;
|
|
}
|
|
|
|
sub check_ceph_inited {
|
|
my ($noerr) = @_;
|
|
|
|
return undef if !check_ceph_installed($noerr);
|
|
|
|
if (! -f $pve_ceph_cfgpath) {
|
|
die "pveceph configuration not initialized\n" if !$noerr;
|
|
return undef;
|
|
}
|
|
|
|
return 1;
|
|
}
|
|
|
|
sub check_ceph_enabled {
|
|
my ($noerr) = @_;
|
|
|
|
return undef if !check_ceph_inited($noerr);
|
|
|
|
if (! -f $ceph_cfgpath) {
|
|
die "pveceph configuration not enabled\n" if !$noerr;
|
|
return undef;
|
|
}
|
|
|
|
return 1;
|
|
}
|
|
|
|
sub parse_ceph_config {
|
|
my ($filename) = @_;
|
|
|
|
$filename = $pve_ceph_cfgpath if !$filename;
|
|
|
|
my $cfg = {};
|
|
|
|
return $cfg if ! -f $filename;
|
|
|
|
my $fh = IO::File->new($filename, "r") ||
|
|
die "unable to open '$filename' - $!\n";
|
|
|
|
my $section;
|
|
|
|
while (defined(my $line = <$fh>)) {
|
|
$line =~ s/[;#].*$//;
|
|
$line =~ s/^\s+//;
|
|
$line =~ s/\s+$//;
|
|
next if !$line;
|
|
|
|
$section = $1 if $line =~ m/^\[(\S+)\]$/;
|
|
if (!$section) {
|
|
warn "no section - skip: $line\n";
|
|
next;
|
|
}
|
|
|
|
if ($line =~ m/^(.*?\S)\s*=\s*(\S.*)$/) {
|
|
$cfg->{$section}->{$1} = $2;
|
|
}
|
|
|
|
}
|
|
|
|
return $cfg;
|
|
}
|
|
|
|
sub write_ceph_config {
|
|
my ($cfg) = @_;
|
|
|
|
my $out = '';
|
|
|
|
my $cond_write_sec = sub {
|
|
my $re = shift;
|
|
|
|
foreach my $section (keys %$cfg) {
|
|
next if $section !~ m/^$re$/;
|
|
$out .= "[$section]\n";
|
|
foreach my $key (sort keys %{$cfg->{$section}}) {
|
|
$out .= "\t $key = $cfg->{$section}->{$key}\n";
|
|
}
|
|
$out .= "\n";
|
|
}
|
|
};
|
|
|
|
&$cond_write_sec('global');
|
|
&$cond_write_sec('client');
|
|
&$cond_write_sec('mds');
|
|
&$cond_write_sec('mds\..*');
|
|
&$cond_write_sec('mon');
|
|
&$cond_write_sec('osd');
|
|
&$cond_write_sec('mon\..*');
|
|
&$cond_write_sec('osd\..*');
|
|
|
|
PVE::Tools::file_set_contents($pve_ceph_cfgpath, $out);
|
|
}
|
|
|
|
sub setup_pve_symlinks {
|
|
# fail if we find a real file instead of a link
|
|
if (-f $ceph_cfgpath) {
|
|
my $lnk = readlink($ceph_cfgpath);
|
|
die "file '$ceph_cfgpath' already exists\n"
|
|
if !$lnk || $lnk ne $pve_ceph_cfgpath;
|
|
} else {
|
|
symlink($pve_ceph_cfgpath, $ceph_cfgpath) ||
|
|
die "unable to create symlink '$ceph_cfgpath' - $!\n";
|
|
}
|
|
}
|
|
|
|
sub ceph_service_cmd {
|
|
# ceph daemons does not call 'setsid', so we do that ourself
|
|
# (fork_worker send KILL to whole process group)
|
|
PVE::Tools::run_command(['setsid', 'service', 'ceph', '-c', $pve_ceph_cfgpath, @_]);
|
|
}
|
|
|
|
sub list_disks {
|
|
my $disklist = {};
|
|
|
|
my $fd = IO::File->new("/proc/mounts", "r") ||
|
|
die "unable to open /proc/mounts - $!\n";
|
|
|
|
my $mounted = {};
|
|
|
|
while (defined(my $line = <$fd>)) {
|
|
my ($dev, $path, $fstype) = split(/\s+/, $line);
|
|
next if !($dev && $path && $fstype);
|
|
next if $dev !~ m|^/dev/|;
|
|
my $real_dev = abs_path($dev);
|
|
$mounted->{$real_dev} = $path;
|
|
}
|
|
close($fd);
|
|
|
|
my $dev_is_mounted = sub {
|
|
my ($dev) = @_;
|
|
return $mounted->{$dev};
|
|
};
|
|
|
|
my $dir_is_empty = sub {
|
|
my ($dir) = @_;
|
|
|
|
my $dh = IO::Dir->new ($dir);
|
|
return 1 if !$dh;
|
|
|
|
while (defined(my $tmp = $dh->read)) {
|
|
next if $tmp eq '.' || $tmp eq '..';
|
|
$dh->close;
|
|
return 0;
|
|
}
|
|
$dh->close;
|
|
return 1;
|
|
};
|
|
|
|
my $journal_uuid = '45b0969e-9b03-4f30-b4c6-b4b80ceff106';
|
|
|
|
my $journalhash = {};
|
|
dir_glob_foreach('/dev/disk/by-parttypeuuid', "$journal_uuid\..+", sub {
|
|
my ($entry) = @_;
|
|
my $real_dev = abs_path("/dev/disk/by-parttypeuuid/$entry");
|
|
$journalhash->{$real_dev} = 1;
|
|
});
|
|
|
|
dir_glob_foreach('/sys/block', '.*', sub {
|
|
my ($dev) = @_;
|
|
|
|
return if $dev eq '.';
|
|
return if $dev eq '..';
|
|
|
|
return if $dev =~ m|^ram\d+$|; # skip ram devices
|
|
return if $dev =~ m|^loop\d+$|; # skip loop devices
|
|
return if $dev =~ m|^md\d+$|; # skip md devices
|
|
return if $dev =~ m|^dm-.*$|; # skip dm related things
|
|
return if $dev =~ m|^fd\d+$|; # skip Floppy
|
|
return if $dev =~ m|^sr\d+$|; # skip CDs
|
|
|
|
my $devdir = "/sys/block/$dev/device";
|
|
return if ! -d $devdir;
|
|
|
|
my $size = file_read_firstline("/sys/block/$dev/size");
|
|
return if !$size;
|
|
|
|
$size = $size * 512;
|
|
|
|
my $info = `udevadm info --path /sys/block/$dev --query all`;
|
|
return if !$info;
|
|
|
|
return if $info !~ m/^E: DEVTYPE=disk$/m;
|
|
return if $info =~ m/^E: ID_CDROM/m;
|
|
|
|
my $serial = 'unknown';
|
|
if ($info =~ m/^E: ID_SERIAL_SHORT=(\S+)$/m) {
|
|
$serial = $1;
|
|
}
|
|
|
|
my $gpt = 0;
|
|
if ($info =~ m/^E: ID_PART_TABLE_TYPE=gpt$/m) {
|
|
$gpt = 1;
|
|
}
|
|
|
|
# detect SSD (fixme - currently only works for ATA disks)
|
|
my $rpm = 7200; # default guess
|
|
if ($info =~ m/^E: ID_ATA_ROTATION_RATE_RPM=(\d+)$/m) {
|
|
$rpm = $1;
|
|
}
|
|
|
|
my $vendor = file_read_firstline("$devdir/vendor") || 'unknown';
|
|
my $model = file_read_firstline("$devdir/model") || 'unknown';
|
|
|
|
my $used;
|
|
|
|
$used = 'LVM' if !&$dir_is_empty("/sys/block/$dev/holders");
|
|
|
|
$used = 'mounted' if &$dev_is_mounted("/dev/$dev");
|
|
|
|
$disklist->{$dev} = {
|
|
vendor => $vendor,
|
|
model => $model,
|
|
size => $size,
|
|
serial => $serial,
|
|
gpt => $gpt,
|
|
rmp => $rpm,
|
|
};
|
|
|
|
my $osdid = -1;
|
|
|
|
my $journal_count = 0;
|
|
|
|
my $found_partitions;
|
|
my $found_lvm;
|
|
my $found_mountpoints;
|
|
dir_glob_foreach("/sys/block/$dev", "$dev.+", sub {
|
|
my ($part) = @_;
|
|
|
|
$found_partitions = 1;
|
|
|
|
if (my $mp = &$dev_is_mounted("/dev/$part")) {
|
|
$found_mountpoints = 1;
|
|
if ($mp =~ m|^/var/lib/ceph/osd/ceph-(\d+)$|) {
|
|
$osdid = $1;
|
|
}
|
|
}
|
|
if (!&$dir_is_empty("/sys/block/$dev/$part/holders")) {
|
|
$found_lvm = 1;
|
|
}
|
|
$journal_count++ if $journalhash->{"/dev/$part"};
|
|
});
|
|
|
|
$used = 'mounted' if $found_mountpoints && !$used;
|
|
$used = 'LVM' if $found_lvm && !$used;
|
|
$used = 'partitions' if $found_partitions && !$used;
|
|
|
|
$disklist->{$dev}->{used} = $used if $used;
|
|
$disklist->{$dev}->{osdid} = $osdid;
|
|
$disklist->{$dev}->{journals} = $journal_count;
|
|
});
|
|
|
|
return $disklist;
|
|
}
|
|
|
|
1;
|