1400 lines
33 KiB
Perl
1400 lines
33 KiB
Perl
package PVE::API2::CephOSD;
|
|
|
|
use strict;
|
|
use warnings;
|
|
use Cwd qw(abs_path);
|
|
|
|
use PVE::SafeSyslog;
|
|
use PVE::Tools qw(extract_param run_command file_get_contents file_read_firstline dir_glob_regex dir_glob_foreach);
|
|
use PVE::Exception qw(raise raise_param_exc);
|
|
use PVE::INotify;
|
|
use PVE::Cluster qw(cfs_lock_file cfs_read_file cfs_write_file);
|
|
use PVE::AccessControl;
|
|
use PVE::Storage;
|
|
use PVE::RESTHandler;
|
|
use PVE::RPCEnvironment;
|
|
use PVE::JSONSchema qw(get_standard_option);
|
|
use PVE::RADOS;
|
|
use PVE::CephTools;
|
|
|
|
use base qw(PVE::RESTHandler);
|
|
|
|
use Data::Dumper; # fixme: remove
|
|
|
|
my $get_osd_status = sub {
|
|
my ($rados, $osdid) = @_;
|
|
|
|
my $stat = $rados->mon_command({ prefix => 'osd dump' });
|
|
|
|
my $osdlist = $stat->{osds} || [];
|
|
|
|
my $osdstat;
|
|
foreach my $d (@$osdlist) {
|
|
$osdstat->{$d->{osd}} = $d if defined($d->{osd});
|
|
}
|
|
if (defined($osdid)) {
|
|
die "no such OSD '$osdid'\n" if !$osdstat->{$osdid};
|
|
return $osdstat->{$osdid};
|
|
}
|
|
|
|
return $osdstat;
|
|
};
|
|
|
|
my $get_osd_usage = sub {
|
|
my ($rados) = @_;
|
|
|
|
my $osdlist = $rados->mon_command({ prefix => 'pg dump',
|
|
dumpcontents => [ 'osds' ]}) || [];
|
|
|
|
my $osdstat;
|
|
foreach my $d (@$osdlist) {
|
|
$osdstat->{$d->{osd}} = $d if defined($d->{osd});
|
|
}
|
|
|
|
return $osdstat;
|
|
};
|
|
|
|
__PACKAGE__->register_method ({
|
|
name => 'index',
|
|
path => '',
|
|
method => 'GET',
|
|
description => "Get Ceph osd list/tree.",
|
|
proxyto => 'node',
|
|
protected => 1,
|
|
parameters => {
|
|
additionalProperties => 0,
|
|
properties => {
|
|
node => get_standard_option('pve-node'),
|
|
},
|
|
},
|
|
# fixme: return a list instead of extjs tree format ?
|
|
returns => {
|
|
type => "object",
|
|
},
|
|
code => sub {
|
|
my ($param) = @_;
|
|
|
|
PVE::CephTools::check_ceph_inited();
|
|
|
|
my $rados = PVE::RADOS->new();
|
|
my $res = $rados->mon_command({ prefix => 'osd tree' });
|
|
|
|
die "no tree nodes found\n" if !($res && $res->{nodes});
|
|
|
|
my $osdhash = &$get_osd_status($rados);
|
|
|
|
my $usagehash = &$get_osd_usage($rados);
|
|
|
|
my $nodes = {};
|
|
my $newnodes = {};
|
|
foreach my $e (@{$res->{nodes}}) {
|
|
$nodes->{$e->{id}} = $e;
|
|
|
|
my $new = {
|
|
id => $e->{id},
|
|
name => $e->{name},
|
|
type => $e->{type}
|
|
};
|
|
|
|
foreach my $opt (qw(status crush_weight reweight)) {
|
|
$new->{$opt} = $e->{$opt} if defined($e->{$opt});
|
|
}
|
|
|
|
if (my $stat = $osdhash->{$e->{id}}) {
|
|
$new->{in} = $stat->{in} if defined($stat->{in});
|
|
}
|
|
|
|
if (my $stat = $usagehash->{$e->{id}}) {
|
|
$new->{total_space} = ($stat->{kb} || 1) * 1024;
|
|
$new->{bytes_used} = ($stat->{kb_used} || 0) * 1024;
|
|
$new->{percent_used} = ($new->{bytes_used}*100)/$new->{total_space};
|
|
if (my $d = $stat->{fs_perf_stat}) {
|
|
$new->{commit_latency_ms} = $d->{commit_latency_ms};
|
|
$new->{apply_latency_ms} = $d->{apply_latency_ms};
|
|
}
|
|
}
|
|
|
|
$newnodes->{$e->{id}} = $new;
|
|
}
|
|
|
|
foreach my $e (@{$res->{nodes}}) {
|
|
my $new = $newnodes->{$e->{id}};
|
|
if ($e->{children} && scalar(@{$e->{children}})) {
|
|
$new->{children} = [];
|
|
$new->{leaf} = 0;
|
|
foreach my $cid (@{$e->{children}}) {
|
|
$nodes->{$cid}->{parent} = $e->{id};
|
|
if ($nodes->{$cid}->{type} eq 'osd' &&
|
|
$e->{type} eq 'host') {
|
|
$newnodes->{$cid}->{host} = $e->{name};
|
|
}
|
|
push @{$new->{children}}, $newnodes->{$cid};
|
|
}
|
|
} else {
|
|
$new->{leaf} = ($e->{id} >= 0) ? 1 : 0;
|
|
}
|
|
}
|
|
|
|
my $rootnode;
|
|
foreach my $e (@{$res->{nodes}}) {
|
|
if (!$nodes->{$e->{id}}->{parent}) {
|
|
$rootnode = $newnodes->{$e->{id}};
|
|
last;
|
|
}
|
|
}
|
|
|
|
die "no root node\n" if !$rootnode;
|
|
|
|
my $data = { root => $rootnode };
|
|
|
|
return $data;
|
|
}});
|
|
|
|
__PACKAGE__->register_method ({
|
|
name => 'createosd',
|
|
path => '',
|
|
method => 'POST',
|
|
description => "Create OSD",
|
|
proxyto => 'node',
|
|
protected => 1,
|
|
parameters => {
|
|
additionalProperties => 0,
|
|
properties => {
|
|
node => get_standard_option('pve-node'),
|
|
dev => {
|
|
description => "Block device name.",
|
|
type => 'string',
|
|
},
|
|
journal_dev => {
|
|
description => "Block device name for journal.",
|
|
optional => 1,
|
|
type => 'string',
|
|
},
|
|
fstype => {
|
|
description => "File system type.",
|
|
type => 'string',
|
|
enum => ['xfs', 'ext4', 'btrfs'],
|
|
default => 'xfs',
|
|
optional => 1,
|
|
},
|
|
},
|
|
},
|
|
returns => { type => 'string' },
|
|
code => sub {
|
|
my ($param) = @_;
|
|
|
|
my $rpcenv = PVE::RPCEnvironment::get();
|
|
|
|
my $authuser = $rpcenv->get_user();
|
|
|
|
PVE::CephTools::check_ceph_inited();
|
|
|
|
PVE::CephTools::setup_pve_symlinks();
|
|
|
|
my $journal_dev;
|
|
|
|
if ($param->{journal_dev} && ($param->{journal_dev} ne $param->{dev})) {
|
|
$journal_dev = PVE::CephTools::verify_blockdev_path($param->{journal_dev});
|
|
}
|
|
|
|
$param->{dev} = PVE::CephTools::verify_blockdev_path($param->{dev});
|
|
|
|
my $disklist = PVE::CephTools::list_disks();
|
|
|
|
my $devname = $param->{dev};
|
|
$devname =~ s|/dev/||;
|
|
|
|
my $diskinfo = $disklist->{$devname};
|
|
die "unable to get device info for '$devname'\n"
|
|
if !$diskinfo;
|
|
|
|
die "device '$param->{dev}' is in use\n"
|
|
if $diskinfo->{used};
|
|
|
|
my $rados = PVE::RADOS->new();
|
|
my $monstat = $rados->mon_command({ prefix => 'mon_status' });
|
|
die "unable to get fsid\n" if !$monstat->{monmap} || !$monstat->{monmap}->{fsid};
|
|
|
|
my $fsid = $monstat->{monmap}->{fsid};
|
|
$fsid = $1 if $fsid =~ m/^([0-9a-f\-]+)$/;
|
|
|
|
my $ceph_bootstrap_osd_keyring = PVE::CephTools::get_config('ceph_bootstrap_osd_keyring');
|
|
|
|
if (! -f $ceph_bootstrap_osd_keyring) {
|
|
my $bindata = $rados->mon_command({ prefix => 'auth get', entity => 'client.bootstrap-osd', format => 'plain' });
|
|
PVE::Tools::file_set_contents($ceph_bootstrap_osd_keyring, $bindata);
|
|
};
|
|
|
|
my $worker = sub {
|
|
my $upid = shift;
|
|
|
|
my $fstype = $param->{fstype} || 'xfs';
|
|
|
|
print "create OSD on $param->{dev} ($fstype)\n";
|
|
|
|
my $ccname = PVE::CephTools::get_config('ccname');
|
|
|
|
my $cmd = ['ceph-disk', 'prepare', '--zap-disk', '--fs-type', $fstype,
|
|
'--cluster', $ccname, '--cluster-uuid', $fsid ];
|
|
|
|
if ($journal_dev) {
|
|
print "using device '$journal_dev' for journal\n";
|
|
push @$cmd, '--journal-dev', $param->{dev}, $journal_dev;
|
|
} else {
|
|
push @$cmd, $param->{dev};
|
|
}
|
|
|
|
run_command($cmd);
|
|
};
|
|
|
|
return $rpcenv->fork_worker('cephcreateosd', $devname, $authuser, $worker);
|
|
}});
|
|
|
|
__PACKAGE__->register_method ({
|
|
name => 'destroyosd',
|
|
path => '{osdid}',
|
|
method => 'DELETE',
|
|
description => "Destroy OSD",
|
|
proxyto => 'node',
|
|
protected => 1,
|
|
parameters => {
|
|
additionalProperties => 0,
|
|
properties => {
|
|
node => get_standard_option('pve-node'),
|
|
osdid => {
|
|
description => 'OSD ID',
|
|
type => 'integer',
|
|
},
|
|
cleanup => {
|
|
description => "If set, we remove partition table entries.",
|
|
type => 'boolean',
|
|
optional => 1,
|
|
default => 0,
|
|
},
|
|
},
|
|
},
|
|
returns => { type => 'string' },
|
|
code => sub {
|
|
my ($param) = @_;
|
|
|
|
my $rpcenv = PVE::RPCEnvironment::get();
|
|
|
|
my $authuser = $rpcenv->get_user();
|
|
|
|
PVE::CephTools::check_ceph_inited();
|
|
|
|
my $osdid = $param->{osdid};
|
|
|
|
my $rados = PVE::RADOS->new();
|
|
my $osdstat = &$get_osd_status($rados, $osdid);
|
|
|
|
die "osd is in use (in == 1)\n" if $osdstat->{in};
|
|
#&$run_ceph_cmd(['osd', 'out', $osdid]);
|
|
|
|
die "osd is still runnung (up == 1)\n" if $osdstat->{up};
|
|
|
|
my $osdsection = "osd.$osdid";
|
|
|
|
my $worker = sub {
|
|
my $upid = shift;
|
|
|
|
# reopen with longer timeout
|
|
$rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
|
|
|
|
print "destroy OSD $osdsection\n";
|
|
|
|
eval { PVE::CephTools::ceph_service_cmd('stop', $osdsection); };
|
|
warn $@ if $@;
|
|
|
|
print "Remove $osdsection from the CRUSH map\n";
|
|
$rados->mon_command({ prefix => "osd crush remove", name => $osdsection, format => 'plain' });
|
|
|
|
print "Remove the $osdsection authentication key.\n";
|
|
$rados->mon_command({ prefix => "auth del", entity => $osdsection, format => 'plain' });
|
|
|
|
print "Remove OSD $osdsection\n";
|
|
$rados->mon_command({ prefix => "osd rm", ids => [ $osdsection ], format => 'plain' });
|
|
|
|
# try to unmount from standard mount point
|
|
my $mountpoint = "/var/lib/ceph/osd/ceph-$osdid";
|
|
|
|
my $remove_partition = sub {
|
|
my ($disklist, $part) = @_;
|
|
|
|
return if !$part || (! -b $part );
|
|
|
|
foreach my $real_dev (keys %$disklist) {
|
|
my $diskinfo = $disklist->{$real_dev};
|
|
next if !$diskinfo->{gpt};
|
|
if ($part =~ m|^/dev/${real_dev}(\d+)$|) {
|
|
my $partnum = $1;
|
|
print "remove partition $part (disk '/dev/${real_dev}', partnum $partnum)\n";
|
|
eval { run_command(['/sbin/sgdisk', '-d', $partnum, "/dev/${real_dev}"]); };
|
|
warn $@ if $@;
|
|
last;
|
|
}
|
|
}
|
|
};
|
|
|
|
my $journal_part;
|
|
my $data_part;
|
|
|
|
if ($param->{cleanup}) {
|
|
my $jpath = "$mountpoint/journal";
|
|
$journal_part = abs_path($jpath);
|
|
|
|
if (my $fd = IO::File->new("/proc/mounts", "r")) {
|
|
while (defined(my $line = <$fd>)) {
|
|
my ($dev, $path, $fstype) = split(/\s+/, $line);
|
|
next if !($dev && $path && $fstype);
|
|
next if $dev !~ m|^/dev/|;
|
|
if ($path eq $mountpoint) {
|
|
$data_part = abs_path($dev);
|
|
last;
|
|
}
|
|
}
|
|
close($fd);
|
|
}
|
|
}
|
|
|
|
print "Unmount OSD $osdsection from $mountpoint\n";
|
|
eval { run_command(['umount', $mountpoint]); };
|
|
if (my $err = $@) {
|
|
warn $err;
|
|
} elsif ($param->{cleanup}) {
|
|
my $disklist = PVE::CephTools::list_disks();
|
|
&$remove_partition($disklist, $journal_part);
|
|
&$remove_partition($disklist, $data_part);
|
|
}
|
|
};
|
|
|
|
return $rpcenv->fork_worker('cephdestroyosd', $osdsection, $authuser, $worker);
|
|
}});
|
|
|
|
__PACKAGE__->register_method ({
|
|
name => 'in',
|
|
path => '{osdid}/in',
|
|
method => 'POST',
|
|
description => "ceph osd in",
|
|
proxyto => 'node',
|
|
protected => 1,
|
|
parameters => {
|
|
additionalProperties => 0,
|
|
properties => {
|
|
node => get_standard_option('pve-node'),
|
|
osdid => {
|
|
description => 'OSD ID',
|
|
type => 'integer',
|
|
},
|
|
},
|
|
},
|
|
returns => { type => "null" },
|
|
code => sub {
|
|
my ($param) = @_;
|
|
|
|
PVE::CephTools::check_ceph_inited();
|
|
|
|
my $osdid = $param->{osdid};
|
|
|
|
my $rados = PVE::RADOS->new();
|
|
|
|
my $osdstat = &$get_osd_status($rados, $osdid); # osd exists?
|
|
|
|
my $osdsection = "osd.$osdid";
|
|
|
|
$rados->mon_command({ prefix => "osd in", ids => [ $osdsection ], format => 'plain' });
|
|
|
|
return undef;
|
|
}});
|
|
|
|
__PACKAGE__->register_method ({
|
|
name => 'out',
|
|
path => '{osdid}/out',
|
|
method => 'POST',
|
|
description => "ceph osd out",
|
|
proxyto => 'node',
|
|
protected => 1,
|
|
parameters => {
|
|
additionalProperties => 0,
|
|
properties => {
|
|
node => get_standard_option('pve-node'),
|
|
osdid => {
|
|
description => 'OSD ID',
|
|
type => 'integer',
|
|
},
|
|
},
|
|
},
|
|
returns => { type => "null" },
|
|
code => sub {
|
|
my ($param) = @_;
|
|
|
|
PVE::CephTools::check_ceph_inited();
|
|
|
|
my $osdid = $param->{osdid};
|
|
|
|
my $rados = PVE::RADOS->new();
|
|
|
|
my $osdstat = &$get_osd_status($rados, $osdid); # osd exists?
|
|
|
|
my $osdsection = "osd.$osdid";
|
|
|
|
$rados->mon_command({ prefix => "osd out", ids => [ $osdsection ], format => 'plain' });
|
|
|
|
return undef;
|
|
}});
|
|
|
|
package PVE::API2::Ceph;
|
|
|
|
use strict;
|
|
use warnings;
|
|
use File::Basename;
|
|
use File::Path;
|
|
use POSIX qw (LONG_MAX);
|
|
use Cwd qw(abs_path);
|
|
use IO::Dir;
|
|
use UUID;
|
|
use Net::IP;
|
|
|
|
use PVE::SafeSyslog;
|
|
use PVE::Tools qw(extract_param run_command file_get_contents file_read_firstline dir_glob_regex dir_glob_foreach);
|
|
use PVE::Exception qw(raise raise_param_exc);
|
|
use PVE::INotify;
|
|
use PVE::Cluster qw(cfs_lock_file cfs_read_file cfs_write_file);
|
|
use PVE::AccessControl;
|
|
use PVE::Storage;
|
|
use PVE::RESTHandler;
|
|
use PVE::RPCEnvironment;
|
|
use PVE::JSONSchema qw(get_standard_option);
|
|
use JSON;
|
|
use PVE::RADOS;
|
|
use PVE::CephTools;
|
|
|
|
use base qw(PVE::RESTHandler);
|
|
|
|
use Data::Dumper; # fixme: remove
|
|
|
|
my $pve_osd_default_journal_size = 1024*5;
|
|
|
|
__PACKAGE__->register_method ({
|
|
subclass => "PVE::API2::CephOSD",
|
|
path => 'osd',
|
|
});
|
|
|
|
__PACKAGE__->register_method ({
|
|
name => 'index',
|
|
path => '',
|
|
method => 'GET',
|
|
description => "Directory index.",
|
|
permissions => { user => 'all' },
|
|
parameters => {
|
|
additionalProperties => 0,
|
|
properties => {
|
|
node => get_standard_option('pve-node'),
|
|
},
|
|
},
|
|
returns => {
|
|
type => 'array',
|
|
items => {
|
|
type => "object",
|
|
properties => {},
|
|
},
|
|
links => [ { rel => 'child', href => "{name}" } ],
|
|
},
|
|
code => sub {
|
|
my ($param) = @_;
|
|
|
|
my $result = [
|
|
{ name => 'init' },
|
|
{ name => 'mon' },
|
|
{ name => 'osd' },
|
|
{ name => 'pools' },
|
|
{ name => 'stop' },
|
|
{ name => 'start' },
|
|
{ name => 'status' },
|
|
{ name => 'crush' },
|
|
{ name => 'config' },
|
|
{ name => 'log' },
|
|
{ name => 'disks' },
|
|
];
|
|
|
|
return $result;
|
|
}});
|
|
|
|
__PACKAGE__->register_method ({
|
|
name => 'disks',
|
|
path => 'disks',
|
|
method => 'GET',
|
|
description => "List local disks.",
|
|
proxyto => 'node',
|
|
protected => 1,
|
|
parameters => {
|
|
additionalProperties => 0,
|
|
properties => {
|
|
node => get_standard_option('pve-node'),
|
|
type => {
|
|
description => "Only list specific types of disks.",
|
|
type => 'string',
|
|
enum => ['unused', 'journal_disks'],
|
|
optional => 1,
|
|
},
|
|
},
|
|
},
|
|
returns => {
|
|
type => 'array',
|
|
items => {
|
|
type => "object",
|
|
properties => {
|
|
dev => { type => 'string' },
|
|
used => { type => 'string', optional => 1 },
|
|
gpt => { type => 'boolean' },
|
|
size => { type => 'integer' },
|
|
osdid => { type => 'integer' },
|
|
vendor => { type => 'string', optional => 1 },
|
|
model => { type => 'string', optional => 1 },
|
|
serial => { type => 'string', optional => 1 },
|
|
},
|
|
},
|
|
# links => [ { rel => 'child', href => "{}" } ],
|
|
},
|
|
code => sub {
|
|
my ($param) = @_;
|
|
|
|
PVE::CephTools::check_ceph_inited();
|
|
|
|
my $disks = PVE::CephTools::list_disks();
|
|
|
|
my $res = [];
|
|
foreach my $dev (keys %$disks) {
|
|
my $d = $disks->{$dev};
|
|
if ($param->{type}) {
|
|
if ($param->{type} eq 'journal_disks') {
|
|
next if $d->{osdid} >= 0;
|
|
next if !$d->{gpt};
|
|
} elsif ($param->{type} eq 'unused') {
|
|
next if $d->{used};
|
|
} else {
|
|
die "internal error"; # should not happen
|
|
}
|
|
}
|
|
|
|
$d->{dev} = "/dev/$dev";
|
|
push @$res, $d;
|
|
}
|
|
|
|
return $res;
|
|
}});
|
|
|
|
__PACKAGE__->register_method ({
|
|
name => 'config',
|
|
path => 'config',
|
|
method => 'GET',
|
|
description => "Get Ceph configuration.",
|
|
parameters => {
|
|
additionalProperties => 0,
|
|
properties => {
|
|
node => get_standard_option('pve-node'),
|
|
},
|
|
},
|
|
returns => { type => 'string' },
|
|
code => sub {
|
|
my ($param) = @_;
|
|
|
|
PVE::CephTools::check_ceph_inited();
|
|
|
|
my $path = PVE::CephTools::get_config('pve_ceph_cfgpath');
|
|
return PVE::Tools::file_get_contents($path);
|
|
|
|
}});
|
|
|
|
__PACKAGE__->register_method ({
|
|
name => 'listmon',
|
|
path => 'mon',
|
|
method => 'GET',
|
|
description => "Get Ceph monitor list.",
|
|
proxyto => 'node',
|
|
protected => 1,
|
|
parameters => {
|
|
additionalProperties => 0,
|
|
properties => {
|
|
node => get_standard_option('pve-node'),
|
|
},
|
|
},
|
|
returns => {
|
|
type => 'array',
|
|
items => {
|
|
type => "object",
|
|
properties => {
|
|
name => { type => 'string' },
|
|
addr => { type => 'string' },
|
|
},
|
|
},
|
|
links => [ { rel => 'child', href => "{name}" } ],
|
|
},
|
|
code => sub {
|
|
my ($param) = @_;
|
|
|
|
PVE::CephTools::check_ceph_inited();
|
|
|
|
my $res = [];
|
|
|
|
my $cfg = PVE::CephTools::parse_ceph_config();
|
|
|
|
my $monhash = {};
|
|
foreach my $section (keys %$cfg) {
|
|
my $d = $cfg->{$section};
|
|
if ($section =~ m/^mon\.(\S+)$/) {
|
|
my $monid = $1;
|
|
if ($d->{'mon addr'} && $d->{'host'}) {
|
|
$monhash->{$monid} = {
|
|
addr => $d->{'mon addr'},
|
|
host => $d->{'host'},
|
|
name => $monid,
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
eval {
|
|
my $rados = PVE::RADOS->new();
|
|
my $monstat = $rados->mon_command({ prefix => 'mon_status' });
|
|
my $mons = $monstat->{monmap}->{mons};
|
|
foreach my $d (@$mons) {
|
|
next if !defined($d->{name});
|
|
$monhash->{$d->{name}}->{rank} = $d->{rank};
|
|
$monhash->{$d->{name}}->{addr} = $d->{addr};
|
|
if (grep { $_ eq $d->{rank} } @{$monstat->{quorum}}) {
|
|
$monhash->{$d->{name}}->{quorum} = 1;
|
|
}
|
|
}
|
|
};
|
|
warn $@ if $@;
|
|
|
|
return PVE::RESTHandler::hash_to_array($monhash, 'name');
|
|
}});
|
|
|
|
__PACKAGE__->register_method ({
|
|
name => 'init',
|
|
path => 'init',
|
|
method => 'POST',
|
|
description => "Create initial ceph default configuration and setup symlinks.",
|
|
proxyto => 'node',
|
|
protected => 1,
|
|
parameters => {
|
|
additionalProperties => 0,
|
|
properties => {
|
|
node => get_standard_option('pve-node'),
|
|
network => {
|
|
description => "Use specific network for all ceph related traffic",
|
|
type => 'string', format => 'CIDR',
|
|
optional => 1,
|
|
maxLength => 128,
|
|
},
|
|
size => {
|
|
description => 'Number of replicas per object',
|
|
type => 'integer',
|
|
default => 2,
|
|
optional => 1,
|
|
minimum => 1,
|
|
maximum => 3,
|
|
},
|
|
pg_bits => {
|
|
description => "Placement group bits, used to specify the default number of placement groups (Note: 'osd pool default pg num' does not work for deafult pools)",
|
|
type => 'integer',
|
|
default => 6,
|
|
optional => 1,
|
|
minimum => 6,
|
|
maximum => 14,
|
|
},
|
|
},
|
|
},
|
|
returns => { type => 'null' },
|
|
code => sub {
|
|
my ($param) = @_;
|
|
|
|
PVE::CephTools::check_ceph_installed();
|
|
|
|
# simply load old config if it already exists
|
|
my $cfg = PVE::CephTools::parse_ceph_config();
|
|
|
|
if (!$cfg->{global}) {
|
|
|
|
my $fsid;
|
|
my $uuid;
|
|
|
|
UUID::generate($uuid);
|
|
UUID::unparse($uuid, $fsid);
|
|
|
|
$cfg->{global} = {
|
|
'fsid' => $fsid,
|
|
'auth supported' => 'cephx',
|
|
'auth cluster required' => 'cephx',
|
|
'auth service required' => 'cephx',
|
|
'auth client required' => 'cephx',
|
|
'filestore xattr use omap' => 'true',
|
|
'osd journal size' => $pve_osd_default_journal_size,
|
|
'osd pool default min size' => 1,
|
|
};
|
|
|
|
# this does not work for default pools
|
|
#'osd pool default pg num' => $pg_num,
|
|
#'osd pool default pgp num' => $pg_num,
|
|
}
|
|
|
|
$cfg->{global}->{keyring} = '/etc/pve/priv/$cluster.$name.keyring';
|
|
$cfg->{osd}->{keyring} = '/var/lib/ceph/osd/ceph-$id/keyring';
|
|
|
|
$cfg->{global}->{'osd pool default size'} = $param->{size} if $param->{size};
|
|
|
|
if ($param->{pg_bits}) {
|
|
$cfg->{global}->{'osd pg bits'} = $param->{pg_bits};
|
|
$cfg->{global}->{'osd pgp bits'} = $param->{pg_bits};
|
|
}
|
|
|
|
if ($param->{network}) {
|
|
$cfg->{global}->{'public network'} = $param->{network};
|
|
$cfg->{global}->{'cluster network'} = $param->{network};
|
|
}
|
|
|
|
PVE::CephTools::write_ceph_config($cfg);
|
|
|
|
PVE::CephTools::setup_pve_symlinks();
|
|
|
|
return undef;
|
|
}});
|
|
|
|
my $find_node_ip = sub {
|
|
my ($cidr) = @_;
|
|
|
|
my $net = Net::IP->new($cidr) || die Net::IP::Error() . "\n";
|
|
my $id = $net->version == 6 ? 'address6' : 'address';
|
|
|
|
my $config = PVE::INotify::read_file('interfaces');
|
|
my $ifaces = $config->{ifaces};
|
|
|
|
foreach my $iface (keys %$ifaces) {
|
|
my $d = $config->{$iface};
|
|
next if !$d->{$id};
|
|
my $a = Net::IP->new($d->{$id});
|
|
next if !$a;
|
|
return $d->{$id} if $net->overlaps($a);
|
|
}
|
|
|
|
die "unable to find local address within network '$cidr'\n";
|
|
};
|
|
|
|
__PACKAGE__->register_method ({
|
|
name => 'createmon',
|
|
path => 'mon',
|
|
method => 'POST',
|
|
description => "Create Ceph Monitor",
|
|
proxyto => 'node',
|
|
protected => 1,
|
|
parameters => {
|
|
additionalProperties => 0,
|
|
properties => {
|
|
node => get_standard_option('pve-node'),
|
|
},
|
|
},
|
|
returns => { type => 'string' },
|
|
code => sub {
|
|
my ($param) = @_;
|
|
|
|
PVE::CephTools::check_ceph_inited();
|
|
|
|
PVE::CephTools::setup_pve_symlinks();
|
|
|
|
my $rpcenv = PVE::RPCEnvironment::get();
|
|
|
|
my $authuser = $rpcenv->get_user();
|
|
|
|
my $cfg = PVE::CephTools::parse_ceph_config();
|
|
|
|
my $moncount = 0;
|
|
|
|
my $monaddrhash = {};
|
|
|
|
foreach my $section (keys %$cfg) {
|
|
next if $section eq 'global';
|
|
my $d = $cfg->{$section};
|
|
if ($section =~ m/^mon\./) {
|
|
$moncount++;
|
|
if ($d->{'mon addr'}) {
|
|
$monaddrhash->{$d->{'mon addr'}} = $section;
|
|
}
|
|
}
|
|
}
|
|
|
|
my $monid;
|
|
for (my $i = 0; $i < 7; $i++) {
|
|
if (!$cfg->{"mon.$i"}) {
|
|
$monid = $i;
|
|
last;
|
|
}
|
|
}
|
|
die "unable to find usable monitor id\n" if !defined($monid);
|
|
|
|
my $monsection = "mon.$monid";
|
|
my $ip;
|
|
if (my $pubnet = $cfg->{global}->{'public network'}) {
|
|
$ip = &$find_node_ip($pubnet);
|
|
} else {
|
|
$ip = PVE::Cluster::remote_node_ip($param->{node});
|
|
}
|
|
|
|
my $monaddr = "$ip:6789";
|
|
my $monname = $param->{node};
|
|
|
|
die "monitor '$monsection' already exists\n" if $cfg->{$monsection};
|
|
die "monitor address '$monaddr' already in use by '$monaddrhash->{$monaddr}'\n"
|
|
if $monaddrhash->{$monaddr};
|
|
|
|
my $worker = sub {
|
|
my $upid = shift;
|
|
|
|
my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
|
|
|
|
if (! -f $pve_ckeyring_path) {
|
|
run_command("ceph-authtool $pve_ckeyring_path --create-keyring " .
|
|
"--gen-key -n client.admin");
|
|
}
|
|
|
|
my $pve_mon_key_path = PVE::CephTools::get_config('pve_mon_key_path');
|
|
if (! -f $pve_mon_key_path) {
|
|
run_command("cp $pve_ckeyring_path $pve_mon_key_path.tmp");
|
|
run_command("ceph-authtool $pve_mon_key_path.tmp -n client.admin --set-uid=0 " .
|
|
"--cap mds 'allow' " .
|
|
"--cap osd 'allow *' " .
|
|
"--cap mon 'allow *'");
|
|
run_command("ceph-authtool $pve_mon_key_path.tmp --gen-key -n mon. --cap mon 'allow *'");
|
|
run_command("mv $pve_mon_key_path.tmp $pve_mon_key_path");
|
|
}
|
|
|
|
my $ccname = PVE::CephTools::get_config('ccname');
|
|
|
|
my $mondir = "/var/lib/ceph/mon/$ccname-$monid";
|
|
-d $mondir && die "monitor filesystem '$mondir' already exist\n";
|
|
|
|
my $monmap = "/tmp/monmap";
|
|
|
|
eval {
|
|
mkdir $mondir;
|
|
|
|
if ($moncount > 0) {
|
|
my $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
|
|
my $mapdata = $rados->mon_command({ prefix => 'mon getmap', format => 'plain' });
|
|
PVE::Tools::file_set_contents($monmap, $mapdata);
|
|
} else {
|
|
run_command("monmaptool --create --clobber --add $monid $monaddr --print $monmap");
|
|
}
|
|
|
|
run_command("ceph-mon --mkfs -i $monid --monmap $monmap --keyring $pve_mon_key_path");
|
|
};
|
|
my $err = $@;
|
|
unlink $monmap;
|
|
if ($err) {
|
|
File::Path::remove_tree($mondir);
|
|
die $err;
|
|
}
|
|
|
|
$cfg->{$monsection} = {
|
|
'host' => $monname,
|
|
'mon addr' => $monaddr,
|
|
};
|
|
|
|
PVE::CephTools::write_ceph_config($cfg);
|
|
|
|
PVE::CephTools::ceph_service_cmd('start', $monsection);
|
|
};
|
|
|
|
return $rpcenv->fork_worker('cephcreatemon', $monsection, $authuser, $worker);
|
|
}});
|
|
|
|
__PACKAGE__->register_method ({
|
|
name => 'destroymon',
|
|
path => 'mon/{monid}',
|
|
method => 'DELETE',
|
|
description => "Destroy Ceph monitor.",
|
|
proxyto => 'node',
|
|
protected => 1,
|
|
parameters => {
|
|
additionalProperties => 0,
|
|
properties => {
|
|
node => get_standard_option('pve-node'),
|
|
monid => {
|
|
description => 'Monitor ID',
|
|
type => 'integer',
|
|
},
|
|
},
|
|
},
|
|
returns => { type => 'string' },
|
|
code => sub {
|
|
my ($param) = @_;
|
|
|
|
my $rpcenv = PVE::RPCEnvironment::get();
|
|
|
|
my $authuser = $rpcenv->get_user();
|
|
|
|
PVE::CephTools::check_ceph_inited();
|
|
|
|
my $cfg = PVE::CephTools::parse_ceph_config();
|
|
|
|
my $monid = $param->{monid};
|
|
my $monsection = "mon.$monid";
|
|
|
|
my $rados = PVE::RADOS->new();
|
|
my $monstat = $rados->mon_command({ prefix => 'mon_status' });
|
|
my $monlist = $monstat->{monmap}->{mons};
|
|
|
|
die "no such monitor id '$monid'\n"
|
|
if !defined($cfg->{$monsection});
|
|
|
|
my $ccname = PVE::CephTools::get_config('ccname');
|
|
|
|
my $mondir = "/var/lib/ceph/mon/$ccname-$monid";
|
|
-d $mondir || die "monitor filesystem '$mondir' does not exist on this node\n";
|
|
|
|
die "can't remove last monitor\n" if scalar(@$monlist) <= 1;
|
|
|
|
my $worker = sub {
|
|
my $upid = shift;
|
|
|
|
# reopen with longer timeout
|
|
$rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
|
|
|
|
$rados->mon_command({ prefix => "mon remove", name => $monid, format => 'plain' });
|
|
|
|
eval { PVE::CephTools::ceph_service_cmd('stop', $monsection); };
|
|
warn $@ if $@;
|
|
|
|
delete $cfg->{$monsection};
|
|
PVE::CephTools::write_ceph_config($cfg);
|
|
File::Path::remove_tree($mondir);
|
|
};
|
|
|
|
return $rpcenv->fork_worker('cephdestroymon', $monsection, $authuser, $worker);
|
|
}});
|
|
|
|
__PACKAGE__->register_method ({
|
|
name => 'stop',
|
|
path => 'stop',
|
|
method => 'POST',
|
|
description => "Stop ceph services.",
|
|
proxyto => 'node',
|
|
protected => 1,
|
|
parameters => {
|
|
additionalProperties => 0,
|
|
properties => {
|
|
node => get_standard_option('pve-node'),
|
|
service => {
|
|
description => 'Ceph service name.',
|
|
type => 'string',
|
|
optional => 1,
|
|
pattern => '(mon|mds|osd)\.[A-Za-z0-9]{1,32}',
|
|
},
|
|
},
|
|
},
|
|
returns => { type => 'string' },
|
|
code => sub {
|
|
my ($param) = @_;
|
|
|
|
my $rpcenv = PVE::RPCEnvironment::get();
|
|
|
|
my $authuser = $rpcenv->get_user();
|
|
|
|
PVE::CephTools::check_ceph_inited();
|
|
|
|
my $cfg = PVE::CephTools::parse_ceph_config();
|
|
scalar(keys %$cfg) || die "no configuration\n";
|
|
|
|
my $worker = sub {
|
|
my $upid = shift;
|
|
|
|
my $cmd = ['stop'];
|
|
if ($param->{service}) {
|
|
push @$cmd, $param->{service};
|
|
}
|
|
|
|
PVE::CephTools::ceph_service_cmd(@$cmd);
|
|
};
|
|
|
|
return $rpcenv->fork_worker('srvstop', $param->{service} || 'ceph',
|
|
$authuser, $worker);
|
|
}});
|
|
|
|
__PACKAGE__->register_method ({
|
|
name => 'start',
|
|
path => 'start',
|
|
method => 'POST',
|
|
description => "Start ceph services.",
|
|
proxyto => 'node',
|
|
protected => 1,
|
|
parameters => {
|
|
additionalProperties => 0,
|
|
properties => {
|
|
node => get_standard_option('pve-node'),
|
|
service => {
|
|
description => 'Ceph service name.',
|
|
type => 'string',
|
|
optional => 1,
|
|
pattern => '(mon|mds|osd)\.[A-Za-z0-9]{1,32}',
|
|
},
|
|
},
|
|
},
|
|
returns => { type => 'string' },
|
|
code => sub {
|
|
my ($param) = @_;
|
|
|
|
my $rpcenv = PVE::RPCEnvironment::get();
|
|
|
|
my $authuser = $rpcenv->get_user();
|
|
|
|
PVE::CephTools::check_ceph_inited();
|
|
|
|
my $cfg = PVE::CephTools::parse_ceph_config();
|
|
scalar(keys %$cfg) || die "no configuration\n";
|
|
|
|
my $worker = sub {
|
|
my $upid = shift;
|
|
|
|
my $cmd = ['start'];
|
|
if ($param->{service}) {
|
|
push @$cmd, $param->{service};
|
|
}
|
|
|
|
PVE::CephTools::ceph_service_cmd(@$cmd);
|
|
};
|
|
|
|
return $rpcenv->fork_worker('srvstart', $param->{service} || 'ceph',
|
|
$authuser, $worker);
|
|
}});
|
|
|
|
__PACKAGE__->register_method ({
|
|
name => 'status',
|
|
path => 'status',
|
|
method => 'GET',
|
|
description => "Get ceph status.",
|
|
proxyto => 'node',
|
|
protected => 1,
|
|
parameters => {
|
|
additionalProperties => 0,
|
|
properties => {
|
|
node => get_standard_option('pve-node'),
|
|
},
|
|
},
|
|
returns => { type => 'object' },
|
|
code => sub {
|
|
my ($param) = @_;
|
|
|
|
PVE::CephTools::check_ceph_enabled();
|
|
|
|
my $rados = PVE::RADOS->new();
|
|
return $rados->mon_command({ prefix => 'status' });
|
|
}});
|
|
|
|
__PACKAGE__->register_method ({
|
|
name => 'lspools',
|
|
path => 'pools',
|
|
method => 'GET',
|
|
description => "List all pools.",
|
|
proxyto => 'node',
|
|
protected => 1,
|
|
parameters => {
|
|
additionalProperties => 0,
|
|
properties => {
|
|
node => get_standard_option('pve-node'),
|
|
},
|
|
},
|
|
returns => {
|
|
type => 'array',
|
|
items => {
|
|
type => "object",
|
|
properties => {
|
|
pool => { type => 'integer' },
|
|
pool_name => { type => 'string' },
|
|
size => { type => 'integer' },
|
|
},
|
|
},
|
|
links => [ { rel => 'child', href => "{pool_name}" } ],
|
|
},
|
|
code => sub {
|
|
my ($param) = @_;
|
|
|
|
PVE::CephTools::check_ceph_inited();
|
|
|
|
my $rados = PVE::RADOS->new();
|
|
|
|
my $stats = {};
|
|
my $res = $rados->mon_command({ prefix => 'df' });
|
|
my $total = $res->{stats}->{total_space} || 0;
|
|
$total = $total * 1024;
|
|
foreach my $d (@{$res->{pools}}) {
|
|
next if !$d->{stats};
|
|
next if !defined($d->{id});
|
|
$stats->{$d->{id}} = $d->{stats};
|
|
}
|
|
|
|
$res = $rados->mon_command({ prefix => 'osd dump' });
|
|
|
|
my $data = [];
|
|
foreach my $e (@{$res->{pools}}) {
|
|
my $d = {};
|
|
foreach my $attr (qw(pool pool_name size min_size pg_num crush_ruleset)) {
|
|
$d->{$attr} = $e->{$attr} if defined($e->{$attr});
|
|
}
|
|
if (my $s = $stats->{$d->{pool}}) {
|
|
$d->{bytes_used} = $s->{bytes_used};
|
|
$d->{percent_used} = ($d->{bytes_used}*$d->{size}*100)/$total
|
|
if $d->{size} && $total;
|
|
}
|
|
push @$data, $d;
|
|
}
|
|
|
|
|
|
return $data;
|
|
}});
|
|
|
|
__PACKAGE__->register_method ({
|
|
name => 'createpool',
|
|
path => 'pools',
|
|
method => 'POST',
|
|
description => "Create POOL",
|
|
proxyto => 'node',
|
|
protected => 1,
|
|
parameters => {
|
|
additionalProperties => 0,
|
|
properties => {
|
|
node => get_standard_option('pve-node'),
|
|
name => {
|
|
description => "The name of the pool. It must be unique.",
|
|
type => 'string',
|
|
},
|
|
size => {
|
|
description => 'Number of replicas per object',
|
|
type => 'integer',
|
|
default => 2,
|
|
optional => 1,
|
|
minimum => 1,
|
|
maximum => 3,
|
|
},
|
|
min_size => {
|
|
description => 'Minimum number of replicas per object',
|
|
type => 'integer',
|
|
default => 1,
|
|
optional => 1,
|
|
minimum => 1,
|
|
maximum => 3,
|
|
},
|
|
pg_num => {
|
|
description => "Number of placement groups.",
|
|
type => 'integer',
|
|
default => 64,
|
|
optional => 1,
|
|
minimum => 8,
|
|
maximum => 32768,
|
|
},
|
|
crush_ruleset => {
|
|
description => "The ruleset to use for mapping object placement in the cluster.",
|
|
type => 'integer',
|
|
minimum => 0,
|
|
maximum => 32768,
|
|
default => 0,
|
|
optional => 1,
|
|
},
|
|
},
|
|
},
|
|
returns => { type => 'null' },
|
|
code => sub {
|
|
my ($param) = @_;
|
|
|
|
PVE::CephTools::check_ceph_inited();
|
|
|
|
my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
|
|
|
|
die "not fully configured - missing '$pve_ckeyring_path'\n"
|
|
if ! -f $pve_ckeyring_path;
|
|
|
|
my $pg_num = $param->{pg_num} || 64;
|
|
my $size = $param->{size} || 2;
|
|
my $min_size = $param->{min_size} || 1;
|
|
my $ruleset = $param->{crush_ruleset} || 0;
|
|
my $rados = PVE::RADOS->new();
|
|
|
|
$rados->mon_command({
|
|
prefix => "osd pool create",
|
|
pool => $param->{name},
|
|
pg_num => int($pg_num),
|
|
# this does not work for unknown reason
|
|
# properties => ["size=$size", "min_size=$min_size", "crush_ruleset=$ruleset"],
|
|
format => 'plain',
|
|
});
|
|
|
|
$rados->mon_command({
|
|
prefix => "osd pool set",
|
|
pool => $param->{name},
|
|
var => 'min_size',
|
|
val => $min_size,
|
|
format => 'plain',
|
|
});
|
|
|
|
$rados->mon_command({
|
|
prefix => "osd pool set",
|
|
pool => $param->{name},
|
|
var => 'size',
|
|
val => $size,
|
|
format => 'plain',
|
|
});
|
|
|
|
if (defined($param->{crush_ruleset})) {
|
|
$rados->mon_command({
|
|
prefix => "osd pool set",
|
|
pool => $param->{name},
|
|
var => 'crush_ruleset',
|
|
val => $param->{crush_ruleset},
|
|
format => 'plain',
|
|
});
|
|
}
|
|
|
|
return undef;
|
|
}});
|
|
|
|
__PACKAGE__->register_method ({
|
|
name => 'destroypool',
|
|
path => 'pools/{name}',
|
|
method => 'DELETE',
|
|
description => "Destroy pool",
|
|
proxyto => 'node',
|
|
protected => 1,
|
|
parameters => {
|
|
additionalProperties => 0,
|
|
properties => {
|
|
node => get_standard_option('pve-node'),
|
|
name => {
|
|
description => "The name of the pool. It must be unique.",
|
|
type => 'string',
|
|
},
|
|
},
|
|
},
|
|
returns => { type => 'null' },
|
|
code => sub {
|
|
my ($param) = @_;
|
|
|
|
PVE::CephTools::check_ceph_inited();
|
|
|
|
my $rados = PVE::RADOS->new();
|
|
# fixme: '--yes-i-really-really-mean-it'
|
|
$rados->mon_command({
|
|
prefix => "osd pool delete",
|
|
pool => $param->{name},
|
|
pool2 => $param->{name},
|
|
sure => '--yes-i-really-really-mean-it',
|
|
format => 'plain',
|
|
});
|
|
|
|
return undef;
|
|
}});
|
|
|
|
|
|
__PACKAGE__->register_method ({
|
|
name => 'crush',
|
|
path => 'crush',
|
|
method => 'GET',
|
|
description => "Get OSD crush map",
|
|
proxyto => 'node',
|
|
protected => 1,
|
|
parameters => {
|
|
additionalProperties => 0,
|
|
properties => {
|
|
node => get_standard_option('pve-node'),
|
|
},
|
|
},
|
|
returns => { type => 'string' },
|
|
code => sub {
|
|
my ($param) = @_;
|
|
|
|
PVE::CephTools::check_ceph_inited();
|
|
|
|
# this produces JSON (difficult to read for the user)
|
|
# my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1);
|
|
|
|
my $txt = '';
|
|
|
|
my $mapfile = "/var/tmp/ceph-crush.map.$$";
|
|
my $mapdata = "/var/tmp/ceph-crush.txt.$$";
|
|
|
|
my $rados = PVE::RADOS->new();
|
|
|
|
eval {
|
|
my $bindata = $rados->mon_command({ prefix => 'osd getcrushmap', format => 'plain' });
|
|
PVE::Tools::file_set_contents($mapfile, $bindata);
|
|
run_command(['crushtool', '-d', $mapfile, '-o', $mapdata]);
|
|
$txt = PVE::Tools::file_get_contents($mapdata);
|
|
};
|
|
my $err = $@;
|
|
|
|
unlink $mapfile;
|
|
unlink $mapdata;
|
|
|
|
die $err if $err;
|
|
|
|
return $txt;
|
|
}});
|
|
|
|
__PACKAGE__->register_method({
|
|
name => 'log',
|
|
path => 'log',
|
|
method => 'GET',
|
|
description => "Read ceph log",
|
|
proxyto => 'node',
|
|
permissions => {
|
|
check => ['perm', '/nodes/{node}', [ 'Sys.Syslog' ]],
|
|
},
|
|
protected => 1,
|
|
parameters => {
|
|
additionalProperties => 0,
|
|
properties => {
|
|
node => get_standard_option('pve-node'),
|
|
start => {
|
|
type => 'integer',
|
|
minimum => 0,
|
|
optional => 1,
|
|
},
|
|
limit => {
|
|
type => 'integer',
|
|
minimum => 0,
|
|
optional => 1,
|
|
},
|
|
},
|
|
},
|
|
returns => {
|
|
type => 'array',
|
|
items => {
|
|
type => "object",
|
|
properties => {
|
|
n => {
|
|
description=> "Line number",
|
|
type=> 'integer',
|
|
},
|
|
t => {
|
|
description=> "Line text",
|
|
type => 'string',
|
|
}
|
|
}
|
|
}
|
|
},
|
|
code => sub {
|
|
my ($param) = @_;
|
|
|
|
my $rpcenv = PVE::RPCEnvironment::get();
|
|
my $user = $rpcenv->get_user();
|
|
my $node = $param->{node};
|
|
|
|
my $logfile = "/var/log/ceph/ceph.log";
|
|
my ($count, $lines) = PVE::Tools::dump_logfile($logfile, $param->{start}, $param->{limit});
|
|
|
|
$rpcenv->set_result_attrib('total', $count);
|
|
|
|
return $lines;
|
|
}});
|
|
|
|
|