5
0
mirror of git://git.proxmox.com/git/pve-storage.git synced 2024-12-22 13:34:16 +03:00

Cephfs storage plugin

- ability to mount through kernel and fuse client
 - allow mount options
 - get MONs from ceph config if not in storage.cfg
 - allow the use of ceph config with fuse client
 - Delete secret on cephfs storage creation

Signed-off-by: Alwin Antreich <a.antreich@proxmox.com>
This commit is contained in:
Alwin Antreich 2018-07-04 12:43:31 +02:00 committed by Thomas Lamprecht
parent 3e47917203
commit e34ce14443
6 changed files with 301 additions and 1 deletions

View File

@ -28,6 +28,7 @@ use PVE::Storage::NFSPlugin;
use PVE::Storage::CIFSPlugin;
use PVE::Storage::ISCSIPlugin;
use PVE::Storage::RBDPlugin;
use PVE::Storage::CephFSPlugin;
use PVE::Storage::SheepdogPlugin;
use PVE::Storage::ISCSIDirectPlugin;
use PVE::Storage::GlusterfsPlugin;
@ -46,6 +47,7 @@ PVE::Storage::NFSPlugin->register();
PVE::Storage::CIFSPlugin->register();
PVE::Storage::ISCSIPlugin->register();
PVE::Storage::RBDPlugin->register();
PVE::Storage::CephFSPlugin->register();
PVE::Storage::SheepdogPlugin->register();
PVE::Storage::ISCSIDirectPlugin->register();
PVE::Storage::GlusterfsPlugin->register();

194
PVE/Storage/CephFSPlugin.pm Normal file
View File

@ -0,0 +1,194 @@
package PVE::Storage::CephFSPlugin;
use strict;
use warnings;
use IO::File;
use Net::IP;
use File::Path;
use PVE::Tools qw(run_command);
use PVE::ProcFSTools;
use PVE::Storage::Plugin;
use PVE::JSONSchema qw(get_standard_option);
use PVE::Storage::CephTools;
use base qw(PVE::Storage::Plugin);
sub cephfs_is_mounted {
my ($scfg, $storeid, $mountdata) = @_;
my $cmd_option = PVE::Storage::CephTools::ceph_connect_option($scfg, $storeid);
my $configfile = $cmd_option->{ceph_conf};
my $server = $cmd_option->{mon_host} // PVE::Storage::CephTools::get_monaddr_list($configfile);
my $subdir = $scfg->{subdir} // '/';
my $mountpoint = $scfg->{path};
my $source = "$server:$subdir";
$mountdata = PVE::ProcFSTools::parse_proc_mounts() if !$mountdata;
return $mountpoint if grep {
$_->[2] =~ m#^ceph|fuse\.ceph-fuse# &&
$_->[0] =~ m#^\Q$source\E|ceph-fuse$# &&
$_->[1] eq $mountpoint
} @$mountdata;
warn "A filesystem is already mounted on $mountpoint\n"
if grep { $_->[1] eq $mountpoint } @$mountdata;
return undef;
}
sub cephfs_mount {
my ($scfg, $storeid) = @_;
my $cmd;
my $mountpoint = $scfg->{path};
my $subdir = $scfg->{subdir} // '/';
my $cmd_option = PVE::Storage::CephTools::ceph_connect_option($scfg, $storeid);
my $configfile = $cmd_option->{ceph_conf};
my $secretfile = $cmd_option->{keyring};
my $server = $cmd_option->{mon_host} // PVE::Storage::CephTools::get_monaddr_list($configfile);
# fuse -> client-enforced quotas (kernel doesn't), updates w/ ceph-fuse pkg
# kernel -> better performance, less frequent updates
if ($scfg->{fuse}) {
# FIXME: ceph-fuse client complains about missing ceph.conf or keyring if
# not provided on its default locations but still connects. Fix upstream??
$cmd = ['/usr/bin/ceph-fuse', '-n', "client.$cmd_option->{userid}", '-m', $server];
push @$cmd, '--keyfile', $secretfile if defined($secretfile);
push @$cmd, '-r', $subdir if !($subdir =~ m|^/$|);
push @$cmd, $mountpoint;
push @$cmd, '--conf', $configfile if defined($configfile);
}else {
my $source = "$server:$subdir";
$cmd = ['/bin/mount', '-t', 'ceph', $source, $mountpoint, '-o', "name=$cmd_option->{userid}"];
push @$cmd, '-o', "secretfile=$secretfile" if defined($secretfile);
}
if ($scfg->{options}) {
push @$cmd, '-o', $scfg->{options};
}
run_command($cmd, errmsg => "mount error");
}
# Configuration
sub type {
return 'cephfs';
}
sub plugindata {
return {
content => [ { vztmpl => 1, iso => 1, backup => 1},
{ backup => 1 }],
};
}
sub properties {
return {
fuse => {
description => "Mount CephFS through FUSE.",
type => 'boolean',
},
subdir => {
description => "Subdir to mount.",
type => 'string', format => 'pve-storage-path',
},
};
}
sub options {
return {
path => { fixed => 1 },
monhost => { optional => 1},
nodes => { optional => 1 },
subdir => { optional => 1 },
disable => { optional => 1 },
options => { optional => 1 },
username => { optional => 1 },
content => { optional => 1 },
format => { optional => 1 },
mkdir => { optional => 1 },
fuse => { optional => 1 },
bwlimit => { optional => 1 },
};
}
sub check_config {
my ($class, $sectionId, $config, $create, $skipSchemaCheck) = @_;
$config->{path} = "/mnt/pve/$sectionId" if $create && !$config->{path};
return $class->SUPER::check_config($sectionId, $config, $create, $skipSchemaCheck);
}
# Storage implementation
sub on_add_hook {
my ($class, $storeid, $scfg, %param) = @_;
return if defined($scfg->{monhost}); # nothing to do if not pve managed ceph
PVE::Storage::CephTools::ceph_create_keyfile($scfg->{type}, $storeid);
}
sub on_delete_hook {
my ($class, $storeid, $scfg) = @_;
return if defined($scfg->{monhost}); # nothing to do if not pve managed ceph
PVE::Storage::CephTools::ceph_remove_keyfile($scfg->{type}, $storeid);
}
sub status {
my ($class, $storeid, $scfg, $cache) = @_;
$cache->{mountdata} = PVE::ProcFSTools::parse_proc_mounts()
if !$cache->{mountdata};
return undef if !cephfs_is_mounted($scfg, $storeid, $cache->{mountdata});
return $class->SUPER::status($storeid, $scfg, $cache);
}
sub activate_storage {
my ($class, $storeid, $scfg, $cache) = @_;
$cache->{mountdata} = PVE::ProcFSTools::parse_proc_mounts()
if !$cache->{mountdata};
my $path = $scfg->{path};
if (!cephfs_is_mounted($scfg, $storeid, $cache->{mountdata})) {
# NOTE: only call mkpath when not mounted (avoid hang
# when cephfs is offline
mkpath $path if !(defined($scfg->{mkdir}) && !$scfg->{mkdir});
die "unable to activate storage '$storeid' - " .
"directory '$path' does not exist\n" if ! -d $path;
cephfs_mount($scfg, $storeid);
}
$class->SUPER::activate_storage($storeid, $scfg, $cache);
}
sub deactivate_storage {
my ($class, $storeid, $scfg, $cache) = @_;
$cache->{mountdata} = PVE::ProcFSTools::parse_proc_mounts()
if !$cache->{mountdata};
my $path = $scfg->{path};
if (cephfs_is_mounted($scfg, $storeid, $cache->{mountdata})) {
my $cmd = ['/bin/umount', $path];
run_command($cmd, errmsg => 'umount error');
}
}
1;

View File

@ -34,6 +34,64 @@ my $ceph_check_keyfile = sub {
return undef;
};
my $parse_ceph_file = sub {
my ($filename) = @_;
my $cfg = {};
return $cfg if ! -f $filename;
my $content = PVE::Tools::file_get_contents($filename);
my @lines = split /\n/, $content;
my $section;
foreach my $line (@lines) {
$line =~ s/[;#].*$//;
$line =~ s/^\s+//;
$line =~ s/\s+$//;
next if !$line;
$section = $1 if $line =~ m/^\[(\S+)\]$/;
if (!$section) {
warn "no section - skip: $line\n";
next;
}
if ($line =~ m/^(.*?\S)\s*=\s*(\S.*)$/) {
$cfg->{$section}->{$1} = $2;
}
}
return $cfg;
};
my $ceph_get_key = sub {
my ($keyfile, $username) = @_;
my $key = $parse_ceph_file->($keyfile);
my $secret = $key->{"client.$username"}->{key};
return $secret;
};
sub get_monaddr_list {
my ($configfile) = shift;
my $server;
if (!defined($configfile)) {
warn "No ceph config specified\n";
return;
}
my $config = $parse_ceph_file->($configfile);
@$server = sort map { $config->{$_}->{'mon addr'} } grep {/mon/} %{$config};
return join(',', @$server);
};
sub hostlist {
my ($list_text, $separator) = @_;
@ -85,4 +143,48 @@ sub ceph_connect_option {
}
sub ceph_create_keyfile {
my ($type, $storeid) = @_;
my $extension = 'keyring';
$extension = 'secret' if ($type eq 'cephfs');
my $ceph_admin_keyring = '/etc/pve/priv/ceph.client.admin.keyring';
my $ceph_storage_keyring = "/etc/pve/priv/ceph/${storeid}.$extension";
die "ceph authx keyring file for storage '$storeid' already exists!\n"
if -e $ceph_storage_keyring;
if (-e $ceph_admin_keyring) {
eval {
if ($type eq 'rbd') {
mkdir '/etc/pve/priv/ceph';
PVE::Tools::file_copy($ceph_admin_keyring, $ceph_storage_keyring);
} elsif ($type eq 'cephfs') {
my $secret = $ceph_get_key->($ceph_admin_keyring, 'admin');
mkdir '/etc/pve/priv/ceph';
PVE::Tools::file_set_contents($ceph_storage_keyring, $secret, 0400);
}
};
if (my $err = $@) {
unlink $ceph_storage_keyring;
die "failed to copy ceph authx $extension for storage '$storeid': $err\n";
}
} else {
warn "$ceph_admin_keyring not found, authentication is disabled.\n";
}
}
sub ceph_remove_keyfile {
my ($type, $storeid) = @_;
my $extension = 'keyring';
$extension = 'secret' if ($type eq 'cephfs');
my $ceph_storage_keyring = "/etc/pve/priv/ceph/${storeid}.$extension";
if (-f $ceph_storage_keyring) {
unlink($ceph_storage_keyring) or warn "removing keyring of storage failed: $!\n";
}
}
1;

View File

@ -1,4 +1,4 @@
SOURCES=Plugin.pm DirPlugin.pm LVMPlugin.pm NFSPlugin.pm CIFSPlugin.pm ISCSIPlugin.pm RBDPlugin.pm CephTools.pm SheepdogPlugin.pm ISCSIDirectPlugin.pm GlusterfsPlugin.pm ZFSPoolPlugin.pm ZFSPlugin.pm DRBDPlugin.pm LvmThinPlugin.pm
SOURCES=Plugin.pm DirPlugin.pm LVMPlugin.pm NFSPlugin.pm CIFSPlugin.pm ISCSIPlugin.pm CephFSPlugin.pm RBDPlugin.pm CephTools.pm SheepdogPlugin.pm ISCSIDirectPlugin.pm GlusterfsPlugin.pm ZFSPoolPlugin.pm ZFSPlugin.pm DRBDPlugin.pm LvmThinPlugin.pm
.PHONY: install
install:

View File

@ -24,6 +24,7 @@ our @SHARED_STORAGE = (
'nfs',
'cifs',
'rbd',
'cephfs',
'sheepdog',
'iscsidirect',
'glusterfs',

1
debian/control vendored
View File

@ -28,6 +28,7 @@ Depends: cstream,
udev,
smbclient,
ceph-common,
ceph-fuse,
cifs-utils,
${perl:Depends},
Description: Proxmox VE storage management library