diff --git a/.gitignore b/.gitignore index 3c41e90836..739c281c6a 100644 --- a/.gitignore +++ b/.gitignore @@ -66,6 +66,11 @@ src/fireedge/yarn.lock src/onedb/local/ src/onedb/shared/ +# Enterprise part of onecfg +src/onecfg/lib/ee.rb +src/onecfg/lib/ee/ +src/onecfg/share/migrators/ + share/esx-fw-vnc/*.rpm share/esx-fw-vnc/.vagrant* diff --git a/install.sh b/install.sh index 4c71b8fefb..afac872670 100755 --- a/install.sh +++ b/install.sh @@ -258,7 +258,10 @@ SHARE_DIRS="$SHARE_LOCATION/examples \ $SHARE_LOCATION/ssh \ $SHARE_LOCATION/start-scripts \ $SHARE_LOCATION/conf \ - $SHARE_LOCATION/context" + $SHARE_LOCATION/context \ + $SHARE_LOCATION/onecfg + $SHARE_LOCATION/onecfg/augeas \ + $SHARE_LOCATION/onecfg/etc" ETC_DIRS="$ETC_LOCATION/vmm_exec \ $ETC_LOCATION/hm \ @@ -301,7 +304,15 @@ LIB_DIRS="$LIB_LOCATION/ruby \ $LIB_LOCATION/oneprovision/lib/provider \ $LIB_LOCATION/oneprovision/lib/provision/resources \ $LIB_LOCATION/oneprovision/lib/provision/resources/virtual \ - $LIB_LOCATION/oneprovision/lib/provision/resources/physical" + $LIB_LOCATION/oneprovision/lib/provision/resources/physical + $LIB_LOCATION/onecfg/lib \ + $LIB_LOCATION/onecfg/lib/common \ + $LIB_LOCATION/onecfg/lib/common/helpers \ + $LIB_LOCATION/onecfg/lib/common/logger \ + $LIB_LOCATION/onecfg/lib/config \ + $LIB_LOCATION/onecfg/lib/config/type \ + $LIB_LOCATION/onecfg/lib/config/type/augeas \ + $LIB_LOCATION/onecfg/lib/config/type/yaml" VAR_DIRS="$VAR_LOCATION/remotes \ $VAR_LOCATION/remotes/etc \ @@ -736,6 +747,20 @@ INSTALL_ONEPROVISION_FILES=( ONEPROVISION_LIB_PROVISION_TEMPLATE_FILES:$LIB_LOCATION/oneprovision/lib/provision_template ) +INSTALL_ONECFG_FILES=( + ONECFG_BIN_FILES:$BIN_LOCATION + ONECFG_LIB_FILES:$LIB_LOCATION/onecfg/lib + ONECFG_LIB_COMMON_FILES:$LIB_LOCATION/onecfg/lib/common + ONECFG_LIB_COMMON_HELPERS_FILES:$LIB_LOCATION/onecfg/lib/common/helpers + ONECFG_LIB_COMMON_LOGGER_FILES:$LIB_LOCATION/onecfg/lib/common/logger + ONECFG_LIB_CONFIG_FILES:$LIB_LOCATION/onecfg/lib/config + ONECFG_LIB_CONFIG_TYPE_FILES:$LIB_LOCATION/onecfg/lib/config/type + ONECFG_LIB_CONFIG_TYPE_AUGEAS_FILES:$LIB_LOCATION/onecfg/lib/config/type/augeas + ONECFG_LIB_CONFIG_TYPE_YAML_FILES:$LIB_LOCATION/onecfg/lib/config/type/yaml + ONECFG_SHARE_AUGEAS_FILES:$SHARE_LOCATION/onecfg/augeas + ONECFG_SHARE_ETC_FILES:$SHARE_LOCATION/onecfg/etc +) + INSTALL_SUNSTONE_RUBY_FILES=( RUBY_OPENNEBULA_LIB_FILES:$LIB_LOCATION/ruby/opennebula RUBY_OPENNEBULA_LIB_FLOW_FILES:$LIB_LOCATION/ruby/opennebula/flow @@ -2666,6 +2691,38 @@ ONEFLOW_LIB_STRATEGY_FILES="src/flow/lib/strategy/straight.rb" ONEFLOW_LIB_MODELS_FILES="src/flow/lib/models/role.rb \ src/flow/lib/models/service.rb" +#----------------------------------------------------------------------------- +# Onecfg files +#----------------------------------------------------------------------------- + +ONECFG_BIN_FILES="src/onecfg/bin/onecfg" + +ONECFG_LIB_FILES="src/onecfg/lib/onecfg.rb + src/onecfg/lib/common.rb \ + src/onecfg/lib/config.rb \ + src/onecfg/lib/exception.rb \ + src/onecfg/lib/settings.rb \ + src/onecfg/lib/version.rb" +ONECFG_LIB_COMMON_FILES="src/onecfg/lib/common/backup.rb" +ONECFG_LIB_COMMON_HELPERS_FILES="src/onecfg/lib/common/helpers/onecfg_helper.rb" +ONECFG_LIB_COMMON_LOGGER_FILES="src/onecfg/lib/common/logger/cli_logger.rb" +ONECFG_LIB_CONFIG_FILES="src/onecfg/lib/config/exception.rb \ + src/onecfg/lib/config/fsops.rb \ + src/onecfg/lib/config/type.rb \ + src/onecfg/lib/config/utils.rb" +ONECFG_LIB_CONFIG_TYPE_FILES="src/onecfg/lib/config/type/augeas.rb \ + src/onecfg/lib/config/type/base.rb \ + src/onecfg/lib/config/type/simple.rb \ + src/onecfg/lib/config/type/yaml.rb" +ONECFG_LIB_CONFIG_TYPE_AUGEAS_FILES="src/onecfg/lib/config/type/augeas/one.rb \ + src/onecfg/lib/config/type/augeas/shell.rb" +ONECFG_LIB_CONFIG_TYPE_YAML_FILES="src/onecfg/lib/config/type/yaml/strict.rb" + +ONECFG_SHARE_AUGEAS_FILES="src/onecfg/share/augeas/oned.aug \ + src/onecfg/share/augeas/test_oned.aug" +ONECFG_SHARE_ETC_FILES="src/onecfg/share/etc/files.yaml" + + #----------------------------------------------------------------------------- # OneHem files #----------------------------------------------------------------------------- @@ -2887,14 +2944,16 @@ elif [ "$SUNSTONE_DEV" = "no" ]; then ${INSTALL_ONEGATE_FILES[@]} \ ${INSTALL_ONEFLOW_FILES[@]} \ ${INSTALL_ONEHEM_FILES[@]} \ - ${INSTALL_ONEPROVISION_FILES[@]}" + ${INSTALL_ONEPROVISION_FILES[@]} \ + ${INSTALL_ONECFG_FILES[@]}" elif [ "$FIREEDGE_DEV" = "no" ]; then INSTALL_SET="${INSTALL_FILES[@]} \ ${INSTALL_FIREEDGE_MINIFIED_DIRS[@]}\ ${INSTALL_ONEGATE_FILES[@]} \ ${INSTALL_ONEFLOW_FILES[@]} \ ${INSTALL_ONEHEM_FILES[@]} \ - ${INSTALL_ONEPROVISION_FILES[@]}" + ${INSTALL_ONEPROVISION_FILES[@]} \ + ${INSTALL_ONECFG_FILES[@]}" else INSTALL_SET="${INSTALL_FILES[@]} \ ${INSTALL_SUNSTONE_FILES[@]} ${INSTALL_SUNSTONE_PUBLIC_DEV_DIR[@]}\ @@ -2902,7 +2961,8 @@ else ${INSTALL_ONEGATE_FILES[@]} \ ${INSTALL_ONEFLOW_FILES[@]} \ ${INSTALL_ONEHEM_FILES[@]} \ - ${INSTALL_ONEPROVISION_FILES[@]}" + ${INSTALL_ONEPROVISION_FILES[@]} \ + ${INSTALL_ONECFG_FILES[@]}" fi for i in ${INSTALL_SET[@]}; do diff --git a/share/install_gems/CentOS7/Gemfile.lock b/share/install_gems/CentOS7/Gemfile.lock index 05e3661c2d..87c33be978 100644 --- a/share/install_gems/CentOS7/Gemfile.lock +++ b/share/install_gems/CentOS7/Gemfile.lock @@ -12,28 +12,28 @@ GEM xml-simple (>= 1.0.12) augeas (0.6.4) aws-eventstream (1.1.0) - aws-partitions (1.375.0) - aws-sdk-cloudwatch (1.44.0) - aws-sdk-core (~> 3, >= 3.99.0) + aws-partitions (1.391.0) + aws-sdk-cloudwatch (1.46.0) + aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) - aws-sdk-core (3.107.0) + aws-sdk-core (3.109.2) aws-eventstream (~> 1, >= 1.0.2) aws-partitions (~> 1, >= 1.239.0) aws-sigv4 (~> 1.1) jmespath (~> 1.0) - aws-sdk-ec2 (1.195.0) - aws-sdk-core (~> 3, >= 3.99.0) + aws-sdk-ec2 (1.209.0) + aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) - aws-sdk-kms (1.38.0) - aws-sdk-core (~> 3, >= 3.99.0) + aws-sdk-kms (1.39.0) + aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) - aws-sdk-s3 (1.81.0) - aws-sdk-core (~> 3, >= 3.104.3) + aws-sdk-s3 (1.84.1) + aws-sdk-core (~> 3, >= 3.109.0) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.1) aws-sigv4 (1.2.2) aws-eventstream (~> 1, >= 1.0.2) - azure_mgmt_compute (0.19.3) + azure_mgmt_compute (0.20.0) ms_rest_azure (~> 0.12.0) azure_mgmt_monitor (0.17.6) ms_rest_azure (~> 0.12.0) @@ -44,17 +44,14 @@ GEM azure_mgmt_storage (0.22.0) ms_rest_azure (~> 0.12.0) builder (3.2.4) - chunky_png (1.3.12) + chunky_png (1.3.14) concurrent-ruby (1.1.7) configparser (0.1.7) - curb (0.9.10) + curb (0.9.11) daemons (1.3.1) - dalli (2.7.10) + dalli (2.7.11) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - em-websocket (0.3.8) - addressable (>= 2.1.1) - eventmachine (>= 0.12.9) eventmachine (1.2.7) faraday (0.17.3) multipart-post (>= 1.2, < 3) @@ -68,6 +65,8 @@ GEM ffi-rzmq-core (>= 1.0.7) ffi-rzmq-core (1.0.7) ffi + git (1.7.0) + rchardet (~> 1.8) gnuplot (2.6.2) hashie (3.6.0) highline (1.7.10) @@ -82,7 +81,7 @@ GEM memcache-client (1.8.5) mime-types (3.3.1) mime-types-data (~> 3.2015) - mime-types-data (3.2020.0512) + mime-types-data (3.2020.1104) mini_portile2 (2.1.0) minitest (5.11.3) ms_rest (0.7.6) @@ -113,20 +112,17 @@ GEM json (>= 1.8) nokogiri (~> 1.5) optimist (~> 3.0) + rchardet (1.8.0) rotp (5.1.0) addressable (~> 2.5) rqrcode (0.10.1) chunky_png (~> 1.0) scrub_rb (1.0.1) - sequel (5.36.0) + sequel (5.38.0) sinatra (1.4.8) rack (~> 1.5) rack-protection (~> 1.4) tilt (>= 1.3, < 3) - sinatra-websocket (0.3.1) - em-websocket (~> 0.3.6) - eventmachine - thin (>= 1.3.1, < 2.0.0) sqlite3 (1.4.2) thin (1.7.2) daemons (~> 1.0, >= 1.0.9) @@ -137,7 +133,7 @@ GEM timeliness (0.3.10) treetop (1.6.11) polyglot (~> 0.3) - tzinfo (1.2.7) + tzinfo (1.2.8) thread_safe (~> 0.1) unf (0.1.4) unf_ext @@ -172,6 +168,7 @@ DEPENDENCIES faraday (~> 0.15) faraday_middleware (~> 0.12) ffi-rzmq (~> 2.0.7) + git (~> 1.5) gnuplot highline (~> 1.7) i18n (~> 0.9) @@ -193,7 +190,6 @@ DEPENDENCIES scrub_rb sequel sinatra - sinatra-websocket sqlite3 thin treetop (>= 1.6.3) diff --git a/share/install_gems/CentOS8/Gemfile.lock b/share/install_gems/CentOS8/Gemfile.lock index 3b81f6f86f..70481a8665 100644 --- a/share/install_gems/CentOS8/Gemfile.lock +++ b/share/install_gems/CentOS8/Gemfile.lock @@ -6,36 +6,34 @@ GEM minitest (~> 5.1) thread_safe (~> 0.3, >= 0.3.4) tzinfo (~> 1.1) - addressable (2.7.0) - public_suffix (>= 2.0.2, < 5.0) amazon-ec2 (0.9.17) xml-simple (>= 1.0.12) android_key_attestation (0.3.0) augeas (0.6.4) awrence (1.1.1) aws-eventstream (1.1.0) - aws-partitions (1.375.0) - aws-sdk-cloudwatch (1.44.0) - aws-sdk-core (~> 3, >= 3.99.0) + aws-partitions (1.391.0) + aws-sdk-cloudwatch (1.46.0) + aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) - aws-sdk-core (3.107.0) + aws-sdk-core (3.109.2) aws-eventstream (~> 1, >= 1.0.2) aws-partitions (~> 1, >= 1.239.0) aws-sigv4 (~> 1.1) jmespath (~> 1.0) - aws-sdk-ec2 (1.195.0) - aws-sdk-core (~> 3, >= 3.99.0) + aws-sdk-ec2 (1.209.0) + aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) - aws-sdk-kms (1.38.0) - aws-sdk-core (~> 3, >= 3.99.0) + aws-sdk-kms (1.39.0) + aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) - aws-sdk-s3 (1.81.0) - aws-sdk-core (~> 3, >= 3.104.3) + aws-sdk-s3 (1.84.1) + aws-sdk-core (~> 3, >= 3.109.0) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.1) aws-sigv4 (1.2.2) aws-eventstream (~> 1, >= 1.0.2) - azure_mgmt_compute (0.19.3) + azure_mgmt_compute (0.20.0) ms_rest_azure (~> 0.12.0) azure_mgmt_monitor (0.17.6) ms_rest_azure (~> 0.12.0) @@ -48,20 +46,17 @@ GEM bindata (2.4.8) builder (3.2.4) cbor (0.5.9.6) - chunky_png (1.3.12) + chunky_png (1.3.14) concurrent-ruby (1.1.7) configparser (0.1.7) cose (1.2.0) cbor (~> 0.5.9) openssl-signature_algorithm (~> 1.0) - curb (0.9.10) + curb (0.9.11) daemons (1.3.1) - dalli (2.7.10) + dalli (2.7.11) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - em-websocket (0.3.8) - addressable (>= 2.1.1) - eventmachine (>= 0.12.9) eventmachine (1.2.7) faraday (0.17.3) multipart-post (>= 1.2, < 3) @@ -75,6 +70,8 @@ GEM ffi-rzmq-core (>= 1.0.7) ffi-rzmq-core (1.0.7) ffi + git (1.7.0) + rchardet (~> 1.8) gnuplot (2.6.2) hashie (4.1.0) highline (1.7.10) @@ -90,7 +87,7 @@ GEM memcache-client (1.8.5) mime-types (3.3.1) mime-types-data (~> 3.2015) - mime-types-data (3.2020.0512) + mime-types-data (3.2020.1104) mini_portile2 (2.4.0) minitest (5.14.2) ms_rest (0.7.6) @@ -125,7 +122,8 @@ GEM json (>= 1.8) nokogiri (~> 1.5) optimist (~> 3.0) - rotp (6.1.0) + rchardet (1.8.0) + rotp (6.2.0) rqrcode (1.1.2) chunky_png (~> 1.0) rqrcode_core (~> 0.1) @@ -134,16 +132,12 @@ GEM safety_net_attestation (0.4.0) jwt (~> 2.0) securecompare (1.0.0) - sequel (5.36.0) + sequel (5.38.0) sinatra (2.1.0) mustermann (~> 1.0) rack (~> 2.2) rack-protection (= 2.1.0) tilt (~> 2.0) - sinatra-websocket (0.3.1) - em-websocket (~> 0.3.6) - eventmachine - thin (>= 1.3.1, < 2.0.0) sqlite3 (1.4.2) thin (1.7.2) daemons (~> 1.0, >= 1.0.9) @@ -157,7 +151,7 @@ GEM openssl-signature_algorithm (~> 1.0) treetop (1.6.11) polyglot (~> 0.3) - tzinfo (1.2.7) + tzinfo (1.2.8) thread_safe (~> 0.1) unf (0.1.4) unf_ext @@ -181,7 +175,7 @@ GEM tpm-key_attestation (~> 0.10.0) xml-simple (1.1.5) xmlrpc (0.3.0) - zendesk_api (1.27.0) + zendesk_api (1.28.0) faraday (>= 0.9.0, < 2.0.0) hashie (>= 3.5.2, < 5.0.0) inflection @@ -209,6 +203,7 @@ DEPENDENCIES faraday (~> 0.15) faraday_middleware (~> 0.12) ffi-rzmq (~> 2.0.7) + git (~> 1.5) gnuplot highline (~> 1.7) i18n (~> 0.9) @@ -229,7 +224,6 @@ DEPENDENCIES rqrcode sequel sinatra - sinatra-websocket sqlite3 thin treetop (>= 1.6.3) diff --git a/share/install_gems/Debian10/Gemfile.lock b/share/install_gems/Debian10/Gemfile.lock index 3b81f6f86f..70481a8665 100644 --- a/share/install_gems/Debian10/Gemfile.lock +++ b/share/install_gems/Debian10/Gemfile.lock @@ -6,36 +6,34 @@ GEM minitest (~> 5.1) thread_safe (~> 0.3, >= 0.3.4) tzinfo (~> 1.1) - addressable (2.7.0) - public_suffix (>= 2.0.2, < 5.0) amazon-ec2 (0.9.17) xml-simple (>= 1.0.12) android_key_attestation (0.3.0) augeas (0.6.4) awrence (1.1.1) aws-eventstream (1.1.0) - aws-partitions (1.375.0) - aws-sdk-cloudwatch (1.44.0) - aws-sdk-core (~> 3, >= 3.99.0) + aws-partitions (1.391.0) + aws-sdk-cloudwatch (1.46.0) + aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) - aws-sdk-core (3.107.0) + aws-sdk-core (3.109.2) aws-eventstream (~> 1, >= 1.0.2) aws-partitions (~> 1, >= 1.239.0) aws-sigv4 (~> 1.1) jmespath (~> 1.0) - aws-sdk-ec2 (1.195.0) - aws-sdk-core (~> 3, >= 3.99.0) + aws-sdk-ec2 (1.209.0) + aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) - aws-sdk-kms (1.38.0) - aws-sdk-core (~> 3, >= 3.99.0) + aws-sdk-kms (1.39.0) + aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) - aws-sdk-s3 (1.81.0) - aws-sdk-core (~> 3, >= 3.104.3) + aws-sdk-s3 (1.84.1) + aws-sdk-core (~> 3, >= 3.109.0) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.1) aws-sigv4 (1.2.2) aws-eventstream (~> 1, >= 1.0.2) - azure_mgmt_compute (0.19.3) + azure_mgmt_compute (0.20.0) ms_rest_azure (~> 0.12.0) azure_mgmt_monitor (0.17.6) ms_rest_azure (~> 0.12.0) @@ -48,20 +46,17 @@ GEM bindata (2.4.8) builder (3.2.4) cbor (0.5.9.6) - chunky_png (1.3.12) + chunky_png (1.3.14) concurrent-ruby (1.1.7) configparser (0.1.7) cose (1.2.0) cbor (~> 0.5.9) openssl-signature_algorithm (~> 1.0) - curb (0.9.10) + curb (0.9.11) daemons (1.3.1) - dalli (2.7.10) + dalli (2.7.11) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - em-websocket (0.3.8) - addressable (>= 2.1.1) - eventmachine (>= 0.12.9) eventmachine (1.2.7) faraday (0.17.3) multipart-post (>= 1.2, < 3) @@ -75,6 +70,8 @@ GEM ffi-rzmq-core (>= 1.0.7) ffi-rzmq-core (1.0.7) ffi + git (1.7.0) + rchardet (~> 1.8) gnuplot (2.6.2) hashie (4.1.0) highline (1.7.10) @@ -90,7 +87,7 @@ GEM memcache-client (1.8.5) mime-types (3.3.1) mime-types-data (~> 3.2015) - mime-types-data (3.2020.0512) + mime-types-data (3.2020.1104) mini_portile2 (2.4.0) minitest (5.14.2) ms_rest (0.7.6) @@ -125,7 +122,8 @@ GEM json (>= 1.8) nokogiri (~> 1.5) optimist (~> 3.0) - rotp (6.1.0) + rchardet (1.8.0) + rotp (6.2.0) rqrcode (1.1.2) chunky_png (~> 1.0) rqrcode_core (~> 0.1) @@ -134,16 +132,12 @@ GEM safety_net_attestation (0.4.0) jwt (~> 2.0) securecompare (1.0.0) - sequel (5.36.0) + sequel (5.38.0) sinatra (2.1.0) mustermann (~> 1.0) rack (~> 2.2) rack-protection (= 2.1.0) tilt (~> 2.0) - sinatra-websocket (0.3.1) - em-websocket (~> 0.3.6) - eventmachine - thin (>= 1.3.1, < 2.0.0) sqlite3 (1.4.2) thin (1.7.2) daemons (~> 1.0, >= 1.0.9) @@ -157,7 +151,7 @@ GEM openssl-signature_algorithm (~> 1.0) treetop (1.6.11) polyglot (~> 0.3) - tzinfo (1.2.7) + tzinfo (1.2.8) thread_safe (~> 0.1) unf (0.1.4) unf_ext @@ -181,7 +175,7 @@ GEM tpm-key_attestation (~> 0.10.0) xml-simple (1.1.5) xmlrpc (0.3.0) - zendesk_api (1.27.0) + zendesk_api (1.28.0) faraday (>= 0.9.0, < 2.0.0) hashie (>= 3.5.2, < 5.0.0) inflection @@ -209,6 +203,7 @@ DEPENDENCIES faraday (~> 0.15) faraday_middleware (~> 0.12) ffi-rzmq (~> 2.0.7) + git (~> 1.5) gnuplot highline (~> 1.7) i18n (~> 0.9) @@ -229,7 +224,6 @@ DEPENDENCIES rqrcode sequel sinatra - sinatra-websocket sqlite3 thin treetop (>= 1.6.3) diff --git a/share/install_gems/Debian9/Gemfile.lock b/share/install_gems/Debian9/Gemfile.lock index 8f56a8b19b..fec62d544e 100644 --- a/share/install_gems/Debian9/Gemfile.lock +++ b/share/install_gems/Debian9/Gemfile.lock @@ -6,34 +6,32 @@ GEM minitest (~> 5.1) thread_safe (~> 0.3, >= 0.3.4) tzinfo (~> 1.1) - addressable (2.7.0) - public_suffix (>= 2.0.2, < 5.0) amazon-ec2 (0.9.17) xml-simple (>= 1.0.12) augeas (0.6.4) aws-eventstream (1.1.0) - aws-partitions (1.375.0) - aws-sdk-cloudwatch (1.44.0) - aws-sdk-core (~> 3, >= 3.99.0) + aws-partitions (1.391.0) + aws-sdk-cloudwatch (1.46.0) + aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) - aws-sdk-core (3.107.0) + aws-sdk-core (3.109.2) aws-eventstream (~> 1, >= 1.0.2) aws-partitions (~> 1, >= 1.239.0) aws-sigv4 (~> 1.1) jmespath (~> 1.0) - aws-sdk-ec2 (1.195.0) - aws-sdk-core (~> 3, >= 3.99.0) + aws-sdk-ec2 (1.209.0) + aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) - aws-sdk-kms (1.38.0) - aws-sdk-core (~> 3, >= 3.99.0) + aws-sdk-kms (1.39.0) + aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) - aws-sdk-s3 (1.81.0) - aws-sdk-core (~> 3, >= 3.104.3) + aws-sdk-s3 (1.84.1) + aws-sdk-core (~> 3, >= 3.109.0) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.1) aws-sigv4 (1.2.2) aws-eventstream (~> 1, >= 1.0.2) - azure_mgmt_compute (0.19.3) + azure_mgmt_compute (0.20.0) ms_rest_azure (~> 0.12.0) azure_mgmt_monitor (0.17.6) ms_rest_azure (~> 0.12.0) @@ -44,17 +42,14 @@ GEM azure_mgmt_storage (0.22.0) ms_rest_azure (~> 0.12.0) builder (3.2.4) - chunky_png (1.3.12) + chunky_png (1.3.14) concurrent-ruby (1.1.7) configparser (0.1.7) - curb (0.9.10) + curb (0.9.11) daemons (1.3.1) - dalli (2.7.10) + dalli (2.7.11) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - em-websocket (0.3.8) - addressable (>= 2.1.1) - eventmachine (>= 0.12.9) eventmachine (1.2.7) faraday (0.17.3) multipart-post (>= 1.2, < 3) @@ -68,6 +63,8 @@ GEM ffi-rzmq-core (>= 1.0.7) ffi-rzmq-core (1.0.7) ffi + git (1.7.0) + rchardet (~> 1.8) gnuplot (2.6.2) hashie (4.1.0) highline (1.7.10) @@ -82,7 +79,7 @@ GEM memcache-client (1.8.5) mime-types (3.3.1) mime-types-data (~> 3.2015) - mime-types-data (3.2020.0512) + mime-types-data (3.2020.1104) mini_portile2 (2.4.0) minitest (5.14.2) ms_rest (0.7.6) @@ -115,22 +112,19 @@ GEM json (>= 1.8) nokogiri (~> 1.5) optimist (~> 3.0) - rotp (6.1.0) + rchardet (1.8.0) + rotp (6.2.0) rqrcode (1.1.2) chunky_png (~> 1.0) rqrcode_core (~> 0.1) rqrcode_core (0.1.2) ruby2_keywords (0.0.2) - sequel (5.36.0) + sequel (5.38.0) sinatra (2.1.0) mustermann (~> 1.0) rack (~> 2.2) rack-protection (= 2.1.0) tilt (~> 2.0) - sinatra-websocket (0.3.1) - em-websocket (~> 0.3.6) - eventmachine - thin (>= 1.3.1, < 2.0.0) sqlite3 (1.4.2) thin (1.7.2) daemons (~> 1.0, >= 1.0.9) @@ -141,7 +135,7 @@ GEM timeliness (0.3.10) treetop (1.6.11) polyglot (~> 0.3) - tzinfo (1.2.7) + tzinfo (1.2.8) thread_safe (~> 0.1) unf (0.1.4) unf_ext @@ -154,7 +148,7 @@ GEM vsphere-automation-cis (~> 0.4.6) vsphere-automation-runtime (~> 0.4.6) xml-simple (1.1.5) - zendesk_api (1.27.0) + zendesk_api (1.28.0) faraday (>= 0.9.0, < 2.0.0) hashie (>= 3.5.2, < 5.0.0) inflection @@ -182,6 +176,7 @@ DEPENDENCIES faraday (~> 0.15) faraday_middleware (~> 0.12) ffi-rzmq (~> 2.0.7) + git (~> 1.5) gnuplot highline (~> 1.7) i18n (~> 0.9) @@ -202,7 +197,6 @@ DEPENDENCIES rqrcode sequel sinatra - sinatra-websocket sqlite3 thin treetop (>= 1.6.3) diff --git a/share/install_gems/Fedora32/Gemfile.lock b/share/install_gems/Fedora32/Gemfile.lock index 299bfedf4a..e5648815ca 100644 --- a/share/install_gems/Fedora32/Gemfile.lock +++ b/share/install_gems/Fedora32/Gemfile.lock @@ -6,36 +6,34 @@ GEM minitest (~> 5.1) thread_safe (~> 0.3, >= 0.3.4) tzinfo (~> 1.1) - addressable (2.7.0) - public_suffix (>= 2.0.2, < 5.0) amazon-ec2 (0.9.17) xml-simple (>= 1.0.12) android_key_attestation (0.3.0) augeas (0.6.4) awrence (1.1.1) aws-eventstream (1.1.0) - aws-partitions (1.375.0) - aws-sdk-cloudwatch (1.44.0) - aws-sdk-core (~> 3, >= 3.99.0) + aws-partitions (1.391.0) + aws-sdk-cloudwatch (1.46.0) + aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) - aws-sdk-core (3.107.0) + aws-sdk-core (3.109.2) aws-eventstream (~> 1, >= 1.0.2) aws-partitions (~> 1, >= 1.239.0) aws-sigv4 (~> 1.1) jmespath (~> 1.0) - aws-sdk-ec2 (1.195.0) - aws-sdk-core (~> 3, >= 3.99.0) + aws-sdk-ec2 (1.209.0) + aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) - aws-sdk-kms (1.38.0) - aws-sdk-core (~> 3, >= 3.99.0) + aws-sdk-kms (1.39.0) + aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) - aws-sdk-s3 (1.81.0) - aws-sdk-core (~> 3, >= 3.104.3) + aws-sdk-s3 (1.84.1) + aws-sdk-core (~> 3, >= 3.109.0) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.1) aws-sigv4 (1.2.2) aws-eventstream (~> 1, >= 1.0.2) - azure_mgmt_compute (0.19.3) + azure_mgmt_compute (0.20.0) ms_rest_azure (~> 0.12.0) azure_mgmt_monitor (0.17.6) ms_rest_azure (~> 0.12.0) @@ -48,20 +46,17 @@ GEM bindata (2.4.8) builder (3.2.4) cbor (0.5.9.6) - chunky_png (1.3.12) + chunky_png (1.3.14) concurrent-ruby (1.1.7) configparser (0.1.7) cose (1.2.0) cbor (~> 0.5.9) openssl-signature_algorithm (~> 1.0) - curb (0.9.10) + curb (0.9.11) daemons (1.3.1) - dalli (2.7.10) + dalli (2.7.11) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - em-websocket (0.3.8) - addressable (>= 2.1.1) - eventmachine (>= 0.12.9) eventmachine (1.2.7) faraday (0.17.3) multipart-post (>= 1.2, < 3) @@ -75,6 +70,8 @@ GEM ffi-rzmq-core (>= 1.0.7) ffi-rzmq-core (1.0.7) ffi + git (1.7.0) + rchardet (~> 1.8) gnuplot (2.6.2) hashie (4.1.0) highline (1.7.10) @@ -90,7 +87,7 @@ GEM memcache-client (1.8.5) mime-types (3.3.1) mime-types-data (~> 3.2015) - mime-types-data (3.2020.0512) + mime-types-data (3.2020.1104) mini_portile2 (2.4.0) minitest (5.14.2) ms_rest (0.7.6) @@ -125,7 +122,8 @@ GEM json (>= 1.8) nokogiri (~> 1.5) optimist (~> 3.0) - rotp (6.1.0) + rchardet (1.8.0) + rotp (6.2.0) rqrcode (1.1.2) chunky_png (~> 1.0) rqrcode_core (~> 0.1) @@ -134,16 +132,12 @@ GEM safety_net_attestation (0.4.0) jwt (~> 2.0) securecompare (1.0.0) - sequel (5.36.0) + sequel (5.38.0) sinatra (2.1.0) mustermann (~> 1.0) rack (~> 2.2) rack-protection (= 2.1.0) tilt (~> 2.0) - sinatra-websocket (0.3.1) - em-websocket (~> 0.3.6) - eventmachine - thin (>= 1.3.1, < 2.0.0) sqlite3 (1.4.2) thin (1.7.2) daemons (~> 1.0, >= 1.0.9) @@ -157,7 +151,7 @@ GEM openssl-signature_algorithm (~> 1.0) treetop (1.6.11) polyglot (~> 0.3) - tzinfo (1.2.7) + tzinfo (1.2.8) thread_safe (~> 0.1) unf (0.1.4) unf_ext @@ -181,7 +175,7 @@ GEM tpm-key_attestation (~> 0.10.0) xml-simple (1.1.5) xmlrpc (0.3.0) - zendesk_api (1.27.0) + zendesk_api (1.28.0) faraday (>= 0.9.0, < 2.0.0) hashie (>= 3.5.2, < 5.0.0) inflection @@ -209,6 +203,7 @@ DEPENDENCIES faraday (~> 0.15) faraday_middleware (~> 0.12) ffi-rzmq (~> 2.0.7) + git (~> 1.5) gnuplot highline (~> 1.7) i18n (~> 0.9) @@ -229,7 +224,6 @@ DEPENDENCIES rqrcode sequel sinatra - sinatra-websocket sqlite3 thin treetop (>= 1.6.3) @@ -241,7 +235,7 @@ DEPENDENCIES zendesk_api RUBY VERSION - ruby 2.7.1p83 + ruby 2.7.2p137 BUNDLED WITH 1.17.3 diff --git a/share/install_gems/Fedora33/Gemfile.lock b/share/install_gems/Fedora33/Gemfile.lock index 0d6bde3d95..e5648815ca 100644 --- a/share/install_gems/Fedora33/Gemfile.lock +++ b/share/install_gems/Fedora33/Gemfile.lock @@ -12,22 +12,22 @@ GEM augeas (0.6.4) awrence (1.1.1) aws-eventstream (1.1.0) - aws-partitions (1.385.0) - aws-sdk-cloudwatch (1.45.0) + aws-partitions (1.391.0) + aws-sdk-cloudwatch (1.46.0) aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) - aws-sdk-core (3.109.1) + aws-sdk-core (3.109.2) aws-eventstream (~> 1, >= 1.0.2) aws-partitions (~> 1, >= 1.239.0) aws-sigv4 (~> 1.1) jmespath (~> 1.0) - aws-sdk-ec2 (1.202.0) + aws-sdk-ec2 (1.209.0) aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) aws-sdk-kms (1.39.0) aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) - aws-sdk-s3 (1.83.1) + aws-sdk-s3 (1.84.1) aws-sdk-core (~> 3, >= 3.109.0) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.1) @@ -46,13 +46,13 @@ GEM bindata (2.4.8) builder (3.2.4) cbor (0.5.9.6) - chunky_png (1.3.13) + chunky_png (1.3.14) concurrent-ruby (1.1.7) configparser (0.1.7) cose (1.2.0) cbor (~> 0.5.9) openssl-signature_algorithm (~> 1.0) - curb (0.9.10) + curb (0.9.11) daemons (1.3.1) dalli (2.7.11) domain_name (0.5.20190701) @@ -70,6 +70,8 @@ GEM ffi-rzmq-core (>= 1.0.7) ffi-rzmq-core (1.0.7) ffi + git (1.7.0) + rchardet (~> 1.8) gnuplot (2.6.2) hashie (4.1.0) highline (1.7.10) @@ -85,7 +87,7 @@ GEM memcache-client (1.8.5) mime-types (3.3.1) mime-types-data (~> 3.2015) - mime-types-data (3.2020.0512) + mime-types-data (3.2020.1104) mini_portile2 (2.4.0) minitest (5.14.2) ms_rest (0.7.6) @@ -120,6 +122,7 @@ GEM json (>= 1.8) nokogiri (~> 1.5) optimist (~> 3.0) + rchardet (1.8.0) rotp (6.2.0) rqrcode (1.1.2) chunky_png (~> 1.0) @@ -129,7 +132,7 @@ GEM safety_net_attestation (0.4.0) jwt (~> 2.0) securecompare (1.0.0) - sequel (5.37.0) + sequel (5.38.0) sinatra (2.1.0) mustermann (~> 1.0) rack (~> 2.2) @@ -148,7 +151,7 @@ GEM openssl-signature_algorithm (~> 1.0) treetop (1.6.11) polyglot (~> 0.3) - tzinfo (1.2.7) + tzinfo (1.2.8) thread_safe (~> 0.1) unf (0.1.4) unf_ext @@ -200,6 +203,7 @@ DEPENDENCIES faraday (~> 0.15) faraday_middleware (~> 0.12) ffi-rzmq (~> 2.0.7) + git (~> 1.5) gnuplot highline (~> 1.7) i18n (~> 0.9) @@ -231,7 +235,7 @@ DEPENDENCIES zendesk_api RUBY VERSION - ruby 2.7.1p83 + ruby 2.7.2p137 BUNDLED WITH 1.17.3 diff --git a/share/install_gems/Gemfile b/share/install_gems/Gemfile index 2f8c3d8633..838b9105de 100644 --- a/share/install_gems/Gemfile +++ b/share/install_gems/Gemfile @@ -138,3 +138,8 @@ group :vmware do gem 'rbvmomi', '~> 2.2.0' end + +group :onecfg do + gem 'git', '~> 1.5' + gem 'augeas', '~> 0.6' +end diff --git a/share/install_gems/Ubuntu1604/Gemfile.lock b/share/install_gems/Ubuntu1604/Gemfile.lock index 19c5b4d59b..cf94ae5d66 100644 --- a/share/install_gems/Ubuntu1604/Gemfile.lock +++ b/share/install_gems/Ubuntu1604/Gemfile.lock @@ -6,34 +6,32 @@ GEM minitest (~> 5.1) thread_safe (~> 0.3, >= 0.3.4) tzinfo (~> 1.1) - addressable (2.7.0) - public_suffix (>= 2.0.2, < 5.0) amazon-ec2 (0.9.17) xml-simple (>= 1.0.12) augeas (0.6.4) aws-eventstream (1.1.0) - aws-partitions (1.375.0) - aws-sdk-cloudwatch (1.44.0) - aws-sdk-core (~> 3, >= 3.99.0) + aws-partitions (1.391.0) + aws-sdk-cloudwatch (1.46.0) + aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) - aws-sdk-core (3.107.0) + aws-sdk-core (3.109.2) aws-eventstream (~> 1, >= 1.0.2) aws-partitions (~> 1, >= 1.239.0) aws-sigv4 (~> 1.1) jmespath (~> 1.0) - aws-sdk-ec2 (1.195.0) - aws-sdk-core (~> 3, >= 3.99.0) + aws-sdk-ec2 (1.209.0) + aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) - aws-sdk-kms (1.38.0) - aws-sdk-core (~> 3, >= 3.99.0) + aws-sdk-kms (1.39.0) + aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) - aws-sdk-s3 (1.81.0) - aws-sdk-core (~> 3, >= 3.104.3) + aws-sdk-s3 (1.84.1) + aws-sdk-core (~> 3, >= 3.109.0) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.1) aws-sigv4 (1.2.2) aws-eventstream (~> 1, >= 1.0.2) - azure_mgmt_compute (0.19.3) + azure_mgmt_compute (0.20.0) ms_rest_azure (~> 0.12.0) azure_mgmt_monitor (0.17.6) ms_rest_azure (~> 0.12.0) @@ -44,17 +42,14 @@ GEM azure_mgmt_storage (0.22.0) ms_rest_azure (~> 0.12.0) builder (3.2.4) - chunky_png (1.3.12) + chunky_png (1.3.14) concurrent-ruby (1.1.7) configparser (0.1.7) - curb (0.9.10) + curb (0.9.11) daemons (1.3.1) - dalli (2.7.10) + dalli (2.7.11) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - em-websocket (0.3.8) - addressable (>= 2.1.1) - eventmachine (>= 0.12.9) eventmachine (1.2.7) faraday (0.17.3) multipart-post (>= 1.2, < 3) @@ -68,6 +63,8 @@ GEM ffi-rzmq-core (>= 1.0.7) ffi-rzmq-core (1.0.7) ffi + git (1.7.0) + rchardet (~> 1.8) gnuplot (2.6.2) hashie (4.1.0) highline (1.7.10) @@ -82,7 +79,7 @@ GEM memcache-client (1.8.5) mime-types (3.3.1) mime-types-data (~> 3.2015) - mime-types-data (3.2020.0512) + mime-types-data (3.2020.1104) mini_portile2 (2.4.0) minitest (5.14.2) ms_rest (0.7.6) @@ -115,22 +112,19 @@ GEM json (>= 1.8) nokogiri (~> 1.5) optimist (~> 3.0) - rotp (6.1.0) + rchardet (1.8.0) + rotp (6.2.0) rqrcode (1.1.2) chunky_png (~> 1.0) rqrcode_core (~> 0.1) rqrcode_core (0.1.2) ruby2_keywords (0.0.2) - sequel (5.36.0) + sequel (5.38.0) sinatra (2.1.0) mustermann (~> 1.0) rack (~> 2.2) rack-protection (= 2.1.0) tilt (~> 2.0) - sinatra-websocket (0.3.1) - em-websocket (~> 0.3.6) - eventmachine - thin (>= 1.3.1, < 2.0.0) sqlite3 (1.4.2) thin (1.7.2) daemons (~> 1.0, >= 1.0.9) @@ -141,7 +135,7 @@ GEM timeliness (0.3.10) treetop (1.6.11) polyglot (~> 0.3) - tzinfo (1.2.7) + tzinfo (1.2.8) thread_safe (~> 0.1) unf (0.1.4) unf_ext @@ -154,7 +148,7 @@ GEM vsphere-automation-cis (~> 0.4.6) vsphere-automation-runtime (~> 0.4.6) xml-simple (1.1.5) - zendesk_api (1.27.0) + zendesk_api (1.28.0) faraday (>= 0.9.0, < 2.0.0) hashie (>= 3.5.2, < 5.0.0) inflection @@ -182,6 +176,7 @@ DEPENDENCIES faraday (~> 0.15) faraday_middleware (~> 0.12) ffi-rzmq (~> 2.0.7) + git (~> 1.5) gnuplot highline (~> 1.7) i18n (~> 0.9) @@ -202,7 +197,6 @@ DEPENDENCIES rqrcode sequel sinatra - sinatra-websocket sqlite3 thin treetop (>= 1.6.3) diff --git a/share/install_gems/Ubuntu1804/Gemfile.lock b/share/install_gems/Ubuntu1804/Gemfile.lock index 90df19f34c..77b0f8b49a 100644 --- a/share/install_gems/Ubuntu1804/Gemfile.lock +++ b/share/install_gems/Ubuntu1804/Gemfile.lock @@ -6,36 +6,34 @@ GEM minitest (~> 5.1) thread_safe (~> 0.3, >= 0.3.4) tzinfo (~> 1.1) - addressable (2.7.0) - public_suffix (>= 2.0.2, < 5.0) amazon-ec2 (0.9.17) xml-simple (>= 1.0.12) android_key_attestation (0.3.0) augeas (0.6.4) awrence (1.1.1) aws-eventstream (1.1.0) - aws-partitions (1.375.0) - aws-sdk-cloudwatch (1.44.0) - aws-sdk-core (~> 3, >= 3.99.0) + aws-partitions (1.391.0) + aws-sdk-cloudwatch (1.46.0) + aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) - aws-sdk-core (3.107.0) + aws-sdk-core (3.109.2) aws-eventstream (~> 1, >= 1.0.2) aws-partitions (~> 1, >= 1.239.0) aws-sigv4 (~> 1.1) jmespath (~> 1.0) - aws-sdk-ec2 (1.195.0) - aws-sdk-core (~> 3, >= 3.99.0) + aws-sdk-ec2 (1.209.0) + aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) - aws-sdk-kms (1.38.0) - aws-sdk-core (~> 3, >= 3.99.0) + aws-sdk-kms (1.39.0) + aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) - aws-sdk-s3 (1.81.0) - aws-sdk-core (~> 3, >= 3.104.3) + aws-sdk-s3 (1.84.1) + aws-sdk-core (~> 3, >= 3.109.0) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.1) aws-sigv4 (1.2.2) aws-eventstream (~> 1, >= 1.0.2) - azure_mgmt_compute (0.19.3) + azure_mgmt_compute (0.20.0) ms_rest_azure (~> 0.12.0) azure_mgmt_monitor (0.17.6) ms_rest_azure (~> 0.12.0) @@ -48,20 +46,17 @@ GEM bindata (2.4.8) builder (3.2.4) cbor (0.5.9.6) - chunky_png (1.3.12) + chunky_png (1.3.14) concurrent-ruby (1.1.7) configparser (0.1.7) cose (1.2.0) cbor (~> 0.5.9) openssl-signature_algorithm (~> 1.0) - curb (0.9.10) + curb (0.9.11) daemons (1.3.1) - dalli (2.7.10) + dalli (2.7.11) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - em-websocket (0.3.8) - addressable (>= 2.1.1) - eventmachine (>= 0.12.9) eventmachine (1.2.7) faraday (0.17.3) multipart-post (>= 1.2, < 3) @@ -75,6 +70,8 @@ GEM ffi-rzmq-core (>= 1.0.7) ffi-rzmq-core (1.0.7) ffi + git (1.7.0) + rchardet (~> 1.8) gnuplot (2.6.2) hashie (4.1.0) highline (1.7.10) @@ -90,7 +87,7 @@ GEM memcache-client (1.8.5) mime-types (3.3.1) mime-types-data (~> 3.2015) - mime-types-data (3.2020.0512) + mime-types-data (3.2020.1104) mini_portile2 (2.4.0) minitest (5.14.2) ms_rest (0.7.6) @@ -125,7 +122,8 @@ GEM json (>= 1.8) nokogiri (~> 1.5) optimist (~> 3.0) - rotp (6.1.0) + rchardet (1.8.0) + rotp (6.2.0) rqrcode (1.1.2) chunky_png (~> 1.0) rqrcode_core (~> 0.1) @@ -134,16 +132,12 @@ GEM safety_net_attestation (0.4.0) jwt (~> 2.0) securecompare (1.0.0) - sequel (5.36.0) + sequel (5.38.0) sinatra (2.1.0) mustermann (~> 1.0) rack (~> 2.2) rack-protection (= 2.1.0) tilt (~> 2.0) - sinatra-websocket (0.3.1) - em-websocket (~> 0.3.6) - eventmachine - thin (>= 1.3.1, < 2.0.0) sqlite3 (1.4.2) thin (1.7.2) daemons (~> 1.0, >= 1.0.9) @@ -157,7 +151,7 @@ GEM openssl-signature_algorithm (~> 1.0) treetop (1.6.11) polyglot (~> 0.3) - tzinfo (1.2.7) + tzinfo (1.2.8) thread_safe (~> 0.1) unf (0.1.4) unf_ext @@ -181,7 +175,7 @@ GEM tpm-key_attestation (~> 0.10.0) xml-simple (1.1.5) xmlrpc (0.3.0) - zendesk_api (1.27.0) + zendesk_api (1.28.0) faraday (>= 0.9.0, < 2.0.0) hashie (>= 3.5.2, < 5.0.0) inflection @@ -209,6 +203,7 @@ DEPENDENCIES faraday (~> 0.15) faraday_middleware (~> 0.12) ffi-rzmq (~> 2.0.7) + git (~> 1.5) gnuplot highline (~> 1.7) i18n (~> 0.9) @@ -229,7 +224,6 @@ DEPENDENCIES rqrcode sequel sinatra - sinatra-websocket sqlite3 thin treetop (>= 1.6.3) diff --git a/share/install_gems/Ubuntu2004/Gemfile.lock b/share/install_gems/Ubuntu2004/Gemfile.lock index e7eeac336a..d234dad933 100644 --- a/share/install_gems/Ubuntu2004/Gemfile.lock +++ b/share/install_gems/Ubuntu2004/Gemfile.lock @@ -6,36 +6,34 @@ GEM minitest (~> 5.1) thread_safe (~> 0.3, >= 0.3.4) tzinfo (~> 1.1) - addressable (2.7.0) - public_suffix (>= 2.0.2, < 5.0) amazon-ec2 (0.9.17) xml-simple (>= 1.0.12) android_key_attestation (0.3.0) augeas (0.6.4) awrence (1.1.1) aws-eventstream (1.1.0) - aws-partitions (1.375.0) - aws-sdk-cloudwatch (1.44.0) - aws-sdk-core (~> 3, >= 3.99.0) + aws-partitions (1.391.0) + aws-sdk-cloudwatch (1.46.0) + aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) - aws-sdk-core (3.107.0) + aws-sdk-core (3.109.2) aws-eventstream (~> 1, >= 1.0.2) aws-partitions (~> 1, >= 1.239.0) aws-sigv4 (~> 1.1) jmespath (~> 1.0) - aws-sdk-ec2 (1.195.0) - aws-sdk-core (~> 3, >= 3.99.0) + aws-sdk-ec2 (1.209.0) + aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) - aws-sdk-kms (1.38.0) - aws-sdk-core (~> 3, >= 3.99.0) + aws-sdk-kms (1.39.0) + aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) - aws-sdk-s3 (1.81.0) - aws-sdk-core (~> 3, >= 3.104.3) + aws-sdk-s3 (1.84.1) + aws-sdk-core (~> 3, >= 3.109.0) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.1) aws-sigv4 (1.2.2) aws-eventstream (~> 1, >= 1.0.2) - azure_mgmt_compute (0.19.3) + azure_mgmt_compute (0.20.0) ms_rest_azure (~> 0.12.0) azure_mgmt_monitor (0.17.6) ms_rest_azure (~> 0.12.0) @@ -48,20 +46,17 @@ GEM bindata (2.4.8) builder (3.2.4) cbor (0.5.9.6) - chunky_png (1.3.12) + chunky_png (1.3.14) concurrent-ruby (1.1.7) configparser (0.1.7) cose (1.2.0) cbor (~> 0.5.9) openssl-signature_algorithm (~> 1.0) - curb (0.9.10) + curb (0.9.11) daemons (1.3.1) - dalli (2.7.10) + dalli (2.7.11) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - em-websocket (0.3.8) - addressable (>= 2.1.1) - eventmachine (>= 0.12.9) eventmachine (1.2.7) faraday (0.17.3) multipart-post (>= 1.2, < 3) @@ -75,6 +70,8 @@ GEM ffi-rzmq-core (>= 1.0.7) ffi-rzmq-core (1.0.7) ffi + git (1.7.0) + rchardet (~> 1.8) gnuplot (2.6.2) hashie (4.1.0) highline (1.7.10) @@ -90,7 +87,7 @@ GEM memcache-client (1.8.5) mime-types (3.3.1) mime-types-data (~> 3.2015) - mime-types-data (3.2020.0512) + mime-types-data (3.2020.1104) mini_portile2 (2.4.0) minitest (5.14.2) ms_rest (0.7.6) @@ -125,7 +122,8 @@ GEM json (>= 1.8) nokogiri (~> 1.5) optimist (~> 3.0) - rotp (6.1.0) + rchardet (1.8.0) + rotp (6.2.0) rqrcode (1.1.2) chunky_png (~> 1.0) rqrcode_core (~> 0.1) @@ -134,16 +132,12 @@ GEM safety_net_attestation (0.4.0) jwt (~> 2.0) securecompare (1.0.0) - sequel (5.36.0) + sequel (5.38.0) sinatra (2.1.0) mustermann (~> 1.0) rack (~> 2.2) rack-protection (= 2.1.0) tilt (~> 2.0) - sinatra-websocket (0.3.1) - em-websocket (~> 0.3.6) - eventmachine - thin (>= 1.3.1, < 2.0.0) sqlite3 (1.4.2) thin (1.7.2) daemons (~> 1.0, >= 1.0.9) @@ -157,7 +151,7 @@ GEM openssl-signature_algorithm (~> 1.0) treetop (1.6.11) polyglot (~> 0.3) - tzinfo (1.2.7) + tzinfo (1.2.8) thread_safe (~> 0.1) unf (0.1.4) unf_ext @@ -181,7 +175,7 @@ GEM tpm-key_attestation (~> 0.10.0) xml-simple (1.1.5) xmlrpc (0.3.0) - zendesk_api (1.27.0) + zendesk_api (1.28.0) faraday (>= 0.9.0, < 2.0.0) hashie (>= 3.5.2, < 5.0.0) inflection @@ -209,6 +203,7 @@ DEPENDENCIES faraday (~> 0.15) faraday_middleware (~> 0.12) ffi-rzmq (~> 2.0.7) + git (~> 1.5) gnuplot highline (~> 1.7) i18n (~> 0.9) @@ -229,7 +224,6 @@ DEPENDENCIES rqrcode sequel sinatra - sinatra-websocket sqlite3 thin treetop (>= 1.6.3) diff --git a/share/install_gems/Ubuntu2010/Gemfile.lock b/share/install_gems/Ubuntu2010/Gemfile.lock index afdc6fb033..5509bffc8e 100644 --- a/share/install_gems/Ubuntu2010/Gemfile.lock +++ b/share/install_gems/Ubuntu2010/Gemfile.lock @@ -12,22 +12,22 @@ GEM augeas (0.6.4) awrence (1.1.1) aws-eventstream (1.1.0) - aws-partitions (1.385.0) - aws-sdk-cloudwatch (1.45.0) + aws-partitions (1.391.0) + aws-sdk-cloudwatch (1.46.0) aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) - aws-sdk-core (3.109.1) + aws-sdk-core (3.109.2) aws-eventstream (~> 1, >= 1.0.2) aws-partitions (~> 1, >= 1.239.0) aws-sigv4 (~> 1.1) jmespath (~> 1.0) - aws-sdk-ec2 (1.202.0) + aws-sdk-ec2 (1.209.0) aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) aws-sdk-kms (1.39.0) aws-sdk-core (~> 3, >= 3.109.0) aws-sigv4 (~> 1.1) - aws-sdk-s3 (1.83.1) + aws-sdk-s3 (1.84.1) aws-sdk-core (~> 3, >= 3.109.0) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.1) @@ -46,13 +46,13 @@ GEM bindata (2.4.8) builder (3.2.4) cbor (0.5.9.6) - chunky_png (1.3.12) + chunky_png (1.3.14) concurrent-ruby (1.1.7) configparser (0.1.7) cose (1.2.0) cbor (~> 0.5.9) openssl-signature_algorithm (~> 1.0) - curb (0.9.10) + curb (0.9.11) daemons (1.3.1) dalli (2.7.11) domain_name (0.5.20190701) @@ -70,6 +70,8 @@ GEM ffi-rzmq-core (>= 1.0.7) ffi-rzmq-core (1.0.7) ffi + git (1.7.0) + rchardet (~> 1.8) gnuplot (2.6.2) hashie (4.1.0) highline (1.7.10) @@ -85,7 +87,7 @@ GEM memcache-client (1.8.5) mime-types (3.3.1) mime-types-data (~> 3.2015) - mime-types-data (3.2020.0512) + mime-types-data (3.2020.1104) mini_portile2 (2.4.0) minitest (5.14.2) ms_rest (0.7.6) @@ -120,6 +122,7 @@ GEM json (>= 1.8) nokogiri (~> 1.5) optimist (~> 3.0) + rchardet (1.8.0) rotp (6.2.0) rqrcode (1.1.2) chunky_png (~> 1.0) @@ -129,7 +132,7 @@ GEM safety_net_attestation (0.4.0) jwt (~> 2.0) securecompare (1.0.0) - sequel (5.37.0) + sequel (5.38.0) sinatra (2.1.0) mustermann (~> 1.0) rack (~> 2.2) @@ -148,7 +151,7 @@ GEM openssl-signature_algorithm (~> 1.0) treetop (1.6.11) polyglot (~> 0.3) - tzinfo (1.2.7) + tzinfo (1.2.8) thread_safe (~> 0.1) unf (0.1.4) unf_ext @@ -200,6 +203,7 @@ DEPENDENCIES faraday (~> 0.15) faraday_middleware (~> 0.12) ffi-rzmq (~> 2.0.7) + git (~> 1.5) gnuplot highline (~> 1.7) i18n (~> 0.9) diff --git a/src/onecfg/bin/onecfg b/src/onecfg/bin/onecfg new file mode 100755 index 0000000000..3d88ae5fb2 --- /dev/null +++ b/src/onecfg/bin/onecfg @@ -0,0 +1,66 @@ +#!/usr/bin/env ruby + +# -------------------------------------------------------------------------- # +# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +ONE_LOCATION = ENV['ONE_LOCATION'] + +if ONE_LOCATION + LIB_LOCATION = ONE_LOCATION + '/lib' + RUBY_LIB_LOCATION = ONE_LOCATION + '/lib/ruby' + GEMS_LOCATION = ONE_LOCATION + '/share/gems' +else + LIB_LOCATION = '/usr/lib/one' + RUBY_LIB_LOCATION = '/usr/lib/one/ruby' + GEMS_LOCATION = '/usr/share/one/gems' +end + +if File.directory?(GEMS_LOCATION) + $LOAD_PATH.reject! {|l| l =~ /vendor_ruby/ } + require 'rubygems' + Gem.use_paths(File.realpath(GEMS_LOCATION)) +end + +$LOAD_PATH << RUBY_LIB_LOCATION +$LOAD_PATH << RUBY_LIB_LOCATION + '/cli' +$LOAD_PATH << LIB_LOCATION + '/onecfg/lib' + +# Supported patch modes +SUPPORTED_PATCH_MODES = [:skip, :force, :replace] + +require 'git' +require 'tmpdir' +require 'yaml' + +require 'cli/command_parser' +require 'cli/one_helper' + +require 'onecfg' + +CommandParser::CmdParser.new(ARGV) do + usage '`onecfg` [] []' + + # onecfg helper + helper = OneCfgHelper.new + + # public command should go here + + begin + require 'ee' + OneCfg::EE::Commands.load_commands(self, ARGV, helper) + rescue LoadError + end +end diff --git a/src/onecfg/lib/common.rb b/src/onecfg/lib/common.rb new file mode 100644 index 0000000000..e23dce5c71 --- /dev/null +++ b/src/onecfg/lib/common.rb @@ -0,0 +1,27 @@ +# -------------------------------------------------------------------------- # +# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +module OneCfg + + # Common module + module Common + end + +end + +require 'common/backup' +require 'common/helpers/onecfg_helper' +require 'common/logger/cli_logger' diff --git a/src/onecfg/lib/common/backup.rb b/src/onecfg/lib/common/backup.rb new file mode 100644 index 0000000000..13d506da71 --- /dev/null +++ b/src/onecfg/lib/common/backup.rb @@ -0,0 +1,202 @@ +# -------------------------------------------------------------------------- # +# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +require 'date' +require 'English' + +# rubocop:disable Style/ClassAndModuleChildren +module OneCfg::Common + + # File backup module + module Backup + + # Make a directory backup + # + # @param dir [String] Dir path + # @param backup [String] Backup path + # + # @return [String] Path to the backup + def self.backup(dir, backup = nil) + unless File.directory?(dir) + raise OneCfg::Exception::FileNotFound, + "Directory '#{dir}' to backup doesn't exist." + end + + # rubocop:disable Style/FormatStringToken + backup ||= format('%s/%s%s_%i', + OneCfg::BACKUP_DIR, + DateTime.now.strftime('%Y-%m-%d_%H:%M:%S'), + dir.split('/').join('_'), + $PID) + # rubocop:enable Style/FormatStringToken + + OneCfg::LOG.debug("Backing up #{dir} in #{backup}") + rsync(dir, backup) + + backup + end + + # Restore the directory backup + # + # @param backup [String] Backup path + # @param dir [String] Dir path + def self.restore(backup, dir) + unless File.exist?(backup) + raise OneCfg::Exception::FileNotFound, + "Backup location '#{backup}' doesn't exist." + end + + OneCfg::LOG.debug("Restoring #{backup} to #{dir}") + rsync(backup, dir) + end + + # Backup all dirs at once. Directories are specified + # NON-PREFIXED, with optional prefix argument (default: /). + # + # @param dirs [Array] NON-PREFIXED dirs to backup + # @param backup [String] Backup path + # @param prefix [String] Prefix to prepend source dirs with + # + # @return [String] Path to the backup + # + # Example: + # ['/etc/one, '/var/lib/one/remotes'], nil, '/'] + # ['/etc/one, '/var/lib/one/remotes'], nil, '/tmp/prefix'] + def self.backup_dirs(dirs, backup = nil, prefix = '/') + # rubocop:disable Style/FormatStringToken + backup ||= format('%s/%s_%i', + OneCfg::BACKUP_DIR, + DateTime.now.strftime('%Y-%m-%d_%H:%M:%S'), + $PID) + # rubocop:enable Style/FormatStringToken + + OneCfg::LOG.debug("Backing up multiple dirs into '#{backup}'") + + dirs.each do |dir| + src = OneCfg::Config::Utils.prefixed(dir, prefix) + dst = File.join(backup, dir) + + backup(src, dst) + end + + # TODO: hmmm + versions = OneCfg::EE::Config::Versions.new + cfg_version = versions.cfg_version + + if cfg_version + File.open("#{backup}/version", 'w') do |file| + file.write(cfg_version) + end + end + + backup + end + + # Restore all dirs at once. Directories are specified + # NON-PREFIXED, with optional prefix argument (default: /). + # + # @param backup [String] Backup path + # @param dirs [Array] NON-PREFIXED dirs to restore + # @param prefix [String] Prefix to prepend target dirs with + # + # @return [String] Path to the backup + # + # Example: + # '/var/backup/xxxx', ['/etc/one', /var/lib/one/remotes'], '/' + # '/var/backup/xxxx', ['/etc/one', /var/lib/one/remotes'], '/tmp/prefix' + def self.restore_dirs(backup, dirs, prefix = '/') + OneCfg::LOG.debug("Restoring multiple dirs from '#{backup}'") + + dirs.each do |dir| + src = File.join(backup, dir) + dst = OneCfg::Config::Utils.prefixed(dir, prefix) + + restore(src, dst) + end + end + + # Sync content of both directories + # + # @param source [String] Source dir path + # @param target [String] Target dir path + def self.rsync(source, target) + unless ::File.exist?(target) + FileUtils.mkdir_p(target) + end + + # trigger rsync + cmd = "rsync -acvh #{source}/ #{target} --delete --delete-after" + + OneCfg::LOG.ddebug("Synchronizing '#{source}' to " \ + "'#{target}' with command: #{cmd}") + + _o, e, rtn = OneCfg::Config::Type::Base.run_shell_command(cmd) + + # handle errors + return if rtn.success? + + OneCfg::LOG.ddebug('Failure on synchronization ' \ + "due to - '#{e}'") + + raise OneCfg::Exception::Generic, # TODO: better exception + 'Data synchronization failed' + end + + end + +end +# rubocop:enable Style/ClassAndModuleChildren + +# +# Create directory tar +# +# @param dir [String] Dir path +# @param tar_location [String] Tar path +# def tar(dir, tar_location = nil) +# if tar_location.nil? +# tar_location = "/tmp/#{rand(36**8).to_s(36)}" +# end +# +# cmd = 'tar --selinux --acls --xattrs ' \ +# "-zcvf #{tar_location} -C #{dir} ./" +# _o, e, rtn = OneCfg::Config::Type::Base.run_shell_command(cmd) +# +# unless rtn.exitstatus == 0 +# raise OneCfg::Config::GenericException, e +# end +# +# tar_location +# end +# +# Untar file into directory +# +# @param tar_location [String] Tar path +# @param untar_location [String] Untar directory path +# +# @return [String] Untar path +# def untar(tar_location, untar_location = nil) +# untar_location ||= Dir.mktmpdir +# +# cmd = "tar -xf #{tar_location} -C #{untar_location}" +# _o, e, rtn = OneCfg::Config::Type::Base.run_shell_command(cmd) +# +# unless rtn.exitstatus == 0 +# raise OneCfg::Config::GenericException, e +# end +# +# untar_location +# end +# diff --git a/src/onecfg/lib/common/helpers/onecfg_helper.rb b/src/onecfg/lib/common/helpers/onecfg_helper.rb new file mode 100644 index 0000000000..75b96c7b4a --- /dev/null +++ b/src/onecfg/lib/common/helpers/onecfg_helper.rb @@ -0,0 +1,83 @@ +# -------------------------------------------------------------------------- # +# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +require 'tempfile' +require 'yaml' + +# OneCfg helper +class OneCfgHelper + + # Check path operation mode is correct + # + # @param options [Hash] CLI options + def check_modes(modes) + modes.each do |mode| + unless SUPPORTED_PATCH_MODES.include?(mode) + warn "Unsupported mode '#{mode}'" + exit(-1) + end + end + end + + # Parse patch file modes string argument providing default mode, + # per-file, per-file with version modes. Data handled as Hash: + # + # { file => { version => [modes] } } + # + # Notes: + # - Nil file applies to all files without overrides + # - Nil version applies to all files versions without overrides + # + # Example: + # { + # nil => { # nil = all files + # nil => [:skip, :force, :replace] # nil = all versions + # }, + # "/etc/one/oned.conf" => { + # nil => [:replace], # nil = all versions + # "5.0" => [:skip] + # } + # } + # + # @param modes [String] Patch modes per file + # + # @return [Hash] Parsed modes + def parse_patch_modes(modes) + ret = {} + + modes.each do |mode| + mode.split(';').each do |m| + mode, file, version = m.split(':', 3) + + # TODO: how about 'skip::5.8.0' ??? + + ret[file] ||= {} + ret[file][version] ||= [] + ret[file][version] << mode.split(',').map {|v| v.to_sym } + ret[file][version].flatten! + ret[file][version].uniq! + + check_modes(ret[file][version]) + end + end + + ret + rescue StandardError + STDERR.puts('ERROR: Wrong modes format, check documentation.') + exit(-1) + end + +end diff --git a/src/onecfg/lib/common/logger/cli_logger.rb b/src/onecfg/lib/common/logger/cli_logger.rb new file mode 100644 index 0000000000..8fa0bae946 --- /dev/null +++ b/src/onecfg/lib/common/logger/cli_logger.rb @@ -0,0 +1,131 @@ +# -------------------------------------------------------------------------- # +# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +require 'logger' +require 'singleton' + +# Singleton OneCfg Logger +# VH-TODO: CLI logger and file logger, we might want to log on both +# VH-TODO: Support different verbose levels for different loggers, e.g. on CLI +# we log based on user preference -d/-D, but into log we can log on specified +# level (e.g., debug) always +module OneCfg + + module Common + + # CLI logger + class CliLogger + + include Singleton + + attr_reader :logger + attr_accessor :custom_level + + # Class constructor + def initialize + @logger = Logger.new(STDERR) + @logger.level = Logger::WARN + @custom_level = Logger::UNKNOWN + end + + # Gets the logger + # + # @param options [Key-Value Object] CLI options + def self.get_logger(options) + instance.logger.formatter = proc do |severity, _d, _p, msg| + "#{severity.ljust(5)} : #{msg}\n" + end + + if options.key? :debug + instance.logger.level = Logger::DEBUG + elsif options.key? :verbose + instance.logger.level = Logger::INFO + elsif options.key? :ddebug + instance.logger.level = Logger::DEBUG + instance.custom_level = :ddebug + elsif options.key? :dddebug + instance.logger.level = Logger::DEBUG + instance.custom_level = :dddebug + else + instance.logger.level = Logger::WARN + end + end + + # Shows a debug message + # + # @param msg [String] Message to show + def self.debug(msg) + instance.logger.debug(msg) + end + + # Shows a debug-debug message + # + # @param msg [String] Message to show + def self.ddebug(msg) + return unless instance.custom_level == :ddebug + + instance.logger.debug(msg) # TODO + end + + # Shows a debug-debug-debug message + # + # @param msg [String] Message to show + def self.dddebug(msg) + return unless instance.custom_level == :dddebug + + instance.logger.debug(msg) # TODO + end + + # Shows an error message + # + # @param msg [String] Message to show + def self.error(msg) + instance.logger.error(msg) + end + + # Shows an fatal error message + # + # @param msg [String] Message to show + def self.fatal(msg) + instance.logger.fatal(msg) + end + + # Shows an info message + # + # @param msg [String] Message to show + def self.info(msg) + instance.logger.info(msg) + end + + # Shows a warning message + # + # @param msg [String] Message to show + def self.warn(msg) + instance.logger.warn(msg) + end + + # Shows an unknown message + # + # @param msg [String] Message to show + def self.unknown(msg) + instance.logger.unknown(msg) + end + + end + + end + +end diff --git a/src/onecfg/lib/config.rb b/src/onecfg/lib/config.rb new file mode 100644 index 0000000000..17f5d53304 --- /dev/null +++ b/src/onecfg/lib/config.rb @@ -0,0 +1,42 @@ +# -------------------------------------------------------------------------- # +# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +require 'config/type' +require 'config/type/base' +require 'config/type/simple' +require 'config/type/augeas' +require 'config/type/augeas/one' +require 'config/type/augeas/shell' +require 'config/type/yaml' +require 'config/type/yaml/strict' + +require 'config/utils' +require 'config/exception' +require 'config/fsops' + +begin + require 'ee/config' +rescue LoadError +end + +module OneCfg + + # OneCfg::Config module + module Config + + end + +end diff --git a/src/onecfg/lib/config/exception.rb b/src/onecfg/lib/config/exception.rb new file mode 100644 index 0000000000..4f04d678c8 --- /dev/null +++ b/src/onecfg/lib/config/exception.rb @@ -0,0 +1,164 @@ +# -------------------------------------------------------------------------- # +# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +# rubocop:disable Style/ClassAndModuleChildren +# rubocop:disable Lint/UselessMethodDefinition +module OneCfg::Config::Exception + + # OneCfg critical error, which shouldn't happen + class FatalError < OneCfg::Exception::Generic + + def initialize(text) + super(text) + end + + end + + # Unsupported OpenNebula version exception + class UnsupportedVersion < OneCfg::Exception::Generic + + def initialize(text) + super(text) + end + + end + + # Invalid file structure. + class StructureError < OneCfg::Exception::Generic + + def initialize(text) + super(text) + end + + end + + # OneCfg config exception when content is not initialized + class NoContent < OneCfg::Exception::Generic + + def initialize + super('Uninitialized content') + end + + end + + ### Patch Exceptions ############################# + + # Generic Patch Exception + class PatchException < OneCfg::Exception::Generic + + # Failed diff operation + attr_accessor :data + + def initialize(text) + super(text) + end + + end + + # Patch exception when value can't be placed on desired place + class PatchPathNotFound < PatchException + + attr_reader :path + + def initialize(path) + if path.empty? + path_str = '(top level)' + else + path_str = path.join('/') + end + + super("Patch path '#{path_str}' not found") + + @path = path + end + + end + + # Patch exception when we would create or remove multiple + # configuration parameters on place where where it doesn't + # make sense. + class PatchInvalidMultiple < PatchException + + attr_reader :path + + def initialize(path) + super("Wrong multiple use of parameter '#{path}'") + + @path = path + end + + end + + # Patch exception when we found unexpected unspecified data structure. + class PatchUnexpectedData < PatchException + + attr_reader :type + attr_reader :expected + + def initialize(type) + @type = type + + if defined?(@expected) + super("Expected #{@expected} but got #{@type}") + else + super("Unexpected data type #{type} found") + end + end + + end + + # Patch exception when we found different data type on + # a place where Hash was expected. + class PatchExpectedHash < PatchUnexpectedData + + def initialize(type) + @expected = 'Hash' + + super(type) + end + + end + + # Patch exception when we found different data type on + # a place where Array was expected. + class PatchExpectedArray < PatchUnexpectedData + + def initialize(type) + @expected = 'Array' + + super(type) + end + + end + + # Value we expected on specified structure index to + # use for old value remove or as context for new + # value, wasn't found. + class PatchValueNotFound < PatchException + + attr_reader :value + + def initialize(value) + super("Value '#{value}' not found") + + @value = value + end + + end + +end +# rubocop:enable Style/ClassAndModuleChildren +# rubocop:enable Lint/UselessMethodDefinition diff --git a/src/onecfg/lib/config/fsops.rb b/src/onecfg/lib/config/fsops.rb new file mode 100644 index 0000000000..76347b4780 --- /dev/null +++ b/src/onecfg/lib/config/fsops.rb @@ -0,0 +1,231 @@ +# -------------------------------------------------------------------------- # +# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +require 'tmpdir' + +# rubocop:disable Style/ClassAndModuleChildren +module OneCfg::Config + + # Management file class + class FileOperation + + # @prefix directory where all operations are done + # @unprivileged true to skip some operations + attr_reader :prefix, :unprivileged + + # Class constructor + # + # @param prefix [String] Directory where all operations are done + # @param unprivileged [Boolean] True to skip some operations + def initialize(prefix = '/', unprivileged = false) + @prefix = prefix + @unprivileged = unprivileged + end + + # Create a new directory if it doesn't exist + # + # @param name [String] Directory path + def mkdir(path) + target = prefixed(path) + + if exist?(target) + dddebug('mkdir', [path], 'Already exists') + else + dddebug('mkdir', [path]) + FileUtils.mkdir_p(target) + end + end + + # Change file ownership + # + # @param path [String] File path + # @param owner [String] New owner + # @param group [String] New group (by default the same as owner) + def chown(path, owner, group = owner) + target = prefixed(path) + + if @unprivileged + dddebug('chown', + [path, owner, group], + 'Skipped on unprivileged run') + else + # TODO: recursive? + dddebug('chown', [path, owner, group]) + FileUtils.chown_R(owner, group, target) + end + end + + # Change file permissions + # + # @param path [String] File path + # @param mode [String] New permissions + def chmod(path, mode) + target = prefixed(path) + dddebug('chmod', [path, mode]) + FileUtils.chmod(mode, target) + end + + # Move file or directory from src location to dst location + # + # @param src [String] Source path + # @param dst [String] Destination path + def move(src, dst) + dddebug('move', [src, dst]) + + glob(src, false).each do |f| + begin + # glob returns relative paths inside the prefix. + # We have to prefix bouth source and dst. + pre_f = prefixed(f) + pre_dst = prefixed(dst) + + OneCfg::LOG.dddebug("Move #{pre_f} " \ + "into #{pre_dst}") + + FileUtils.mv(pre_f, pre_dst) + rescue StandardError => e + # skip files which are the same as destination + # TODO: WTF is error 22 !!! + next if e.errno == 22 + + raise e + end + end + end + + # Check if the given path exists + # + # @param path [String] File path + def exist?(path) + ::File.exist?(prefixed(path)) + end + + # Check if the given path is a file + # + # @param path [String] File path + def file?(path) + ::File.file?(prefixed(path)) + end + + # Check if the given path is a directory + # + # @param path [String] Directory path + def directory?(path) + ::File.directory?(prefixed(path)) + end + + # Delete the file + # + # @param path [String] File path + def delete(path) + dddebug('delete', [path]) + + FileUtils.rm_r(prefixed(path)) + end + + # Get directory files + # + # @param path [String] Directory path + # @param deep [Boolean] True to read recursively all files + def glob(path, deep = true) + dddebug('glob', [path, deep]) + + target = prefixed(path) + + if ::File.directory?(target) && deep + ret = Dir["#{target}/**/**"] + elsif ::File.directory?(target) + ret = Dir["#{target}/**"] + else + ret = Dir[target] + end + + # remove prefix and duplicate // + ret.map do |f| + unprefixed(f) + end + end + + # Read file content + # + # @param path [String] File path + # + # @return [String] File content + def file_read(path) + ret = '' + + dddebug('file_read', [path]) + ::File.open(prefixed(path), 'r') do |file| + ret += file.read + end + + ret + end + + # Write content into file + # + # @param path [String] File path + # @param content [String] File content + # @param append [Boolean] true to append the content + # false to replace + def file_write(path, content, append = false) + dddebug('file_write', [path, '...', append]) + + append == true ? mode = 'a' : mode = 'w' + + ::File.open(prefixed(path), mode) do |file| + file.write(content) + end + end + + # Get path with prefix + # + # @param path [String] Path to add prefix + # + # @return [String] Prefixed path + def prefixed(path) + OneCfg::Config::Utils.prefixed(path, @prefix) + end + + # Get path without prefix + # + # @param path [String] Path to remove prefix + # + # @return [String] Unprefixed path + def unprefixed(path) + OneCfg::Config::Utils.unprefixed(path, @prefix) + end + + private + + # Log debug^3 message about + # + # @param method [String] Method name + # @param args [Array] Method arguments + # @param comment [String] Optional comment + def dddebug(method, args, comment = nil) + text = self.class.name.split('::').last + ' ' + text << "[prefix=#{prefix}] " + text << "#{method}(#{args.join(',')})" + text << " ... #{comment}" if comment + + OneCfg::LOG.dddebug(text) + end + + end + +end +# rubocop:enable Style/ClassAndModuleChildren diff --git a/src/onecfg/lib/config/type.rb b/src/onecfg/lib/config/type.rb new file mode 100644 index 0000000000..9d4d63b8cd --- /dev/null +++ b/src/onecfg/lib/config/type.rb @@ -0,0 +1,28 @@ +# -------------------------------------------------------------------------- # +# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +module OneCfg + + module Config + + # Config::Type module + module Type + + end + + end + +end diff --git a/src/onecfg/lib/config/type/augeas.rb b/src/onecfg/lib/config/type/augeas.rb new file mode 100644 index 0000000000..fedc64e699 --- /dev/null +++ b/src/onecfg/lib/config/type/augeas.rb @@ -0,0 +1,359 @@ +# -------------------------------------------------------------------------- # +# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +gem 'augeas', '~> 0.6' +require 'augeas' + +# rubocop:disable Style/ClassAndModuleChildren +module OneCfg::Config::Type + + # Augeas class + class Augeas < Base + + # @lens Augeas lens file + # @load_path Augeas custom load path + attr_accessor :lens, :load_path + + # Class constructor + # + # @param name [String] File name + # @param lens [String] Augeas lens file + # @param load_path [String] Augeas custom load path + def initialize(name, lens, load_path = nil) + @lens = lens + @work_file = nil + @load_path = load_path + + super(name) + end + + # TODO: destructor to delete @work_file + + # Load file content + # + # @param name [String] Custom file name + # + # @return [Object] Read content + def load(name = @name) + reset + + # clone file to own place + @work_file = Tempfile.new('onescape-') + @work_file.close + FileUtils.cp(name, @work_file.path) + + work_file_dir = File.dirname(@work_file.path) + work_file_name = File.basename(@work_file.path) + + # TODO: Consider :no_stdinc (but if :loadpath) + aug = ::Augeas.create(:no_modl_autoload => true, + :no_load => true, + :root => work_file_dir, + :loadpath => @load_path) + + aug.clear_transforms + aug.transform(:lens => @lens, :incl => work_file_name) + aug.context = "/files/#{work_file_name}" + aug.load + + # validate there was no Augeas error + if aug.exists("/augeas#{aug.context}/error") + raise OneCfg::Exception::FileReadError, + 'Failed to parse file' + end + + # validate we parsed file, even empty + # file should have a node in a tree + begin + aug.match(aug.context) + rescue ::Augeas::NoMatchError + raise OneCfg::Exception::FileReadError, + 'Failed to parse file' + end + + @content = aug + + @content + end + + # Save content into a file + # + # @param name [String] Custom file name + def save(name = @name) + raise OneCfg::Config::Exception::NoContent if @content.nil? + + # unless @content.class == ::Augeas + # raise Exception, 'Invalid content' + # end + + # NOTE, it will save only if there are pending changes + @content.save # TODO: save! + + FileUtils.cp(@work_file.path, name) + end + + # Diff between 2 configuration classes. + # + # Method is not implemented for generic Augeas class, specific + # configuration type class (e.g., Augeas::ONE or Augeas::Shell) + # must be used !!! + # + # @param cfg[OneCfg::Config::Base] Configuration to diff with + def diff(_cfg) + method_not_implemented(__method__) + end + + # Get list of keys (node names) from Augeas tree + # + # @param nested [Boolean] Indicates if there are sub levels + # e.g: A = [B = "C"] + # + # @return [Array] List of keys + def get_keys(nested) + # IMPORTANT: we depend on this order where values inside + # the sections are at the end of returned list, so that we can + # reverse order when upper casing the node names in oned.conf. + # + # E.g.: + # LOG + # MONITORING_INTERVAL + # MARKET_MAD_CONF[3] + # AUTH_MAD_CONF[1] + # AUTH_MAD_CONF[2] + # AUTH_MAD + # SESSION_EXPIRATION_TIME + # VM_USE_OPERATIONS + # ... + # ... << after all top level nodes, continues nested nodes >> + # ... + # AUTH_MAD_CONF[2]/NAME + # AUTH_MAD_CONF[2]/PASSWORD_CHANGE + # AUTH_MAD_CONF[2]/DRIVER_MANAGED_GROUPS + # AUTH_MAD_CONF[2]/MAX_TOKEN_TIME + ret = @content.match('*') + ret += @content.match('*/*') if nested + + context = "#{@content.context}/" + + ret.map! {|v| v.sub(context, '') } + ret.reject! {|v| v.include?('#comment') } + + ret + end + + # Get list of all values for the path + def get_values(path) + ret = [] + + content.match(path).each do |key| + ret << content.get(key) + end + + ret + end + + # Get complete content tree from Augeas as Hash. Tree doesn't + # contain metadata for comments and shell exports. + # + # TBD + # + # @return [Hash] Hash with Augeas content + # rubocop:disable Naming/AccessorMethodName + def get_tree + ret = {} + + keys = get_keys + + keys.each do |key| + path1, path2 = key.split('/', 2) + + # get and strip indexes from each path component + name1 = strip_index(path1) + name2 = strip_index(path2) + + idx1 = get_index(path1) + idx2 = get_index(path2) + + val = content.get(key) + # val_addr = [val, key] # addresable value + val_addr = val + + # Cases + # + # a) unique sections + # LOG = nil + # LOG/SYSTEM = file + # LOG/DEBUG_LEVEL = 3 + # + # b) unique entries + # MONITORING_THREADS = 50 + # + # c) multiple entries + # MONITORING_INTERVAL[1] = 60 + # MONITORING_INTERVAL[2] = 61 + # + # d) multiple sections + # AUTH_MAD_CONF[1] = nil + # AUTH_MAD_CONF[1]/NAME = core + # AUTH_MAD_CONF[1]/PASSWORD_CHANGE = YES + # AUTH_MAD_CONF[1]/DRIVER_MANAGED_GROUPS = NO + # AUTH_MAD_CONF[1]/MAX_TOKEN_TIME = -1 + # + # e) multiple sections with multiple entries + # AUTH_MAD_CONF[1] = nil + # AUTH_MAD_CONF[1]/NAME = core + # AUTH_MAD_CONF[1]/PASSWORD_CHANGE = YES + # AUTH_MAD_CONF[1]/DRIVER_MANAGED_GROUPS = NO + # AUTH_MAD_CONF[1]/MAX_TOKEN_TIME[1] = -1 + # AUTH_MAD_CONF[1]/MAX_TOKEN_TIME[2] = -1 + + # Create main unique and multiple sections + # Entry without any value opens a section, e.g.: + # LOG = nil + # AUTH_MAD_CONF[1] = nil + # AUTH_MAD_CONF[2] = nil + if val.nil? + if path1 && path2.nil? + ret[name1] = {} unless ret.key?(name1) + + ret[name1][idx1] = {} unless ret[name1].key?(idx1) + else + raise OneCfg::Config::Exception::StructureError, + 'Double nested sections unsupported' + end + + # Create main unique and multiple entries. + # Based on existence of index, we create value or array, e.g.: + # MONITORING_THREADS = 50 + # MONITORING_INTERVAL[1] = 60 + # MONITORING_INTERVAL[2] = 61 + elsif path1 && path2.nil? + if idx1 + ret[name1] = [] unless ret.key?(name1) + ret[name1] << val_addr + else + ret[name1] = [val_addr] + end + + # Create nested unique and multiple entries. + # First we detect a base section structure (e.g., LOG or + # AUTH_MAD_CONF[1]), then we create entires as value or + # array as in top main. E.g.: + # AUTH_MAD_CONF[1]/DRIVER_MANAGED_GROUPS = NO + # AUTH_MAD_CONF[1]/MAX_TOKEN_TIME[1] = -1 + # AUTH_MAD_CONF[1]/MAX_TOKEN_TIME[2] = -1 + else + base = ret[name1][idx1] + + if idx2 + base[name2] = [] unless base.key?(name2) + base[name2] << val_addr + else + base[name2] = [val_addr] + end + end + end + + ret + end + # rubocop:enable Naming/AccessorMethodName + + # Get complete key/values tree from Augeas as a hash. + # Tree doesn't contain metadata about comments or + # shell exports. + # + # @return [Hash] Hash with path keys and values + def old_get_tree + ret = {} + + keys = get_keys + + keys.each do |key| + # Don't create separate keys for sections, + # e.g. avoid "TM_MAD_CONF[5]" hash keys + subkey_found = false + subkey_prefix = "#{key}/" + + keys.each do |k| + if k.start_with?(subkey_prefix) + subkey_found = true + break + end + end + + ret[key] = content.get(key) unless subkey_found + end + + ret + end + + # Unquote string. Removes leading and trailing double quotes ("). + # + # @param str [String] String + # + # @return [String] Unquoted string + def self.unquote(str) + return unless str + + str.chomp('"').reverse.chomp('"').reverse + end + + # Quote string. Adds double quotes around. + # + # @param str [String] String + # + # @return [String] Quoted string + def self.quote(str) + return unless str + + '"' + unquote(str) + '"' + end + + private + + # Returns index from last component of Augeas path. E.g., + # for AUTH_MAD_CONF[1]/MAX_TOKEN_TIME[2] returns string '2'. + # + # @param str [String] String + # + # @return [String] Path index + def get_index(str) + return unless str + + match = str.match(/\[([0-9]+)\]$/) + + return unless match + + match[1] + end + + # Removes index from last component of Augeas path. E.g., + # for AUTH_MAD_CONF[1]/MAX_TOKEN_TIME[2] + # get AUTH_MAD_CONF[1]/MAX_TOKEN_TIME + # + # @param str [String] String + # + # @return [String] Path index + def strip_index(str) + return unless str + + str.sub(/\[[0-9]+\]$/, '') + end + + end + +end +# rubocop:enable Style/ClassAndModuleChildren diff --git a/src/onecfg/lib/config/type/augeas/one.rb b/src/onecfg/lib/config/type/augeas/one.rb new file mode 100644 index 0000000000..bd723f9b43 --- /dev/null +++ b/src/onecfg/lib/config/type/augeas/one.rb @@ -0,0 +1,904 @@ +# -------------------------------------------------------------------------- # +# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +# rubocop:disable Style/ClassAndModuleChildren +module OneCfg::Config::Type + + # One Augeas class + class Augeas::ONE < Augeas + + # Patch Safe Default Modes + PATCH_SAFE_MODES = [:skip] + + # Sections with their unique identification parameters. + SECTIONS = { + 'IM_MAD' => 'NAME', + 'VM_MAD' => 'NAME', + 'VM_HOOK' => 'NAME', + 'VNET_HOOK' => 'NAME', + 'USER_HOOK' => 'NAME', + 'GROUP_HOOK' => 'NAME', + 'IMAGE_HOOK' => 'NAME', + 'HOST_HOOK' => 'NAME', + 'TM_MAD_CONF' => 'NAME', + 'DS_MAD_CONF' => 'NAME', + 'MARKET_MAD_CONF' => 'NAME', + 'AUTH_MAD_CONF' => 'NAME', + 'VN_MAD_CONF' => 'NAME' + # 'TM_MAD' => 'EXECUTABLE', + # 'AUTH_MAD' => 'EXECUTABLE', + } + + # Parameters which are expected to be specified MULTIPLE + # times in the main section of configuration file. + MULTIPLE = %w[ + VM_RESTRICTED_ATTR + IMAGE_RESTRICTED_ATTR + VNET_RESTRICTED_ATTR + USER_RESTRICTED_ATTR + GROUP_RESTRICTED_ATTR + HOST_ENCRYPTED_ATTR + VNET_ENCRYPTED_ATTR + DATASTORE_ENCRYPTED_ATTR + CLUSTER_ENCRYPTED_ATTR + INHERIT_DATASTORE_ATTR + INHERIT_IMAGE_ATTR + INHERIT_VNET_ATTR + ] + + # Class constructor + # + # @param name [String] File name + # @param load_path [String] directories for modules + def initialize(name = nil, load_path = OneCfg::LENS_DIR) + super(name, 'Oned.lns', load_path) + end + + # Load file content + # + # @param name [String] Custom file name + # + # @return [Object] Read content + def load(name = @name) + super(name) + + upcase_keys + + # any structure error is fatal for load + begin + validate + rescue StandardError + reset + raise + end + + @content + end + + # Validates the content, raises exception for serious problems. + # + # @return [Boolean] True if OK. + def validate + # rubocop:disable Lint/UselessAssignment + tree = get_tree + # rubocop:enable Lint/UselessAssignment + + # TODO: validate here for parameters specified multiple times + # tree.keys.each do |section| + # ... + # end + + true + end + + # Get list of keys (node names) from Augeas tree + # + # @return [Array] List of keys + # rubocop:disable Naming/AccessorMethodName + def get_keys + super(true) + end + # rubocop:enable Naming/AccessorMethodName + + # Check if content of both configuration objects is similar + # + # @param cfg [OneCfg::Config::Type::Base] Configuration to compare + # + # @return [Boolean] True if content is similar + def similar?(cfg) + return(true) if same?(cfg) + + diff(cfg).nil? + end + + # Get complete content tree from Augeas as Hash. Tree doesn't + # contain metadata for comments and shell exports. + # + # For known SECTIONS, the numeric index of subsection is replaced + # by unique string identification taken from subsection key field. + # + # @return [Hash] Hash with Augeas content + # rubocop:disable Naming/AccessorMethodName + def get_tree + tree = super + + # Transform number based indexes in nested sections by + # unique name identification based on the section content + # (e.g., usually the "NAME" value from inside the section). E.g. + # AUTH_MAD_CONF['1']['MAX_TOKEN_TIME'] --> + # AUTH_MAD_CONF['core']['MAX_TOKEN_TIME'] + tree.keys.each do |section| + # we check every element of our tree, which + # contains Hash structure inside + next unless tree[section].is_a? Hash + + # If section is not described in known SECTIONS, it must + # contain only unindexed (nil) subsection identification. + # If it contains multiple or indexed subsections, the file + # is **semantically wrong**. + # + # E.g., good example: + # "VXLAN_IDS"=> + # {nil=>{"START"=>["\"2\"", "VXLAN_IDS[1]/START"]}} + # + # bad example: + # "VXLAN_IDS"=> + # {"1"=>{"START"=>["\"2\"", "VXLAN_IDS[1]/START"]}, + # "2"=>{"START"=>["\"2\"", "VXLAN_IDS[2]/START"]}}, + unless SECTIONS.key?(section) + next if tree[section].keys.length == 1 && + tree[section].keys.include?(nil) + + raise OneCfg::Config::Exception::StructureError, + "Invalid multiple sections of #{section}" + end + + id = SECTIONS[section] + + tree[section].keys.each do |idx| + # Section doesn't have key inside we use for unique + # identification. E.g., TM_MAD_CONF without NAME + unless tree[section][idx].key?(id) + raise OneCfg::Config::Exception::StructureError, + "Missing #{id} identification for " \ + "#{section}[#{idx}]" + end + + # Section can't have multiple keys which are used for + # unique identification. E.g., TM_MAD_CONF with 2x NAME + unless tree[section][idx][id].is_a?(Array) && + tree[section][idx][id].length == 1 + raise OneCfg::Config::Exception::StructureError, + "Multiple #{id} identifications for " \ + "#{section}[#{idx}]" + end + + # TODO: unquote??? + # new_idx = unquote(tree[section][idx][id][0]) + new_idx = tree[section][idx][id][0] + + # Section is already defined. There is a section + # with same key value multiple times. E.g., + # 2x TM_MAD_CONF with same NAME=ceph + if tree[section].key?(new_idx) + raise OneCfg::Config::Exception::StructureError, + 'Duplicate identification for ' \ + "#{section}[#{new_idx}]" + end + + tree[section][new_idx] = tree[section].delete(idx) + end + end + + tree + end + # rubocop:enable Naming/AccessorMethodName + + # Get diff between 2 configuration files + # + # @param cfg [Augeas::ONE] Configuration to diff with + # + # @return [Array, nil] Array with diff if files are not + # identical. nil if files are identical. Exception on error. + def diff(cfg) + tree1 = get_tree + tree2 = cfg.get_tree + + ret = diff_subtree(tree1, tree2, []) + ret.flatten! + + ret.empty? ? nil : ret + end + + ################################################################## + # Private Methods + ################################################################## + + private + + # Walks through the tree1 and tree2 data structures + # starting the provided path, compares them and returns + # array of differences. + # + # @param tree1 [Object] Data structure 1 to compare + # @param tree2 [Object] Data structure 2 to compare + # @param path [Array] Path from the tree top + # + # @return [Array] Array of Hashes with diff objects + def diff_subtree(tree1, tree2, path) + ret = [] + + keys = (tree1.keys + tree2.keys).uniq + + keys.each do |key| + if tree1.key?(key) && tree2.key?(key) + val1 = tree1[key] + val2 = tree2[key] + + if val1.class == val2.class + # rubocop:disable Style/CaseLikeIf + if val1.is_a? Hash + ret << diff_subtree(val1, val2, path + [key]) + elsif val1.is_a? Array + ret << diff_subtree_array(val1, val2, path + [key]) + else + # this should never happen + raise OneCfg::Config::Exception::FatalError, + 'Types to compare are not Hash nor Array ' \ + "(but #{val1.class})" + end + # rubocop:enable Style/CaseLikeIf + else + # changed data structure + ret << { + 'path' => path.compact, + 'key' => key, + 'old' => tree1[key], + 'state' => 'rm', + 'extra' => {} + } + + # create path + ret << diff_subtree_ins(tree2[key], path + [key]) + end + + elsif tree1.key?(key) + # drop path + ret << diff_subtree_rm(tree1[key], path + [key]) + + # ret << { + # 'path' => path, + # 'key' => key, + # 'old' => tree1[key], # TODO: too complex data there + # 'state' => 'rm', + # 'extra' => {} + # } + else + # create path + ret << diff_subtree_ins(tree2[key], path + [key]) + end + end + + ret + end + + # On a subtrees locations provided in tree1 and tree2, + # we expect and compare Arrays with values, e.g. + # all merged values of VM_RESTRICTED_ATTR. Returns list + # of diff (Hash) operations. + # + # @param tree1 [Object] Data structure 1 to compare + # @param tree2 [Object] Data structure 2 to compare + # @param path [Array] Path from the tree top + # + # @return [Array] Array of Hashes with diff objects + def diff_subtree_array(tree1, tree2, path) + ret = [] + + # TODO? + # For top level multiple parameters, we ignore the order of values, + # as mostly all the values are taken and merged and order is not + # important. In deeper levels, the multiple parameters are mostly + # an error, redundant option incorrectly used. In deeper levels, + # we take care of order. + ### + # strict = (path.length > 1) + # strict = false + + if OneCfg::Config::Utils.deep_compare(tree1, tree2) + return(ret) + end + + is_m = multiple?(path) + + # We have change in simple parameters, e.g. + # MONITORING_INTERVAL = "60" + if tree1.length == 1 && tree2.length == 1 && !is_m + ret << { + 'path' => path[0..-2].compact, + 'key' => path[-1], + 'value' => tree2[0], + 'old' => tree1[0], + 'state' => 'set', + 'extra' => {} + } + + # We have change in multiple parameters, e.g. + # VM_RESTRICTED_ATTR = "CONTEXT/FILES" + # VM_RESTRICTED_ATTR = "NIC/MAC" + else + found_m = (tree1.length > 1) || (tree2.length > 1) + + if found_m && !is_m + p = path.flatten.compact.join('/') + + # TODO: Move into validate (basename can be different file) + OneCfg::LOG.warn("File #{basename}: " \ + "Parameter #{p} specified " \ + 'multiple times.') + end + + # --- Multiple parameters reduced to single one + # We reduce unexpectedly multiple parameter (found_m && !is_m) + # back to single occurance. We handle this differently. Example: + # - tree1: TM_MAD_SYSTEM="X", TM_MAD_SYSTEM="Y" + # - tree2: TM_MAD_SYSTEM="Z" + # Instead of dropping olds and creating new, we drop all extra + # occurances, and change first one with set. Patch would be: + # - rm Y + # - set Z (old X) + if found_m && !is_m && tree2.length == 1 + if tree1.include?(tree2[0]) + # If tree2 contains value from tree1, we only + # drop old values found in tree1. No insert/set + # need to happen. + (tree1 - tree2).uniq.each do |val| + ret << { + 'path' => path[0..-2].compact, + 'key' => path[-1], + 'old' => val, + 'state' => 'rm', + 'extra' => { 'multiple' => found_m || is_m } + } + end + else + # If tree2 contains different value, we go by index + # and drop values from tree1 from index 1 and up, + # and set new value for index 0. See example with X,Y,Z. + tree1[1..-1].uniq.each do |val| + ret << { + 'path' => path[0..-2].compact, + 'key' => path[-1], + 'old' => val, + 'state' => 'rm', + 'extra' => { 'multiple' => found_m || is_m } + } + end + + ret << { + 'path' => path[0..-2].compact, + 'key' => path[-1], + 'value' => tree2[0], + 'old' => tree1[0], + 'state' => 'set', + 'extra' => {} # multiple?? + } + end + else + (tree1 + tree2).uniq.each do |val| + # Parameter with value was removed + if tree1.include?(val) && !tree2.include?(val) + ret << { + 'path' => path[0..-2].compact, + 'key' => path[-1], + 'old' => val, + 'state' => 'rm', + 'extra' => { 'multiple' => found_m || is_m } + } + + # New parameter value was added + elsif tree2.include?(val) && !tree1.include?(val) + ret << { + 'path' => path[0..-2].compact, + 'key' => path[-1], + 'value' => val, + 'state' => 'ins', + 'extra' => { 'multiple' => found_m || is_m } + } + end + end + end + end + + ret + end + + # Drops subtree. For sections, remove whole path identifying + # the subtree. For values of multiple parameters remove each + # individual value. + # + # @param tree [Object] Tree structure to remove + # @param path [Array] Path from the tree top + # + # @return [Array] Array of Hashes with diff objects + def diff_subtree_rm(tree, path) + ret = [] + + # rubocop:disable Style/CaseLikeIf + if tree.is_a? Hash + # drop each parameter and value + # TODO: drop section key as last + # TODO: consider if we really want to drop each parameter + ### + # tree.each_key do |key| + # ret << diff_subtree_rm(tree[key], path + [key]) + # end + + # final path drop + ret << { + # Just path, no key inside... + # 'path' => path[0..-2].compact, + # 'key' => path[-1], + 'path' => path.compact, + 'key' => nil, + 'state' => 'rm', + 'extra' => {} + } + elsif tree.is_a? Array + found_m = (tree.length > 1) + + if found_m && !multiple?(path) + p = path.flatten.compact.join('/') + + # TODO: Move into validate (basename can be different file) + OneCfg::LOG.warn("File #{basename}: " \ + "Parameter #{p} specified " \ + 'multiple times.') + end + + tree.each do |val| + ret << { + 'path' => path[0..-2].compact, + 'key' => path[-1], + 'old' => val, + 'state' => 'rm', + 'extra' => { 'multiple' => found_m } # '|| is_m' ??? + } + end + else + # this should never happen + raise OneCfg::Config::Exception::FatalError, + 'Type of subtree is not Hash nor Array ' \ + "(but #{tree.class})" + end + # rubocop:enable Style/CaseLikeIf + + ret + end + + # Inserts subtree. For sections, recursively insert all + # parameters with their values. For regular parameters, + # insert each individual value. + # + # @param tree [Object] Tree structure to insert + # @param path [Array] Path from the tree top + # + # @return [Array] Array of Hashes with diff objects + def diff_subtree_ins(tree, path) + ret = [] + + # rubocop:disable Style/CaseLikeIf + # For a subsection (Hash), we process individual parameters. + # E.g., for tree["DS_MAD_CONF"]["ceph"] we go through + # {"NAME"=>["\"ceph\""], + # "REQUIRED_ATTRS"=>["\"DISK_TYPE,BRIDGE_LIST\""], + # "PERSISTENT_ONLY"=>["\"NO\""], + # "MARKETPLACE_ACTIONS"=>["\"export\""]} + if tree.is_a? Hash + keys = tree.keys + + # --- Reprioritize section identifying parameter + # If subtree is a known multiple section, we check if parameter + # inside used to uniquely identify the section (usually "NAME") + # is listed on first place. If not, we move it in the array. + # E.g., for some TM_MAD_CONF change + # - from: ['LN_TARGET', 'NAME', 'SHARED'] + # - to: ['NAME', 'LN_TARGET', 'SHARED'] + section_key = SECTIONS[path[0]] + + if keys.length > 1 && + !section_key.nil? && + keys.include?(section_key) && + keys[0] != section_key + + keys.insert(0, keys.delete(section_key)) + end + + keys.each do |key| + ret << diff_subtree_ins(tree[key], path + [key]) + end + + # For end values (Array), we add each individual value. + # E.g., for tree["VM_RESTRICTED_ATTR"] we process array of values + # ["\"CONTEXT/FILES\"", + # "\"NIC/MAC\"", + # "\"NIC/VLAN_ID\"", + # "\"NIC/BRIDGE\"", + # "\"NIC_DEFAULT/MAC\"", ...] + elsif tree.is_a? Array + tree.each do |val| + ret << { + 'path' => path[0..-2].compact, + 'key' => path[-1], + 'value' => val, + 'state' => 'ins', + 'extra' => { 'multiple' => tree.length > 1 } + } + end + else + # this should never happen + raise OneCfg::Config::Exception::FatalError, + 'Type of subtree is not Hash nor Array ' \ + "(but #{tree.class})" + end + # rubocop:enable Style/CaseLikeIf + + ret + end + + # Apply single diff/patch operation on current content. Finds + # the path in Augeas tree and triggers the method handling + # particular apply (patch) action (set/ins/rm). + # + # @param data [Hash] Single diff operation data + # @param mode [Array] Patch modes (see patch method) + # + # @return [Hash] Patch status + def apply_diff_op(data, mode) + ret = { :status => false } + + path = nil + + # Based on diff path (and parameter key name) we propose + # full Augeas key path/name. For new sections, we construct + # a path so that the section for them is created. + if data['path'].length == 2 + # second path component uniquely identifies location, + # but needs to be transformed into Augeas specific path + idx = SECTIONS[data['path'][0]] + + if idx.nil? + # this should never happen + raise OneCfg::Config::Exception::FatalError, + 'Unknown index key name for section ' \ + "#{data['path'][0]}" + end + + # prepare just section part of path to check if + # there is some matching element already, or we + # have to create a new one + path = [ + data['path'][0], + '[' + idx + " = '" + data['path'][1] + "']" + ].flatten.compact + + found = content.match(path.join) + + if found.empty? + # If diff key is section key (idx), allow to create + # new section. We specify index 0, but Augeas puts + # it automatically on next available index number. + # All other operations on the section are already + # addressed by section unique identifier. + if idx == data['key'] + path = [ + data['path'][0], + '[0]' + ].flatten.compact + + elsif mode.include?(:skip) + ret[:mode] = :skip + + return ret + else + raise OneCfg::Config::Exception::PatchPathNotFound, + data['path'] + end + + # We found multiple matching sections, we can't uniquely + # address the change. This is a fatal problem of the file + # content we can't just skip. + elsif found.length > 1 + raise OneCfg::Config::Exception::PatchPathNotFound, + data['path'] + end + + # add section parameter name and merge path + if data['key'] + path << '/' \ + << data['key'] + end + + path = path.join + + elsif data['path'].length < 2 + path = [ + data['path'], + data['key'] + ].flatten.compact.join('/') + else + # this should never happen + raise OneCfg::Config::Exception::FatalError, + 'Diff path can not have more than 2 components ' \ + "(got #{data['path'].length})" + end + + case data['state'] + when 'rm' + ret = apply_diff_op_rm(path, data, mode) + when 'ins' + ret = apply_diff_op_ins(path, data, mode) + when 'set' + ret = apply_diff_op_set(path, data, mode) + else + raise OneCfg::Config::Exception::FatalError, + "Invalid patch action '#{data['state']}'" + end + + ret + end + + # Appply single diff/patch "rm" operation. + # + # @param path [String] Augeas path + # @param data [Hash] Single diff operation data + # @param mode [Array] Patch modes (see patch method) + # + # @return [Hash] Patch status + def apply_diff_op_rm(path, data, _mode) + ret = { :status => false } + + is_m = multiple?([data['path'], data['key']]) + + # We also respect what diff considered as multiple + # parameter occurances. But, we don't follow this strictly. + if data['extra'].key?('multiple') + is_m ||= data['extra']['multiple'] + end + + # Parameters, which are unique (doesn't repeat), we blindly + # drop the path. For the recurrent parameters, we drop + # **by value**. + if is_m + idx = get_values(path).index(data['old']) + + if idx + # We find old value in current Augeas values, the order + # of array values respects order in Augeas tree, only + # index from 0, not from 1. + ret_rm = content.rm(path + '[' + (idx + 1).to_s + ']') + + ret[:status] = (ret_rm > 0) + end + else + ret_rm = content.rm(path) + + # NOTE, here, it can happen than that we remove multiple + # entries (ret_rm>1) in Augeas tree. It doesn't have to + # be error, as we might be dropping whole section at once. + + ret[:status] = (ret_rm > 0) + end + + ret + end + + # Appply single diff/patch "ins" operation. + # + # @param path [String] Augeas path + # @param data [Hash] Single diff operation data + # @param mode [Array] Patch modes (see patch method) + # + # @return [Hash] Patch status + def apply_diff_op_ins(path, data, mode) + ret = { :status => false } + + found = get_values(path) + + if found.empty? + ret[:status] = true + + content.set(path, data['value']) + else + is_m = multiple?([data['path'], data['key']]) + + # We also respect what diff considered as multiple + # parameter occurances. But, we don't follow this strictly. + if data['extra'].key?('multiple') + is_m ||= data['extra']['multiple'] + end + + if is_m + unless found.include?(data['value']) + content.set("#{path}[0]", data['value']) + + ret[:status] = true + end + elsif found.length == 1 + if found[0] != data['value'] && mode.include?(:replace) + content.set(path, data['value']) + + ret[:status] = true + ret[:mode] = :replace + ret[:old] = found[0] + end + elsif !found.include?(data['value']) + if mode.include?(:replace) + # Configuration file has multiple parameters, but + # we didn't expect this parameter to exist already and + # even shouldn't be multiple. On replace, we drop + # all indexes >1 and replace base path at the end + (found.length - 1).times do + content.rm("#{path}[last()]") + end + + content.set(path, data['value']) + + ret[:status] = true + ret[:mode] = :replace + ret[:old] = found + + elsif mode.include?(:skip) + ret[:mode] = :skip + else + raise OneCfg::Config::Exception::PatchInvalidMultiple, + path + end + end + end + + ret + end + + # Appply single diff/patch "set" operation. + # + # @param path [String] Augeas path + # @param data [Hash] Single diff operation data + # @param mode [Array] Patch modes (see patch method) + # + # @return [Hash] Patch status + def apply_diff_op_set(path, data, mode) + ret = { :status => false } + + found = get_values(path) + + # rubocop:disable Style/EmptyElse + if found[0] == data['old'] + if found.length > 1 + # Even if we want to "set" unique parameter, there + # can be multiple params created wrongly by user. So we + # try to test and set only first one. Nothing else + # matters anyway... + content.set("#{path}[1]", data['value']) + else + content.set(path, data['value']) + end + + ret[:status] = true + + elsif mode.include?(:replace) + if found.length > 1 + (found.length - 1).times do + content.rm("#{path}[last()]") + end + end + + content.set(path, data['value']) + + ret[:status] = true + ret[:mode] = :replace + ret[:old] = found + else + # TODO, ??? exception ??? + end + # rubocop:enable Style/EmptyElse + + ret + end + + # Get key for hintings + # + # @param diff [Hash] Element with diff information + # + # @return [String] Formatted key + def hinting_key(data) + full_path = [data['path'], data['key']].flatten.compact + + if full_path.empty? + '(top)' + else + # if first element of path is section with known identifier, + # merge first 2 elements into Augeas-like path expression + # rubocop:disable Style/FormatStringToken + if SECTIONS.key?(full_path[0]) + full_path[0] = format("%s[%s = '%s']", + full_path[0], + SECTIONS[full_path[0]], + full_path.delete_at(1)) + end + # rubocop:enable Style/FormatStringToken + + full_path.join('/') + end + end + + # Upcase node names in the Augeas content object to avoid + # problems with case sensitivity. Change is directly in + # the Augeas tree. + # + # no parameters + # no return value + def upcase_keys + get_keys.reverse.each do |key| + # upper case last part of the path + parts = key.split('/', 2) + parts[-1].upcase! + new_key = parts.join('/') + + # TODO: on duplicate tm_mad_system breaks the order + # rubocop:disable Style/Next + if key != new_key + if new_key.end_with?(']') + # if we are changing casing on last indexed [] + # node we have to drop the index as the new + # index will be assigned automatically by + # Augeas, for example: + # tm_mad_conf[2] -> TM_MAD_CONF + # tm_mad_conf/tm_mad_system[2] -> TM_MAD_SYSTEM + parts[-1].sub!(/\[[0-9]+\]$/, '') + new_key = parts.join('/') + end + + # Rename only if paths differ and expect + # the change to happen on just single node, + # not more or less. Otherwise it's a BUG! + changed = @content.rename(key, parts[-1]) + + if changed != 1 + raise OneCfg::Config::Exception::StructureError, + "Upcase of '#{key}' to '#{new_key}' updated " \ + "#{changed} nodes instead of expected 1" + end + end + # rubocop:enable Style/Next + end + end + + # Returns if configuration parameter is expected to appear multiple + # times (e.g., VM_RESTRICTED_ATTR). Doesn't apply to multiple + # sections. + # + # @param path [Array] Path as array of path components + # + # @return [Boolean] True or False if multiple. + def multiple?(path) + MULTIPLE.include?(path.flatten.compact[0]) + end + + end + +end +# rubocop:enable Style/ClassAndModuleChildren diff --git a/src/onecfg/lib/config/type/augeas/shell.rb b/src/onecfg/lib/config/type/augeas/shell.rb new file mode 100644 index 0000000000..6f4fab781f --- /dev/null +++ b/src/onecfg/lib/config/type/augeas/shell.rb @@ -0,0 +1,439 @@ +# -------------------------------------------------------------------------- # +# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +# rubocop:disable Style/ClassAndModuleChildren +module OneCfg::Config::Type + + # Shell Augeas class + class Augeas::Shell < Augeas + + # Patch Safe Default Modes + # Empty array, means there is no default mode + PATCH_SAFE_MODES = [] + + # Class constructor + # + # @param name [String] File name + # @param load_path [String] directories for modules + def initialize(name = nil, load_path = nil) + super(name, 'Shellvars.lns', load_path) + + @strict = false + end + + # Load file content + # + # @param name [String] Custom file name + # + # @return [Object] Read content + def load(name = @name) + super(name) + + # any structure error is fatal for load + begin + validate + rescue StandardError + reset + raise + end + + @content + end + + # Validates the content, raises exception for serious problems. + # + # @return [Boolean] True if OK. + def validate + tree = get_tree + + tree.each do |key, val| + next if val.length <= 1 + + OneCfg::LOG.warn("File #{basename}: " \ + "Variable #{key} specified " \ + 'multiple times. Only last ' \ + 'value will be respected.') + end + + true + end + + # Get list of keys (node names) from Augeas tree + # + # @return [Array] List of keys + # rubocop:disable Naming/AccessorMethodName + def get_keys + keys = super(false) + + keys.reject! {|v| v.start_with?('@export') } + + keys + end + # rubocop:enable Naming/AccessorMethodName + + # Check if content of both configuration objects is similar + # + # @param cfg [OneCfg::Config::Base] Configuration to compare + # + # @return [Boolean] True if content is similar + def similar?(cfg) + return(true) if same?(cfg) + + tree1 = get_tree + tree2 = cfg.get_tree + keys1 = tree1.keys.sort + keys2 = tree2.keys.sort + + return(false) if keys1.size != keys2.size + + # NOTE, It should be enough just to check emptiness of "diff" + # structure, but it's so easy to do it here. Also, we can take + # this as a validation of "diff" method. + + # Take keys on both sides and compare content ... + # Variables, values and exports must be all the same. + keys1.zip(keys2) do |k1, k2| + return(false) if k1 != k2 + return(false) if tree1[k1].last != tree2[k2].last + return(false) if exported_key?(k1) != cfg.exported_key?(k2) + end + + true + end + + # Get diff between 2 configuration files + # + # @param cfg [Augeas::Shell] Configuration to diff with + # + # @return [Array, nil] Array with diff if files are not + # identical. nil if files are identical. Exception on error. + def diff(cfg) + ret = [] + + tree1 = get_tree + tree2 = cfg.get_tree + keys1 = tree1.keys.sort + keys2 = tree2.keys.sort + + (keys1 + keys2).uniq.each do |key| + val1 = tree1[key].last if tree1.key?(key) + val2 = tree2[key].last if tree2.key?(key) + exp1 = exported_key?(key) + exp2 = cfg.exported_key?(key) + + next if val1 == val2 && exp1 == exp2 + + # change variable (content or export flag) + if !val1.nil? && !val2.nil? + op = { + 'path' => [], + 'key' => key, + 'state' => 'set', + 'extra' => {} + } + + # changed value + if val1 != val2 + op['value'] = val2 + op['old'] = val1 + end + + # changed export flag + if exp1 != exp2 + op['extra']['export'] = exp2 + end + + ret << op + + # delete variable + elsif !val1.nil? + ret << { + 'path' => [], + 'key' => key, + 'old' => val1, + 'state' => 'rm', + 'extra' => {} + } + + # insert new variable + elsif !val2.nil? + op = { + 'path' => [], + 'key' => key, + 'value' => val2, + 'state' => 'ins', + 'extra' => { 'export' => exp2 } + } + + # TODO: before/after context + + ret << op + end + end + + ret.empty? ? nil : ret + end + + # Check if key with variable is exported inside configuration + # + # @param key [String] Variable + # + # @return [Boolean] True if variable is exported + def exported_key?(key) + ret = nil + + begin + # find exports within assignments, e.g.: + # export KEY=VALUES + ret = @content.exists("#{key}/export") + + # find exports outside assignments, e.g.: + # export KEY1 KEY2 KEY3 + # rubocop:disable Style/OrAssignment + unless ret + # We might get Augeas key with index if multiple + # variables with same name is defined, but in @export, + # there is a list of raw unindexed variable names. + # E.g., for 2x "export LIBVIRT_URI" + # LIBVIRT_URI[1]/export + # LIBVIRT_URI[2]/export + # but, for "export LIBVIRT_URI LIBVIRT_URI" + # @export/1 LIBVIRT_URI + # @export/2 LIBVIRT_URI + ret = @content.exists("@export/*[.='#{strip_index(key)}']") + end + # rubocop:enable Style/OrAssignment + rescue ::Augeas::MultipleMatchesError + ret = true + end + + ret + end + + # Exports or unexports variable. + # + # @param key [String] Variable name + # @param new [Boolean] True if export + def export_key(key, new) + state = exported_key?(key) + + if new && !state + content.set("#{key}/export", nil) + + elsif !new && state + content.rm("#{key}/export") + content.rm("@export/*[.='#{key}']") + + # all @exports without any children must be dropped + content.match('@export').reverse_each do |node| + content.rm(node) if content.match("#{node}/*").empty? + end + end + end + + ################################################################## + # Private Methods + ################################################################## + + private + + # Appply single diff/patch operation. Based on type + # trigger inidividual delete / insert / set actions. + # + # @param data [Hash] Single diff operation data + # @param mode [Array] Patch modes (see patch method) + # + # @return [Hash] Patch status + def apply_diff_op(data, mode) + ret = { :status => false } + + case data['state'] + when 'rm' + ret = apply_diff_op_rm(data, mode) + when 'ins' + ret = apply_diff_op_ins(data, mode) + when 'set' + ret = apply_diff_op_set(data, mode) + else + raise OneCfg::Config::Exception::FatalError, + "Invalid patch action '#{data['state']}'" + end + + ret + end + + # Appply single diff/patch "rm" operation. + # + # @param data [Hash] Single diff operation data + # @param mode [Array] Patch modes (see patch method) + # + # @return [Hash] Patch status + def apply_diff_op_rm(data, _mode) + ret = { :status => false } + + key, _val, _old, _exp = extract_diff_op(data) + + # delete all matching variables and unexport + content.match(key).length.times do + ret[:status] = (content.rm("#{key}[last()]") > 0) + end + + export_key(key, false) + + ret + end + + # Appply single diff/patch "ins" operation. + # + # @param data [Hash] Single diff operation data + # @param mode [Array] Patch modes (see patch method) + # + # @return [Hash] Patch status + def apply_diff_op_ins(data, mode) + ret = { :status => false } + + key, val, _old, exp = extract_diff_op(data) + + found = get_values(key) + + # New variables are easily set with required export status. + if found.empty? + content.set(key, val) + export_key(key, exp) + ret[:status] = true + + # It can happen that variable is already in the file placed + # manually by the user. We don't touch it, unless "replace" + # mode specified. + elsif found[-1] != val && mode.include?(:replace) + # drop all variable occurances, leave only last one + (found.length - 1).times do + content.rm("#{key}[1]") + end + + content.set(key, val) + export_key(key, exp) + + ret[:status] = true + ret[:mode] = :replace + ret[:old] = found + end + + ret + end + + # Appply single diff/patch "set" operation. + # + # @param data [Hash] Single diff operation data + # @param mode [Array] Patch modes (see patch method) + # + # @return [Hash] Patch status + def apply_diff_op_set(data, mode) + ret = { :status => false } + + key, val, old, exp = extract_diff_op(data) + + # change in value + if val && old + found = get_values(key) + + # Set new value only if user didn't change it (respect + # last found value), or if user forced replace. + if found[-1] == old || mode.include?(:replace) + # drop all variable occurances, leave only last one + (found.length - 1).times do + content.rm("#{key}[1]") + end + + content.set(key, val) + + ret[:status] = true + ret[:mode] = :replace if found[-1] != old + ret[:old] = found + end + end + + # change in export status + unless exp.nil? + # Set export state only if it's different + # to current state. + # rubocop:disable Style/SoleNestedConditional + if exp != exported_key?(key) + export_key(key, exp) + ret[:status] = true + end + # rubocop:enable Style/SoleNestedConditional + end + + ret + end + + # Get value for hintings + # + # @param data [Hash] Element with diff information + # + # @return [String] Formatted value + def hinting_value(data) + ret = nil + + if data.key?('value') + ret = data['value'].inspect + elsif data.key?('old') + ret = data['old'].inspect + end + + ret + end + + # Get extra metadata for hintings + # + # @param data [Hash] Element with diff information + # + # @return [String] Formatted value + def hinting_extra(data) + return unless data.key?('extra') + + ret = '' + + if data['state'] == 'set' && data['extra'].key?('export') + if data['extra']['export'] + ret << ' export' + else + ret << ' unexport' + end + end + + ret.strip + end + + # Returns all details from single diff operation metadata + # + # @param data [Hash] Single diff operation data + # + # @return [Array] Array of with [key, value, old, export] + def extract_diff_op(data) + [ + data['key'], + data['value'], + data['old'], + data['extra']['export'] + ] + end + + end + +end +# rubocop:enable Style/ClassAndModuleChildren diff --git a/src/onecfg/lib/config/type/base.rb b/src/onecfg/lib/config/type/base.rb new file mode 100644 index 0000000000..dc32bb9773 --- /dev/null +++ b/src/onecfg/lib/config/type/base.rb @@ -0,0 +1,569 @@ +# -------------------------------------------------------------------------- # +# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +require 'fileutils' +require 'open3' + +# rubocop:disable Style/ClassAndModuleChildren +module OneCfg::Config::Type + + # Base configuration file + class Base + + # @name File name + # @content File content + # @strict Configuration file respects order + attr_accessor :name + attr_accessor :content + attr_reader :strict + + # Class constructor + # + # @param name [String] File name (optional) + def initialize(name = nil) + @strict = false + + reset + + # For unspecified file names we'll generate random + # name. The file should be automatically deleted + # on object destruction. + if name.nil? + tmpfile = Tempfile.new('onescape-') + tmpfile.close + @name = tmpfile.path + else + @name = name + end + end + + # Returns file base name + # + # @return [String] Base name + def basename + ::File.basename(@name) + end + + # Clears content stored in the memory + # + # No parameters + # No return + def reset + @content = nil + end + + # Load file content + # + # @param name [String] Custom file name + def load(_name = @name) + method_not_implemented(__method__) + end + + # Save content into a file + # + # @param name [String] Custom file name + def save(_name = @name) + method_not_implemented(__method__) + end + + # Validates the content, raises exception in case of problem. + # + # @return [Boolean] True if OK. + def validate + !@content.nil? + end + + # Delete the file + def delete + ::File.delete(@name) + end + + # Check if file exists + # + # @param name [String] Custom file name + def exist?(name = @name) + ::File.exist?(name) + end + + # Copy content from other configuration object + # + # @param cfg [OneCfg::Config::Base] Source config. object + def copy(cfg) + # TODO: check compatible classes??? + if cfg.content.nil? + reset + else + Tempfile.open('onescape-') do |tmp| + tmp.close + + cfg.save(tmp.path) + load(tmp.path) + end + end + end + + # TODO: copy base names + + # Render configuration content as string + # + # @return [String] String configuration + def to_s + rtn = nil + + Tempfile.open('onescape-') do |tmp| + tmp.close + + save(tmp.path) + + file_operation(tmp.path, 'r') do |file| + rtn = file.read + end + end + + rtn + end + + # Check if content of both configuration objects is same + # + # @param cfg [OneCfg::Config::Base] Configuration to compare with + # + # @return [Boolean] True if content is same + def same?(cfg) + # TODO: check compatible classes + if content.nil? && cfg.content.nil? + true + elsif content.nil? || cfg.content.nil? + false + else + file_diff(cfg).nil? + end + end + + # Check if content of both configuration objects is similar + # + # @param cfg [OneCfg::Config::Base] Configuration to compare with + # + # @return [Boolean] True if content is similar + def similar?(cfg) + same?(cfg) + end + + # Get diff between 2 rendered configuration files + # + # @param cfg[OneCfg::Config::Base] Configuration to diff with + # + # @return [String, nil] String with diff if files are not + # identical. nil if files are identical. Exception on error. + def diff(cfg) + data = file_diff(cfg) + + if data.nil? + nil + else + # TODO: split hunks + [{ + 'path' => [], + 'key' => nil, + 'value' => data, # or extra? + 'state' => 'set', + 'extra' => {} + }] + end + end + + # Patches object based on provided diff data. + # + # @param data [Array] Diff data + # @param mode [Array] Patch modes + # :dummy - dummy mode, don't aply changes to content + # :skip - skip operation if + # - path can't be followed + # - path doesn't contain expected structure (Hash) + # - element to change is already missing + # :force - force application on most suitable place + # :replace - replace changed values with proposed new ones + # + # @return [[Boolean, Array]] - returns 2 items array + # [Boolean] patch status, true if at least 1 patch op. happened + # [Array] report with details about each patch operation + def patch(data, mode = []) + return if data.nil? + + mode ||= [] + + # TODO: rename :none to :dummy + if mode.include?(:none) || mode.include?(:dummy) + # In dummy (no operation) mode, we spawn temporary + # object with same content and try the patch operation + # on them. + dummy = self.class.new + dummy.copy(self) + ret = dummy.patch(data, mode - [:none, :dummy]) + + # Save is just for sure to check the configuration + # object content is left in a state which can be + # dumped into a file. Concern are the Augeas classes. + dummy.save + + return ret + end + + ret = false + rep = [] + + # We backup current state into a temporary object (file) + # with copy of our content. In case of exception, we + # copy back from the backup object. + backup = self.class.new + backup.copy(self) + + data.each do |diff_op| + begin + patch_status = apply_diff_op(diff_op, mode) + + ret ||= patch_status[:status] + rep << patch_status + + # TODO: rescue on any exception + rescue StandardError => e + # restore content from backup object + copy(backup) + + if e.is_a?(OneCfg::Config::Exception::PatchException) + # enrich patch exception with extra + # diff operation data + e.data = diff_op + end + + raise e + end + end + + [ret, rep] + end + + # Get human readable hints based on diff + # + # @param diff [Array] Array with diff information + def hintings(data, report = []) + return if data.nil? + + ret = [] + report ||= [] + + data.zip(report).each do |d, r| + key = hinting_key(d) + value = hinting_value(d) + extra = hinting_extra(d) + + r_status = '' + r_mode = '' + + if r && r.key?(:status) + if r[:mode] + r_mode = "#{r[:mode]} " + end + + if r[:status] + r_status = "[OK] #{r_mode}" + else + r_status = "[--] #{r_mode}" + end + end + + case d['state'] + when 'ins' + ret << "#{r_status}ins #{key} = #{value} #{extra}" + when 'set' + if value.nil? + ret << "#{r_status}set #{key} #{extra}" + else + ret << "#{r_status}set #{key} = #{value} #{extra}" + end + when 'rm' + if value.nil? + ret << "#{r_status}rm #{key} #{extra}" + else + ret << "#{r_status}rm #{key} = #{value} #{extra}" + end + else + ret << "unknown operation #{state}" + end + end + + ret + end + + # Run shell commands + # + # @param cmd [String] Command to execute + # @param stdin [Sting] Standard input + # + # @return [Array] Array with output, error and return code + def self.run_shell_command((*cmd), stdin = nil) + rtn = nil + + if Hash == cmd.last + opts = cmd.pop.dup + else + opts = {} + end + + binmode = opts.delete(:binmode) + + Open3.popen3(*cmd, opts) do |i, o, e, t| + i.puts stdin unless stdin.nil? + + if binmode + i.binmode + o.binmode + e.binmode + end + + out_reader = Thread.new { o.read } + err_reader = Thread.new { e.read } + + begin + i.close + rescue IOError => e + raise e.message + end + + rtn = [out_reader.value, err_reader.value, t.value] + end + + rtn + end + + ################################################################## + # Private Methods + ################################################################## + + private + + # Apply single diff/patch operation wrapper. + # + # @param data [Hash] Single diff operation data + # @param mode [Array] Patch modes (see patch method) + # + # @return [Hash] Patch status + def apply_diff_op(data, mode) + ret = { :status => false } + + case data['state'] + when 'set' + ret = apply_diff_op_set(data, mode) + else + raise OneCfg::Config::Exception::FatalError, + "Invalid patch action '#{diff_op['state']}'" + end + + ret + end + + # Appply single diff/patch "set" operation. + # + # @param data [Hash] Single diff operation data + # @param mode [Array] Patch modes (see patch method) + # + # @return [Hash] Patch status + def apply_diff_op_set(data, mode) + ret = { :status => false } + + begin + ret[:status] = file_patch(data['value']) + rescue OneCfg::Config::Exception::PatchException + # rubocop:disable Style/GuardClause + if mode.include?(:skip) + ret[:mode] = :skip + else + raise + end + # rubocop:enable Style/GuardClause + end + + ret + end + + # Get key for hintings + # + # @param diff [Hash] Element with diff information + # + # @return [String] Formatted key + def hinting_key(data) + full_path = [data['path'], data['key']].flatten.compact + + if full_path.empty? + '(top)' + else + full_path.join('/') + end + end + + # Get value for hintings + # + # @param data [Hash] Element with diff information + # + # @return [String] Formatted value + def hinting_value(data) + ret = '??? UNKNOWN ???' + + if data.key?('value') + ret = data['value'].inspect + elsif data.key?('old') + ret = data['old'].inspect + end + + ret + end + + # Get extra metadata for hintings + # + # @param data [Hash] Element with diff information + # + # @return [String] Formatted value + def hinting_extra(_data) + '' + end + + # Provide a mechanism to duplicate object + # + # @param cfg [OneCfg::Config::Base] Configuration object to copy + def initialize_dup(cfg) + # TODO: implement elegant object copy + raise TypeError, "can't dup #{cfg.class}" + end + + # Provide a mechanism to clone object + # + # @param cfg [OneCfg::Config::Base] Configuration object to copy + def initialize_clone(cfg) + # TODO: implement elegant object copy + raise TypeError, "can't clone #{cfg.class}" + end + + # Provide a mechanism to copy object + # + # @param cfg [OneCfg::Config::Base] Configuration object to copy + def initialize_copy(cfg) + # TODO: implement elegant object copy + raise TypeError, "can't copy #{cfg.class}" + end + + # Get diff between 2 rendered configuration files + # + # @param cfg [OneCfg::Config::Base] Configuration to diff with + # + # @return [String, nil] String with diff if files are not + # identical. nil if files are identical. Exception on error. + def file_diff(cfg) + file_a = Tempfile.new('onescape-') + file_b = Tempfile.new('onescape-') + + file_a.close + file_b.close + + save(file_a.path) + cfg.save(file_b.path) + + cmd = 'diff -ud --label old --label new ' \ + "#{file_a.path} #{file_b.path}" + + out, _err, rtn = Base.run_shell_command(cmd) + + # Exit status is (based on diff manual page) + # 0 if inputs are the same, + # 1 if different, + # 2 if trouble + case rtn.exitstatus + when 0 + nil + when 1 + out + else + msg = 'Error generating diff' + + raise OneCfg::Exception::Generic, msg + end + ensure + # TODO: is this better? + file_a.unlink if defined?(file_a) + file_b.unlink if defined?(file_b) + end + + # Apply diff to the configuration object + # + # @param diff [String] Diff to apply + def file_patch(data) + return if data.nil? + + Tempfile.open('onescape-') do |tmp| + tmp.close + save(tmp.path) + + out, err, rtn = Base.run_shell_command("patch -f #{tmp.path}", + data) + + unless rtn.success? + OneCfg::LOG.debug('Patch command failed with output - ' \ + "#{out.tr("\n", ' ')} " \ + "#{err.tr("\n", ' ')}") + + raise OneCfg::Config::Exception::PatchException, + 'Patch command failed' + end + + load(tmp.path) + end + + true + end + + # Get the diff between 1 objects and patch current object + # + # @param cfg1 [OneCfg::Config::Base] First configuration obj. + # @param cfg2 [OneCfg::Config::Base] Second configuration obj. + # + # VH: disabling for now, might be confusing which obj. + # is source and dest. + # def patch_diff(cfg1, cfg2) + # diff = cfg1.diff(cfg2) + # patch(diff) + # end + + # Raise an exception in case method is not implemented + # + # @param method [String] Name of the method + def method_not_implemented(method) + raise "#{method} is not implemented" + end + + # Make some files operations + # + # @param name [String] File name + # @param operation [Char] File operation: r, w + def file_operation(name, operation) + file = File.open(name, operation) + + yield(file) + + file.close + end + + end + +end +# rubocop:enable Style/ClassAndModuleChildren diff --git a/src/onecfg/lib/config/type/simple.rb b/src/onecfg/lib/config/type/simple.rb new file mode 100644 index 0000000000..d20c45b9cc --- /dev/null +++ b/src/onecfg/lib/config/type/simple.rb @@ -0,0 +1,65 @@ +# -------------------------------------------------------------------------- # +# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +require 'tempfile' + +# rubocop:disable Style/ClassAndModuleChildren +module OneCfg::Config::Type + + # Simple file class + class Simple < Base + + # Patch Safe Default Modes + # Empty array, means there is no default mode + PATCH_SAFE_MODES = [] + + # Class constructor + # + # @param name [String] File name (optional) + def initialize(name = nil) + super(name) + end + + # Load file content + # + # @param name [String] Custom file name + # + # @return [Object] Read content + def load(name = @name) + reset + + file_operation(name, 'r') do |file| + @content = file.read + end + + @content + end + + # Save content into a file + # + # @param name [String] Custom file name + def save(name = @name) + raise OneCfg::Config::Exception::NoContent if @content.nil? + + file_operation(name, 'w') do |file| + file.write(@content) + end + end + + end + +end +# rubocop:enable Style/ClassAndModuleChildren diff --git a/src/onecfg/lib/config/type/yaml.rb b/src/onecfg/lib/config/type/yaml.rb new file mode 100644 index 0000000000..92f582dcef --- /dev/null +++ b/src/onecfg/lib/config/type/yaml.rb @@ -0,0 +1,680 @@ +# -------------------------------------------------------------------------- # +# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +require 'yaml' + +# rubocop:disable Style/ClassAndModuleChildren +module OneCfg::Config::Type + + # Yaml class + class Yaml < Base + + # Patch Safe Default Modes + # Empty array, means there is no default mode + PATCH_SAFE_MODES = [] + + # Class constructor + # + # @param name [String] File name + def initialize(name = nil) + super(name) + end + + # Load file content + # + # @param name [String] Custom file name + # + # @return [Object] Read content + def load(name = @name) + reset + + @content = YAML.load_file(name) + + @content + end + + # Save content into a file + # + # @param name [String] Custom file name + def save(name = @name) + raise OneCfg::Config::Exception::NoContent if @content.nil? + + file_operation(name, 'w') {|file| file.write(to_s) } + end + + # Render configuration content as yaml + # + # @return [String] Yaml format configuration + def to_s + @content.to_yaml(:indentation => 4) + end + + # Check if content of both configuration objects is similar + # + # @param cfg [OneCfg::Config::Type::Base] Configuration to compare + # + # @return [Boolean] True if content is similar + def similar?(cfg) + return(true) if same?(cfg) + + return(false) unless diff(cfg).nil? + + true + end + + def diff(cfg) + tree1 = @content + tree2 = cfg.content + + ret = diff_subtree(tree1, tree2, []) + ret.flatten! + + ret.empty? ? nil : ret + end + + ################################################################## + # Private Methods + ################################################################## + + private + + # Walks through the tree1 and tree2 data structures + # starting the provided path, compares them and returns + # array of differences. + # + # @param tree1 [Object] Data structure 1 to compare + # @param tree2 [Object] Data structure 2 to compare + # @param path [Array] Path in the tree to compare + # + # @return [Array] Array of Hashes with diff objects + # rubocop:disable Metrics/BlockNesting + def diff_subtree(tree1, tree2, path) + ret = [] + + # compatible classes + if tree1.class == tree2.class + # both subtrees are Hash + if tree1.is_a? Hash + keys = (tree1.keys + tree2.keys).uniq + + keys.each do |key| + if tree1.key?(key) && tree2.key?(key) + ret << diff_subtree(tree1[key], + tree2[key], + path + [key]) + + elsif tree1.key?(key) + # delete hash key + ret << { + 'path' => path, + 'key' => key, + 'old' => tree1[key], + 'state' => 'rm', + 'extra' => {} + } + else + # insert new hash key + ret << { + 'path' => path, + 'key' => key, + 'value' => tree2[key], + 'state' => 'ins', + 'extra' => {} + } + end + end + + # both subtrees are Array + elsif tree1.is_a? Array + if @strict + idx = 0 + idx1 = 0 + idx2 = 0 + + while (idx1 < tree1.length) || (idx2 < tree2.length) + val1 = tree1[idx1] + val2 = tree2[idx2] + + # We need to be sure we are comparing values + # still inside the arrays and not valid nil + # value with item outside the array range. + if (idx1 < tree1.length) && (idx2 < tree2.length) + if OneCfg::Config::Utils.deep_compare(val1, + val2, + @strict) + idx += 1 + idx1 += 1 + idx2 += 1 + else + # Inserting values: + # 1 = A, B, C, D, E, F + # 2 = A, B, X, C, Y, D, E, F + # INSERT X, idx 2 + # INSERT X, idx 4 + # when on pos 2, forward lookup for 'C' + # in tree2, find on pos 3, so add new + # 'X' on pos 2, idx2++ + # + # Deleting values: + # 1 = A, B, C, D, E, F + # 2 = A, B, E, F + # DELETE C, idx 2 + # DELETE D, idx 2 + # when on pos 2, forward lookup for 'C' + # in tree, don't find any, so delete + # 'C' from pos 2, idx1++ + + # forward lookup for val1 + fwd_found = false + fwd_idx2 = idx2 + 1 + + # rubocop:disable Layout/LineLength + while (fwd_idx2 < tree2.length) && !fwd_found + if OneCfg::Config::Utils.deep_compare(tree2[fwd_idx2], val1, @strict) + fwd_found = true + else + fwd_idx2 += 1 + end + end + # rubocop:enable Layout/LineLength + + if fwd_found + # insert array item + ret << { + 'path' => path, + 'key' => nil, + 'value' => val2, + 'old' => val1, + 'state' => 'ins', + 'extra' => { 'index' => idx } + } + + idx += 1 + idx2 += 1 + + else + # delete array item + ret << { + 'path' => path, + 'key' => nil, + 'old' => val1, + 'state' => 'rm', + 'extra' => { 'index' => idx } + } + + idx1 += 1 + end + end + + # Process remaining array items on tree1 + # by dropping them (not found on tree2) + elsif idx1 < tree1.length + # delete array item + ret << { + 'path' => path, + 'key' => nil, + 'old' => val1, + 'state' => 'rm', + 'extra' => { 'index' => idx } + } + + idx1 += 1 + + # Process remaining new array items on tree2 + # by creating them. + else + # insert array item + ret << { + 'path' => path, + 'key' => nil, + 'value' => val2, + 'old' => val1, + 'state' => 'ins', + 'extra' => { 'index' => idx } + } + + idx += 1 + idx2 += 1 + end + end + else + values = (tree1 + tree2).uniq + + values.each do |val| + di1 = OneCfg::Config::Utils.deep_include?( + tree1, val, @strict + ) + + di2 = OneCfg::Config::Utils.deep_include?( + tree2, val, @strict + ) + + if di1 && di2 + # skip + nil + + elsif di1 + # delete array item + ret << { + 'path' => path, + 'key' => nil, + 'old' => val, + 'state' => 'rm', + 'extra' => {} + } + else + # insert array item + ret << { + 'path' => path, + 'key' => nil, + 'value' => val, + 'state' => 'ins', + 'extra' => {} + } + end + end + end + + # both subtrees are Nil + elsif tree1.is_a? NilClass + # skip + nil + + # both subtrees are of some other type + else + unless OneCfg::Config::Utils.deep_compare(tree1, + tree2, + @strict) + # set new value + ret << { + 'path' => path[0..-2], + 'key' => path[-1], + 'value' => tree2, + 'old' => tree1, + 'state' => 'set', + 'extra' => {} + } + end + end + + # Tree1 and tree2 are not same classes. We can't compare + # them so we take tree2 as new value to set as-is. + else + # set new value + ret << { + 'path' => path[0..-2], + 'key' => path[-1], + 'value' => tree2, + 'old' => tree1, + 'state' => 'set', + 'extra' => {} + } + end + + ret + end + # rubocop:enable Metrics/BlockNesting + + # Apply single diff/patch operation. Find working subtree + # based on the patch path. Then trigger inidividual + # delete / insert / set action on this subtree. + # + # @param data [Hash] Single diff operation data + # @param mode [Array] Patch modes (see patch method) + # + # @return [Hash] Patch status + def apply_diff_op(data, mode) + ret = { :status => false } + + # Follow the path and recursively go deep into the tree, + # If we don't find structure corresponding to the path, + # trigger exception (default) or skip operation (skip mode). + tree = @content + data['path'].each do |p| + if tree.is_a?(Hash) && tree.key?(p) + tree = tree[p] + elsif mode.include?(:skip) + ret[:mode] = :skip + + return ret + else + raise OneCfg::Config::Exception::PatchPathNotFound, + data['path'] + end + end + + case data['state'] + when 'rm' + ret = apply_diff_op_rm(tree, data, mode) + when 'ins' + ret = apply_diff_op_ins(tree, data, mode) + when 'set' + ret = apply_diff_op_set(tree, data, mode) + else + raise OneCfg::Config::Exception::FatalError, + "Invalid patch action '#{data['state']}'" + end + + ret + end + + # Appply single diff/patch "rm" operation. + # + # @param tree [Object] Subtree to work on + # @param data [Hash] Single diff operation data + # @param mode [Array] Patch modes (see patch method) + # + # @return [Hash] Patch status + def apply_diff_op_rm(tree, data, mode) + ret = { :status => false } + + # TODO: check if variable hasn't changed???? + # delete key from Hash + if data['key'] + if tree.is_a? Hash + ret[:status] = tree.key?(data['key']) + tree.delete(data['key']) + + elsif mode.include?(:skip) + ret[:mode] = :skip + else + raise OneCfg::Config::Exception::PatchExpectedHash, + tree.class + end + + # delete Array element by index + # rubocop:disable Layout/LineLength + # rubocop:disable Layout/CommentIndentation + elsif @strict && data['extra'] && data['extra']['index'] + if tree.is_a? Array + tree_idx_value = tree[data['extra']['index']] + + # try to drop by index + if OneCfg::Config::Utils.deep_compare(tree_idx_value, data['old'], @strict) + tree.delete_at(data['extra']['index']) + ret[:status] = true + +# TODO, ----> maybe here we don't need a force drop???, simply skip by value... + # or, force drop by value + elsif mode.include?(:force) + idx = OneCfg::Config::Utils.deep_index(tree, data['old'], @strict) + + if idx + ret[:status] = true + ret[:mode] = :force + tree.delete_at(idx) + end + + # TODO: is skip right condition? + elsif mode.include?(:skip) + ret[:mode] = :skip + + # or, fail on value not found + # TODO: else???? + else + raise OneCfg::Config::Exception::PatchValueNotFound, + data['old'] + end + + elsif mode.include?(:skip) + ret[:mode] = :skip + + # if not in skip mode, fail on unexpected data str. + else + raise OneCfg::Config::Exception::PatchExpectedArray, + tree.class + end +# TODO, <--------------------------------------------------------------------------------- + + # delete Array element by value + else + # TODO: merge with above? + if [Hash, Array].include? tree.class + idx = OneCfg::Config::Utils.deep_index(tree, + data['old'], + @strict) + + if idx + ret[:status] = true + tree.delete_at(idx) + end + elsif mode.include?(:skip) + ret[:mode] = :skip + else + raise OneCfg::Config::Exception::PatchUnexpectedData, + tree.class + end + end + # rubocop:enable Layout/LineLength + # rubocop:enable Layout/CommentIndentation + + ret + end + + # Appply single diff/patch "ins" operation. + # + # @param tree [Object] Subtree to work on + # @param data [Hash] Single diff operation data + # @param mode [Array] Patch modes (see patch method) + # + # @return [Hash] Patch status + def apply_diff_op_ins(tree, data, mode) + ret = { :status => false } + + # insert new key into a Hash + if data['key'] + if tree.is_a? Hash + if tree.key?(data['key']) + dc = OneCfg::Config::Utils.deep_compare( + tree[data['key']], + data['value'], + @strict + ) + + if !dc && mode.include?(:replace) + ret[:status] = true + ret[:mode] = :replace + ret[:old] = tree[data['key']] + tree[data['key']] = data['value'] + end + else + tree[data['key']] = data['value'] + ret[:status] = true + end + elsif mode.include?(:skip) + ret[:mode] = :skip + else + raise OneCfg::Config::Exception::PatchExpectedHash, + tree.class + end + + # insert new Array elemented by index + elsif @strict && data['extra'] && data['extra']['index'] + idx = data['extra']['index'] + + if tree.is_a? Array + # If element already exists (manually added by user), + # we won't touch it and e.g., fix the index. But, + # it might break the indexes and expected values + # there and application must be forced. + if !OneCfg::Config::Utils.deep_include?(tree, + data['value'], + @strict) + tree_idx_value = tree[idx] + + # if we find on idx expected data, insert by idx + if OneCfg::Config::Utils.deep_compare(tree_idx_value, + data['old'], + @strict) + + # Proposed idx could be after the end of array + # and we could be easily comparing nils. Better + # check and only append if index is outside. + if idx <= tree.length + tree.insert(idx, data['value']) + else + tree.push(data['value']) + end + + ret[:status] = true + + elsif mode.include?(:force) + ret[:status] = true + ret[:mode] = :force + + # On diff proposed index (idx) we expected some + # value (data['old']), but it is not there anymore. + # We try to guess new index by searching the array + # for the element with expected old content. If we + # find the element, we place new item on that index + # instead of the diff proposed index. + guess_idx = OneCfg::Config::Utils.deep_index( + tree, + data['old'], + @strict + ) + + # TODO: this automagic placement should be done + # based on 2 values (before / after), not just + # 1 (old). + # we tolerate only +/- 1 guessed index difference + if !guess_idx.nil? && (guess_idx - idx).abs <= 1 + tree.insert(guess_idx, data['value']) + # rubocop:disable Layout/CommentIndentation + # rubocop:disable Layout/LineLength + # OneCfg::LOG.debug("Guessed placement (#{guess_idx} instead of #{idx})") + # rubocop:enable Layout/CommentIndentation + # rubocop:enable Layout/LineLength + + # try to place on same index as proposed + # by diff if it's still inside range of Array + elsif idx <= tree.length + tree.insert(idx, data['value']) + + # otherwise, append at the end + else + tree.push(data['value']) + end + + # We probably shouldn't allow skipping now on this + # error, as the context placement is currently very + # lame and will allow too many changes not to be + # applied due to out poor implementation. + # --- + # elsif mode.include?(:skip) + # ret[:mode] = :skip + + else + raise OneCfg::Config::Exception::PatchValueNotFound, + data['old'] + end + end + elsif mode.include?(:skip) + ret[:mode] = :skip + else + raise OneCfg::Config::Exception::PatchExpectedArray, + tree.class + end + + # simply append Array element + else + if tree.is_a? Array + # append only if element doesn't exist already + unless OneCfg::Config::Utils.deep_include?(tree, + data['value'], + @strict) + tree.push(data['value']) + ret[:status] = true + end + elsif mode.include?(:skip) + ret[:mode] = :skip + else + raise OneCfg::Config::Exception::PatchExpectedArray, + tree.class + end + end + + ret + end + + # Appply single diff/patch "set" operation. + # + # @param tree [Object] Subtree to work on + # @param data [Hash] Single diff operation data + # @param mode [Array] Patch modes (see patch method) + # + # @return [Hash] Patch status + def apply_diff_op_set(tree, data, mode) + ret = { :status => false } + + if data['key'] + if tree.is_a? Hash + if tree.key?(data['key']) + # if old data weren't changed, we can replace by new + if OneCfg::Config::Utils.deep_compare(tree[data['key']], + data['old'], + @strict) + tree[data['key']] = data['value'] + ret[:status] = true + + elsif mode.include?(:replace) + ret[:status] = true + ret[:mode] = :replace + ret[:old] = tree[data['key']] + tree[data['key']] = data['value'] + end + + # Key existed in the past, but user deleted it. + # We can put it back (replace), skip or fail. + elsif mode.include?(:replace) + ret[:status] = true + ret[:mode] = :replace + tree[data['key']] = data['value'] + + elsif mode.include?(:skip) + ret[:mode] = :skip + else + raise OneCfg::Config::Exception::PatchValueNotFound, + data['key'] + end + + elsif mode.include?(:skip) + ret[:mode] = :skip + else + raise OneCfg::Config::Exception::PatchExpectedHash, + tree.class + end + + elsif data['path'].empty? + # WARNING: this is a dirty workaround which allows us + # to modify root path and completely main root data + # structure Array <-> Hash. We can't just do this on + # 'tree', as it would apply only in this method context! + ret[:status] = true + @content = data['value'] + else + # this should never happen + raise OneCfg::Config::Exception::FatalError, + 'Invalid patch without "key" or "path"' + end + + ret + end + + end + +end +# rubocop:enable Style/ClassAndModuleChildren diff --git a/src/onecfg/lib/config/type/yaml/strict.rb b/src/onecfg/lib/config/type/yaml/strict.rb new file mode 100644 index 0000000000..445b124463 --- /dev/null +++ b/src/onecfg/lib/config/type/yaml/strict.rb @@ -0,0 +1,38 @@ +# -------------------------------------------------------------------------- # +# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +# rubocop:disable Style/ClassAndModuleChildren +module OneCfg::Config::Type + + # Yaml strict class, order in arrays matters + class Yaml::Strict < Yaml + + # Patch Safe Default Modes + PATCH_SAFE_MODES = [:force] + + # Class constructor + # + # @param name [String] File name + def initialize(name = nil) + super(name) + + @strict = true + end + + end + +end +# rubocop:enable Style/ClassAndModuleChildren diff --git a/src/onecfg/lib/config/utils.rb b/src/onecfg/lib/config/utils.rb new file mode 100644 index 0000000000..68a5bdf70b --- /dev/null +++ b/src/onecfg/lib/config/utils.rb @@ -0,0 +1,173 @@ +# -------------------------------------------------------------------------- # +# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +module OneCfg + + module Config + + # Config::Utils module + module Utils + + # Deep comparison of data structures + # + # @param val1 [Object] First value + # @param val2 [Object] Second value + # @param strict [Boolean] Strict ordered Array values + # + # @return [Boolean] True if values are same + # + # Note: Strict order applied only on Arrays, which are + # used as values. Strict order doesn't apply on Array + # used as a Hash key!!! + def self.deep_compare(val1, val2, strict = false) + # values are not of the same type + return(false) unless val1.class == val2.class + + # rubocop:disable Style/CaseLikeIf + # rubocop:disable Style/RedundantReturn + if val1.is_a? Array + # arrays are not of the same length + return(false) unless val1.length == val2.length + + idx1 = 0 + idxs = 0.step(val1.length - 1, 1).to_a + + while idx1 < val1.length + # in strict mode, we compare same indexes, + # in non-strict mode, we try to find an element + if strict + ret = deep_compare(val1[idx1], val2[idx1], + strict) + + idxs.delete(idx1) + + # rubocop:disable Style/IdenticalConditionalBranches + return(false) unless ret + # rubocop:enable Style/IdenticalConditionalBranches + else + ret = false + + # we iterate over indexes we didn't go through + # yet, if we find matching index, we drop it + # from next runs + idxs.each do |idx2| + ret = deep_compare(val1[idx1], val2[idx2], + strict) + + if ret + idxs.delete(idx2) + break + end + end + + # rubocop:disable Style/IdenticalConditionalBranches + return(false) unless ret + # rubocop:enable Style/IdenticalConditionalBranches + end + + idx1 += 1 + end + + # at the end, the indexes to check must be empty + return(idxs.empty?) + + elsif val1.is_a? Hash + keys1 = val1.keys.sort_by {|k| k.to_s } + keys2 = val2.keys.sort_by {|k| k.to_s } + + # hashes must have same keys + return(false) unless keys1.length == keys2.length + return(false) unless deep_compare(keys1, keys2, strict) + + keys1.each do |key| + ret = deep_compare(val1[key], val2[key], strict) + + return(false) unless ret + end + + return(true) + else + return(val1 == val2) + end + # rubocop:enable Style/CaseLikeIf + # rubocop:enable Style/RedundantReturn + end + + # Detects existence of element in array by deep comparison + # and returns true/false based on the search. + # + # @param tree [Array] First value + # @param val [Object] Value to check + # @param strict [Boolean] Strict ordered Array values + # + # @return [Boolean] True if value exists in array + def self.deep_include?(tree, val, strict = false) + tree.each do |tree_val| + return(true) if deep_compare(tree_val, val, strict) + end + + false + end + + # Detects existence of element in array by deep comparison + # and returns its index or nil. + # + # @param tree [Array] First value + # @param val [Object] Value to check + # @param strict [Boolean] Strict ordered Array values + # + # @return [Integer,Nil] Value index or nil. + def self.deep_index(tree, val, strict = false) + tree.each_with_index do |tree_val, idx| + return(idx) if deep_compare(tree_val, val, strict) + end + + nil + end + + # Get path with prefix. + # + # @param path [String] Path + # @param prefix [String] Prefix + # + # @return [String] Prefixed path. + def self.prefixed(path, prefix) + if prefix == '/' + path + else + if prefix[-1] == '/' || path[0] == '/' + "#{prefix}#{path}" + else + "#{prefix}/#{path}" + end + end + end + + # Get path without prefix. + # + # @param path [String] Path + # @param prefix [String] Prefix + # + # @return [String] unprefixed path + def self.unprefixed(path, prefix) + path.sub(%r{^#{prefix}/*}, '/') + end + + end + + end + +end diff --git a/src/onecfg/lib/exception.rb b/src/onecfg/lib/exception.rb new file mode 100644 index 0000000000..1c0fb29258 --- /dev/null +++ b/src/onecfg/lib/exception.rb @@ -0,0 +1,64 @@ +# -------------------------------------------------------------------------- # +# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +module OneCfg + + module Exception + + # Generic OneCfg exception + class Generic < RuntimeError + + attr_reader :text + + def initialize(text = nil) + @text = text + super(text) + end + + end + + # OneCfg config exception when file not found + class FileNotFound < Generic + + def initialize(text = 'File not found') + super(text) + end + + end + + # OneCfg config exception on file read + # rubocop:disable Lint/UselessMethodDefinition + class FileReadError < Generic + + def initialize(text) + super(text) + end + + end + + # OneCfg config exception on file write + class FileWriteError < Generic + + def initialize(text) + super(text) + end + + end + # rubocop:enable Lint/UselessMethodDefinition + + end + +end diff --git a/src/onecfg/lib/onecfg.rb b/src/onecfg/lib/onecfg.rb new file mode 100644 index 0000000000..46c582fe91 --- /dev/null +++ b/src/onecfg/lib/onecfg.rb @@ -0,0 +1,68 @@ +# -------------------------------------------------------------------------- # +# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +require 'exception' +require 'common' +require 'settings' +require 'version' +require 'config' + +begin + require 'ee' +rescue LoadError +end + +# OneCfg main module +module OneCfg + + ONE_LOCATION = ENV['ONE_LOCATION'] + + # Global directories + # TODO: improve + if ONE_LOCATION + BIN_DIR = File.join(ONE_LOCATION, 'bin') + ETC_DIR = '/tmp/onescape/etc' + BACKUP_DIR = '/tmp/onescape/backups' + SHARE_DIR = ONE_LOCATION + '/share/onecfg' + + [ETC_DIR, BACKUP_DIR].each do |d| + OneCfg::LOG.warn("Using local state in #{d}") + FileUtils.mkdir_p(d) + end + else + BIN_DIR = '/usr/bin' + ETC_DIR = '/etc/onescape' + BACKUP_DIR = '/var/lib/one/backups/config' + SHARE_DIR = '/usr/share/one/onecfg' + end + + LOG = OneCfg::Common::CliLogger + + # Project local directories + CONF_DIR = File.join(SHARE_DIR, 'etc') + MIGR_DIR = File.join(SHARE_DIR, 'migrators') + LENS_DIR = File.join(SHARE_DIR, 'augeas') + + # Individual files + FILES_CFG = File.join(CONF_DIR, 'files.yaml') + CONFIG_CFG = File.join(ETC_DIR, 'config.yaml') + + # Configuration management releated constants + CONFIG_BACKUP_DIRS = ['/etc/one', '/var/lib/one/remotes'] + CONFIG_UPDATE_DIRS = ['/etc/one', '/var/lib/one/remotes/etc'] + CONFIG_LOCAL_FIX_DIRS = { '/etc' => '/etc/one', '/var' => '/var/lib/one' } + +end diff --git a/src/onecfg/lib/settings.rb b/src/onecfg/lib/settings.rb new file mode 100644 index 0000000000..feac749141 --- /dev/null +++ b/src/onecfg/lib/settings.rb @@ -0,0 +1,79 @@ +# -------------------------------------------------------------------------- # +# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +module OneCfg + + # Settings class + class Settings + + # @defaults Default content for new file + # @mode Operation modes (:auto_load, :read_only, ...) + # @name File name + # @content Content data + attr_reader :defaults + attr_reader :mode + attr_accessor :name + attr_accessor :content + + def initialize(name, mode = [:auto_load]) + @name = name + @content = nil + @mode = mode + @defaults ||= {} + + load if @mode.include?(:auto_load) + end + + # Load settings from file + def load + reset + + if ::File.exist?(@name) + @content = YAML.load_file(@name) + end + rescue StandardError => e + OneCfg::LOG.error("Can't load settings from '#{@name}' " \ + "due to '#{e.message}'") + raise + end + + # Save settings into file + def save + if @mode.include?(:read_only) + raise OneCfg::Exception::FileWriteError, + "Settings file #{@name} is read-only!" + end + + begin + # TODO: use temporary file and rename + File.open(@name, 'w') do |f| + f.write(@content.to_yaml) + end + rescue StandardError => e + OneCfg::LOG.error("Can't save settings from '#{@name}' " \ + "due to '#{e.message}'") + raise + end + end + + # Replace content by defaults + def reset + @content = @defaults.dup + end + + end + +end diff --git a/src/onecfg/lib/version.rb b/src/onecfg/lib/version.rb new file mode 100644 index 0000000000..53a5b03f19 --- /dev/null +++ b/src/onecfg/lib/version.rb @@ -0,0 +1,21 @@ +# -------------------------------------------------------------------------- # +# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +module OneCfg + + VERSION = '5.13.80' + +end diff --git a/src/onecfg/share/augeas/oned.aug b/src/onecfg/share/augeas/oned.aug new file mode 100644 index 0000000000..1501f74806 --- /dev/null +++ b/src/onecfg/share/augeas/oned.aug @@ -0,0 +1,78 @@ +module Oned = + autoload xfm + +(* Version: 1.4 *) + +(* Change log: *) +(* 1.4: Allow space after section *) +(* 1.3: Allow escaped quotes in values *) +(* 1.2: Include /etc/one/monitord.conf *) + +(* primitives *) +let sep = del /[ \t]*=[ \t]*/ " = " +let eol = del /\n/ "\n" +let opt_space = del /[ \t]*/ "" +let opt_space_nl = del /[ \t\n]*/ "\n" +let opt_nl_indent = del /[ \t\n]*/ "\n " +let comma = del /,/ "," +let left_br = del /\[/ "[" +let right_br = del /\]/ "]" + +(* Regexes *) +(* Match everyhting within quotes, allow escape quote *) +let re_quoted_str = /"(\\\\[\\\\"]|[^\\\\"])*"/ + +(* Match everything except spaces, quote("), l-bracket([) and num-sign(#) *) +let re_value_str = /[^ \t\n"\[#]+/ + +(* Match everything except spaces, quote("), num-sign(#) and comma(,) *) +let re_section_value_str = /[^ \t\n"#,]+/ + +(* Store either after-value comment or full-line comment *) +let comment = [ label "#comment" . store /#[^\n]*/ ] +let comment_eol = comment . eol + + +(* Simple words *) +let name = key /[A-Za-z_0-9]+/ +let re_simple_value = re_quoted_str | re_value_str + + +(* Top level entry like `PORT = 2633` *) +let top_level_entry = name . sep . store re_simple_value +let top_level_line = opt_space + . [ top_level_entry . opt_space . (comment)? ] + . eol + + +(* Section lens for section like `LOG = [ ... ]` *) +let section_value = re_quoted_str | re_section_value_str +let section_entry = [ name . sep . store section_value ] +let section_entry_list = + ( section_entry . opt_space . comma . opt_nl_indent + | comment_eol . opt_space )* + . section_entry . opt_space_nl + . ( comment_eol )* + +let section = opt_space + . [ name . sep + . left_br + . opt_nl_indent + . section_entry_list + . right_br ] + . opt_space + . eol + +let empty_line = [ del /[ \t]*\n/ "\n" ] + +(* Main lens *) +let lns = ( top_level_line | comment_eol | section | empty_line )* + + +(* Variable: filter *) +let filter = incl "/etc/one/oned.conf" + . incl "/etc/one/sched.conf" + . incl "/etc/one/monitord.conf" + . incl "/etc/one/vmm_exec/vmm_exec_kvm.conf" + +let xfm = transform lns filter diff --git a/src/onecfg/share/augeas/test_oned.aug b/src/onecfg/share/augeas/test_oned.aug new file mode 100644 index 0000000000..d635e91f39 --- /dev/null +++ b/src/onecfg/share/augeas/test_oned.aug @@ -0,0 +1,184 @@ +module Test_oned = + + test Oned.lns get +"ENTRY = 123 +" =? + + test Oned.lns get +"ENTRY = \"MANAGE ABC\" +" =? + + test Oned.lns get +"TM_MAD_CONF = [NAME=123] +" =? + + test Oned.lns get " +A = [ NAME=123 ] +" =? + + test Oned.lns get +"A = [ +NAME=123 +] +" = ? + + test Oned.lns get +"A = [ +NAME=123, NAME2=2 +] +" = ? + + test Oned.lns get + +"#abc +LOG = [ + SYSTEM = \"file\", + DEBUG_LEVEL = 3 +] +" =? + + test Oned.lns get +"A=1 +A=1 +B=2 # comment +# abc +# + + C=[ + A=\"B\", + A=\"B\",#abc + # abc + X=\"Y\", + A=123 +] +" =? + + test Oned.lns get +"C=[ + A=123, #abc + B=223# abc +] +" +=? + test Oned.lns get +"TM_MAD = [ + EXECUTABLE = \"one_tm\", + ARGUMENTS = \"-t 15 -d dummy,lvm,shared,fs_lvm,qcow2,ssh,ceph,dev,vcenter,iscsi_libvirt\" +] +INHERIT_DATASTORE_ATTR = \"CEPH_HOST\" +" +=? + +test Oned.lns get +"LOG = [ + SYSTEM = \"file\", + DEBUG_LEVEL = 3 +] + +MONITORING_INTERVAL_HOST = 180 +MONITORING_INTERVAL_VM = 180 +MONITORING_INTERVAL_DATASTORE = 300 +MONITORING_INTERVAL_MARKET = 600 +MONITORING_THREADS = 50 + +SCRIPTS_REMOTE_DIR=/var/tmp/one +PORT = 2633 +LISTEN_ADDRESS = \"0.0.0.0\" +DB = [ BACKEND = \"sqlite\" ] + +VNC_PORTS = [ + START = 5900 +] + +FEDERATION = [ + MODE = \"STANDALONE\", + ZONE_ID = 0, + SERVER_ID = -1, + MASTER_ONED = \"\" +] + +RAFT = [ + LIMIT_PURGE = 100000, + LOG_RETENTION = 500000, + LOG_PURGE_TIMEOUT = 600, + ELECTION_TIMEOUT_MS = 2500, + BROADCAST_TIMEOUT_MS = 500, + XMLRPC_TIMEOUT_MS = 450 +] + +DEFAULT_COST = [ + CPU_COST = 0, + MEMORY_COST = 0, + DISK_COST = 0 +] + +NETWORK_SIZE = 254 + +MAC_PREFIX = \"02:00\" + +VLAN_IDS = [ + START = \"2\", + RESERVED = \"0, 1, 4095\" +] + +VXLAN_IDS = [ + START = \"2\" +] + +DATASTORE_CAPACITY_CHECK = \"yes\" + +DEFAULT_DEVICE_PREFIX = \"hd\" +DEFAULT_CDROM_DEVICE_PREFIX = \"hd\" + +DEFAULT_IMAGE_TYPE = \"OS\" +IM_MAD = [ + NAME = \"collectd\", + EXECUTABLE = \"collectd\", + ARGUMENTS = \"-p 4124 -f 5 -t 50 -i 60\" ] + +IM_MAD = [ + NAME = \"kvm\", + SUNSTONE_NAME = \"KVM\", + EXECUTABLE = \"one_im_ssh\", + ARGUMENTS = \"-r 3 -t 15 -w 90 kvm\" ] + +IM_MAD = [ + NAME = \"vcenter\", + SUNSTONE_NAME = \"VMWare vCenter\", + EXECUTABLE = \"one_im_sh\", + ARGUMENTS = \"-c -t 15 -r 0 vcenter\" ] + +IM_MAD = [ + NAME = \"ec2\", + SUNSTONE_NAME = \"Amazon EC2\", + EXECUTABLE = \"one_im_sh\", + ARGUMENTS = \"-c -t 1 -r 0 -w 600 ec2\" ] + +VM_MAD = [ + NAME = \"kvm\", + SUNSTONE_NAME = \"KVM\", + EXECUTABLE = \"one_vmm_exec\", + ARGUMENTS = \"-t 15 -r 0 kvm\", + DEFAULT = \"vmm_exec/vmm_exec_kvm.conf\", + TYPE = \"kvm\", + KEEP_SNAPSHOTS = \"no\", + IMPORTED_VMS_ACTIONS = \"terminate, terminate-hard, hold, release, suspend, + resume, delete, reboot, reboot-hard, resched, unresched, disk-attach, + disk-detach, nic-attach, nic-detach, snapshot-create, snapshot-delete\" +] +" = ? + + + test Oned.lns get +"PASSWORD = \"open\\\"nebula\" +" =? + + test Oned.lns get +"DB = [ + PASSWORD = \"open\\\"nebula\" +] +" =? + + test Oned.lns get +" NIC = [ model=\"virtio\" ] +" =? diff --git a/src/onecfg/share/etc/files.yaml b/src/onecfg/share/etc/files.yaml new file mode 100644 index 0000000000..906d05d4ce --- /dev/null +++ b/src/onecfg/share/etc/files.yaml @@ -0,0 +1,333 @@ +--- + +# -------------------------------------------------------------------------- # +# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +# This file describes all known OpenNebula +# configurations, their format (class) and ownership with +# permissions of newly created files. At the end, there are +# catch all entries to detect any new unclassified file. + +### Dir: /etc/one/ ######################################### + +- name: /etc/one/vcenter_driver.default + class: Simple + owner: root + group: oneadmin + mode: '0640' + +- name: /etc/one/ec2_driver.default + class: Simple + owner: root + group: oneadmin + mode: '0640' + +- name: /etc/one/az_driver.default + class: Simple + owner: root + group: oneadmin + mode: '0640' + +- name: /etc/one/packet_driver.default + class: Simple + owner: root + group: oneadmin + mode: '0640' + +- name: /etc/one/auth/ldap_auth.conf + class: Yaml::Strict + owner: root + group: oneadmin + mode: '0640' + +- name: /etc/one/auth/server_x509_auth.conf + class: Yaml + owner: root + group: oneadmin + mode: '0640' + +- name: /etc/one/auth/x509_auth.conf + class: Yaml + owner: root + group: oneadmin + mode: '0640' + +- name: /etc/one/az_driver.conf + class: Yaml + owner: root + group: oneadmin + mode: '0640' + +- name: /etc/one/cli/*.yaml + class: Yaml::Strict + owner: root + group: root + mode: '0644' + +- name: /etc/one/defaultrc + class: Augeas::Shell + owner: root + group: oneadmin + mode: '0640' + +- name: /etc/one/ec2_driver.conf + class: Yaml + owner: root + group: oneadmin + mode: '0640' + +- name: /etc/one/ec2query_templates/*.erb + class: Simple + owner: root + group: oneadmin + mode: '0640' + +- name: /etc/one/econe.conf + class: Yaml + owner: root + group: oneadmin + mode: '0640' + +- name: /etc/one/fireedge-server.conf + class: Yaml + owner: root + group: oneadmin + mode: '0640' + +- name: /etc/one/monitord.conf + class: Augeas::ONE + owner: root + group: oneadmin + mode: '0640' + +- name: /etc/one/hm/hmrc + class: Augeas::Shell + owner: root + group: oneadmin + mode: '0640' + +- name: /etc/one/oned.conf + class: Augeas::ONE + owner: root + group: oneadmin + mode: '0640' + +- name: /etc/one/oneflow-server.conf + class: Yaml + owner: root + group: oneadmin + mode: '0640' + +- name: /etc/one/onegate-server.conf + class: Yaml + owner: root + group: oneadmin + mode: '0640' + +- name: /etc/one/sched.conf + class: Augeas::ONE + owner: root + group: oneadmin + mode: '0640' + +- name: /etc/one/sunstone-logos.yaml + class: Yaml::Strict + owner: root + group: oneadmin + mode: '0640' + +- name: /etc/one/sunstone-server.conf + class: Yaml + owner: root + group: oneadmin + mode: '0640' + +- name: /etc/one/onehem-server.conf + class: Yaml + owner: root + group: oneadmin + mode: '0640' + +- name: /etc/one/sunstone-views/**/*.yaml + class: Yaml + owner: root + group: oneadmin + mode: '0640' + +- name: /etc/one/sunstone-views.yaml + class: Yaml + owner: root + group: oneadmin + mode: '0640' + +- name: /etc/one/tmrc + class: Augeas::Shell + owner: root + group: oneadmin + mode: '0640' + +- name: /etc/one/vcenter_driver.conf + class: Yaml + owner: root + group: oneadmin + mode: '0640' + +- name: /etc/one/vmm_exec/vmm_exec_kvm.conf + class: Augeas::ONE + owner: root + group: oneadmin + mode: '0640' + +- name: /etc/one/vmm_exec/vmm_execrc + class: Augeas::Shell + owner: root + group: oneadmin + mode: '0640' + +- name: /etc/one/vmm_exec/vmm_exec_vcenter.conf + class: Augeas::ONE + owner: root + group: oneadmin + mode: '0640' + + +### Dir: /var/lib/one/remotes/etc/ ######################### + +- name: /var/lib/one/remotes/etc/datastore/datastore.conf + class: Augeas::Shell + owner: oneadmin + group: oneadmin + mode: '0640' + +- name: /var/lib/one/remotes/etc/datastore/ceph/ceph.conf + class: Augeas::Shell + owner: oneadmin + group: oneadmin + mode: '0640' + +- name: /var/lib/one/remotes/etc/datastore/fs/fs.conf + class: Augeas::Shell + owner: oneadmin + group: oneadmin + mode: '0640' + +- name: /var/lib/one/remotes/etc/im/kvm-probes.d/pci.conf + class: Yaml + owner: oneadmin + group: oneadmin + mode: '0640' + +- name: /var/lib/one/remotes/etc/im/kvm-probes.d/probe_db.conf + class: Yaml + owner: oneadmin + group: oneadmin + mode: '0640' + +- name: /var/lib/one/remotes/etc/im/lxd-probes.d/pci.conf + class: Yaml + owner: oneadmin + group: oneadmin + mode: '0640' + +- name: /var/lib/one/remotes/etc/im/lxd-probes.d/probe_db.conf + class: Yaml + owner: oneadmin + group: oneadmin + mode: '0640' + +- name: /var/lib/one/remotes/etc/im/firecracker-probes.d/probe_db.conf + class: Yaml + owner: oneadmin + group: oneadmin + mode: '0640' + +- name: /var/lib/one/remotes/etc/market/http/http.conf + class: Augeas::Shell + owner: oneadmin + group: oneadmin + mode: '0640' + +- name: /var/lib/one/remotes/etc/tm/fs_lvm/fs_lvm.conf + class: Augeas::Shell + owner: oneadmin + group: oneadmin + mode: '0640' + +- name: /var/lib/one/remotes/etc/tm/ssh/sshrc + class: Augeas::Shell + owner: oneadmin + group: oneadmin + mode: '0640' + +- name: /var/lib/one/remotes/etc/vmm/kvm/kvmrc + class: Augeas::Shell + owner: oneadmin + group: oneadmin + mode: '0640' + +- name: /var/lib/one/remotes/etc/vmm/lxd/lxdrc + class: Yaml + owner: oneadmin + group: oneadmin + mode: '0640' + +- name: /var/lib/one/remotes/etc/vmm/firecracker/firecrackerrc + class: Yaml + owner: oneadmin + group: oneadmin + mode: '0640' + +- name: /var/lib/one/remotes/etc/vmm/vcenter/vcenterrc + class: Yaml + owner: oneadmin + group: oneadmin + mode: '0640' + +- name: /var/lib/one/remotes/etc/vnm/OpenNebulaNetwork.conf + class: Yaml + owner: oneadmin + group: oneadmin + mode: '0640' + + +### Dir: /var/lib/one/remotes/ (ONE < 5.6) ################# + +- name: /var/lib/one/remotes/datastore/ceph/ceph.conf + class: Augeas::Shell + owner: oneadmin + group: oneadmin + mode: '0640' + old: true + +- name: /var/lib/one/remotes/vmm/kvm/kvmrc + class: Augeas::Shell + owner: oneadmin + group: oneadmin + mode: '0640' + old: true + +- name: /var/lib/one/remotes/vnm/OpenNebulaNetwork.conf + class: Yaml + owner: oneadmin + group: oneadmin + mode: '0640' + old: true + + +### Catch all unclassified files above and fail ############ + +- name: '/etc/one/**/**' +- name: '/var/lib/one/remotes/etc/**/**'