F #5780: Improve logs and messages on provision (#2168)

src/fireedge/src/client/components/DebugLog/utils.js

@@ -64,10 +64,15 @@ export const concatNewMessageToLog = (log, message = {}) => {
 
   const { data, command, commandId } = message
 
-  return {
-    ...log,
-    [command]: {
+  if (log?.[command]?.[commandId] !== undefined) {
+    log[command][commandId]?.push(data)
+  } else if (log?.[command] !== undefined) {
+    log[command][commandId] = [data]
+  } else {
+    log[command] = {
       [commandId]: [...(log?.[command]?.[commandId] ?? []), data],
-    },
+    }
   }
+
+  return { ...log }
 }

src/fireedge/src/client/containers/Providers/Create.js

@@ -49,7 +49,10 @@ function ProviderCreateForm() {
   const { enqueueSuccess, enqueueError } = useGeneralApi()
 
   const [createProvider] = useCreateProviderMutation()
-  const [updateProvider] = useUpdateProviderMutation()
+  const [
+    updateProvider,
+    { isSuccess: successUpdate, originalArgs: { id: updatedProviderId } = {} },
+  ] = useUpdateProviderMutation()
 
   const { data: providerConfig, error: errorConfig } =
     useGetProviderConfigQuery(undefined, { refetchOnMountOrArgChange: false })
@@ -71,7 +74,6 @@ function ProviderCreateForm() {
 
       if (id !== undefined) {
         await updateProvider({ id, data: submitData })
-        enqueueSuccess(`Provider updated - ID: ${id}`)
       } else {
         if (!isValidProviderTemplate(submitData, providerConfig)) {
           enqueueError(
@@ -81,7 +83,7 @@ function ProviderCreateForm() {
         }
 
         const responseId = await createProvider({ data: submitData }).unwrap()
-        enqueueSuccess(`Provider created - ID: ${responseId}`)
+        responseId && enqueueSuccess(`Provider created - ID: ${responseId}`)
       }
 
       history.push(PATH.PROVIDERS.LIST)
@@ -93,6 +95,11 @@ function ProviderCreateForm() {
     id && getConnection(id)
   }, [])
 
+  useEffect(() => {
+    successUpdate &&
+      enqueueSuccess(`Provider updated - ID: ${updatedProviderId}`)
+  }, [successUpdate])
+
   if (errorConfig || errorConnection || errorProvider) {
     return <Redirect to={PATH.PROVIDERS.LIST} />
   }

src/fireedge/src/client/containers/Providers/index.js

@@ -49,8 +49,14 @@ function Providers() {
 
   const { enqueueSuccess } = useGeneralApi()
   const { data: providerConfig } = useGetProviderConfigQuery()
-  const [deleteProvider, { isLoading: isDeleting }] =
-    useDeleteProviderMutation()
+  const [
+    deleteProvider,
+    {
+      isLoading: isDeleting,
+      isSuccess: successDelete,
+      originalArgs: { id: deletedProviderId } = {},
+    },
+  ] = useDeleteProviderMutation()
 
   const {
     refetch,
@@ -68,10 +74,14 @@ function Providers() {
     try {
       hide()
       await deleteProvider({ id })
-      enqueueSuccess(`Provider deleted - ID: ${id}`)
     } catch {}
   }
 
+  useEffect(() => {
+    successDelete &&
+      enqueueSuccess(`Provider deleted - ID: ${deletedProviderId}`)
+  }, [successDelete])
+
   return (
     <>
       <ListHeader

src/fireedge/src/client/containers/Provisions/DialogInfo/datastores.js

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and       *
  * limitations under the License.                                            *
  * ------------------------------------------------------------------------- */
-import { memo } from 'react'
+import { memo, useEffect } from 'react'
 import PropTypes from 'prop-types'
 
 import { Trash as DeleteIcon } from 'iconoir-react'
@@ -33,8 +33,14 @@ const Datastores = memo(
   ({ id }) => {
     const { enqueueSuccess } = useGeneralApi()
 
-    const [removeResource, { isLoading: loadingRemove }] =
-      useRemoveResourceMutation()
+    const [
+      removeResource,
+      {
+        isLoading: loadingRemove,
+        isSuccess: successRemove,
+        originalArgs: { id: deletedDatastoreId } = {},
+      },
+    ] = useRemoveResourceMutation()
     const { data } = useGetProvisionQuery(id)
 
     const provisionDatastores =
@@ -42,6 +48,11 @@ const Datastores = memo(
         (datastore) => +datastore.id
       ) ?? []
 
+    useEffect(() => {
+      successRemove &&
+        enqueueSuccess(`Datastore deleted - ID: ${deletedDatastoreId}`)
+    }, [successRemove])
+
     return (
       <DatastoresTable
         disableGlobalSort
@@ -76,7 +87,6 @@ const Datastores = memo(
                     id: datastore.ID,
                     resource: 'datastore',
                   })
-                  enqueueSuccess(`Datastore deleted - ID: ${datastore.ID}`)
                 }}
               />
             }

src/fireedge/src/client/containers/Provisions/DialogInfo/hosts.js

@@ -40,7 +40,7 @@ import { T } from 'client/constants'
 
 const Hosts = memo(({ id }) => {
   const [amount, setAmount] = useState(() => 1)
-  const { enqueueSuccess, enqueueInfo } = useGeneralApi()
+  const { enqueueInfo } = useGeneralApi()
 
   const [addHost, { isLoading: loadingAddHost }] =
     useAddHostToProvisionMutation()
@@ -77,7 +77,7 @@ const Hosts = memo(({ id }) => {
             isSubmitting={loadingAddHost}
             onClick={async () => {
               addHost({ id, amount })
-              enqueueSuccess(`Host added ${amount}x`)
+              enqueueInfo(`Adding ${amount} Host${amount > 1 ? 's' : ''}`)
             }}
           />
         </Stack>
@@ -125,7 +125,7 @@ const Hosts = memo(({ id }) => {
                       id: host.ID,
                       resource: 'host',
                     })
-                    enqueueSuccess(`Host deleted - ID: ${host.ID}`)
+                    enqueueInfo(`Deleting Host - ID:${host.ID}`)
                   }}
                 />
               </>

src/fireedge/src/client/features/OneApi/provision.js

@@ -220,7 +220,7 @@ const provisionApi = oneApi.injectEndpoints({
        * @returns {object} Object of document deleted
        * @throws Fails when response isn't code 200
        */
-      query: ({ provision: _, ...params }) => {
+      query: (params) => {
         const name = Actions.PROVISION_DELETE_RESOURCE
         const command = { name, ...Commands[name] }
 

src/fireedge/src/server/routes/api/oneprovision/provision/helpers.js

@@ -0,0 +1,295 @@
+/* ------------------------------------------------------------------------- *
+ * Copyright 2002-2022, OpenNebula Project, OpenNebula Systems               *
+ *                                                                           *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may   *
+ * not use this file except in compliance with the License. You may obtain   *
+ * a copy of the License at                                                  *
+ *                                                                           *
+ * http://www.apache.org/licenses/LICENSE-2.0                                *
+ *                                                                           *
+ * Unless required by applicable law or agreed to in writing, software       *
+ * distributed under the License is distributed on an "AS IS" BASIS,         *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  *
+ * See the License for the specific language governing permissions and       *
+ * limitations under the License.                                            *
+ * ------------------------------------------------------------------------- */
+const btoa = require('btoa')
+const { parse } = require('yaml')
+const { v4 } = require('uuid')
+const { DateTime } = require('luxon')
+const { publish } = require('server/utils/server')
+const {
+  httpResponse,
+  existsFile,
+  rotateBySize,
+  executeCommand,
+  executeCommandAsync,
+} = require('server/utils/server')
+const { defaults, httpCodes } = require('server/utils/constants')
+const {
+  findRecursiveFolder,
+  getSpecificConfig,
+} = require('server/routes/api/oneprovision/utils')
+
+const {
+  defaultCommandProvision,
+  defaultEmptyFunction,
+  defaultRegexpStartJSON,
+  defaultRegexpEndJSON,
+  defaultRegexpSplitLine,
+  defaultSizeRotate,
+} = defaults
+const { ok, internalServerError } = httpCodes
+const relName = 'provision-mapping'
+const ext = 'yml'
+const logFile = {
+  name: 'stdouterr',
+  ext: 'log',
+}
+const appendError = '.ERROR'
+
+/**
+ * Execute command Async and emit in WS.
+ *
+ * @param {string} command - command to execute
+ * @param {object} actions - external functions when command emit in stderr, stdout and finalize
+ * @param {Function} actions.err - emit when have stderr
+ * @param {Function} actions.out - emit when have stdout
+ * @param {Function} actions.close - emit when finalize
+ * @param {object} dataForLog - data
+ * @param {number} dataForLog.id - data id
+ * @param {string} dataForLog.command - data command
+ * @returns {boolean} check if emmit data
+ */
+const executeWithEmit = (command = [], actions = {}, dataForLog = {}) => {
+  if (
+    !(
+      command &&
+      Array.isArray(command) &&
+      command.length > 0 &&
+      actions &&
+      dataForLog
+    )
+  ) {
+    return
+  }
+
+  const { err: externalErr, out: externalOut, close: externalClose } = actions
+  const err =
+    externalErr && typeof externalErr === 'function'
+      ? externalErr
+      : defaultEmptyFunction
+  const out =
+    externalOut && typeof externalOut === 'function'
+      ? externalOut
+      : defaultEmptyFunction
+  const close =
+    externalClose && typeof externalClose === 'function'
+      ? actions.close
+      : defaultEmptyFunction
+
+  // data for log
+  const id = (dataForLog && dataForLog.id) || ''
+  const commandName = (dataForLog && dataForLog.command) || ''
+
+  let lastLine = ''
+  const uuid = v4()
+
+  let pendingMessages = ''
+
+  /**
+   * Emit data of command.
+   *
+   * @param {string} message - line of command CLI
+   * @param {Function} callback - function when recieve a information
+   */
+  const emit = (message, callback = defaultEmptyFunction) => {
+    /**
+     * Publisher data to WS.
+     *
+     * @param {string} line - command CLI line
+     */
+    const publisher = (line = '') => {
+      const resposeData = callback(line, uuid) || {
+        id,
+        data: line,
+        command: commandName,
+        commandId: uuid,
+      }
+      publish(defaultCommandProvision, resposeData)
+    }
+
+    message
+      .toString()
+      .split(defaultRegexpSplitLine)
+      .forEach((line) => {
+        if (line) {
+          if (
+            (defaultRegexpStartJSON.test(line) &&
+              defaultRegexpEndJSON.test(line)) ||
+            (!defaultRegexpStartJSON.test(line) &&
+              !defaultRegexpEndJSON.test(line) &&
+              pendingMessages.length === 0)
+          ) {
+            lastLine = line
+            publisher(lastLine)
+          } else if (
+            (defaultRegexpStartJSON.test(line) &&
+              !defaultRegexpEndJSON.test(line)) ||
+            (!defaultRegexpStartJSON.test(line) &&
+              !defaultRegexpEndJSON.test(line) &&
+              pendingMessages.length > 0)
+          ) {
+            pendingMessages += line
+          } else {
+            lastLine = pendingMessages + line
+            publisher(lastLine)
+            pendingMessages = ''
+          }
+        }
+      })
+  }
+
+  executeCommandAsync(
+    defaultCommandProvision,
+    command,
+    getSpecificConfig('oneprovision_prepend_command'),
+    {
+      err: (message) => {
+        emit(message, err)
+      },
+      out: (message) => {
+        emit(message, out)
+      },
+      close: (success) => {
+        close(success, lastLine)
+      },
+    }
+  )
+
+  return true
+}
+
+/**
+ * Find log data.
+ *
+ * @param {string} id - id of provision
+ * @param {boolean} fullPath - if need return the path of log
+ * @returns {object} data of log
+ */
+const logData = (id, fullPath = false) => {
+  let rtn = false
+  if (!Number.isInteger(parseInt(id, 10))) {
+    return rtn
+  }
+  const basePath = `${global.paths.CPI}/provision`
+  const relFile = `${basePath}/${relName}`
+  const relFileYML = `${relFile}.${ext}`
+  const find = findRecursiveFolder(basePath, id)
+
+  /**
+   * Found log.
+   *
+   * @param {string} path - path of log
+   * @param {string} uuid - uuid of log
+   */
+  const rtnFound = (path = '', uuid) => {
+    if (!path) {
+      return
+    }
+
+    const stringPath = `${path}/${logFile.name}.${logFile.ext}`
+    existsFile(
+      stringPath,
+      (filedata) => {
+        rotateBySize(stringPath, defaultSizeRotate)
+        rtn = { uuid, log: filedata.split(defaultRegexpSplitLine) }
+        if (fullPath) {
+          rtn.fullPath = stringPath
+        }
+      },
+      defaultEmptyFunction
+    )
+  }
+
+  if (find) {
+    rtnFound(find)
+  } else {
+    // Temporal provision
+    existsFile(
+      relFileYML,
+      (filedata) => {
+        const fileData = parse(filedata) || {}
+        if (!fileData[id]) {
+          return
+        }
+
+        const findPending = findRecursiveFolder(basePath, fileData[id])
+        if (findPending) {
+          rtnFound(findPending, fileData[id])
+        } else {
+          const findError = findRecursiveFolder(
+            basePath,
+            fileData[id] + appendError
+          )
+          findError && rtnFound(findError, fileData[id])
+        }
+      },
+      defaultEmptyFunction
+    )
+  }
+
+  return rtn
+}
+
+/**
+ * Execute Command sync and return http response.
+ *
+ * @param {any[]} params - params for command.
+ * @returns {object} httpResponse
+ */
+const addResourceSync = (params) => {
+  if (!(params && Array.isArray(params))) {
+    return
+  }
+
+  const executedCommand = executeCommand(
+    defaultCommandProvision,
+    params,
+    getSpecificConfig('oneprovision_prepend_command')
+  )
+  try {
+    const response = executedCommand.success ? ok : internalServerError
+
+    return httpResponse(
+      response,
+      executedCommand.data ? JSON.parse(executedCommand.data) : params.id
+    )
+  } catch (error) {
+    return httpResponse(internalServerError, '', executedCommand.data)
+  }
+}
+
+/**
+ * Executing line for provision logs.
+ *
+ * @param {string} message - message to log
+ * @returns {string} line message stringify
+ */
+const executingMessage = (message = '') =>
+  JSON.stringify({
+    timestamp: DateTime.now().toFormat('yyyy-MM-dd HH:mm:ss ZZZ'),
+    severity: 'DEBUG',
+    message: btoa(message),
+  })
+
+module.exports = {
+  executeWithEmit,
+  logData,
+  addResourceSync,
+  executingMessage,
+  relName,
+  ext,
+  logFile,
+  appendError,
+}

src/fireedge/src/server/routes/api/oneprovision/provision/routes.js

@@ -140,7 +140,7 @@ module.exports = {
       },
     },
     [PROVISION_DELETE_RESOURCE]: {
-      path: `${basepath}/resource/:resource/:id`,
+      path: `${basepath}/resource/:resource/:id/:provision`,
       httpMethod: DELETE,
       auth: true,
       params: {
@@ -150,6 +150,9 @@ module.exports = {
         id: {
           from: resource,
         },
+        provision: {
+          from: resource,
+        },
       },
     },
     [PROVISION_HOST_ACTION]: {

src/fireedge/src/server/utils/constants/defaults.js

@@ -57,6 +57,7 @@ const defaults = {
   defaultRegexID: /^ID: (?<id>\d+)/,
   defaultRegexpEndJSON: /}$/,
   defaultRegexpSplitLine: /\r|\n/,
+  defaultSizeRotate: '100k',
   defaultAppName: appName,
   defaultConfigErrorMessage: {
     color: 'red',

src/fireedge/src/server/utils/server.js

@@ -36,14 +36,17 @@ const {
   readdirSync,
   statSync,
   removeSync,
+  moveSync,
+  ensureFileSync,
 } = require('fs-extra')
 const { spawnSync, spawn } = require('child_process')
 const events = require('events')
+const { DateTime } = require('luxon')
+const { request: axios } = require('axios')
 const { defaults, httpCodes } = require('server/utils/constants')
 const { messageTerminal } = require('server/utils/general')
 const { validateAuth } = require('server/utils/jwt')
 const { writeInLogger } = require('server/utils/logger')
-const { request: axios } = require('axios')
 
 const eventsEmitter = new events.EventEmitter()
 const {
@@ -304,6 +307,48 @@ const decrypt = (data = '', decryptKey = '', iv = '') => {
   return rtn
 }
 
+const getSize = (limit) => {
+  const size = limit?.toLowerCase?.()?.match(/^((?:0\.)?\d+)([kmg])$/)
+  const limitNumber = parseInt(limit, 10)
+  if (size) {
+    switch (size[2]) {
+      case 'k':
+        return size[1] * 1024
+      case 'm':
+        return size[1] * 1024 ** 2
+      case 'g':
+        return size[1] * 1024 ** 3
+    }
+  } else if (Number.isInteger(limitNumber)) {
+    return limitNumber
+  }
+}
+
+/**
+ * Rotate file by size.
+ *
+ *
+ * @param {string} filepath - file path
+ * @param {number} limit - size to rotate
+ */
+const rotateBySize = (filepath = '', limit) => {
+  try {
+    const fileStats = statSync(filepath)
+    if (fileStats.size >= getSize(limit)) {
+      moveSync(filepath, `${filepath}.${DateTime.now().toSeconds()}`)
+      ensureFileSync(filepath)
+    }
+  } catch (error) {
+    const errorData = (error && error.message) || ''
+    writeInLogger(errorData)
+    messageTerminal({
+      color: 'red',
+      message: 'Error: %s',
+      error: errorData,
+    })
+  }
+}
+
 /**
  * Check if file exist.
  *
@@ -1005,4 +1050,5 @@ module.exports = {
   publish,
   subscriber,
   executeRequest,
+  rotateBySize,
 }