Fixed implicit project id, and try to get name of project if scoped auth used

server/src/tests/services/openstack/fixtures.py

@@ -270,6 +270,7 @@ def random_element(lst: list[T], *args: typing.Any, **kwargs: typing.Any) -> T:
 # The idea behind this is to allow testing the provider, service and deployment classes
 # without the need of a real OpenStack environment
 CLIENT_METHODS_INFO: typing.Final[list[AutoSpecMethodInfo]] = [
+    AutoSpecMethodInfo(client.OpenStackClient.get_project_id, returns=PROJECTS_LIST[0].id),
     AutoSpecMethodInfo(client.OpenStackClient.list_flavors, returns=FLAVORS_LIST),
     AutoSpecMethodInfo(client.OpenStackClient.list_availability_zones, returns=AVAILABILITY_ZONES_LIST),
     AutoSpecMethodInfo(client.OpenStackClient.list_projects, returns=PROJECTS_LIST),
@@ -355,7 +356,7 @@ PROVIDER_VALUES_DICT: typing.Final[gui.ValuesDictType] = {
     'concurrent_creation_limit': 1,
     'concurrent_removal_limit': 1,
     'timeout': 10,
-    'project_id': 'tenant',  # Old name, new is project_id
+    'project_id': '',  # No project_id allowed if using application_credential, it's implicit
     'region': 'region',
     'use_subnets_name': False,
     'https_proxy': 'https_proxy',

server/src/uds/services/OpenStack/openstack/client.py

@@ -117,6 +117,7 @@ class OpenStackClient:  # pylint: disable=too-many-public-methods
     _auth_method: openstack_types.AuthMethod
     _userid: typing.Optional[str]
     _projectid: typing.Optional[str]
+    _project_name: typing.Optional[str]
     _region: typing.Optional[str]
     _timeout: int
     _session: 'requests.Session'
@@ -155,6 +156,7 @@ class OpenStackClient:  # pylint: disable=too-many-public-methods
         self._domain, self._username, self._password = domain or 'Default', username, password
         self._userid = None
         self._projectid = projectid
+        self._project_name = None
         self._region = region
         self._timeout = timeout
         self._auth_method = auth_method
@@ -322,7 +324,9 @@ class OpenStackClient:  # pylint: disable=too-many-public-methods
         # logger.debug('Authenticating...')
         # If credential is cached, use it instead of requesting it again
         if (cached_creds := self.cache.get('auth')) != None:
-            self._authenticated_projectid, self._tokenid, self._userid, self._catalog = cached_creds
+            self._authenticated_projectid, self._projectid, self._tokenid, self._userid, self._catalog = (
+                cached_creds
+            )
             return
 
         data: dict[str, typing.Any]
@@ -381,12 +385,26 @@ class OpenStackClient:  # pylint: disable=too-many-public-methods
         self._tokenid = r.headers['X-Subject-Token']
         # Extract the token id
         token = r.json()['token']
-        # logger.debug('Got token {}'.format(token))
+
+        # Get user id, used for list projects
         self._userid = token['user']['id']
 
+        # If authentication method is application_credential, set projectid to the one in the token
+        # Note that in case of unscoped password, 'project' is not present in the token
+        if 'project' in token:
+            self._authenticated_projectid = self._projectid = token['project']['id']
+            self._project_name = token['project'].get('name', self._projectid)
+
         # For cache, we store the token validity, minus 60 seconds t
-        validity = (dateutil.parser.parse(token['expires_at']).replace(tzinfo=None) - dateutil.parser.parse(token['issued_at']).replace(tzinfo=None)).seconds - 60
-        self.cache.put('auth', (self._authenticated_projectid, self._tokenid, self._userid, self._catalog), validity)
+        validity = (
+            dateutil.parser.parse(token['expires_at']).replace(tzinfo=None)
+            - dateutil.parser.parse(token['issued_at']).replace(tzinfo=None)
+        ).seconds - 60
+        self.cache.put(
+            'auth',
+            (self._authenticated_projectid, self._projectid, self._tokenid, self._userid, self._catalog),
+            validity,
+        )
 
         # logger.debug('The token {} will be valid for {}'.format(self._tokenId, validity))
 
@@ -407,6 +425,10 @@ class OpenStackClient:  # pylint: disable=too-many-public-methods
         if self._authenticated is False or self._projectid != self._authenticated_projectid:
             self.authenticate()
 
+    @auth_required()
+    def get_project_id(self) -> tuple[str, str]:
+        return (self._projectid or '', self._project_name or '')
+
     @auth_required()
     @decorators.cached(prefix='prjs', timeout=consts.cache.EXTREME_CACHE_TIMEOUT, key_helper=cache_key_helper)
     def list_projects(self) -> list[openstack_types.ProjectInfo]:
@@ -757,10 +779,15 @@ class OpenStackClient:  # pylint: disable=too-many-public-methods
             raise Exception('Connection error')
 
         try:
-            for v in r.json()['versions']['values']:
+            values = r.json()['versions']['values']
+        except Exception:
+            raise gen_exceptions.Error('Invalid response from OpenStack (Mayby invalid endpoint?)')
+
+        for v in values:
             if v['id'] >= 'v3.1':
                 # Tries to authenticate
                 try:
+                    self.cache.clear()  # Clear cache, as we are going to authenticate again
                     self.authenticate()
                     # Log some useful information
                     logger.info('Openstack version: %s', v['id'])
@@ -769,9 +796,6 @@ class OpenStackClient:  # pylint: disable=too-many-public-methods
                 except Exception:
                     logger.exception('Authenticating')
                     raise Exception(_('Authentication error'))
-        except Exception:  # Not json
-            # logger.exception('xx')
-            raise Exception('Invalid endpoint (maybe invalid version selected?)')
 
         raise Exception(
             _(
@@ -844,7 +868,7 @@ class OpenStackClient:  # pylint: disable=too-many-public-methods
             if error_message is None and expects_json:
                 error_message = 'Error checking response'
                 logger.error('%s: %s', error_message, response.content)
-            raise Exception(error_message)
+            raise gen_exceptions.Error(error_message)
 
     # Only for testing purposes, not used at runtime
     def t_create_volume(self, name: str, size: int) -> openstack_types.VolumeInfo:

server/src/uds/services/OpenStack/provider.py

@@ -38,7 +38,7 @@ from django.utils.translation import gettext_noop as _
 from uds.core import exceptions, types
 from uds.core.services import ServiceProvider
 from uds.core.ui import gui
-from uds.core.util import validators, fields
+from uds.core.util import validators, fields, decorators
 
 from .openstack import client, sanitized_name, types as openstack_types
 from .service import OpenStackLiveService
@@ -214,9 +214,9 @@ class OpenStackProvider(ServiceProvider):
             self.timeout.value = validators.validate_timeout(self.timeout.value)
             if self.auth_method.value == openstack_types.AuthMethod.APPLICATION_CREDENTIAL:
                 # Ensure that the project_id is provided, so it's bound to the application credential
-                if not self.project_id.value:
+                if self.project_id.value:
                     raise exceptions.ui.ValidationError(
-                        _('Project Id is required when using Application Credential')
+                        _('Project Id not allowed when using Application Credential')
                     )
 
     def api(
@@ -264,6 +264,17 @@ class OpenStackProvider(ServiceProvider):
 
         return types.core.TestResult(True)
 
+    @decorators.cached('prid', timeout=60)
+    def get_project_info(self) -> tuple[str, str]:
+        """
+        Returns the project id and name (if known)
+        """
+        if self.auth_method.value == openstack_types.AuthMethod.APPLICATION_CREDENTIAL:
+            # Authenticate, and
+            return self.api().get_project_id()
+
+        return self.project_id.value, self.project_id.value
+
     @staticmethod
     def test(env: 'environment.Environment', data: 'types.core.ValuesType') -> 'types.core.TestResult':
         """

server/src/uds/services/OpenStack/service.py

@@ -217,10 +217,16 @@ class OpenStackLiveService(DynamicService):
 
         self.region.set_choices(regions)
 
-        if parent and parent.project_id.value:
-            projects = [gui.choice_item(parent.project_id.value, parent.project_id.value)]
-        else:
+        # If project is already selected, we use it, if not, we list all projects
+        projects: list[types.ui.ChoiceItem] = []
+        if parent:
+            project_id, project_name = parent.get_project_info()
+            if project_id:
+                projects = [gui.choice_item(project_id, project_name)]
+
+        if not projects:
             projects = [gui.choice_item(t.id, t.name) for t in api.list_projects()]
+
         self.project.set_choices(projects)
 
         self.prov_uuid.value = self.provider().get_uuid()
@@ -241,21 +247,31 @@ class OpenStackLiveService(DynamicService):
             if i.name == name:
                 yield i.id
 
-    def get_ip(self, caller_instance: typing.Optional['DynamicUserService | DynamicPublication'], vmid: str) -> str:
+    def get_ip(
+        self, caller_instance: typing.Optional['DynamicUserService | DynamicPublication'], vmid: str
+    ) -> str:
         return self.api.get_server_info(vmid).validated().addresses[0].ip
 
-    def get_mac(self, caller_instance: typing.Optional['DynamicUserService | DynamicPublication'], vmid: str) -> str:
+    def get_mac(
+        self, caller_instance: typing.Optional['DynamicUserService | DynamicPublication'], vmid: str
+    ) -> str:
         return self.api.get_server_info(vmid).validated().addresses[0].mac
 
-    def is_running(self, caller_instance: typing.Optional['DynamicUserService | DynamicPublication'], vmid: str) -> bool:
+    def is_running(
+        self, caller_instance: typing.Optional['DynamicUserService | DynamicPublication'], vmid: str
+    ) -> bool:
         return self.api.get_server_info(vmid).validated().power_state.is_running()
 
-    def start(self, caller_instance: typing.Optional['DynamicUserService | DynamicPublication'], vmid: str) -> None:
+    def start(
+        self, caller_instance: typing.Optional['DynamicUserService | DynamicPublication'], vmid: str
+    ) -> None:
         if self.api.get_server_info(vmid).validated().power_state.is_running():
             return
         self.api.start_server(vmid)
 
-    def stop(self, caller_instance: typing.Optional['DynamicUserService | DynamicPublication'], vmid: str) -> None:
+    def stop(
+        self, caller_instance: typing.Optional['DynamicUserService | DynamicPublication'], vmid: str
+    ) -> None:
         if self.api.get_server_info(vmid).validated().power_state.is_stopped():
             return
         self.api.stop_server(vmid)
@@ -264,11 +280,15 @@ class OpenStackLiveService(DynamicService):
     # Note that on openstack, stop is "soft", but may fail to stop if no agent is installed or not responding
     # We can anyway delete de machine even if it is not stopped
 
-    def reset(self, caller_instance: typing.Optional['DynamicUserService | DynamicPublication'], vmid: str) -> None:
+    def reset(
+        self, caller_instance: typing.Optional['DynamicUserService | DynamicPublication'], vmid: str
+    ) -> None:
         # Default is to stop "hard"
         return self.stop(caller_instance, vmid)
 
-    def delete(self, caller_instance: typing.Optional['DynamicUserService | DynamicPublication'], vmid: str) -> None:
+    def delete(
+        self, caller_instance: typing.Optional['DynamicUserService | DynamicPublication'], vmid: str
+    ) -> None:
         """
         Removes the machine, or queues it for removal, or whatever :)
         """

server/src/uds/services/OpenStack/service_fixed.py

@@ -127,11 +127,17 @@ class OpenStackServiceFixed(FixedService):  # pylint: disable=too-many-public-me
 
         self.region.set_choices(regions)
 
-        if parent and parent.project_id.value:
-            tenants = [gui.choice_item(parent.project_id.value, parent.project_id.value)]
-        else:
-            tenants = [gui.choice_item(t.id, t.name) for t in api.list_projects()]
-        self.project.set_choices(tenants)
+        # If project is already selected, we use it, if not, we list all projects
+        projects: list[types.ui.ChoiceItem] = []
+        if parent:
+            project_id, project_name = parent.get_project_info()
+            if project_id:
+                projects = [gui.choice_item(project_id, project_name)]
+
+        if not projects:
+            projects = [gui.choice_item(t.id, t.name) for t in api.list_projects()]
+
+        self.project.set_choices(projects)
 
         self.prov_uuid.value = self.provider().get_uuid()