Adding tunnel server tests and some minor fixes

server/src/server/asgi.py

@@ -4,7 +4,7 @@ ASGI config for server project.
 It exposes the ASGI callable as a module-level variable named ``application``.
 
 For more information on this file, see
-https://docs.djangoproject.com/en/3.2/howto/deployment/asgi/
+https://docs.djangoproject.com/en/4.1/howto/deployment/asgi/
 """
 
 import os

tunnel-server/pytest.ini

@@ -0,0 +1,8 @@
+[pytest]
+addopts = "-s"
+pythonpath = ./src
+python_files = tests.py test_*.py *_tests.py
+log_format = "%(asctime)s %(levelname)s %(message)s"
+log_date_format = "%Y-%m-%d %H:%M:%S"
+log_cli = true
+log_level = info

tunnel-server/src/uds_tunnel/config.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (c) 2020 Virtual Cable S.L.U.
+# Copyright (c) 2022 Virtual Cable S.L.U.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification,
@@ -26,7 +26,7 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 '''
-@author: Adolfo Gómez, dkmaster at dkmon dot com
+Author: Adolfo Gómez, dkmaster at dkmon dot com
 '''
 import hashlib
 import multiprocessing
@@ -38,6 +38,7 @@ from .consts import CONFIGFILE
 
 logger = logging.getLogger(__name__)
 
+
 class ConfigurationType(typing.NamedTuple):
     pidfile: str
     user: str
@@ -51,7 +52,7 @@ class ConfigurationType(typing.NamedTuple):
     listen_port: int
 
     workers: int
-    
+
     ssl_certificate: str
     ssl_certificate_key: str
     ssl_ciphers: str
@@ -64,11 +65,24 @@ class ConfigurationType(typing.NamedTuple):
     allow: typing.Set[str]
 
     use_uvloop: bool
-    
 
-def read() -> ConfigurationType:
-    with open(CONFIGFILE, 'r') as f:
-        config_str = '[uds]\n' + f.read()
+
+def read_config_file(
+    cfg_file: typing.Optional[typing.Union[typing.TextIO, str]] = None
+) -> str:
+    if cfg_file is None:
+        cfg_file = CONFIGFILE
+    if isinstance(cfg_file, str):
+        with open(cfg_file, 'r') as f:
+            return '[uds]\n' + f.read()
+    # path is in fact a file-like object
+    return '[uds]\n' + cfg_file.read()
+
+
+def read(
+    cfg_file: typing.Optional[typing.Union[typing.TextIO, str]] = None
+) -> ConfigurationType:
+    config_str = read_config_file(cfg_file)
 
     cfg = configparser.ConfigParser()
     cfg.read_string(config_str)
@@ -97,7 +111,7 @@ def read() -> ConfigurationType:
             user=uds.get('user', ''),
             log_level=uds.get('loglevel', 'ERROR'),
             log_file=uds.get('logfile', ''),
-            log_size=int(logsize)*1024*1024,
+            log_size=int(logsize) * 1024 * 1024,
             log_number=int(uds.get('lognumber', '3')),
             listen_address=uds.get('address', '0.0.0.0'),
             listen_port=int(uds.get('port', '443')),
@@ -113,6 +127,10 @@ def read() -> ConfigurationType:
             use_uvloop=uds.get('use_uvloop', 'true').lower() == 'true',
         )
     except ValueError as e:
-        raise Exception(f'Mandatory configuration file in incorrect format: {e.args[0]}. Please, revise  {CONFIGFILE}')
+        raise Exception(
+            f'Mandatory configuration file in incorrect format: {e.args[0]}. Please, revise  {CONFIGFILE}'
+        )
     except KeyError as e:
-        raise Exception(f'Mandatory configuration parameter not found: {e.args[0]}. Please, revise {CONFIGFILE}')
+        raise Exception(
+            f'Mandatory configuration parameter not found: {e.args[0]}. Please, revise {CONFIGFILE}'
+        )

tunnel-server/src/uds_tunnel/consts.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (c) 2021 Virtual Cable S.L.U.
+# Copyright (c) 2023 Virtual Cable S.L.U.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification,
@@ -26,7 +26,7 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 '''
-@author: Adolfo Gómez, dkmaster at dkmon dot com
+Author: Adolfo Gómez, dkmaster at dkmon dot com
 '''
 DEBUG = True
 

tunnel-server/src/uds_tunnel/processes.py

@@ -1,3 +1,33 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2022 Virtual Cable S.L.U.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+#    * Redistributions of source code must retain the above copyright notice,
+#      this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright notice,
+#      this list of conditions and the following disclaimer in the documentation
+#      and/or other materials provided with the distribution.
+#    * Neither the name of Virtual Cable S.L. nor the names of its contributors
+#      may be used to endorse or promote products derived from this software
+#      without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+'''
+Author: Adolfo Gómez, dkmaster at dkmon dot com
+'''
 import multiprocessing
 import asyncio
 import sys

tunnel-server/src/uds_tunnel/proxy.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (c) 2020 Virtual Cable S.L.U.
+# Copyright (c) 2022 Virtual Cable S.L.U.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification,
@@ -26,7 +26,7 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 '''
-@author: Adolfo Gómez, dkmaster at dkmon dot com
+Author: Adolfo Gómez, dkmaster at dkmon dot com
 '''
 import asyncio
 import socket

tunnel-server/src/uds_tunnel/stats.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (c) 2020 Virtual Cable S.L.U.
+# Copyright (c) 2022 Virtual Cable S.L.U.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification,
@@ -26,7 +26,7 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 '''
-@author: Adolfo Gómez, dkmaster at dkmon dot com
+Author: Adolfo Gómez, dkmaster at dkmon dot com
 '''
 import multiprocessing
 import socket

tunnel-server/src/uds_tunnel/tunnel.py

@@ -1,9 +1,38 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2022 Virtual Cable S.L.U.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+#    * Redistributions of source code must retain the above copyright notice,
+#      this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright notice,
+#      this list of conditions and the following disclaimer in the documentation
+#      and/or other materials provided with the distribution.
+#    * Neither the name of Virtual Cable S.L. nor the names of its contributors
+#      may be used to endorse or promote products derived from this software
+#      without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+'''
+Author: Adolfo Gómez, dkmaster at dkmon dot com
+'''
 import asyncio
 import typing
 import logging
 
-
-import requests
+import aiohttp
 
 from . import consts
 from . import config
@@ -64,7 +93,7 @@ class TunnelProtocol(asyncio.Protocol):
 
     def process_open(self):
         # Open Command has the ticket behind it
-        
+
         if len(self.cmd) < consts.TICKET_LENGTH + consts.COMMAND_LENGTH:
             return  # Wait for more data to complete OPEN command
 
@@ -81,7 +110,7 @@ class TunnelProtocol(asyncio.Protocol):
 
         async def open_other_side() -> None:
             try:
-                result = await TunnelProtocol.getFromUds(
+                result = await TunnelProtocol.getTicketFromUDS(
                     self.owner.cfg, ticket, self.source
                 )
             except Exception as e:
@@ -246,7 +275,7 @@ class TunnelProtocol(asyncio.Protocol):
             logger.info('TERMINATED %s', self.pretty_source())
 
     @staticmethod
-    def _getUdsUrl(
+    async def _getUdsUrl(
         cfg: config.ConfigurationType,
         ticket: bytes,
         msg: str,
@@ -260,22 +289,18 @@ class TunnelProtocol(asyncio.Protocol):
                 url += '?' + '&'.join(
                     [f'{key}={value}' for key, value in queryParams.items()]
                 )
-            r = requests.get(
-                url,
-                headers={
-                    'content-type': 'application/json',
-                    'User-Agent': f'UDSTunnel/{consts.VERSION}',
-                },
-            )
-            if not r.ok:
-                raise Exception(r.content or 'Invalid Ticket (timed out)')
+            # Requests url with aiohttp
 
-            return r.json()
+            async with aiohttp.ClientSession() as session:
+                async with session.get(url) as r:
+                    if not r.ok:
+                        raise Exception(await r.text())
+                    return await r.json()
         except Exception as e:
             raise Exception(f'TICKET COMMS ERROR: {ticket.decode()} {msg} {e!s}')
 
     @staticmethod
-    async def getFromUds(
+    async def getTicketFromUDS(
         cfg: config.ConfigurationType, ticket: bytes, address: typing.Tuple[str, int]
     ) -> typing.MutableMapping[str, typing.Any]:
         # Sanity checks
@@ -291,17 +316,13 @@ class TunnelProtocol(asyncio.Protocol):
                 continue  # Correctus
             raise Exception(f'TICKET INVALID (char {i} at pos {n})')
 
-        return await asyncio.get_event_loop().run_in_executor(
-            None, TunnelProtocol._getUdsUrl, cfg, ticket, address[0]
-        )
+        return await TunnelProtocol._getUdsUrl(cfg, ticket, address[0])
 
     @staticmethod
     async def notifyEndToUds(
         cfg: config.ConfigurationType, ticket: bytes, counter: stats.Stats
     ) -> None:
-        await asyncio.get_event_loop().run_in_executor(
-            None,
-            TunnelProtocol._getUdsUrl,
+        await TunnelProtocol._getUdsUrl(
             cfg,
             ticket,
             'stop',

tunnel-server/src/udstunnel.conf

@@ -3,7 +3,6 @@
 # Pid file, optional
 # pidfile = /tmp/udstunnel.pid
 user = dkmaster
-group = dkmaster
 
 # Log level, valid are DEBUG, INFO, WARN, ERROR. Defaults to ERROR
 loglevel = DEBUG

tunnel-server/src/udstunnel.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
-# Copyright (c) 2021-2022 Virtual Cable S.L.U.
+# Copyright (c) 2022 Virtual Cable S.L.U.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification,
@@ -27,7 +27,7 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 '''
-@author: Adolfo Gómez, dkmaster at dkmon dot com
+Author: Adolfo Gómez, dkmaster at dkmon dot com
 '''
 import os
 import pwd
@@ -185,15 +185,16 @@ def process_connection(
         client.close()
 
 
-def tunnel_main() -> None:
-    cfg = config.read()
+def tunnel_main(args: 'argparse.Namespace') -> None:
+    cfg = config.read(args.config)
 
     # Try to bind to port as running user
     # Wait for socket incoming connections and spread them
     socket.setdefaulttimeout(
         3.0
     )  # So we can check for stop from time to time and not block forever
-    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    af_inet = socket.AF_INET6 if args.ipv6 or ':' in cfg.listen_address else socket.AF_INET
+    sock = socket.socket(af_inet, socket.SOCK_STREAM)
     sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
     sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
     # We will not reuse port, we only want a UDS tunnel server running on a port
@@ -294,10 +295,24 @@ def main() -> None:
         help='get current detailed stats from RUNNING tunnel',
         action='store_true',
     )
+    # Config file
+    parser.add_argument(
+        '-c',
+        '--config',
+        help=f'Config file to use (default: {consts.CONFIGFILE})',
+        default=consts.CONFIGFILE,
+    )
+    # If force ipv6
+    parser.add_argument(
+        '-6',
+        '--ipv6',
+        help='Force IPv6 for tunnel server',
+        action='store_true',
+    )
     args = parser.parse_args()
 
     if args.tunnel:
-        tunnel_main()
+        tunnel_main(args)
     elif args.rdp:
         pass
     elif args.detailed_stats:

tunnel-server/test/fixtures.py

@@ -0,0 +1,54 @@
+
+TEST_CONFIG='''# Sample UDS tunnel configuration
+
+# Pid file, optional
+pidfile = {pidfile}
+user = {user}
+
+# Log level, valid are DEBUG, INFO, WARN, ERROR. Defaults to ERROR
+loglevel = {loglevel}
+
+# Log file, Defaults to stdout
+logfile = {logfile}
+
+# Max log size before rotating it. Defaults to 32 MB.
+# The value is in MB. You can include or not the M string at end.
+logsize = {logsize}
+
+# Number of backup logs to keep. Defaults to 3
+lognumber = {lognumber}
+
+# Listen address. Defaults to 0.0.0.0
+address = {address}
+
+# Number of workers. Defaults to  0 (means "as much as cores")
+workers = {workers}
+
+# Listening port
+port = 7777
+
+# SSL Related parameters. 
+ssl_certificate = {ssl_certificate}
+ssl_certificate_key = {ssl_certificate_key}
+# ssl_ciphers and ssl_dhparam are optional.
+ssl_ciphers = {ssl_ciphers}
+ssl_dhparam = {ssl_dhparam}
+
+# UDS server location. https NEEDS valid certificate if https
+# Must point to tunnel ticket dispatcher URL, that is under /uds/rest/tunnel/ on tunnel server
+# Valid examples:
+#  http://www.example.com/uds/rest/tunnel/ticket
+#  https://www.example.com:14333/uds/rest/tunnel/ticket
+uds_server = {uds_server}
+uds_token = {uds_token}
+
+# Secret to get access to admin commands (Currently only stats commands). No default for this.
+# Admin commands and only allowed from "allow" ips
+# So, in order to allow this commands, ensure listen address allows connections from localhost
+secret = {secret}
+
+# List of af allowed admin commands ips (Currently only stats commands).
+# Only use IPs, no networks allowed
+# defaults to localhost (change if listen address is different from 0.0.0.0)
+allow = {allow}
+'''

tunnel-server/test/test_config_file.py

@@ -0,0 +1,92 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2022 Virtual Cable S.L.U.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+#    * Redistributions of source code must retain the above copyright notice,
+#      this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright notice,
+#      this list of conditions and the following disclaimer in the documentation
+#      and/or other materials provided with the distribution.
+#    * Neither the name of Virtual Cable S.L. nor the names of its contributors
+#      may be used to endorse or promote products derived from this software
+#      without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+'''
+Author: Adolfo Gómez, dkmaster at dkmon dot com
+'''
+import typing
+import hashlib
+import string
+import io
+import random
+
+from unittest import TestCase
+
+from uds_tunnel import config
+
+from . import fixtures
+
+class TestConfigFile(TestCase):
+    def test_config_file(self) -> None:
+        # Test in-memory configuration files ramdomly created
+        for _ in range(100):
+            values: typing.Mapping[str, typing.Any] = {
+                'pidfile': f'/tmp/uds_tunnel_{random.randint(0, 100)}.pid',  # Random pid file
+                'user': f'user{random.randint(0, 100)}',  # Random user
+                'loglevel': random.choice(['DEBUG', 'INFO', 'WARNING', 'ERROR']),  # Random log level
+                'logfile': f'/tmp/uds_tunnel_{random.randint(0, 100)}.log',  # Random log file
+                'logsize': random.randint(0, 100),  # Random log size
+                'lognumber': random.randint(0, 100),  # Random log number
+                'address': f'{random.randint(0, 255)}.{random.randint(0, 255)}.{random.randint(0, 255)}.{random.randint(0, 255)}',  # Random address
+                'workers': random.randint(1, 100),  # Random workers, 0 will return as many as cpu cores
+                'ssl_certificate': f'/tmp/uds_tunnel_{random.randint(0, 100)}.crt',  # Random ssl certificate
+                'ssl_certificate_key': f'/tmp/uds_tunnel_{random.randint(0, 100)}.key',  # Random ssl certificate key
+                'ssl_ciphers': f'ciphers{random.randint(0, 100)}',  # Random ssl ciphers
+                'ssl_dhparam': f'/tmp/uds_tunnel_{random.randint(0, 100)}.dh',  # Random ssl dhparam
+                'uds_server': f'https://uds_server{random.randint(0, 100)}/some_path',  # Random uds server
+                'uds_token': f'uds_token{random.choices(string.ascii_uppercase + string.digits, k=32)}',  # Random uds token
+                'secret': f'secret{random.randint(0, 100)}',  # Random secret
+                'allow': f'{random.randint(0, 255)}.0.0.0',  # Random allow
+
+            }
+            h = hashlib.sha256()
+            h.update(values.get('secret', '').encode())
+            secret = h.hexdigest()
+            # Generate an in-memory configuration file from fixtures.TEST_CONFIG
+            config_file = io.StringIO(fixtures.TEST_CONFIG.format(**values))
+            # Read it
+            cfg = config.read(config_file)
+            # Ensure data is correct
+            self.assertEqual(cfg.pidfile, values['pidfile'])
+            self.assertEqual(cfg.user, values['user'])
+            self.assertEqual(cfg.log_level, values['loglevel'])
+            self.assertEqual(cfg.log_file, values['logfile'])
+            self.assertEqual(cfg.log_size, values['logsize'] * 1024 * 1024)  # Config file is in MB
+            self.assertEqual(cfg.log_number, values['lognumber'])
+            self.assertEqual(cfg.listen_address, values['address'])
+            self.assertEqual(cfg.workers, values['workers'])
+            self.assertEqual(cfg.ssl_certificate, values['ssl_certificate'])
+            self.assertEqual(cfg.ssl_certificate_key, values['ssl_certificate_key'])
+            self.assertEqual(cfg.ssl_ciphers, values['ssl_ciphers'])
+            self.assertEqual(cfg.ssl_dhparam, values['ssl_dhparam'])
+            self.assertEqual(cfg.uds_server, values['uds_server'])
+            self.assertEqual(cfg.uds_token, values['uds_token'])
+            self.assertEqual(cfg.secret, secret)
+            self.assertEqual(cfg.allow, {values['allow']})
+    
+
+