Merge pull request #2156 from ryanpetrello/fix-iso-dev

fix a few isolated dev issues

Makefile

@@ -219,7 +219,7 @@ init:
 	if [ "$(AWX_GROUP_QUEUES)" == "tower,thepentagon" ]; then \
 		$(MANAGEMENT_COMMAND) provision_instance --hostname=isolated; \
 		$(MANAGEMENT_COMMAND) register_queue --queuename='thepentagon' --hostnames=isolated --controller=tower; \
-		$(MANAGEMENT_COMMAND) generate_isolated_key | ssh -o "StrictHostKeyChecking no" root@isolated 'cat > /root/.ssh/authorized_keys'; \
+		$(MANAGEMENT_COMMAND) generate_isolated_key | ssh -o "StrictHostKeyChecking no" root@isolated 'cat >> /root/.ssh/authorized_keys'; \
 	fi;
 
 # Refresh development environment after pulling new code.
@@ -560,7 +560,7 @@ docker-isolated:
 	TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose.yml -f tools/docker-isolated-override.yml create
 	docker start tools_awx_1
 	docker start tools_isolated_1
-	echo "__version__ = '`python setup.py --version`'" | docker exec -i tools_isolated_1 /bin/bash -c "cat > /venv/awx/lib/python2.7/site-packages/awx.py"
+	echo "__version__ = '`git describe --long | cut -d - -f 1-1`'" | docker exec -i tools_isolated_1 /bin/bash -c "cat > /venv/awx/lib/python2.7/site-packages/awx.py"
 	if [ "`docker exec -i -t tools_isolated_1 cat /root/.ssh/authorized_keys`" == "`docker exec -t tools_awx_1 cat /root/.ssh/id_rsa.pub`" ]; then \
 		echo "SSH keys already copied to isolated instance"; \
 	else \

awx/main/expect/isolated_manager.py

@@ -434,6 +434,7 @@ class IsolatedManager(object):
                 task_result = {}
             if 'capacity_cpu' in task_result and 'capacity_mem' in task_result:
                 cls.update_capacity(instance, task_result, awx_application_version)
+                logger.debug('Isolated instance {} successful heartbeat'.format(instance.hostname))
             elif instance.capacity == 0:
                 logger.debug('Isolated instance {} previously marked as lost, could not re-join.'.format(
                     instance.hostname))

awx/main/management/commands/test_isolated_connection.py

@@ -3,7 +3,6 @@ import shutil
 import subprocess
 import sys
 import tempfile
-from optparse import make_option
 
 from django.conf import settings
 from django.core.management.base import BaseCommand, CommandError
@@ -15,10 +14,9 @@ class Command(BaseCommand):
     """Tests SSH connectivity between a controller and target isolated node"""
     help = 'Tests SSH connectivity between a controller and target isolated node'
 
-    option_list = BaseCommand.option_list + (
-        make_option('--hostname', dest='hostname', type='string',
-                    help='Hostname of an isolated node'),
-    )
+    def add_arguments(self, parser):
+        parser.add_argument('--hostname', dest='hostname', type=str,
+                            help='Hostname of an isolated node')
 
     def handle(self, *args, **options):
         hostname = options.get('hostname')
@@ -30,7 +28,7 @@ class Command(BaseCommand):
             args = [
                 'ansible', 'all', '-i', '{},'.format(hostname), '-u',
                 settings.AWX_ISOLATED_USERNAME, '-T5', '-m', 'shell',
-                '-a', 'hostname', '-vvv'
+                '-a', 'awx-expect -h', '-vvv'
             ]
             if all([
                 getattr(settings, 'AWX_ISOLATED_KEY_GENERATION', False) is True,

awx/main/migrations/0006_v320_release.py

@@ -484,7 +484,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='instance',
             name='last_isolated_check',
-            field=models.DateTimeField(auto_now_add=True, null=True),
+            field=models.DateTimeField(editable=False, null=True),
         ),
         # Migrations that don't change db schema but simply to make Django ORM happy.
         # e.g. Choice updates, help_text updates, etc.

awx/main/models/ha.py

@@ -50,7 +50,6 @@ class Instance(BaseModel):
     last_isolated_check = models.DateTimeField(
         null=True,
         editable=False,
-        auto_now_add=True
     )
     version = models.CharField(max_length=24, blank=True)
     capacity = models.PositiveIntegerField(

awx/main/tasks.py

@@ -226,6 +226,7 @@ def handle_ha_toplogy_worker_ready(sender, **kwargs):
     # Expedite the first hearbeat run so a node comes online quickly.
     cluster_node_heartbeat.apply([])
     apply_cluster_membership_policies.apply([])
+    awx_isolated_heartbeat.apply([])
 
 
 @celeryd_after_setup.connect
@@ -380,7 +381,11 @@ def awx_isolated_heartbeat(self):
     accept_before = nowtime - timedelta(seconds=(poll_interval - 10))
     isolated_instance_qs = Instance.objects.filter(
         rampart_groups__controller__instances__hostname=local_hostname,
+    )
+    isolated_instance_qs = isolated_instance_qs.filter(
         last_isolated_check__lt=accept_before
+    ) | isolated_instance_qs.filter(
+        last_isolated_check=None
     )
     # Fast pass of isolated instances, claiming the nodes to update
     with transaction.atomic():
@@ -883,6 +888,7 @@ class BaseTask(Task):
         stdout_handle = None
 
         try:
+            kwargs['isolated'] = instance.is_isolated()
             self.pre_run_hook(instance, **kwargs)
             if instance.cancel_flag:
                 instance = self.update_model(instance.pk, status='canceled')

awx/playbooks/clean_isolated.yml

@@ -18,7 +18,7 @@
       file: path="{{item}}" state=absent
       register: result
       with_items: "{{cleanup_dirs}}"
-      until: result|succeeded
+      until: result is succeeded
       ignore_errors: yes
       retries: 3
       delay: 5
@@ -26,4 +26,4 @@
     - name: fail if build artifacts were not cleaned
       fail:
         msg: 'Unable to cleanup build artifacts'
-      when: not result|succeeded
+      when: not result is succeeded

tools/docker-isolated-override.yml

@@ -14,6 +14,4 @@ services:
       - "../awx/main/expect:/awx_devel"
       - "../awx/lib:/awx_lib"
       - "/sys/fs/cgroup:/sys/fs/cgroup:ro"
-    tmpfs:
-      - "/tmp:exec"
-      - "/run"
+    privileged: true

tools/docker-isolated/Dockerfile

@@ -27,7 +27,4 @@ RUN ssh-keygen -A
 RUN mkdir -p /root/.ssh
 RUN touch /root/.ssh/authorized_keys
 
-STOPSIGNAL SIGRTMIN+3
-
-
 CMD ["/usr/sbin/init"]