Cache downloaded roles & collections

Populate the cache the first time the job is run for a revision
that needs them, and for future runs for that revision just
copy it into the private directory.

Delete the cache on project deletion.

awx/main/models/projects.py

@@ -194,6 +194,11 @@ class ProjectOptions(models.Model):
             if not check_if_exists or os.path.exists(smart_str(proj_path)):
                 return proj_path
 
+    def get_cache_path(self):
+        local_path = os.path.basename(self.local_path)
+        if local_path:
+            return os.path.join(settings.PROJECTS_ROOT, '.__awx_cache', local_path)
+
     @property
     def playbooks(self):
         results = []
@@ -455,11 +460,12 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
         )
 
     def delete(self, *args, **kwargs):
-        path_to_delete = self.get_project_path(check_if_exists=False)
+        paths_to_delete = (self.get_project_path(check_if_exists=False), self.get_cache_path())
         r = super(Project, self).delete(*args, **kwargs)
-        if self.scm_type and path_to_delete:  # non-manual, concrete path
-            from awx.main.tasks import delete_project_files
-            delete_project_files.delay(path_to_delete)
+        for path_to_delete in paths_to_delete:
+            if self.scm_type and path_to_delete:  # non-manual, concrete path
+                from awx.main.tasks import delete_project_files
+                delete_project_files.delay(path_to_delete)
         return r
 
 

awx/main/tasks.py

@@ -2181,6 +2181,7 @@ class RunProjectUpdate(BaseTask):
             'scm_url': scm_url,
             'scm_branch': scm_branch,
             'scm_clean': project_update.scm_clean,
+            'project_cache': project_update.get_cache_path(),
             'roles_enabled': settings.AWX_ROLES_ENABLED,
             'collections_enabled': settings.AWX_COLLECTIONS_ENABLED,
         })

awx/playbooks/project_update.yml

@@ -1,6 +1,7 @@
 ---
 # The following variables will be set by the runner of this playbook:
 # project_path: PROJECTS_DIR/_local_path_
+# project_cache: CACHE_DIR/.__awx_cache/_local_path_
 # scm_url: https://server/repo
 # insights_url: Insights service URL (from configuration)
 # scm_branch: branch/tag/revision (HEAD if unset)
@@ -116,13 +117,17 @@
   tasks:
 
     - block:
+        - name: set content cache location
+          set_fact:
+            cache_dir: "{{ project_cache }}/{{ scm_branch }}"
+
         - name: detect requirements.yml
           stat:
             path: '{{project_path|quote}}/roles/requirements.yml'
           register: doesRequirementsExist
 
         - name: fetch galaxy roles from requirements.yml
-          command: ansible-galaxy role install -r roles/requirements.yml -p {{roles_destination|quote}}{{ ' -' + 'v' * ansible_verbosity if ansible_verbosity else '' }}
+          command: ansible-galaxy role install -r roles/requirements.yml -p {{ cache_dir }}/requirements_roles {{ ' -' + 'v' * ansible_verbosity if ansible_verbosity else '' }}
           args:
             chdir: "{{project_path|quote}}"
           register: galaxy_result
@@ -132,18 +137,27 @@
             ANSIBLE_FORCE_COLOR: false
             GIT_SSH_COMMAND: "ssh -o StrictHostKeyChecking=no"
 
+        - name: populate job directory with needed roles
+          synchronize:
+            src: "{{ cache_dir }}/requirements_roles"
+            dest: "{{ roles_destination }}"
+
       when: roles_enabled|bool
       tags:
         - install_roles
 
     - block:
+        - name: set content cache location
+          set_fact:
+            cache_dir: "{{ project_cache }}/{{ scm_branch }}"
+
         - name: detect collections/requirements.yml
           stat:
             path: '{{project_path|quote}}/collections/requirements.yml'
           register: doesCollectionRequirementsExist
 
         - name: fetch galaxy collections from collections/requirements.yml
-          command: ansible-galaxy collection install -r collections/requirements.yml -p {{collections_destination|quote}}{{ ' -' + 'v' * ansible_verbosity if ansible_verbosity else '' }}
+          command: ansible-galaxy collection install -r collections/requirements.yml -p {{ cache_dir }}/requirements_collections {{ ' -' + 'v' * ansible_verbosity if ansible_verbosity else '' }}
           args:
             chdir: "{{project_path|quote}}"
           register: galaxy_collection_result
@@ -151,9 +165,14 @@
           changed_when: "'Installing ' in galaxy_collection_result.stdout"
           environment:
             ANSIBLE_FORCE_COLOR: false
-            ANSIBLE_COLLECTIONS_PATHS: "{{ collections_destination }}"
+            ANSIBLE_COLLECTIONS_PATHS: "{{ cache_dir }}/requirements_collections"
             GIT_SSH_COMMAND: "ssh -o StrictHostKeyChecking=no"
 
+        - name: populate job directory with needed collections
+          synchronize:
+            src: "{{ cache_dir }}/requirements_collections"
+            dest: "{{ collections_destination }}"
+
       when:
         - "ansible_version.full is version_compare('2.8', '>=')"
         - collections_enabled|bool