Clean install and test result bases for stale packages (#2399)

* Install colcon clean extension

* Clean install path for packages to rebuild

* Clean test result path for packages to retest

* Set entrypoint for builder stage
to update time stamp used for breaking caches

do so at end of builder stage to capture overlay dependencies

* Set test-result-base for clean package verb

* Reinstate copy test results step

* Set default test-result-base for all verbs

* Drop copy step for test results

* Use parallel executor for testing
to print Processing status to stdout and avoid CI timeouts
but limit parallel-workers to 1 to be like sequential

* Remove entrypoint for builder stage
as lockfiles capture installed dependency versions
and colcon cache can ensure underlay sourse correspond to ws build
thus updates to builder stage layers need not break cache

Author: ruffsl

.circleci/config.yml

@@ -150,6 +150,10 @@ _commands:
                     fi
                     echo BUILD_PACKAGES: $BUILD_PACKAGES
 
+                    colcon clean packages --yes \
+                      --packages-select ${BUILD_PACKAGES} \
+                      --base-select install
+
                     . << parameters.underlay >>/install/setup.sh
                     colcon build \
                       --packages-select ${BUILD_PACKAGES} \
@@ -221,6 +225,10 @@ _commands:
                 | xargs)
               echo TEST_PACKAGES: $TEST_PACKAGES
 
+              colcon clean packages --yes \
+                --packages-select ${TEST_PACKAGES} \
+                --base-select test_result 
+
               . install/setup.sh
               set -o xtrace
               colcon test \
@@ -242,13 +250,6 @@ _commands:
             when: always
         - store_artifacts:
             path: << parameters.workspace >>/log/test
-        - run:
-            name: Copy Test Results
-            working_directory: << parameters.workspace >>
-            command: |
-              mkdir test_results/
-              cp -rH build/*/test_results/* test_results
-            when: always
         - store_test_results:
             path: << parameters.workspace >>/test_results
         - store_artifacts:

.circleci/defaults.yaml

@@ -1,4 +1,16 @@
+_common: &common
+  "test-result-base": "test_results"
+
+"clean.packages":
+  <<: *common
 "build":
+  <<: *common
   "executor": "parallel"
   "parallel-workers": 2
   "symlink-install": true
+"test":
+  <<: *common
+  "executor": "parallel"
+  "parallel-workers": 1
+"test-result":
+  <<: *common

Dockerfile

@@ -58,6 +58,7 @@ RUN apt-get update && \
     && pip3 install \
       fastcov \
       git+https://github.com/ruffsl/colcon-cache.git@c1cedadc1ac6131fe825d075526ed4ae8e1b473c \
+      git+https://github.com/ruffsl/colcon-clean.git@87dee2dd1e47c2b97ac6d8300f76e3f607d19ef6 \
     && rosdep update \
     && rm -rf /var/lib/apt/lists/*