[WIP] GH action update lockfiles

.github/workflows/update-lockfiles.yml

@@ -0,0 +1,82 @@
+name: Submit PRs to update lockfiles
+
+on:
+  workflow_dispatch: # Allows manual triggering from the GitHub UI
+  schedule:
+    - cron: '0 4 * * *' # Daily at 4:00am UTC
+
+jobs:
+  update-lockfiles:
+    if: github.event.repository.fork == true && github.repository == 'coreosbot-releng/coreos-assembler'
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        branch: [main]
+
+    permissions:
+      pull-requests: write # Required to create a pull request
+      contents: write # Required to rebase branches
+
+    steps:
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          # Required for 'peter-evans/create-pull-request' to push to a new branch
+          fetch-depth: 0
+
+      - name: Synchronise the 'lockfiles-update-STREAM' branch with the upstream one
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          BRANCH: ${{ matrix.branch }}
+        run: |
+          TARGET_BRANCH=update-lockfiles-${BRANCH}
+          echo "The target branch is '$TARGET_BRANCH'."
+
+          git remote add upstream https://github.com/coreos/coreos-assembler.git
+          git fetch upstream
+          if git ls-remote --heads origin "$TARGET_BRANCH" | grep -q "$TARGET_BRANCH"; then
+              echo "Branch '$TARGET_BRANCH' exists on origin. Checking it out."
+              git checkout "$TARGET_BRANCH"
+          elif git rev-parse --verify --quiet "$TARGET_BRANCH" > /dev/null; then
+              echo "Branch '$TARGET_BRANCH' exists locally. Checking it out."
+              git checkout "$TARGET_BRANCH"
+          else
+              echo "Branch '$TARGET_BRANCH' does not exist. Creating it from upstream/${BRANCH}."
+              if git rev-parse --verify --quiet "upstream/${BRANCH}" > /dev/null; then
+                  git checkout -b "$TARGET_BRANCH" upstream/${BRANCH}
+                  echo "Successfully created and checked out branch '$TARGET_BRANCH' from upstream/${BRANCH}."
+              else
+                  echo "Error: upstream/${BRANCH} does not exist after fetch. Cannot create new branch."
+                  exit 1
+              fi
+          fi
+          git rebase upstream/${BRANCH}
+          git push --force origin "$TARGET_BRANCH"
+
+      - name: Update the lockfiles
+        working-directory: ci/hermetic
+        run: |
+          git checkout update-lockfiles-${{ matrix.branch }}
+          sudo apt-get update && sudo apt-get install -y python3-dnf
+          pip3 install requests ruamel.yaml
+          ./update_artifacts_lockfile
+          bash -x update_rpms_lockfile
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v6
+        with:
+          token: ${{ secrets.UPDATE_LOCKFILES_PAT }}
+          commit-message: 'feat(automated): Update the lockfiles'
+          title: 'Automated: lockfiles updated'
+          body: |
+            This PR was automatically generated by the 'Submit PRs to update lockfiles' workflow.
+            It updates the lockfiles.
+          branch: update-lockfiles/${{ matrix.branch }}-candidate
+          base: update-lockfiles-${{ matrix.branch }}
+          labels: |
+            update-lockfiles
+          committer: "CoreOS Bot <coreosbot@fedoraproject.org>"
+          author: "CoreOS Bot <coreosbot@fedoraproject.org>"

.gitignore

@@ -9,3 +9,6 @@ __pycache__/
 
 # generated by `make all`
 /bin/
+
+# generated by `hermeto` when pulling the dependencies locally with hermeto
+hermeto-output/

.tekton/coreos-assembler-pull-request.yaml

@@ -34,10 +34,20 @@ spec:
     value: Dockerfile
   - name: path-context
     value: .
+  - name: hermetic
+    value: false
+  - name: prefetch-input
+    value: '[{"type": "rpm", "path": "ci/hermetic"}, {"path": "ci/hermetic", "type": "generic"}]'
+  # Note: to be removed once rpm fully supported
+  # https://github.com/hermetoproject/hermeto?tab=readme-ov-file#package-managers
+  - name: dev-package-managers
+    value: true
+  - name: build-args
+    value: ["NO_NETWORK=1"]
   pipelineRef:
     params:
     - name: bundle
-      value: quay.io/konflux-ci/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:5bc58ee8213aaa3da4c1d67e380007097fbbbfb4dca3d0711777bd2b0d115da1
+      value: quay.io/jcapitao/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:a3993688715cba973af5e7fba95bc91f92673e8491f2524853736161974334fb
     - name: name
       value: docker-build-multi-platform-oci-ta
     - name: kind

.tekton/coreos-assembler-push.yaml

@@ -31,10 +31,20 @@ spec:
     value: Dockerfile
   - name: path-context
     value: .
+  - name: hermetic
+    value: false
+  - name: prefetch-input
+    value: '[{"type": "rpm", "path": "ci/hermetic"}, {"path": "ci/hermetic", "type": "generic"}]'
+  # Note: to be removed once rpm fully supported
+  # https://github.com/hermetoproject/hermeto?tab=readme-ov-file#package-managers
+  - name: dev-package-managers
+    value: true
+  - name: build-args
+    value: ["NO_NETWORK=1"]
   pipelineRef:
     params:
     - name: bundle
-      value: quay.io/konflux-ci/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:5bc58ee8213aaa3da4c1d67e380007097fbbbfb4dca3d0711777bd2b0d115da1
+      value: quay.io/jcapitao/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:a3993688715cba973af5e7fba95bc91f92673e8491f2524853736161974334fb
     - name: name
       value: docker-build-multi-platform-oci-ta
     - name: kind

.tekton/kola-nfs-pull-request.yaml

@@ -9,8 +9,11 @@ metadata:
     pipelinesascode.tekton.dev/cancel-in-progress: "true"
     pipelinesascode.tekton.dev/max-keep-runs: "3"
     pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && target_branch
-      == "main" && ( "./tests/containers/nfs/***".pathChanged() || ".tekton/kola-nfs-pull-request.yaml".pathChanged()
-      || "Containerfile".pathChanged() )
+      == "main" &&
+      ("tests/containers/nfs/***".pathChanged() ||
+       ".tekton/kola-nfs-pull-request.yaml".pathChanged() ||
+       "ci/hermetic/rpms.lock.yaml".pathChanged()
+      )
   creationTimestamp: null
   labels:
     appstudio.openshift.io/application: coreos-assembler
@@ -35,10 +38,18 @@ spec:
     value: Containerfile
   - name: path-context
     value: tests/containers/nfs
+  - name: hermetic
+    value: false
+  - name: prefetch-input
+    value: '[{"type": "rpm", "path": "ci/hermetic"}]'
+  # Note: to be removed once rpm fully supported
+  # https://github.com/hermetoproject/hermeto?tab=readme-ov-file#package-managers
+  - name: dev-package-managers
+    value: true
   pipelineRef:
     params:
     - name: bundle
-      value: quay.io/konflux-ci/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:5bc58ee8213aaa3da4c1d67e380007097fbbbfb4dca3d0711777bd2b0d115da1
+      value: quay.io/jcapitao/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:a3993688715cba973af5e7fba95bc91f92673e8491f2524853736161974334fb
     - name: name
       value: docker-build-multi-platform-oci-ta
     - name: kind

.tekton/kola-nfs-push.yaml

@@ -8,7 +8,10 @@ metadata:
     pipelinesascode.tekton.dev/cancel-in-progress: "false"
     pipelinesascode.tekton.dev/max-keep-runs: "3"
     pipelinesascode.tekton.dev/on-cel-expression: event == "push" && target_branch
-      == "main" && "./tests/containers/nfs/***".pathChanged()
+      == "main" &&
+      ("tests/containers/nfs/***".pathChanged() ||
+       "ci/hermetic/rpms.lock.yaml".pathChanged()
+      )
   creationTimestamp: null
   labels:
     appstudio.openshift.io/application: coreos-assembler
@@ -31,10 +34,18 @@ spec:
     value: Containerfile
   - name: path-context
     value: tests/containers/nfs
+  - name: hermetic
+    value: false
+  - name: prefetch-input
+    value: '[{"type": "rpm", "path": "ci/hermetic"}]'
+  # Note: to be removed once rpm fully supported
+  # https://github.com/hermetoproject/hermeto?tab=readme-ov-file#package-managers
+  - name: dev-package-managers
+    value: true
   pipelineRef:
     params:
     - name: bundle
-      value: quay.io/konflux-ci/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:5bc58ee8213aaa3da4c1d67e380007097fbbbfb4dca3d0711777bd2b0d115da1
+      value: quay.io/jcapitao/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:a3993688715cba973af5e7fba95bc91f92673e8491f2524853736161974334fb
     - name: name
       value: docker-build-multi-platform-oci-ta
     - name: kind

.tekton/kola-tang-pull-request.yaml

@@ -9,7 +9,11 @@ metadata:
     pipelinesascode.tekton.dev/cancel-in-progress: "true"
     pipelinesascode.tekton.dev/max-keep-runs: "3"
     pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && target_branch
-      == "main" && ( "./tests/containers/tang/***".pathChanged() || ".tekton/kola-tang-pull-request.yaml".pathChanged())
+      == "main" &&
+      ("tests/containers/tang/***".pathChanged() ||
+       ".tekton/kola-tang-pull-request.yaml".pathChanged() ||
+       "ci/hermetic/rpms.lock.yaml".pathChanged()
+      )
   creationTimestamp: null
   labels:
     appstudio.openshift.io/application: coreos-assembler
@@ -34,10 +38,18 @@ spec:
     value: ./tests/containers/tang/Containerfile
   - name: path-context
     value: .
+  - name: hermetic
+    value: false
+  - name: prefetch-input
+    value: '[{"type": "rpm", "path": "ci/hermetic"}]'
+  # Note: to be removed once rpm fully supported
+  # https://github.com/hermetoproject/hermeto?tab=readme-ov-file#package-managers
+  - name: dev-package-managers
+    value: true
   pipelineRef:
     params:
     - name: bundle
-      value: quay.io/konflux-ci/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:5bc58ee8213aaa3da4c1d67e380007097fbbbfb4dca3d0711777bd2b0d115da1
+      value: quay.io/jcapitao/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:a3993688715cba973af5e7fba95bc91f92673e8491f2524853736161974334fb
     - name: name
       value: docker-build-multi-platform-oci-ta
     - name: kind

.tekton/kola-tang-push.yaml

@@ -8,7 +8,10 @@ metadata:
     pipelinesascode.tekton.dev/cancel-in-progress: "false"
     pipelinesascode.tekton.dev/max-keep-runs: "3"
     pipelinesascode.tekton.dev/on-cel-expression: event == "push" && target_branch
-      == "main" && "./tests/containers/tang/***".pathChanged()
+      == "main" &&
+      ("tests/containers/tang/***".pathChanged() ||
+       "ci/hermetic/rpms.lock.yaml".pathChanged()
+      )
   creationTimestamp: null
   labels:
     appstudio.openshift.io/application: coreos-assembler
@@ -31,10 +34,18 @@ spec:
     value: ./tests/containers/tang/Containerfile
   - name: path-context
     value: .
+  - name: hermetic
+    value: false
+  - name: prefetch-input
+    value: '[{"type": "rpm", "path": "ci/hermetic"}]'
+  # Note: to be removed once rpm fully supported
+  # https://github.com/hermetoproject/hermeto?tab=readme-ov-file#package-managers
+  - name: dev-package-managers
+    value: true
   pipelineRef:
     params:
     - name: bundle
-      value: quay.io/konflux-ci/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:5bc58ee8213aaa3da4c1d67e380007097fbbbfb4dca3d0711777bd2b0d115da1
+      value: quay.io/jcapitao/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:a3993688715cba973af5e7fba95bc91f92673e8491f2524853736161974334fb
     - name: name
       value: docker-build-multi-platform-oci-ta
     - name: kind

.tekton/kola-targetcli-pull-request.yaml

@@ -9,7 +9,11 @@ metadata:
     pipelinesascode.tekton.dev/cancel-in-progress: "true"
     pipelinesascode.tekton.dev/max-keep-runs: "3"
     pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && target_branch
-      == "main" && ( "./tests/containers/targetcli/***".pathChanged() || ".tekton/kola-targetcli-pull-request.yaml".pathChanged())
+      == "main" &&
+      ("tests/containers/targetcli/***".pathChanged() ||
+       ".tekton/kola-targetcli-pull-request.yaml".pathChanged() ||
+       "ci/hermetic/rpms.lock.yaml".pathChanged()
+      )
   creationTimestamp: null
   labels:
     appstudio.openshift.io/application: coreos-assembler
@@ -34,10 +38,18 @@ spec:
     value: ./tests/containers/targetcli/Containerfile
   - name: path-context
     value: .
+  - name: hermetic
+    value: false
+  - name: prefetch-input
+    value: '[{"type": "rpm", "path": "ci/hermetic"}]'
+  # Note: to be removed once rpm fully supported
+  # https://github.com/hermetoproject/hermeto?tab=readme-ov-file#package-managers
+  - name: dev-package-managers
+    value: true
   pipelineRef:
     params:
     - name: bundle
-      value: quay.io/konflux-ci/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:5bc58ee8213aaa3da4c1d67e380007097fbbbfb4dca3d0711777bd2b0d115da1
+      value: quay.io/jcapitao/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:a3993688715cba973af5e7fba95bc91f92673e8491f2524853736161974334fb
     - name: name
       value: docker-build-multi-platform-oci-ta
     - name: kind

.tekton/kola-targetcli-push.yaml

@@ -8,7 +8,10 @@ metadata:
     pipelinesascode.tekton.dev/cancel-in-progress: "false"
     pipelinesascode.tekton.dev/max-keep-runs: "3"
     pipelinesascode.tekton.dev/on-cel-expression: event == "push" && target_branch
-      == "main" && "./tests/containers/targetcli/***".pathChanged()
+      == "main" &&
+      ("tests/containers/targetcli/***".pathChanged() ||
+       "ci/hermetic/rpms.lock.yaml".pathChanged()
+      )
   creationTimestamp: null
   labels:
     appstudio.openshift.io/application: coreos-assembler
@@ -31,10 +34,18 @@ spec:
     value: ./tests/containers/targetcli/Containerfile
   - name: path-context
     value: .
+  - name: hermetic
+    value: false
+  - name: prefetch-input
+    value: '[{"type": "rpm", "path": "ci/hermetic"}]'
+  # Note: to be removed once rpm fully supported
+  # https://github.com/hermetoproject/hermeto?tab=readme-ov-file#package-managers
+  - name: dev-package-managers
+    value: true
   pipelineRef:
     params:
     - name: bundle
-      value: quay.io/konflux-ci/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:5bc58ee8213aaa3da4c1d67e380007097fbbbfb4dca3d0711777bd2b0d115da1
+      value: quay.io/jcapitao/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:a3993688715cba973af5e7fba95bc91f92673e8491f2524853736161974334fb
     - name: name
       value: docker-build-multi-platform-oci-ta
     - name: kind

Dockerfile

@@ -2,6 +2,8 @@
 # https://github.com/openshift/release/tree/master/ci-operator/config/coreos/coreos-assembler/coreos-coreos-assembler-main.yaml
 FROM quay.io/fedora/fedora:42
 WORKDIR /root/containerbuild
+# This variable is enabled by Konflux to build the container image hermatically.
+ARG NO_NETWORK=0
 
 # Keep this Dockerfile idempotent for local development rebuild use cases.
 USER root

build.sh

@@ -31,6 +31,7 @@ srcdir=$(pwd)
 
 configure_yum_repos() {
     [ "${arch}" == "riscv64" ] && return # No continuous repo for riscv64 yet
+    [ "${NO_NETWORK}" == "1" ] && return
     local version_id
     version_id=$(. /etc/os-release && echo ${VERSION_ID})
     # Add continuous tag for latest build tools and mark as required so we
@@ -46,7 +47,7 @@ install_rpms() {
     # First, a general update; this is best practice.  We also hit an issue recently
     # where qemu implicitly depended on an updated libusbx but didn't have a versioned
     # requires https://bugzilla.redhat.com/show_bug.cgi?id=1625641
-    yum -y distro-sync
+    [ "${NO_NETWORK}" == "0" ] && yum -y distro-sync
 
     # xargs is part of findutils, which may not be installed
     yum -y install /usr/bin/xargs
@@ -104,10 +105,16 @@ install_rpms() {
 # to CoreOS.
 install_ocp_tools() {
     [ "${arch}" == "riscv64" ] && return # No ocp tools for riscv64
-    # If $OCP_VERSION is defined we'll grab that specific version.
-    # Otherwise we'll get the latest.
-    local url="https://mirror.openshift.com/pub/openshift-v4/${arch}/clients/ocp/latest${OCP_VERSION:+-$OCP_VERSION}/openshift-client-linux.tar.gz"
-    curl -L "$url" | tar zxf - oc
+    if [ "${NO_NETWORK}" == "0" ]; then
+        # If $OCP_VERSION is defined we'll grab that specific version.
+        # Otherwise we'll get the latest.
+        local url="https://mirror.openshift.com/pub/openshift-v4/${arch}/clients/ocp/latest${OCP_VERSION:+-$OCP_VERSION}/openshift-client-linux.tar.gz"
+        curl -L "$url" | tar zxf - oc
+    else
+        local oc_archive=""
+        oc_archive=$(find /*/output/deps/generic/ -name "openshift-client-linux-${arch}.tar.gz")
+        tar zxf "$oc_archive" oc
+    fi
     mv oc /usr/bin
 }
 

ci/hermetic/Dockerfile

@@ -0,0 +1 @@
+../../Dockerfile