feat(prowgen): utilize sparse checkout

Author: Prucek

cmd/ci-operator-prowgen/testdata/zz_fixture_postsubmit_TestFromCIOperatorConfigToProwYaml_Custom_test_timeout.yaml

@@ -25,7 +25,8 @@ postsubmits:
     - ^branch$
     decorate: true
     decoration_config:
-      skip_cloning: true
+      sparse_checkout_files:
+      - Dockerfile
     labels:
       ci-operator.openshift.io/is-promotion: "true"
       ci.openshift.io/generator: prowgen

cmd/ci-operator-prowgen/testdata/zz_fixture_postsubmit_TestFromCIOperatorConfigToProwYaml_Input_is_YAML_and_it_is_correctly_processed.yaml

@@ -25,7 +25,8 @@ postsubmits:
     - ^branch$
     decorate: true
     decoration_config:
-      skip_cloning: true
+      sparse_checkout_files:
+      - Dockerfile
     labels:
       ci-operator.openshift.io/is-promotion: "true"
       ci.openshift.io/generator: prowgen

cmd/ci-operator-prowgen/testdata/zz_fixture_postsubmit_TestFromCIOperatorConfigToProwYaml_Using_a_variant_config__one_test_and_images__one_existing_job._Expect_one_presubmit__pre_post_submit_images_jobs._Existing_job_should_not_be_changed..yaml

@@ -27,7 +27,8 @@ postsubmits:
     - ^branch$
     decorate: true
     decoration_config:
-      skip_cloning: true
+      sparse_checkout_files:
+      - Dockerfile
     labels:
       ci-operator.openshift.io/is-promotion: "true"
       ci-operator.openshift.io/variant: rhel

cmd/ci-operator-prowgen/testdata/zz_fixture_postsubmit_TestFromCIOperatorConfigToProwYaml_one_test_and_images__no_previous_jobs._Expect_test_presubmit__pre_post_submit_images_jobs.yaml

@@ -6,7 +6,8 @@ postsubmits:
     - ^branch$
     decorate: true
     decoration_config:
-      skip_cloning: true
+      sparse_checkout_files:
+      - Dockerfile
     labels:
       ci-operator.openshift.io/is-promotion: "true"
       ci.openshift.io/generator: prowgen

cmd/ci-operator-prowgen/testdata/zz_fixture_presubmit_TestFromCIOperatorConfigToProwYaml_Custom_test_timeout.yaml

@@ -8,7 +8,8 @@ presubmits:
     context: ci/prow/images
     decorate: true
     decoration_config:
-      skip_cloning: true
+      sparse_checkout_files:
+      - Dockerfile
     labels:
       ci.openshift.io/generator: prowgen
       pj-rehearse.openshift.io/can-be-rehearsed: "true"
@@ -62,7 +63,8 @@ presubmits:
     context: ci/prow/unit
     decorate: true
     decoration_config:
-      skip_cloning: true
+      sparse_checkout_files:
+      - Dockerfile
       timeout: 8h0m0s
     labels:
       ci.openshift.io/generator: prowgen

cmd/ci-operator-prowgen/testdata/zz_fixture_presubmit_TestFromCIOperatorConfigToProwYaml_Input_is_YAML_and_it_is_correctly_processed.yaml

@@ -8,7 +8,8 @@ presubmits:
     context: ci/prow/images
     decorate: true
     decoration_config:
-      skip_cloning: true
+      sparse_checkout_files:
+      - Dockerfile
     labels:
       ci.openshift.io/generator: prowgen
       pj-rehearse.openshift.io/can-be-rehearsed: "true"
@@ -62,7 +63,8 @@ presubmits:
     context: ci/prow/unit
     decorate: true
     decoration_config:
-      skip_cloning: true
+      sparse_checkout_files:
+      - Dockerfile
     labels:
       ci.openshift.io/generator: prowgen
       pj-rehearse.openshift.io/can-be-rehearsed: "true"

cmd/ci-operator-prowgen/testdata/zz_fixture_presubmit_TestFromCIOperatorConfigToProwYaml_Using_a_variant_config__one_test_and_images__one_existing_job._Expect_one_presubmit__pre_post_submit_images_jobs._Existing_job_should_not_be_changed..yaml

@@ -8,7 +8,8 @@ presubmits:
     context: ci/prow/rhel-images
     decorate: true
     decoration_config:
-      skip_cloning: true
+      sparse_checkout_files:
+      - Dockerfile
     labels:
       ci-operator.openshift.io/variant: rhel
       ci.openshift.io/generator: prowgen
@@ -64,7 +65,8 @@ presubmits:
     context: ci/prow/rhel-unit
     decorate: true
     decoration_config:
-      skip_cloning: true
+      sparse_checkout_files:
+      - Dockerfile
     labels:
       ci-operator.openshift.io/variant: rhel
       ci.openshift.io/generator: prowgen

cmd/ci-operator-prowgen/testdata/zz_fixture_presubmit_TestFromCIOperatorConfigToProwYaml_one_test_and_images__no_previous_jobs._Expect_test_presubmit__pre_post_submit_images_jobs.yaml

@@ -8,7 +8,8 @@ presubmits:
     context: ci/prow/images
     decorate: true
     decoration_config:
-      skip_cloning: true
+      sparse_checkout_files:
+      - Dockerfile
     labels:
       ci.openshift.io/generator: prowgen
       pj-rehearse.openshift.io/can-be-rehearsed: "true"
@@ -62,7 +63,8 @@ presubmits:
     context: ci/prow/unit
     decorate: true
     decoration_config:
-      skip_cloning: true
+      sparse_checkout_files:
+      - Dockerfile
     labels:
       ci.openshift.io/generator: prowgen
       pj-rehearse.openshift.io/can-be-rehearsed: "true"

pkg/prowgen/jobbase.go

@@ -1,6 +1,7 @@
 package prowgen
 
 import (
+	"path"
 	"time"
 
 	utilpointer "k8s.io/utils/pointer"
@@ -20,22 +21,51 @@ type prowJobBaseBuilder struct {
 	testName string
 }
 
-// If any included buildRoot uses from_repository we must not skip cloning
-func skipCloning(configSpec *cioperatorapi.ReleaseBuildConfiguration) bool {
-	buildRoots := configSpec.BuildRootImages
-	if buildRoots == nil {
-		buildRoots = make(map[string]cioperatorapi.BuildRootImageConfiguration)
-	}
-	if configSpec.BuildRootImage != nil {
-		buildRoots[""] = *configSpec.BuildRootImage
+func fromRepositorySet(configSpec *cioperatorapi.ReleaseBuildConfiguration) bool {
+	if configSpec.BuildRootImage != nil && configSpec.BuildRootImage.FromRepository {
+		return true
 	}
-	for _, buildRoot := range buildRoots {
+	for _, buildRoot := range configSpec.BuildRootImages {
 		if buildRoot.FromRepository {
-			return false
+			return true
 		}
 	}
+	return false
+}
 
-	return true
+// sparseCheckoutFiles returns the list of files to include in a sparse checkout
+// when the configuration requires cloning. This includes the in-repo ci-operator
+// configuration file and any Dockerfiles that need to be built.
+// Returns an empty slice if no sparse checkout is needed.
+func sparseCheckoutFiles(configSpec *cioperatorapi.ReleaseBuildConfiguration) []string {
+	seen := map[string]bool{}
+	var result []string
+	add := func(path string) {
+		if !seen[path] {
+			seen[path] = true
+			result = append(result, path)
+		}
+	}
+	if fromRepositorySet(configSpec) {
+		add(cioperatorapi.CIOperatorInrepoConfigFileName)
+	}
+	for _, image := range configSpec.Images.Items {
+		if image.DockerfileLiteral != nil {
+			continue
+		}
+		if image.Ref != "" {
+			continue
+		}
+		dockerfilePath := image.DockerfilePath
+		if dockerfilePath == "" {
+			dockerfilePath = "Dockerfile"
+		}
+		if image.ContextDir != "" {
+			dockerfilePath = path.Join(image.ContextDir, dockerfilePath)
+		}
+		add(dockerfilePath)
+	}
+	return result
 }
 
 func hasNoBuilds(c *cioperatorapi.ReleaseBuildConfiguration, info *ProwgenInfo) bool {
@@ -63,16 +93,20 @@ func NewProwJobBaseBuilder(configSpec *cioperatorapi.ReleaseBuildConfiguration,
 			Agent:  string(prowv1.KubernetesAgent),
 			Labels: map[string]string{},
 			UtilityConfig: prowconfig.UtilityConfig{
-				Decorate: utilpointer.Bool(true),
+				Decorate:         utilpointer.Bool(true),
+				DecorationConfig: &prowv1.DecorationConfig{},
 			},
 		},
 	}
 
-	if skipCloning(configSpec) {
-		b.base.UtilityConfig.DecorationConfig = &prowv1.DecorationConfig{SkipCloning: utilpointer.Bool(true)}
+	sparseFiles := sparseCheckoutFiles(configSpec)
+	shouldSkipCloning := len(sparseFiles) == 0
+	if shouldSkipCloning {
+		b.base.UtilityConfig.DecorationConfig.SkipCloning = utilpointer.Bool(true)
 	} else if info.Config.Private {
-		b.base.UtilityConfig.DecorationConfig = &prowv1.DecorationConfig{OauthTokenSecret: &prowv1.OauthTokenSecret{Key: cioperatorapi.OauthTokenSecretKey, Name: cioperatorapi.OauthTokenSecretName}}
+		b.base.UtilityConfig.DecorationConfig.OauthTokenSecret = &prowv1.OauthTokenSecret{Key: cioperatorapi.OauthTokenSecretKey, Name: cioperatorapi.OauthTokenSecretName}
 	}
+	b.base.UtilityConfig.DecorationConfig.SparseCheckoutFiles = sparseFiles
 
 	if len(info.Variant) > 0 {
 		b.base.Labels[jc.ProwJobLabelVariant] = info.Variant
@@ -91,7 +125,7 @@ func NewProwJobBaseBuilder(configSpec *cioperatorapi.ReleaseBuildConfiguration,
 	b.PodSpec.Add(Variant(info.Variant))
 	if info.Config.Private {
 		// We can reuse Prow's volume with the token if ProwJob itself is cloning the code
-		b.PodSpec.Add(GitHubToken(!skipCloning(configSpec)))
+		b.PodSpec.Add(GitHubToken(!shouldSkipCloning))
 	}
 
 	if configSpec.CanonicalGoRepository != nil {