Add data.credentials support to Buildpack and CNB lifecycles

app/actions/space_diff_manifest.rb

@@ -1,4 +1,5 @@
 require 'presenters/v3/app_manifest_presenter'
+require 'presenters/helpers/censorship'
 require 'messages/app_manifest_message'
 require 'json-diff'
 
@@ -10,6 +11,8 @@ class SpaceDiffManifest
       'sidecars' => 'name'
     }.freeze
 
+    REDACTED_KEYS = %w[credentials cnb-credentials].freeze
+
     class << self
       # rubocop:todo Metrics/CyclomaticComplexity
       def generate_diff(app_manifests, space)
@@ -56,6 +59,8 @@ def generate_diff(app_manifests, space)
             # To preserve backwards compability, we've decided to skip diffs that satisfy this conditon
             next if !nested_attribute_exists && %w[disk_quota disk-quota memory].include?(key)
 
+            value = obfuscate_buildpack_urls(value) if %w[buildpack buildpacks].include?(key)
+
             key_diffs = JsonDiff.diff(
               existing_value,
               value,
@@ -68,6 +73,8 @@ def generate_diff(app_manifests, space)
 
               diff['op'] = 'add' if diff['op'] == 'replace' && diff['was'].nil?
 
+              redact_credentials(diff) if REDACTED_KEYS.include?(key)
+
               json_diff << diff
             end
           end
@@ -79,6 +86,18 @@ def generate_diff(app_manifests, space)
 
       private
 
+      def redact_credentials(diff)
+        diff['value'] = Presenters::Censorship::PRIVATE_DATA_HIDDEN if diff.key?('value')
+        diff['was'] = Presenters::Censorship::PRIVATE_DATA_HIDDEN if diff.key?('was')
+      end
+
+      def obfuscate_buildpack_urls(buildpacks)
+        return CloudController::UrlSecretObfuscator.obfuscate(buildpacks) if buildpacks.is_a?(String)
+        return buildpacks unless buildpacks.is_a?(Array)
+
+        buildpacks.map { |bp| bp.is_a?(String) ? CloudController::UrlSecretObfuscator.obfuscate(bp) : bp }
+      end
+
       # rubocop:todo Metrics/CyclomaticComplexity
       def filter_manifest_app_hash(manifest_app_hash)
         if manifest_app_hash.key? 'sidecars'

app/messages/app_manifest_message.rb

@@ -43,6 +43,7 @@ class AppManifestMessage < BaseMessage
       sidecars
       stack
       timeout
+      credentials
       cnb_credentials
       features
     ]
@@ -75,6 +76,7 @@ def self.underscore_keys(hash)
     validate :validate_docker_enabled!
     validate :validate_cnb_enabled!
     validate :validate_docker_buildpacks_combination!
+    validate :validate_credentials_combination!
     validate :validate_service_bindings_message!, if: ->(record) { record.requested?(:services) }
     validate :validate_env_update_message!,       if: ->(record) { record.requested?(:env) }
     validate :validate_manifest_singular_buildpack_message!, if: ->(record) { record.requested?(:buildpack) }
@@ -126,27 +128,23 @@ def manifest_features_update_message
     def audit_hash
       result = original_yaml.dup
       result['env'] = Presenters::Censorship::PRIVATE_DATA_HIDDEN if result.key?('env')
+      result['credentials'] = Presenters::Censorship::PRIVATE_DATA_HIDDEN if result.key?('credentials')
       result['cnb-credentials'] = Presenters::Censorship::PRIVATE_DATA_HIDDEN if result.key?('cnb-credentials')
       obfuscate_buildpack_urls(result)
       redact_service_parameters(result)
       result
     end
 
     def app_lifecycle_hash
-      lifecycle_type = if requested?(:lifecycle) && @lifecycle == 'cnb'
-                         Lifecycles::CNB
-                       elsif requested?(:lifecycle) && @lifecycle == 'buildpack'
-                         Lifecycles::BUILDPACK
-                       elsif requested?(:docker)
-                         Lifecycles::DOCKER
-                       end
+      lifecycle_type = requested_lifecycle_type
 
       data = {
         buildpacks: requested_buildpacks,
-        stack: @stack,
-        credentials: @cnb_credentials
+        stack: @stack
       }.compact
 
+      data[:credentials] = @credentials || @cnb_credentials if requested?(:credentials) || requested?(:cnb_credentials)
+
       if lifecycle_type == Lifecycles::DOCKER
         lifecycle = {
           type: Lifecycles::DOCKER
@@ -496,6 +494,19 @@ def validate_docker_buildpacks_combination!
       errors.add(:base, 'Cannot specify both buildpack(s) and docker keys')
     end
 
+    def validate_credentials_combination!
+      return unless requested?(:credentials) && requested?(:cnb_credentials)
+
+      errors.add(:base, 'Cannot specify both credentials and cnb-credentials keys')
+    end
+
+    def requested_lifecycle_type
+      return Lifecycles::CNB if requested?(:lifecycle) && @lifecycle == 'cnb'
+      return Lifecycles::BUILDPACK if requested?(:lifecycle) && @lifecycle == 'buildpack'
+
+      Lifecycles::DOCKER if requested?(:docker)
+    end
+
     def validate_features!
       manifest_features_update_message.valid?
       manifest_features_update_message.errors.full_messages.each do |error_message|

app/models/runtime/buildpack_lifecycle_data_model.rb

@@ -9,6 +9,9 @@ class BuildpackLifecycleDataModel < Sequel::Model(:buildpack_lifecycle_data)
     LIFECYCLE_TYPE = Lifecycles::BUILDPACK
 
     set_field_as_encrypted :buildpack_url, salt: :encrypted_buildpack_url_salt, column: :encrypted_buildpack_url
+    set_field_as_encrypted :credentials_json,
+                           salt: :encrypted_credentials_json_salt,
+                           column: :encrypted_credentials_json
 
     many_to_one :droplet,
                 class: '::VCAP::CloudController::DropletModel',
@@ -85,11 +88,23 @@ def first_custom_buildpack_url
       buildpack_lifecycle_buildpacks.find(&:custom?)&.buildpack_url || legacy_buildpack_url
     end
 
+    def credentials
+      return unless credentials_json
+
+      Oj.load(credentials_json)
+    end
+
+    def credentials=(creds)
+      self.credentials_json = creds.nil? ? nil : Oj.dump(creds)
+    end
+
     def to_hash
-      {
+      hash = {
         buildpacks: obfuscated_buildpacks,
         stack: stack
       }
+      hash[:credentials] = Presenters::Censorship::REDACTED_CREDENTIAL unless credentials.nil?
+      hash
     end
 
     def validate

app/presenters/v3/app_manifest_presenters/buildpack_presenter.rb

@@ -1,3 +1,5 @@
+require 'presenters/helpers/censorship'
+
 module VCAP::CloudController
   module Presenters
     module V3
@@ -7,8 +9,9 @@ def to_hash(app:, **_)
             return {} if app.docker?
 
             {
-              buildpacks: app.lifecycle_data.buildpacks.presence,
-              stack: app.lifecycle_data.stack
+              buildpacks: app.lifecycle_data.obfuscated_buildpacks.presence,
+              stack: app.lifecycle_data.stack,
+              credentials: app.lifecycle_data.credentials.nil? ? nil : Presenters::Censorship::REDACTED_CREDENTIAL
             }
           end
         end

app/repositories/app_event_repository.rb

@@ -207,6 +207,7 @@ def obfuscate_buildpack_urls(changes)
 
         lifecycle_data['buildpack'] = CloudController::UrlSecretObfuscator.obfuscate(lifecycle_data['buildpack']) if lifecycle_data['buildpack']
         lifecycle_data['buildpacks'] = lifecycle_data['buildpacks'].map { |b| CloudController::UrlSecretObfuscator.obfuscate(b) } if lifecycle_data['buildpacks']
+        lifecycle_data['credentials'] = Presenters::Censorship::PRIVATE_DATA_HIDDEN if lifecycle_data.key?('credentials')
       end
     end
   end

db/migrations/20261001120000_add_buildpack_lifecycle_credentials.rb

@@ -0,0 +1,18 @@
+Sequel.migration do
+  up do
+    add_column :buildpack_lifecycle_data, :encrypted_credentials_json, String, size: 16_000, if_not_exists: true
+    add_column :buildpack_lifecycle_data, :encrypted_credentials_json_salt, String, size: 255, if_not_exists: true
+    # encryption_key_label and encryption_iterations already exist (added by earlier migrations for buildpack_url encryption)
+
+    # Existing CNB credentials column was created unsized (varchar(255) on MySQL),
+    # too small for an encrypted, base64-encoded multi-host credentials map. Grow it to match.
+    set_column_type :cnb_lifecycle_data, :encrypted_registry_credentials_json, String, size: 16_000
+  end
+
+  down do
+    drop_column :buildpack_lifecycle_data, :encrypted_credentials_json, if_exists: true
+    drop_column :buildpack_lifecycle_data, :encrypted_credentials_json_salt, if_exists: true
+
+    set_column_type :cnb_lifecycle_data, :encrypted_registry_credentials_json, String, size: 255
+  end
+end

docs/v3/source/includes/concepts/_lifecycles.md.erb

@@ -41,6 +41,7 @@ Name | Type | Description
 **type** | _string_ | `buildpack`
 **data.buildpacks** | _list of strings_ | A list of the names of buildpacks, URLs from which they may be downloaded, or `null` to auto-detect a suitable buildpack during staging
 **data.stack** | _string_ | The root filesystem to use with the buildpack, for example `cflinuxfs4`
+**data.credentials** | _object_ | Credentials used to download custom buildpacks from private git repositories, keyed by host. Each entry is a username/password pair. If any buildpack URL embeds credentials, **data.credentials** must be empty; provide credentials in one place only.
 
 For records whose historical lifecycle data is no longer available,
 **data.buildpacks** may be an empty list and **data.stack** may be `null`.
@@ -83,7 +84,7 @@ Name | Type | Description
 ---- | ---- | -----------
 **type** | _string_ | `cnb`
 **data.buildpacks** | _list of strings_ | A list of URLs with either `docker://` or `http(s)://` scheme, pointing to the Cloud Native Buildpack OCI image. <br/>When the scheme is `http(s)://`, an OCI tarball is expected to be present at the specified location.
-**data.credentials** | _object_ | Credentials used to download the configured buildpacks. This can either contain username/password  or a token.
+**data.credentials** | _object_ | Credentials used to download the configured buildpacks, keyed by registry host. Each entry contains either a username/password pair or a token. CNB buildpack URLs must not embed credentials; use **data.credentials** instead.
 **data.stack** | _string_ | The root filesystem to use with the buildpack, for example `cflinuxfs4`
 
 For records whose historical lifecycle data is no longer available,

docs/v3/source/includes/resources/manifests/_object.md.erb

@@ -105,6 +105,8 @@ Name | Type | Description
 **name** | _string_ | Name of the app
 **lifecycle** | _string_ | Type of the lifecycle; valid values are `buildpack`, `cnb`, `docker`. Defaults to `buildpack`, unless the **docker** property is present.
 **buildpacks** | _list of strings_ | a) An empty array, which will automatically select the appropriate default buildpack according to the coding language (incompatible with **lifecycle: cnb**) <br>b) An array of one or more URLs pointing to buildpacks <br>c) An array of one or more installed buildpack names <br>Replaces the legacy `buildpack` field
+**credentials** | _object_ | Credentials used to download custom buildpacks (buildpack lifecycle) or buildpack OCI images (cnb lifecycle) from private git repositories or registries, keyed by host. For the buildpack lifecycle each entry is a username/password pair; for the cnb lifecycle each entry is a username/password pair or a token. Set to `null` to clear previously stored credentials. If any buildpack URL embeds credentials, **credentials** must be empty; provide credentials in one place only.
+**cnb-credentials** | _object_ | Backward-compatible alias for **credentials**. Specifying both **credentials** and **cnb-credentials** is an error.
 **docker** | _object_ | If present, the created app will have Docker lifecycle type; the value of this key is ignored by the API but may be used by clients to source the registry address of the image and credentials, if needed; the [generate manifest endpoint](#generate-a-manifest-for-an-app) will return the registry address of the image and username provided with this key
 **env** | _object_ | A key-value mapping of environment variables to be used for the app when running
 **features** | _object_ | A key-value mapping of feature names to booleans (true = enabled, false = disabled)

lib/cloud_controller/diego/buildpack_entry_generator.rb

@@ -6,14 +6,15 @@ def initialize(blobstore_url_generator, type)
         @type = type
       end
 
-      def buildpack_entries(buildpack_infos, stack_name)
+      def buildpack_entries(buildpack_infos, stack_name, credentials: nil)
         return default_admin_buildpacks(stack_name) if buildpack_infos.empty?
 
         buildpack_infos.map do |buildpack_info|
           if buildpack_info.buildpack_exists_in_db? && buildpack_info.buildpack_enabled?
             admin_buildpack_entry(buildpack_info.buildpack_record).merge(skip_detect: true)
           elsif buildpack_info.buildpack_url
-            { name: 'custom', key: buildpack_info.buildpack_url, url: buildpack_info.buildpack_url, skip_detect: true }
+            url = inject_credentials(buildpack_info.buildpack_url, credentials)
+            { name: 'custom', key: url, url: url, skip_detect: true }
           else
             raise "Unsupported buildpack type: '#{buildpack_info.buildpack}'"
           end
@@ -22,6 +23,25 @@ def buildpack_entries(buildpack_infos, stack_name)
 
       private
 
+      def inject_credentials(url, credentials)
+        return url if credentials.blank?
+
+        credentials = credentials.deep_stringify_keys
+        uri = URI.parse(url)
+        cred = credentials[uri.host]
+        return url if cred.blank?
+
+        username = cred['username']
+        password = cred['password']
+        return url if username.blank?
+
+        uri.user = CGI.escape(username)
+        uri.password = CGI.escape(password) if password.present?
+        uri.to_s
+      rescue URI::InvalidURIError
+        url
+      end
+
       def custom_buildpack_entry(buildpack)
         { name: 'custom', key: buildpack.url, url: buildpack.url }
       end

lib/cloud_controller/diego/cnb/lifecycle_protocol.rb

@@ -30,6 +30,11 @@ def new_lifecycle_data(staging_details)
             lifecycle_data
           end
 
+          # CNB credentials travel via CNB_REGISTRY_CREDS, never in the buildpack URL.
+          def url_credentials_for_staging(_staging_details)
+            nil
+          end
+
           def type
             VCAP::CloudController::Lifecycles::CNB
           end

lib/cloud_controller/diego/lifecycle_protocol.rb

@@ -34,8 +34,12 @@ def lifecycle_data(staging_details)
         lifecycle_data.build_artifacts_cache_download_uri = @blobstore_url_generator.buildpack_cache_download_url(staging_details.package.app_guid, stack)
         lifecycle_data.build_artifacts_cache_upload_uri   = @blobstore_url_generator.buildpack_cache_upload_url(staging_details.package.app_guid, stack)
         lifecycle_data.droplet_upload_uri                 = @blobstore_url_generator.droplet_upload_url(staging_details.staging_guid)
-        lifecycle_data.buildpacks                         = @buildpack_entry_generator.buildpack_entries(staging_details.lifecycle.buildpack_infos, stack)
-        lifecycle_data.stack                              = stack
+        lifecycle_data.buildpacks                         = @buildpack_entry_generator.buildpack_entries(
+          staging_details.lifecycle.buildpack_infos,
+          stack,
+          credentials: url_credentials_for_staging(staging_details)
+        )
+        lifecycle_data.stack = stack
 
         lifecycle_data.message
       rescue Membrane::SchemaValidationError => e
@@ -45,6 +49,11 @@ def lifecycle_data(staging_details)
         raise InvalidDownloadUri.new("Failed to get blobstore download url for package #{staging_details.package.guid}")
       end
 
+      # Buildpack lifecycle injects these into custom buildpack URLs; CNB returns nil (uses CNB_REGISTRY_CREDS).
+      def url_credentials_for_staging(staging_details)
+        staging_details.lifecycle.credentials_for_staging
+      end
+
       private
 
       def droplet_checksum_info(droplet)

lib/cloud_controller/diego/lifecycles/app_base_lifecycle.rb

@@ -1,27 +1,50 @@
+require 'cloud_controller/diego/lifecycles/buildpack_info'
+require 'cloud_controller/diego/lifecycles/buildpack_lifecycle_data_validator'
+require 'fetchers/buildpack_lifecycle_fetcher'
+
 module VCAP::CloudController
   class AppBaseLifecycle
-    def update_lifecycle_data_model(app)
-      if [update_lifecycle_data_buildpacks(app),
-          update_lifecycle_data_stack(app)].any?
-        app.lifecycle_data.save
-      end
-    end
+    def initialize(message, app=nil)
+      @message = message
 
-    def update_lifecycle_data_buildpacks(app)
-      return unless message.buildpack_data.requested?(:buildpacks)
+      db_result = BuildpackLifecycleFetcher.fetch(effective_buildpacks(app), stack, type)
 
-      app.lifecycle_data.buildpacks = buildpacks
+      @validator = validator_class.new(
+        buildpack_infos: db_result[:buildpack_infos],
+        stack: db_result[:stack],
+        credentials: effective_credentials(app),
+        credentials_from_message: message.buildpack_data.requested?(:credentials)
+      )
     end
 
-    def update_lifecycle_data_stack(app)
-      return unless message.buildpack_data.requested?(:stack)
+    delegate :valid?, :errors, to: :validator
+
+    def update_lifecycle_data_model(app)
+      requested = %i[buildpacks stack credentials].select { |field| message.buildpack_data.requested?(field) }
+      return if requested.empty?
 
-      app.lifecycle_data.stack = message.buildpack_data.stack
+      app.lifecycle_data.buildpacks = buildpacks if requested.include?(:buildpacks)
+      app.lifecycle_data.stack = message.buildpack_data.stack if requested.include?(:stack)
+      app.lifecycle_data.credentials = message.buildpack_data.credentials if requested.include?(:credentials)
+      app.lifecycle_data.save
     end
 
     private
 
-    attr_reader :message
+    attr_reader :message, :validator
+
+    def validator_class
+      BuildpackLifecycleDataValidator
+    end
+
+    def effective_buildpacks(app)
+      message.buildpack_data.requested?(:buildpacks) ? buildpacks : (app&.lifecycle_data&.buildpacks || [])
+    end
+
+    def effective_credentials(app)
+      stored = app&.lifecycle_data&.then { |ld| ld.respond_to?(:credentials) ? ld.credentials : nil }
+      message.buildpack_data.requested?(:credentials) ? message.buildpack_data.credentials : stored
+    end
 
     def buildpacks
       message.buildpack_data.requested?(:buildpacks) ? (message.buildpack_data.buildpacks || []) : []