Skip to content

Commit 83f7460

Browse files
committed
feat: add data.credentials support to Buildpack and CNB lifecycles
Makes credential handling consistent across lifecycles by introducing data.credentials for both buildpack and CNB lifecycle types, stored encrypted in the buildpack_lifecycle_data table. - New migration adds encrypted_credentials_json / _salt columns to buildpack_lifecycle_data (previously CNB-only via cnb_lifecycle_data) - BuildpackLifecycleDataModel gains credentials accessors backed by set_field_as_encrypted; to_hash redacts the value - AppBaseLifecycle#update_lifecycle_data_model persists requested lifecycle fields (including credentials, honoring an explicit null to clear); staging reads credentials via LifecycleBase#credentials_for_staging - Buildpack lifecycle injects credentials into the custom buildpack URL for both the url and key fields, so the staging git clone (driven by key via -buildpackOrder) authenticates; CNB does not inject into the URL and relies solely on CNB_REGISTRY_CREDS - Validators enforce XOR between URL-embedded credentials and data.credentials, with a clearer message when the conflicting credentials are already stored - Manifest: unified top-level credentials key for both lifecycles, with cnb-credentials kept as a backward-compatible alias; specifying both is a validation error; an explicit null clears stored credentials - Redaction: audit events, the app manifest output, and the manifest diff all mask credentials and buildpack-URL userinfo (singular and plural keys) - url_has_userinfo? detects password-only userinfo (:token@host) - Docs: document data.credentials for both lifecycles and the manifest credentials / cnb-credentials keys
1 parent de451a5 commit 83f7460

36 files changed

Lines changed: 1042 additions & 104 deletions

‎app/actions/space_diff_manifest.rb‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
require 'presenters/v3/app_manifest_presenter'
2+
require 'presenters/helpers/censorship'
23
require 'messages/app_manifest_message'
34
require 'json-diff'
45

@@ -10,6 +11,8 @@ class SpaceDiffManifest
1011
'sidecars' => 'name'
1112
}.freeze
1213

14+
REDACTED_KEYS = %w[credentials cnb-credentials].freeze
15+
1316
class << self
1417
# rubocop:todo Metrics/CyclomaticComplexity
1518
def generate_diff(app_manifests, space)
@@ -56,6 +59,8 @@ def generate_diff(app_manifests, space)
5659
# To preserve backwards compability, we've decided to skip diffs that satisfy this conditon
5760
next if !nested_attribute_exists && %w[disk_quota disk-quota memory].include?(key)
5861

62+
value = obfuscate_buildpack_urls(value) if %w[buildpack buildpacks].include?(key)
63+
5964
key_diffs = JsonDiff.diff(
6065
existing_value,
6166
value,
@@ -68,6 +73,8 @@ def generate_diff(app_manifests, space)
6873

6974
diff['op'] = 'add' if diff['op'] == 'replace' && diff['was'].nil?
7075

76+
redact_credentials(diff) if REDACTED_KEYS.include?(key)
77+
7178
json_diff << diff
7279
end
7380
end
@@ -79,6 +86,18 @@ def generate_diff(app_manifests, space)
7986

8087
private
8188

89+
def redact_credentials(diff)
90+
diff['value'] = Presenters::Censorship::PRIVATE_DATA_HIDDEN if diff.key?('value')
91+
diff['was'] = Presenters::Censorship::PRIVATE_DATA_HIDDEN if diff.key?('was')
92+
end
93+
94+
def obfuscate_buildpack_urls(buildpacks)
95+
return CloudController::UrlSecretObfuscator.obfuscate(buildpacks) if buildpacks.is_a?(String)
96+
return buildpacks unless buildpacks.is_a?(Array)
97+
98+
buildpacks.map { |bp| bp.is_a?(String) ? CloudController::UrlSecretObfuscator.obfuscate(bp) : bp }
99+
end
100+
82101
# rubocop:todo Metrics/CyclomaticComplexity
83102
def filter_manifest_app_hash(manifest_app_hash)
84103
if manifest_app_hash.key? 'sidecars'

‎app/messages/app_manifest_message.rb‎

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ class AppManifestMessage < BaseMessage
4343
sidecars
4444
stack
4545
timeout
46+
credentials
4647
cnb_credentials
4748
features
4849
]
@@ -75,6 +76,7 @@ def self.underscore_keys(hash)
7576
validate :validate_docker_enabled!
7677
validate :validate_cnb_enabled!
7778
validate :validate_docker_buildpacks_combination!
79+
validate :validate_credentials_combination!
7880
validate :validate_service_bindings_message!, if: ->(record) { record.requested?(:services) }
7981
validate :validate_env_update_message!, if: ->(record) { record.requested?(:env) }
8082
validate :validate_manifest_singular_buildpack_message!, if: ->(record) { record.requested?(:buildpack) }
@@ -126,27 +128,23 @@ def manifest_features_update_message
126128
def audit_hash
127129
result = original_yaml.dup
128130
result['env'] = Presenters::Censorship::PRIVATE_DATA_HIDDEN if result.key?('env')
131+
result['credentials'] = Presenters::Censorship::PRIVATE_DATA_HIDDEN if result.key?('credentials')
129132
result['cnb-credentials'] = Presenters::Censorship::PRIVATE_DATA_HIDDEN if result.key?('cnb-credentials')
130133
obfuscate_buildpack_urls(result)
131134
redact_service_parameters(result)
132135
result
133136
end
134137

135138
def app_lifecycle_hash
136-
lifecycle_type = if requested?(:lifecycle) && @lifecycle == 'cnb'
137-
Lifecycles::CNB
138-
elsif requested?(:lifecycle) && @lifecycle == 'buildpack'
139-
Lifecycles::BUILDPACK
140-
elsif requested?(:docker)
141-
Lifecycles::DOCKER
142-
end
139+
lifecycle_type = requested_lifecycle_type
143140

144141
data = {
145142
buildpacks: requested_buildpacks,
146-
stack: @stack,
147-
credentials: @cnb_credentials
143+
stack: @stack
148144
}.compact
149145

146+
data[:credentials] = @credentials || @cnb_credentials if requested?(:credentials) || requested?(:cnb_credentials)
147+
150148
if lifecycle_type == Lifecycles::DOCKER
151149
lifecycle = {
152150
type: Lifecycles::DOCKER
@@ -496,6 +494,19 @@ def validate_docker_buildpacks_combination!
496494
errors.add(:base, 'Cannot specify both buildpack(s) and docker keys')
497495
end
498496

497+
def validate_credentials_combination!
498+
return unless requested?(:credentials) && requested?(:cnb_credentials)
499+
500+
errors.add(:base, 'Cannot specify both credentials and cnb-credentials keys')
501+
end
502+
503+
def requested_lifecycle_type
504+
return Lifecycles::CNB if requested?(:lifecycle) && @lifecycle == 'cnb'
505+
return Lifecycles::BUILDPACK if requested?(:lifecycle) && @lifecycle == 'buildpack'
506+
507+
Lifecycles::DOCKER if requested?(:docker)
508+
end
509+
499510
def validate_features!
500511
manifest_features_update_message.valid?
501512
manifest_features_update_message.errors.full_messages.each do |error_message|

‎app/models/runtime/buildpack_lifecycle_data_model.rb‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@ class BuildpackLifecycleDataModel < Sequel::Model(:buildpack_lifecycle_data)
99
LIFECYCLE_TYPE = Lifecycles::BUILDPACK
1010

1111
set_field_as_encrypted :buildpack_url, salt: :encrypted_buildpack_url_salt, column: :encrypted_buildpack_url
12+
set_field_as_encrypted :credentials_json,
13+
salt: :encrypted_credentials_json_salt,
14+
column: :encrypted_credentials_json
1215

1316
many_to_one :droplet,
1417
class: '::VCAP::CloudController::DropletModel',
@@ -85,11 +88,23 @@ def first_custom_buildpack_url
8588
buildpack_lifecycle_buildpacks.find(&:custom?)&.buildpack_url || legacy_buildpack_url
8689
end
8790

91+
def credentials
92+
return unless credentials_json
93+
94+
Oj.load(credentials_json)
95+
end
96+
97+
def credentials=(creds)
98+
self.credentials_json = creds.nil? ? nil : Oj.dump(creds)
99+
end
100+
88101
def to_hash
89-
{
102+
hash = {
90103
buildpacks: obfuscated_buildpacks,
91104
stack: stack
92105
}
106+
hash[:credentials] = Presenters::Censorship::REDACTED_CREDENTIAL unless credentials.nil?
107+
hash
93108
end
94109

95110
def validate

‎app/presenters/v3/app_manifest_presenters/buildpack_presenter.rb‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
require 'presenters/helpers/censorship'
2+
13
module VCAP::CloudController
24
module Presenters
35
module V3
@@ -7,8 +9,9 @@ def to_hash(app:, **_)
79
return {} if app.docker?
810

911
{
10-
buildpacks: app.lifecycle_data.buildpacks.presence,
11-
stack: app.lifecycle_data.stack
12+
buildpacks: app.lifecycle_data.obfuscated_buildpacks.presence,
13+
stack: app.lifecycle_data.stack,
14+
credentials: app.lifecycle_data.credentials.nil? ? nil : Presenters::Censorship::REDACTED_CREDENTIAL
1215
}
1316
end
1417
end

‎app/repositories/app_event_repository.rb‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,7 @@ def obfuscate_buildpack_urls(changes)
207207

208208
lifecycle_data['buildpack'] = CloudController::UrlSecretObfuscator.obfuscate(lifecycle_data['buildpack']) if lifecycle_data['buildpack']
209209
lifecycle_data['buildpacks'] = lifecycle_data['buildpacks'].map { |b| CloudController::UrlSecretObfuscator.obfuscate(b) } if lifecycle_data['buildpacks']
210+
lifecycle_data['credentials'] = Presenters::Censorship::PRIVATE_DATA_HIDDEN if lifecycle_data.key?('credentials')
210211
end
211212
end
212213
end
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
Sequel.migration do
2+
up do
3+
# rubocop:disable Migration/IncludeStringSize
4+
add_column :buildpack_lifecycle_data, :encrypted_credentials_json, String, if_not_exists: true
5+
# rubocop:enable Migration/IncludeStringSize
6+
add_column :buildpack_lifecycle_data, :encrypted_credentials_json_salt, String, size: 255, if_not_exists: true
7+
# encryption_key_label and encryption_iterations already exist (added by earlier migrations for buildpack_url encryption)
8+
end
9+
10+
down do
11+
drop_column :buildpack_lifecycle_data, :encrypted_credentials_json, if_exists: true
12+
drop_column :buildpack_lifecycle_data, :encrypted_credentials_json_salt, if_exists: true
13+
end
14+
end

‎docs/v3/source/includes/concepts/_lifecycles.md.erb‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ Name | Type | Description
4141
**type** | _string_ | `buildpack`
4242
**data.buildpacks** | _list of strings_ | A list of the names of buildpacks, URLs from which they may be downloaded, or `null` to auto-detect a suitable buildpack during staging
4343
**data.stack** | _string_ | The root filesystem to use with the buildpack, for example `cflinuxfs4`
44+
**data.credentials** | _object_ | Credentials used to download custom buildpacks from private git repositories, keyed by host. Each entry is a username/password pair. If any buildpack URL embeds credentials, **data.credentials** must be empty; provide credentials in one place only.
4445

4546
For records whose historical lifecycle data is no longer available,
4647
**data.buildpacks** may be an empty list and **data.stack** may be `null`.
@@ -83,7 +84,7 @@ Name | Type | Description
8384
---- | ---- | -----------
8485
**type** | _string_ | `cnb`
8586
**data.buildpacks** | _list of strings_ | A list of URLs with either `docker://` or `http(s)://` scheme, pointing to the Cloud Native Buildpack OCI image. <br/>When the scheme is `http(s)://`, an OCI tarball is expected to be present at the specified location.
86-
**data.credentials** | _object_ | Credentials used to download the configured buildpacks. This can either contain username/password or a token.
87+
**data.credentials** | _object_ | Credentials used to download the configured buildpacks, keyed by registry host. Each entry contains either a username/password pair or a token. CNB buildpack URLs must not embed credentials; use **data.credentials** instead.
8788
**data.stack** | _string_ | The root filesystem to use with the buildpack, for example `cflinuxfs4`
8889

8990
For records whose historical lifecycle data is no longer available,

‎docs/v3/source/includes/resources/manifests/_object.md.erb‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,8 @@ Name | Type | Description
105105
**name** | _string_ | Name of the app
106106
**lifecycle** | _string_ | Type of the lifecycle; valid values are `buildpack`, `cnb`, `docker`. Defaults to `buildpack`, unless the **docker** property is present.
107107
**buildpacks** | _list of strings_ | a) An empty array, which will automatically select the appropriate default buildpack according to the coding language (incompatible with **lifecycle: cnb**) <br>b) An array of one or more URLs pointing to buildpacks <br>c) An array of one or more installed buildpack names <br>Replaces the legacy `buildpack` field
108+
**credentials** | _object_ | Credentials used to download custom buildpacks (buildpack lifecycle) or buildpack OCI images (cnb lifecycle) from private git repositories or registries, keyed by host. For the buildpack lifecycle each entry is a username/password pair; for the cnb lifecycle each entry is a username/password pair or a token. Set to `null` to clear previously stored credentials. If any buildpack URL embeds credentials, **credentials** must be empty; provide credentials in one place only.
109+
**cnb-credentials** | _object_ | Backward-compatible alias for **credentials**. Specifying both **credentials** and **cnb-credentials** is an error.
108110
**docker** | _object_ | If present, the created app will have Docker lifecycle type; the value of this key is ignored by the API but may be used by clients to source the registry address of the image and credentials, if needed; the [generate manifest endpoint](#generate-a-manifest-for-an-app) will return the registry address of the image and username provided with this key
109111
**env** | _object_ | A key-value mapping of environment variables to be used for the app when running
110112
**features** | _object_ | A key-value mapping of feature names to booleans (true = enabled, false = disabled)

‎lib/cloud_controller/diego/buildpack_entry_generator.rb‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,15 @@ def initialize(blobstore_url_generator, type)
66
@type = type
77
end
88

9-
def buildpack_entries(buildpack_infos, stack_name)
9+
def buildpack_entries(buildpack_infos, stack_name, credentials: nil)
1010
return default_admin_buildpacks(stack_name) if buildpack_infos.empty?
1111

1212
buildpack_infos.map do |buildpack_info|
1313
if buildpack_info.buildpack_exists_in_db? && buildpack_info.buildpack_enabled?
1414
admin_buildpack_entry(buildpack_info.buildpack_record).merge(skip_detect: true)
1515
elsif buildpack_info.buildpack_url
16-
{ name: 'custom', key: buildpack_info.buildpack_url, url: buildpack_info.buildpack_url, skip_detect: true }
16+
url = inject_credentials(buildpack_info.buildpack_url, credentials)
17+
{ name: 'custom', key: url, url: url, skip_detect: true }
1718
else
1819
raise "Unsupported buildpack type: '#{buildpack_info.buildpack}'"
1920
end
@@ -22,6 +23,25 @@ def buildpack_entries(buildpack_infos, stack_name)
2223

2324
private
2425

26+
def inject_credentials(url, credentials)
27+
return url if credentials.blank?
28+
29+
host = URI.parse(url).host
30+
cred = credentials[host]
31+
return url if cred.blank?
32+
33+
username = cred['username']
34+
password = cred['password']
35+
return url if username.blank?
36+
37+
uri = URI.parse(url)
38+
uri.user = URI.encode_www_form_component(username)
39+
uri.password = URI.encode_www_form_component(password) if password.present?
40+
uri.to_s
41+
rescue URI::InvalidURIError
42+
url
43+
end
44+
2545
def custom_buildpack_entry(buildpack)
2646
{ name: 'custom', key: buildpack.url, url: buildpack.url }
2747
end

‎lib/cloud_controller/diego/cnb/lifecycle_protocol.rb‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,11 @@ def new_lifecycle_data(staging_details)
3030
lifecycle_data
3131
end
3232

33+
# CNB credentials travel via CNB_REGISTRY_CREDS, never in the buildpack URL.
34+
def url_credentials_for_staging(_staging_details)
35+
nil
36+
end
37+
3338
def type
3439
VCAP::CloudController::Lifecycles::CNB
3540
end

0 commit comments

Comments
 (0)