From 643060ff15095c4151f5f61c080fdfe36f09ebd3 Mon Sep 17 00:00:00 2001 From: ham Date: Sun, 6 Jul 2025 18:48:09 +0900 Subject: [PATCH 1/9] Support Git-style searchPaths with wildcards in AWS S3 buckets (#2812) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #2812 - ListObjectsV2 + AntPathMatcher based matching for *, **, ?, dot-wildcard - auto-ext lookup order (.properties → .json → .yml/.yaml) - directory scan, deduplication, prefix extraction - only active when searchPaths non-empty Signed-off-by: Geonwook Ham Signed-off-by: ham --- .../AwsS3EnvironmentProperties.java | 14 + .../AwsS3EnvironmentRepository.java | 373 +++++++++++++++++- .../AwsS3EnvironmentRepositoryFactory.java | 5 +- .../AwsS3EnvironmentRepositoryTests.java | 305 ++++++++++++++ 4 files changed, 679 insertions(+), 18 deletions(-) diff --git a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentProperties.java b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentProperties.java index 9f20aae979..cfcab639dc 100644 --- a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentProperties.java +++ b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentProperties.java @@ -16,11 +16,15 @@ package org.springframework.cloud.config.server.environment; +import java.util.ArrayList; +import java.util.List; + import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.cloud.config.server.support.EnvironmentRepositoryProperties; /** * @author Clay McCoy + * @author Geonwook Ham */ @ConfigurationProperties("spring.cloud.config.server.awss3") public class AwsS3EnvironmentProperties implements EnvironmentRepositoryProperties { @@ -48,6 +52,16 @@ public class AwsS3EnvironmentProperties implements EnvironmentRepositoryProperti private int order = DEFAULT_ORDER; + private List searchPaths = new ArrayList<>(); + + public List getSearchPaths() { + return searchPaths; + } + + public void setSearchPaths(List searchPaths) { + this.searchPaths = searchPaths; + } + public String getRegion() { return region; } diff --git a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java index 9b26a94b58..cb5617794e 100644 --- a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java +++ b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java @@ -20,8 +20,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Optional; import java.util.Properties; +import java.util.Set; import java.util.function.Consumer; import org.apache.commons.logging.Log; @@ -30,6 +33,11 @@ import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.model.S3Object; import org.springframework.beans.factory.config.YamlProcessor; import org.springframework.beans.factory.config.YamlPropertiesFactoryBean; @@ -38,18 +46,24 @@ import org.springframework.cloud.config.server.config.ConfigServerProperties; import org.springframework.core.Ordered; import org.springframework.core.io.InputStreamResource; +import org.springframework.util.AntPathMatcher; import org.springframework.util.ObjectUtils; +import org.springframework.util.PathMatcher; import org.springframework.util.StringUtils; import static org.springframework.cloud.config.server.environment.AwsS3EnvironmentRepository.PATH_SEPARATOR; + /** * @author Clay McCoy * @author Scott Frederick * @author Daniel Aiken + * @author Geonwook Ham */ public class AwsS3EnvironmentRepository implements EnvironmentRepository, Ordered, SearchPathLocator { + private final PathMatcher pathMatcher = new AntPathMatcher(); + protected static final String PATH_SEPARATOR = "/"; private static final Log LOG = LogFactory.getLog(AwsS3EnvironmentRepository.class); @@ -66,16 +80,24 @@ public class AwsS3EnvironmentRepository implements EnvironmentRepository, Ordere protected int order = Ordered.LOWEST_PRECEDENCE; + private final List searchPaths; + public AwsS3EnvironmentRepository(S3Client s3Client, String bucketName, ConfigServerProperties server) { this(s3Client, bucketName, false, server); } public AwsS3EnvironmentRepository(S3Client s3Client, String bucketName, boolean useApplicationAsDirectory, ConfigServerProperties server) { + this(s3Client, bucketName, useApplicationAsDirectory, server, null); + } + + public AwsS3EnvironmentRepository(S3Client s3Client, String bucketName, boolean useApplicationAsDirectory, + ConfigServerProperties server, List searchPaths) { this.s3Client = s3Client; this.bucketName = bucketName; this.serverProperties = server; this.useApplicationAsDirectory = useApplicationAsDirectory; + this.searchPaths = (searchPaths == null ? Collections.emptyList() : searchPaths); } @Override @@ -97,8 +119,8 @@ public Environment findOne(String specifiedApplication, String specifiedProfiles String[] profileArray = parseProfiles(profiles); List apps = Arrays.asList(StringUtils.commaDelimitedListToStringArray(application.replace(" ", ""))); - Collections.reverse(apps); - if (!apps.contains(serverProperties.getDefaultApplicationName())) { + if (searchPaths.isEmpty() && !apps.contains(serverProperties.getDefaultApplicationName())) { + Collections.reverse(apps); apps = new ArrayList<>(apps); apps.add(serverProperties.getDefaultApplicationName()); } @@ -126,6 +148,17 @@ public Environment findOne(String specifiedApplication, String specifiedProfiles private void addPropertySources(Environment environment, List apps, String[] profiles, List labels) { + if (!this.searchPaths.isEmpty()) { + for (String label : labels) { + for (String profile : profiles) { + for (String app : apps) { + List s3ConfigFiles = getS3ConfigFileWithSearchPaths(app, profile, label); + addPropertySource(environment, s3ConfigFiles); + } + } + } + return; + } for (String label : labels) { // If we have profiles, add property sources with those profiles for (String profile : profiles) { @@ -163,15 +196,21 @@ private void addPropertySourcesForApps(List apps, Consumer addPr } private void addProfileSpecificPropertySource(Environment environment, String app, String profile, String label) { - List s3ConfigFiles = getS3ConfigFile(app, profile, label, this::getS3PropertiesOrJsonConfigFile, - this::getProfileSpecificS3ConfigFileYaml); + if (!searchPaths.isEmpty() && app.equals(serverProperties.getDefaultApplicationName())) { + return; + } + List s3ConfigFiles = searchPaths.isEmpty() + ? getS3ConfigFile(app, profile, label, this::getS3PropertiesOrJsonConfigFile, this::getProfileSpecificS3ConfigFileYaml) + : getS3ConfigFileWithSearchPaths(app, profile, label); addPropertySource(environment, s3ConfigFiles); } - private void addNonProfileSpecificPropertySource(Environment environment, String app, String profile, - String label) { - List s3ConfigFiles = getS3ConfigFile(app, profile, label, - this::getNonProfileSpecificPropertiesOrJsonConfigFile, this::getNonProfileSpecificS3ConfigFileYaml); + private void addNonProfileSpecificPropertySource(Environment environment, String app, String profile, String label) { + List s3ConfigFiles = searchPaths.isEmpty() + ? getS3ConfigFile(app, profile, label, + this::getNonProfileSpecificPropertiesOrJsonConfigFile, + this::getNonProfileSpecificS3ConfigFileYaml) + : Collections.emptyList(); addPropertySource(environment, s3ConfigFiles); } @@ -215,6 +254,204 @@ private List getS3ConfigFile(String application, String profile, S } + private List getS3ConfigFileWithSearchPaths( + String application, String profile, String label) { + + List result = new ArrayList<>(); + Set seenKeys = new LinkedHashSet<>(); + + for (String template : this.searchPaths) { + String pattern = template + .replace("{application}", application) + .replace("{profile}", profile == null ? "" : profile) + .replace("{label}", label == null ? "" : label); + + if (!pathMatcher.isPattern(pattern)) { + boolean fileFound = false; + for (String ext : List.of(".properties", ".json", ".yml", ".yaml")) { + String key = pattern.endsWith(ext) ? pattern : pattern + ext; + if (!seenKeys.add(key)) { + continue; + } + try { + s3Client.headObject(HeadObjectRequest.builder() + .bucket(bucketName) + .key(key) + .build()); + result.addAll(wrapKeyWithConfigFiles(key, application, profile, label)); + fileFound = true; + break; + } + catch (S3Exception e) { + int status = e.statusCode(); + if (status != 404 && status != 403) { + throw e; + } + } + } + if (fileFound) { + continue; + } + + String dirPrefix = pattern.endsWith("/") ? pattern : pattern + "/"; + String token = null; + do { + ListObjectsV2Response resp = s3Client.listObjectsV2( + ListObjectsV2Request.builder() + .bucket(bucketName) + .prefix(dirPrefix) + .continuationToken(token) + .build()); + for (S3Object obj : resp.contents()) { + String key = obj.key(); + if (!hasSupportedExtension(key)) { + continue; + } + if (seenKeys.add(key)) { + result.addAll(wrapKeyWithConfigFiles(key, application, profile, label)); + } + } + token = resp.nextContinuationToken(); + } while (token != null); + + continue; + } + + if (pattern.endsWith(".*")) { + String base = pattern.substring(0, pattern.length() - 2); + for (String ext : List.of(".properties", ".json", ".yml", ".yaml")) { + String key = base + ext; + if (!seenKeys.add(key)) { + continue; + } + try { + s3Client.headObject(HeadObjectRequest.builder() + .bucket(bucketName) + .key(key) + .build()); + result.addAll(wrapKeyWithConfigFiles(key, application, profile, label)); + break; + } + catch (S3Exception e) { + int status = e.statusCode(); + if (status != 404 && status != 403) { + throw e; + } + } + } + continue; + } + + String prefix = extractPrefix(pattern); + String token = null; + do { + ListObjectsV2Response resp = s3Client.listObjectsV2( + ListObjectsV2Request.builder() + .bucket(bucketName) + .prefix(prefix) + .continuationToken(token) + .build()); + for (S3Object obj : resp.contents()) { + String key = obj.key(); + if (!pathMatcher.match(pattern, key) || !hasSupportedExtension(key)) { + continue; + } + if (seenKeys.add(key)) { + result.addAll(wrapKeyWithConfigFiles(key, application, profile, label)); + } + } + token = resp.nextContinuationToken(); + } while (token != null); + } + + return result; + } + + private boolean hasSupportedExtension(String key) { + return key.endsWith(".properties") + || key.endsWith(".json") + || key.endsWith(".yml") + || key.endsWith(".yaml"); + } + + private List wrapKeyWithConfigFiles( + String key, + String application, + String profile, + String label) { + + if (key.endsWith(".yml") || key.endsWith(".yaml")) { + List files = new ArrayList<>(); + files.addAll(getProfileSpecificYamlFromKey(key, application, profile, label)); + files.addAll(getNonProfileSpecificYamlFromKey(key, application, profile, label)); + return files; + } + return createConfigFileFromKey(key, application, profile, label) + .map(Collections::singletonList) + .orElseGet(Collections::emptyList); + } + + + private List getProfileSpecificYamlFromKey( + String key, String application, String profile, String label) { + + YamlConfigFileFromKey config = new YamlConfigFileFromKey( + key, application, profile, label, + bucketName, useApplicationAsDirectory, s3Client, + properties -> YamlS3ConfigFile.profileMatchesActivateProperty(profile, properties) + ? YamlProcessor.MatchStatus.FOUND + : YamlProcessor.MatchStatus.NOT_FOUND + ); + config.setShouldIncludeWithEmptyProperties(false); + return List.of(config); + } + + private List getNonProfileSpecificYamlFromKey( + String key, String application, String profile, String label) { + + YamlConfigFileFromKey config = new YamlConfigFileFromKey( + key, application, profile, label, + bucketName, useApplicationAsDirectory, s3Client, + properties -> !YamlS3ConfigFile.onProfilePropertyExists(properties) + ? YamlProcessor.MatchStatus.FOUND + : YamlProcessor.MatchStatus.NOT_FOUND + ); + return List.of(config); + } + + private String extractPrefix(String pattern) { + int idx = pattern.indexOf('*'); + int q = pattern.indexOf('?'); + if (q != -1 && (idx == -1 || q < idx)) { + idx = q; + } + if (idx <= 0) { + return ""; + } + int slash = pattern.lastIndexOf('/', idx); + return (slash == -1 ? "" : pattern.substring(0, slash + 1)); + } + + + private Optional createConfigFileFromKey(String key, + String application, String profile, String label) { + String ext = key.substring(key.lastIndexOf('.') + 1); + if ("properties".equalsIgnoreCase(ext)) { + return Optional.of(new PropertyConfigFileFromKey( + key, application, profile, label, bucketName, s3Client)); + } + if ("json".equalsIgnoreCase(ext)) { + return Optional.of(new JsonConfigFileFromKey( + key, application, profile, label, bucketName, s3Client)); + } + if ("yml".equalsIgnoreCase(ext) || "yaml".equalsIgnoreCase(ext)) { + return Optional.of(new YamlConfigFileFromKey( + key, application, profile, label, bucketName, s3Client)); + } + return Optional.empty(); + } + + private List getNonProfileSpecificS3ConfigFileYaml(String application, String profile, String label) { List configFiles = new ArrayList<>(); @@ -434,10 +671,18 @@ private String createPropertySourceName(String app, String profile) { class PropertyS3ConfigFile extends S3ConfigFile { + PropertyS3ConfigFile(String application, String profile, String label, + String bucketName, boolean useApplicationAsDirectory, + S3Client s3Client) { + this(application, profile, label, bucketName, useApplicationAsDirectory, s3Client, true); + } + PropertyS3ConfigFile(String application, String profile, String label, String bucketName, - boolean useApplicationAsDirectory, S3Client s3Client) { + boolean useApplicationAsDirectory, S3Client s3Client, boolean callReadImmediately) { super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client); - this.properties = read(); + if (callReadImmediately) { + this.properties = read(); + } } @Override @@ -469,17 +714,18 @@ class YamlS3ConfigFile extends S3ConfigFile { YamlS3ConfigFile(String application, String profile, String label, String bucketName, boolean useApplicationAsDirectory, S3Client s3Client) { - this(application, profile, label, bucketName, useApplicationAsDirectory, s3Client, + this(application, profile, label, bucketName, useApplicationAsDirectory, s3Client, true, new YamlProcessor.DocumentMatcher[] {}); } YamlS3ConfigFile(String application, String profile, String label, String bucketName, - boolean useApplicationAsDirectory, S3Client s3Client, + boolean useApplicationAsDirectory, S3Client s3Client, boolean callReadImmediately, final YamlProcessor.DocumentMatcher... documentMatchers) { super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client); this.documentMatchers = documentMatchers; - this.properties = read(); - + if (callReadImmediately) { + this.properties = read(); + } } protected static boolean profileMatchesActivateProperty(String profile, Properties properties) { @@ -520,7 +766,7 @@ class ProfileSpecificYamlDocumentS3ConfigFile extends YamlS3ConfigFile { ProfileSpecificYamlDocumentS3ConfigFile(String application, String profile, String label, String bucketName, boolean useApplicationAsDirectory, S3Client s3Client) { - super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client, + super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client, true, properties -> profileMatchesActivateProperty(profile, properties) ? YamlProcessor.MatchStatus.FOUND : YamlProcessor.MatchStatus.NOT_FOUND); } @@ -541,7 +787,7 @@ class NonProfileSpecificYamlDocumentS3ConfigFile extends YamlS3ConfigFile { NonProfileSpecificYamlDocumentS3ConfigFile(String application, String profile, String label, String bucketName, boolean useApplicationAsDirectory, S3Client s3Client) { - super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client, + super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client, true, properties -> !onProfilePropertyExists(properties) ? YamlProcessor.MatchStatus.FOUND : YamlProcessor.MatchStatus.NOT_FOUND); } @@ -552,7 +798,7 @@ class ProfileSpecificYamlS3ConfigFile extends YamlS3ConfigFile { ProfileSpecificYamlS3ConfigFile(String application, String profile, String label, String bucketName, boolean useApplicationAsDirectory, S3Client s3Client) { - super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client, + super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client, true, properties -> !onProfilePropertyExists(properties) ? YamlProcessor.MatchStatus.ABSTAIN : profileMatchesActivateProperty(profile, properties) ? YamlProcessor.MatchStatus.FOUND : YamlProcessor.MatchStatus.NOT_FOUND); @@ -570,9 +816,102 @@ class JsonS3ConfigFile extends YamlS3ConfigFile { this.properties = read(); } + JsonS3ConfigFile(String application, String profile, String label, String bucketName, + boolean useApplicationAsDirectory, S3Client s3Client, boolean callReadImmediately) { + super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client, callReadImmediately); + } + @Override protected List getExtensions() { return List.of("json"); } } + +class PropertyConfigFileFromKey extends PropertyS3ConfigFile { + + private final String key; + + PropertyConfigFileFromKey(String key, + String application, + String profile, + String label, + String bucketName, + S3Client s3Client) { + super(application, profile, label, bucketName, false, s3Client, false); + this.key = key; + this.properties = read(); + } + + @Override + public String getName() { + return "s3:" + bucketName + "/" + key; + } + + @Override + protected String buildObjectKeyPrefix() { + return key.substring(0, key.lastIndexOf('.')); + } +} + +class YamlConfigFileFromKey extends YamlS3ConfigFile { + + private final String key; + + YamlConfigFileFromKey(String key, + String application, + String profile, + String label, + String bucketName, + S3Client s3Client) { + super(application, profile, label, bucketName, false, s3Client, false); + this.key = key; + this.properties = read(); + } + + YamlConfigFileFromKey(String key, + String application, + String profile, + String label, + String bucketName, + boolean useApplicationAsDirectory, + S3Client s3Client, + YamlProcessor.DocumentMatcher... matchers) { + super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client, false, matchers); + this.key = key; + this.properties = read(); + } + + @Override + public String getName() { + return "s3:" + bucketName + "/" + key; + } + + @Override + protected String buildObjectKeyPrefix() { + return key.substring(0, key.lastIndexOf('.')); + } +} + +class JsonConfigFileFromKey extends JsonS3ConfigFile { + + private final String key; + + JsonConfigFileFromKey(String key, String application, String profile, + String label, String bucketName, S3Client s3Client) { + super(application, profile, label, bucketName, false, s3Client, false); + this.key = key; + this.properties = read(); + } + + @Override + public String getName() { + return "s3:" + bucketName + "/" + key; + } + + @Override + protected String buildObjectKeyPrefix() { + return key.substring(0, key.lastIndexOf('.')); + } +} + diff --git a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryFactory.java b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryFactory.java index 6781b88a4f..d9a9eaa83c 100644 --- a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryFactory.java +++ b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryFactory.java @@ -23,6 +23,9 @@ import static org.springframework.cloud.config.server.environment.AwsClientBuilderConfigurer.configureClientBuilder; +/** + * @author Geonwook Ham + */ public class AwsS3EnvironmentRepositoryFactory implements EnvironmentRepositoryFactory { @@ -39,7 +42,7 @@ public AwsS3EnvironmentRepository build(AwsS3EnvironmentProperties environmentPr final S3Client client = clientBuilder.build(); AwsS3EnvironmentRepository repository = new AwsS3EnvironmentRepository(client, - environmentProperties.getBucket(), environmentProperties.isUseDirectoryLayout(), server); + environmentProperties.getBucket(), environmentProperties.isUseDirectoryLayout(), server, environmentProperties.getSearchPaths()); repository.setOrder(environmentProperties.getOrder()); return repository; } diff --git a/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryTests.java b/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryTests.java index b62af5403b..64726c86a4 100644 --- a/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryTests.java +++ b/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryTests.java @@ -21,12 +21,16 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Properties; +import java.util.stream.Stream; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestFactory; import org.testcontainers.containers.localstack.LocalStackContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @@ -55,6 +59,7 @@ /** * @author Clay McCoy * @author Matej Nedić + * @author Geonwook Ham */ @Testcontainers @Tag("DockerRequired") @@ -488,6 +493,306 @@ public void getLocationsTest() { "default", "defaultlabel", null, new String[] { "s3://test/defaultlabel" })); } + // 1) Placeholder only + @Test + public void searchPath_placeholderOnly_shouldResolveExactFile() { + List paths = List.of("{label}/{application}-{profile}.yml"); + AwsS3EnvironmentRepository repo = + new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + putFiles("v1/foo-bar.yml", yamlContent); + + Environment env = repo.findOne("foo", "bar", "v1"); + assertThat(env.getPropertySources()).hasSize(1); + assertThat(env.getPropertySources().get(0).getName()) + .contains("v1/foo-bar.yml"); + } + + // 2) Wildcard only (.properties) + @Test + public void searchPath_wildcardOnly_shouldResolveAllProperties() { + List paths = List.of("{label}/common/*.properties"); + AwsS3EnvironmentRepository repo = + new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + putFiles("v1/common/a.properties", "a=1\n"); + putFiles("v1/common/b.properties", "b=2\n"); + + Environment env = repo.findOne("app", "", "v1"); + assertThat(env.getPropertySources()).hasSize(2); + assertThat(env.getPropertySources().get(0).getName()) + .contains("v1/common/a.properties"); + assertThat(env.getPropertySources().get(1).getName()) + .contains("v1/common/b.properties"); + } + + // 3) Placeholder + wildcard combined + @Test + public void searchPath_placeholderAndWildcard_shouldResolveMatchingKeys() { + List paths = List.of( + "{label}/{application}-{profile}.yml", + "{label}/common/*.properties" + ); + AwsS3EnvironmentRepository repo = + new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + putFiles("v1/foo-bar.yml", yamlContent); + putFiles("v1/common/foo.properties", "k=v\n"); + + Environment env = repo.findOne("foo", "bar", "v1"); + assertThat(env.getPropertySources()).hasSize(2); + assertThat(env.getPropertySources().get(0).getName()) + .contains("v1/foo-bar.yml"); + assertThat(env.getPropertySources().get(1).getName()) + .contains("v1/common/foo.properties"); + } + + // 4) Order matters + @Test + public void searchPaths_orderMatters_forPropertySourceOrder() { + List paths = List.of( + "{label}/common/*.properties", + "{label}/{application}-{profile}.yml" + ); + AwsS3EnvironmentRepository repo = + new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + putFiles("v1/common/foo.properties", "k=v\n"); + putFiles("v1/foo-bar.yml", yamlContent); + + Environment env = repo.findOne("foo", "bar", "v1"); + assertThat(env.getPropertySources()).hasSize(2); + assertThat(env.getPropertySources().get(0).getName()) + .contains("v1/common/foo.properties"); + assertThat(env.getPropertySources().get(1).getName()) + .contains("v1/foo-bar.yml"); + } + + // 5) Extension preserved + @TestFactory + public Stream searchPath_extensionPreserved() { + List exts = List.of("yml", "yaml", "properties", "json"); + return exts.stream().map(ext -> DynamicTest.dynamicTest(ext, () -> { + List paths = List.of("{label}/foo-bar." + ext); + AwsS3EnvironmentRepository repo = + new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + + String content = "key=v\n"; + if (ext.equals("yml") || ext.equals("yaml")) { + content = yamlContent; + } + else if (ext.equals("json")) { + content = jsonContent; + } + putFiles("v1/foo-bar." + ext, content); + + Environment env = repo.findOne("foo", "bar", "v1"); + assertThat(env.getPropertySources()).hasSize(1); + assertThat(env.getPropertySources().get(0).getName()) + .contains("v1/foo-bar." + ext); + })); + } + + // 6) Application-as-directory layout + @Test + public void searchPaths_applicationAsDirectory_shouldStillHonorSearchPaths() { + List paths = List.of("{label}/{application}/foo.*"); + AwsS3EnvironmentRepository repo = + new AwsS3EnvironmentRepository(s3Client, "bucket1", true, server, paths); + putFiles("v1/foo/foo.properties", "k=v\n"); + putFiles("v1/foo/foo.json", jsonContent); + + Environment env = repo.findOne("foo", "", "v1"); + assertThat(env.getPropertySources()).hasSize(1); + assertThat(env.getPropertySources().get(0).getName()) + .contains("v1/foo/foo.properties"); + } + + // 7) Multi-document YAML should not be split + @Test + public void multiDocumentYaml_withSearchPaths_shouldNotSplitDocuments() throws IOException { + List paths = List.of("{label}/{application}.yml"); + AwsS3EnvironmentRepository repo = + new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + + String multi = "---\na: 1\n---\nb: 2\n"; + putFiles("lab/app.yml", multi); + + Environment env = repo.findOne("app", "", "lab"); + assertThat(env.getPropertySources()).hasSize(1); + @SuppressWarnings("unchecked") + Map map = (Map) + env.getPropertySources().get(0).getSource(); + assertThat(map).containsEntry("a", 1).containsEntry("b", 2); + } + + // 8) Deduplication across patterns + @Test + public void searchPaths_deduplication_shouldOnlyAddOnce() { + List paths = List.of( + "{label}/foo.yml", + "{label}/{application}.yml" + ); + AwsS3EnvironmentRepository repo = + new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + putFiles("v1/foo.yml", yamlContent); + + Environment env = repo.findOne("foo", "", "v1"); + assertThat(env.getPropertySources()).hasSize(1); + } + + // 9) Literal stops at .properties + @Test + public void searchPaths_literalStopsAtProperties() { + List paths = List.of("{label}/{application}"); + AwsS3EnvironmentRepository repo = + new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + putFiles("v1/app.properties", "foo=bar\nflag=false\n"); + putFiles("v1/app.json", jsonContent); + putFiles("v1/app.yml", yamlContent); + + Environment env = repo.findOne("app", "", "v1"); + assertThat(env.getPropertySources()).hasSize(1); + assertThat(env.getPropertySources().get(0).getName()) + .contains("v1/app.properties"); + } + + // 10) Literal stops at .json + @Test + public void searchPaths_literalStopsAtJson() { + List paths = List.of("{label}/{application}"); + AwsS3EnvironmentRepository repo = + new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + putFiles("v1/app.json", jsonContent); + putFiles("v1/app.yml", yamlContent); + + Environment env = repo.findOne("app", "", "v1"); + assertThat(env.getPropertySources()).hasSize(1); + assertThat(env.getPropertySources().get(0).getName()) + .contains("v1/app.json"); + } + + // 11) Literal stops at .yml + @Test + public void searchPaths_literalStopsAtYaml() { + List paths = List.of("{label}/{application}"); + AwsS3EnvironmentRepository repo = + new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + putFiles("v1/app.yml", yamlContent); + + Environment env = repo.findOne("app", "", "v1"); + assertThat(env.getPropertySources()).hasSize(1); + assertThat(env.getPropertySources().get(0).getName()) + .contains("v1/app.yml"); + } + + // 12) Dot-wildcard auto-extension (.properties preferred) + @Test + public void searchPaths_dotWildcardAutoExtProperties() { + List paths = List.of("{label}/{application}.*"); + AwsS3EnvironmentRepository repo = + new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + putFiles("v1/app.properties", "x=1\n"); + putFiles("v1/app.json", jsonContent); + + Environment env = repo.findOne("app", "", "v1"); + assertThat(env.getPropertySources()).hasSize(1); + assertThat(env.getPropertySources().get(0).getName()) + .contains("v1/app.properties"); + } + + // 13) Literal directory scan + @Test + public void searchPaths_literalDirectoryScan() { + List paths = List.of("{label}/{application}"); + AwsS3EnvironmentRepository repo = + new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + putFiles("v1/app/foo.properties", "a=1\n"); + putFiles("v1/app/sub/bar.yml", "bar: 2\n"); + + Environment env = repo.findOne("app", "", "v1"); + assertThat(env.getPropertySources()).hasSize(2); + assertThat(env.getPropertySources().get(0).getName()) + .contains("v1/app/foo.properties"); + assertThat(env.getPropertySources().get(1).getName()) + .contains("v1/app/sub/bar.yml"); + } + + // 14) Double-wildcard nested directories + @Test + public void searchPaths_doubleWildcardNested() { + List paths = List.of("{label}/**/*.properties"); + AwsS3EnvironmentRepository repo = + new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + putFiles("v1/foo/a.properties", "a=1\n"); + putFiles("v1/foo/sub/b.properties", "b=2\n"); + + Environment env = repo.findOne("app", "", "v1"); + assertThat(env.getPropertySources()).hasSize(2); + assertThat(env.getPropertySources().get(0).getName()) + .contains("v1/foo/a.properties"); + assertThat(env.getPropertySources().get(1).getName()) + .contains("v1/foo/sub/b.properties"); + } + + // 15) Single-character wildcard + @Test + public void searchPaths_singleCharacterWildcard_shouldMatchExactlyOneChar() { + List paths = List.of("{label}/data-?.yml"); + AwsS3EnvironmentRepository repo = + new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + putFiles("lab/data-a.yml", "a:1\n"); + putFiles("lab/data-b.yml", "b:2\n"); + putFiles("lab/data-10.yml", "x:3\n"); // should not match + + Environment env = repo.findOne("app", "", "lab"); + assertThat(env.getPropertySources()).hasSize(2); + assertThat(env.getPropertySources().get(0).getName()) + .contains("lab/data-a.yml"); + assertThat(env.getPropertySources().get(1).getName()) + .contains("lab/data-b.yml"); + } + + // 16) Wildcard at start (empty prefix) + @Test + public void searchPaths_wildcardAtStart_prefixExtractionEmpty_shouldMatchAll() { + List paths = List.of("{label}/*.yml"); + AwsS3EnvironmentRepository repo = + new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + putFiles("lab/app.yml", yamlContent); + putFiles("lab/other.yml", yamlContent); + + Environment env = repo.findOne("app", "", "lab"); + assertThat(env.getPropertySources()).hasSize(2); + } + + // 17) Empty label uses defaultLabel + @Test + public void searchPaths_withEmptyLabel_shouldUseDefaultLabel() { + server.setDefaultLabel("main"); + List paths = List.of("{label}/foo-bar.yml"); + AwsS3EnvironmentRepository repo = + new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + putFiles("main/foo-bar.yml", yamlContent); + + Environment env = repo.findOne("foo", "", ""); + assertThat(env.getPropertySources()).hasSize(1); + assertThat(env.getPropertySources().get(0).getName()) + .contains("main/foo-bar.yml"); + } + + // 18) Multiple labels applied in reverse order + @Test + public void searchPaths_multipleLabels_shouldApplyForEachLabelInReverseOrder() { + List paths = List.of("{label}/{application}.yml"); + AwsS3EnvironmentRepository repo = + new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + putFiles("lab1/app.yml", yamlContent); + putFiles("lab2/app.yml", yamlContent); + + Environment env = repo.findOne("app", "", "lab1,lab2"); + assertThat(env.getPropertySources().get(0).getName()) + .contains("lab2/app.yml"); + assertThat(env.getPropertySources().get(1).getName()) + .contains("lab1/app.yml"); + } + private String putFiles(String fileName, String propertyContent) { toBeRemoved.add(fileName); return s3Client From daa088a82e1d306a162213294c438ebc4753ddec Mon Sep 17 00:00:00 2001 From: Geonwook Ham Date: Tue, 2 Jun 2026 21:35:07 +0900 Subject: [PATCH 2/9] Refactor S3 env repo path/profile handling (#2812) Fixes #2812 Rework AwsS3EnvironmentRepository path resolution and profile handling. Reverse the app list before checking/appending the default application and only append the default when it is missing. Simplify property-source iteration and invoke negated-profile property source logic only when searchPaths is empty to avoid duplicate sources. Normalize search path patterns by resolving null label/profile vars, collapsing duplicate slashes and trimming leading '/', and optimize extension probing so existing extensions aren't re-probed. Signed-off-by: Geonwook Ham --- .../AwsS3EnvironmentRepository.java | 47 ++++++++++--------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java index a8e2fc9bc5..6752f51380 100644 --- a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java +++ b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java @@ -120,8 +120,8 @@ public Environment findOne(String specifiedApplication, String specifiedProfiles String[] profileArray = parseProfiles(profiles); List apps = Arrays.asList(StringUtils.commaDelimitedListToStringArray(application.replace(" ", ""))); - if (searchPaths.isEmpty() && !apps.contains(serverProperties.getDefaultApplicationName())) { - Collections.reverse(apps); + Collections.reverse(apps); + if (!apps.contains(serverProperties.getDefaultApplicationName())) { apps = new ArrayList<>(apps); apps.add(serverProperties.getDefaultApplicationName()); } @@ -149,22 +149,10 @@ public Environment findOne(String specifiedApplication, String specifiedProfiles private void addPropertySources(Environment environment, List apps, String[] profiles, List labels) { - if (!this.searchPaths.isEmpty()) { - for (String label : labels) { - for (String profile : profiles) { - for (String app : apps) { - List s3ConfigFiles = getS3ConfigFileWithSearchPaths(app, profile, label); - addPropertySource(environment, s3ConfigFiles); - } - } - } - return; - } for (String label : labels) { - // If we have profiles, add property sources with those profiles for (String profile : profiles) { addPropertySourcesForApps(apps, - app -> addProfileSpecificPropertySource(environment, app, profile, label)); + app -> addProfileSpecificPropertySource(environment, app, profile, label)); } } @@ -176,8 +164,10 @@ private void addPropertySources(Environment environment, List apps, Stri // Even with no profiles, negated profile documents (e.g. on-profile: // "!my-profile") should be included because no profile is active, // so all negations are satisfied - addPropertySourcesForApps(apps, + if (this.searchPaths.isEmpty()) { + addPropertySourcesForApps(apps, app -> addNegatedProfilePropertySource(environment, app, profiles, label)); + } } } else { @@ -196,8 +186,10 @@ private void addPropertySources(Environment environment, List apps, Stri // Handle documents with negated profile expressions (e.g. on-profile: // "!my-profile") // once per label rather than once per profile to avoid duplicates - addPropertySourcesForApps(apps, + if (this.searchPaths.isEmpty()) { + addPropertySourcesForApps(apps, app -> addNegatedProfilePropertySource(environment, app, profiles, label)); + } } } } @@ -297,16 +289,25 @@ private List getS3ConfigFileWithSearchPaths( List result = new ArrayList<>(); Set seenKeys = new LinkedHashSet<>(); - for (String template : this.searchPaths) { + for (String template : this.searchPaths) {String resolvedLabel = (label == null ? "" : label); + String resolvedProfile = (profile == null ? "" : profile); + String pattern = template .replace("{application}", application) - .replace("{profile}", profile == null ? "" : profile) - .replace("{label}", label == null ? "" : label); + .replace("{profile}", resolvedProfile) + .replace("{label}", resolvedLabel); + + pattern = StringUtils.trimLeadingCharacter(pattern.replaceAll("/{2,}", "/"), '/'); if (!pathMatcher.isPattern(pattern)) { boolean fileFound = false; - for (String ext : List.of(".properties", ".json", ".yml", ".yaml")) { - String key = pattern.endsWith(ext) ? pattern : pattern + ext; + List extensionsToProbe = (pattern.endsWith(".properties") || pattern.endsWith(".json") + || pattern.endsWith(".yml") || pattern.endsWith(".yaml")) + ? List.of("") + : List.of(".properties", ".json", ".yml", ".yaml"); + + for (String ext : extensionsToProbe) { + String key = pattern + ext; if (!seenKeys.add(key)) { continue; } @@ -991,4 +992,4 @@ protected String buildObjectKeyPrefix() { public boolean isShouldIncludeWithEmptyProperties() { return false; } -} \ No newline at end of file +} From 2d7d2d4b60420ccfbdbff1fb22e3bf6fb13d8f6d Mon Sep 17 00:00:00 2001 From: Geonwook Ham Date: Tue, 2 Jun 2026 21:52:20 +0900 Subject: [PATCH 3/9] Use separate ConfigServerProperties in test (#2812) Fixes #2812 In AwsS3EnvironmentRepositoryTests, avoid mutating a shared 'server' instance by creating a new ConfigServerProperties (serverWithDefaultLabel) and setting its defaultLabel before constructing AwsS3EnvironmentRepository. This prevents side-effects on shared test state and ensures the repository gets the intended default label for the test. Signed-off-by: Geonwook Ham --- .../environment/AwsS3EnvironmentRepositoryTests.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryTests.java b/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryTests.java index b6e1af733a..cc894d741d 100644 --- a/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryTests.java +++ b/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryTests.java @@ -1039,10 +1039,13 @@ public void searchPaths_wildcardAtStart_prefixExtractionEmpty_shouldMatchAll() { // 17) Empty label uses defaultLabel @Test public void searchPaths_withEmptyLabel_shouldUseDefaultLabel() { - server.setDefaultLabel("main"); + ConfigServerProperties serverWithDefaultLabel = new ConfigServerProperties(); + serverWithDefaultLabel.setDefaultLabel("main"); + List paths = List.of("{label}/foo-bar.yml"); AwsS3EnvironmentRepository repo = - new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + new AwsS3EnvironmentRepository(s3Client, "bucket1", false, serverWithDefaultLabel, paths); + putFiles("main/foo-bar.yml", yamlContent); Environment env = repo.findOne("foo", "", ""); From 5b2ac69f1cc4b0ffc5ddcc8c6a07d3feade55da6 Mon Sep 17 00:00:00 2001 From: Geonwook Ham Date: Tue, 2 Jun 2026 23:04:42 +0900 Subject: [PATCH 4/9] Potential fix for pull request finding The for loop header and the first statement are on the same line (for (...) {String resolvedLabel = ...), which is likely to fail formatting checks (and is hard to read). Split the loop body onto its own lines. Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Signed-off-by: Geonwook Ham --- .../config/server/environment/AwsS3EnvironmentRepository.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java index 6752f51380..a3dbfd4a59 100644 --- a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java +++ b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java @@ -289,7 +289,8 @@ private List getS3ConfigFileWithSearchPaths( List result = new ArrayList<>(); Set seenKeys = new LinkedHashSet<>(); - for (String template : this.searchPaths) {String resolvedLabel = (label == null ? "" : label); + for (String template : this.searchPaths) { + String resolvedLabel = (label == null ? "" : label); String resolvedProfile = (profile == null ? "" : profile); String pattern = template From d17d7273f2809420a04aad3ae7c9e954e8ac8519 Mon Sep 17 00:00:00 2001 From: Geonwook Ham Date: Tue, 2 Jun 2026 23:38:48 +0900 Subject: [PATCH 5/9] Refactor S3 search paths and negated profiles (spring-cloud#2812) Fixes spring-cloud#2812 Generalize getS3ConfigFileWithSearchPaths by adding a Function keyWrapper to handle different key-wrapping strategies (profile-specific and negated-profile cases). Implement wrapKeyWithNegatedConfigFiles to support YAML documents with negated spring.config.activate.on-profile expressions and wire it into property source discovery. Clean up formatting, streamline S3 client calls, and simplify create/wrapper methods. Also remove explicit setOrder in AwsS3EnvironmentRepositoryFactory (return repository directly) and update tests for minor formatting/usage changes. Signed-off-by: Geonwook Ham --- .../AwsS3EnvironmentRepository.java | 226 ++++++++---------- .../AwsS3EnvironmentRepositoryFactory.java | 6 +- .../AwsS3EnvironmentRepositoryTests.java | 145 ++++------- 3 files changed, 154 insertions(+), 223 deletions(-) diff --git a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java index a3dbfd4a59..0d1c50ae97 100644 --- a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java +++ b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java @@ -26,6 +26,7 @@ import java.util.Properties; import java.util.Set; import java.util.function.Consumer; +import java.util.function.Function; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -54,7 +55,6 @@ import static org.springframework.cloud.config.server.environment.AwsS3EnvironmentRepository.PATH_SEPARATOR; - /** * @author Clay McCoy * @author Scott Frederick @@ -93,7 +93,7 @@ public AwsS3EnvironmentRepository(S3Client s3Client, String bucketName, boolean } public AwsS3EnvironmentRepository(S3Client s3Client, String bucketName, boolean useApplicationAsDirectory, - ConfigServerProperties server, List searchPaths) { + ConfigServerProperties server, List searchPaths) { this.s3Client = s3Client; this.bucketName = bucketName; this.serverProperties = server; @@ -152,7 +152,7 @@ private void addPropertySources(Environment environment, List apps, Stri for (String label : labels) { for (String profile : profiles) { addPropertySourcesForApps(apps, - app -> addProfileSpecificPropertySource(environment, app, profile, label)); + app -> addProfileSpecificPropertySource(environment, app, profile, label)); } } @@ -164,10 +164,8 @@ private void addPropertySources(Environment environment, List apps, Stri // Even with no profiles, negated profile documents (e.g. on-profile: // "!my-profile") should be included because no profile is active, // so all negations are satisfied - if (this.searchPaths.isEmpty()) { - addPropertySourcesForApps(apps, + addPropertySourcesForApps(apps, app -> addNegatedProfilePropertySource(environment, app, profiles, label)); - } } } else { @@ -186,10 +184,8 @@ private void addPropertySources(Environment environment, List apps, Stri // Handle documents with negated profile expressions (e.g. on-profile: // "!my-profile") // once per label rather than once per profile to avoid duplicates - if (this.searchPaths.isEmpty()) { - addPropertySourcesForApps(apps, + addPropertySourcesForApps(apps, app -> addNegatedProfilePropertySource(environment, app, profiles, label)); - } } } } @@ -200,7 +196,9 @@ private void addPropertySourcesForApps(List apps, Consumer addPr private void addNegatedProfilePropertySource(Environment environment, String app, String[] allProfiles, String label) { - List s3ConfigFiles = getNegatedProfileS3ConfigFileYaml(app, allProfiles, label); + List s3ConfigFiles = this.searchPaths.isEmpty() + ? getNegatedProfileS3ConfigFileYaml(app, allProfiles, label) : getS3ConfigFileWithSearchPaths(app, null, + label, key -> wrapKeyWithNegatedConfigFiles(key, app, allProfiles, label)); addPropertySource(environment, s3ConfigFiles); } @@ -225,21 +223,19 @@ private List getNegatedProfileS3ConfigFileYaml(String application, } private void addProfileSpecificPropertySource(Environment environment, String app, String profile, String label) { - if (!searchPaths.isEmpty() && app.equals(serverProperties.getDefaultApplicationName())) { - return; - } List s3ConfigFiles = searchPaths.isEmpty() - ? getS3ConfigFile(app, profile, label, this::getS3PropertiesOrJsonConfigFile, this::getProfileSpecificS3ConfigFileYaml) - : getS3ConfigFileWithSearchPaths(app, profile, label); + ? getS3ConfigFile(app, profile, label, this::getS3PropertiesOrJsonConfigFile, + this::getProfileSpecificS3ConfigFileYaml) + : getS3ConfigFileWithSearchPaths(app, profile, label, + key -> wrapKeyWithConfigFiles(key, app, profile, label)); addPropertySource(environment, s3ConfigFiles); } - private void addNonProfileSpecificPropertySource(Environment environment, String app, String profile, String label) { - List s3ConfigFiles = searchPaths.isEmpty() - ? getS3ConfigFile(app, profile, label, - this::getNonProfileSpecificPropertiesOrJsonConfigFile, - this::getNonProfileSpecificS3ConfigFileYaml) - : Collections.emptyList(); + private void addNonProfileSpecificPropertySource(Environment environment, String app, String profile, + String label) { + List s3ConfigFiles = searchPaths.isEmpty() ? getS3ConfigFile(app, profile, label, + this::getNonProfileSpecificPropertiesOrJsonConfigFile, this::getNonProfileSpecificS3ConfigFileYaml) + : Collections.emptyList(); addPropertySource(environment, s3ConfigFiles); } @@ -283,8 +279,8 @@ private List getS3ConfigFile(String application, String profile, S } - private List getS3ConfigFileWithSearchPaths( - String application, String profile, String label) { + private List getS3ConfigFileWithSearchPaths(String application, String profile, String label, + Function> keyWrapper) { List result = new ArrayList<>(); Set seenKeys = new LinkedHashSet<>(); @@ -293,8 +289,7 @@ private List getS3ConfigFileWithSearchPaths( String resolvedLabel = (label == null ? "" : label); String resolvedProfile = (profile == null ? "" : profile); - String pattern = template - .replace("{application}", application) + String pattern = template.replace("{application}", application) .replace("{profile}", resolvedProfile) .replace("{label}", resolvedLabel); @@ -303,9 +298,8 @@ private List getS3ConfigFileWithSearchPaths( if (!pathMatcher.isPattern(pattern)) { boolean fileFound = false; List extensionsToProbe = (pattern.endsWith(".properties") || pattern.endsWith(".json") - || pattern.endsWith(".yml") || pattern.endsWith(".yaml")) - ? List.of("") - : List.of(".properties", ".json", ".yml", ".yaml"); + || pattern.endsWith(".yml") || pattern.endsWith(".yaml")) ? List.of("") + : List.of(".properties", ".json", ".yml", ".yaml"); for (String ext : extensionsToProbe) { String key = pattern + ext; @@ -313,11 +307,8 @@ private List getS3ConfigFileWithSearchPaths( continue; } try { - s3Client.headObject(HeadObjectRequest.builder() - .bucket(bucketName) - .key(key) - .build()); - result.addAll(wrapKeyWithConfigFiles(key, application, profile, label)); + s3Client.headObject(HeadObjectRequest.builder().bucket(bucketName).key(key).build()); + result.addAll(keyWrapper.apply(key)); fileFound = true; break; } @@ -335,23 +326,23 @@ private List getS3ConfigFileWithSearchPaths( String dirPrefix = pattern.endsWith("/") ? pattern : pattern + "/"; String token = null; do { - ListObjectsV2Response resp = s3Client.listObjectsV2( - ListObjectsV2Request.builder() - .bucket(bucketName) - .prefix(dirPrefix) - .continuationToken(token) - .build()); + ListObjectsV2Response resp = s3Client.listObjectsV2(ListObjectsV2Request.builder() + .bucket(bucketName) + .prefix(dirPrefix) + .continuationToken(token) + .build()); for (S3Object obj : resp.contents()) { String key = obj.key(); if (!hasSupportedExtension(key)) { continue; } if (seenKeys.add(key)) { - result.addAll(wrapKeyWithConfigFiles(key, application, profile, label)); + result.addAll(keyWrapper.apply(key)); } } token = resp.nextContinuationToken(); - } while (token != null); + } + while (token != null); continue; } @@ -364,11 +355,8 @@ private List getS3ConfigFileWithSearchPaths( continue; } try { - s3Client.headObject(HeadObjectRequest.builder() - .bucket(bucketName) - .key(key) - .build()); - result.addAll(wrapKeyWithConfigFiles(key, application, profile, label)); + s3Client.headObject(HeadObjectRequest.builder().bucket(bucketName).key(key).build()); + result.addAll(keyWrapper.apply(key)); break; } catch (S3Exception e) { @@ -384,40 +372,33 @@ private List getS3ConfigFileWithSearchPaths( String prefix = extractPrefix(pattern); String token = null; do { - ListObjectsV2Response resp = s3Client.listObjectsV2( - ListObjectsV2Request.builder() - .bucket(bucketName) - .prefix(prefix) - .continuationToken(token) - .build()); + ListObjectsV2Response resp = s3Client.listObjectsV2(ListObjectsV2Request.builder() + .bucket(bucketName) + .prefix(prefix) + .continuationToken(token) + .build()); for (S3Object obj : resp.contents()) { String key = obj.key(); if (!pathMatcher.match(pattern, key) || !hasSupportedExtension(key)) { continue; } if (seenKeys.add(key)) { - result.addAll(wrapKeyWithConfigFiles(key, application, profile, label)); + result.addAll(keyWrapper.apply(key)); } } token = resp.nextContinuationToken(); - } while (token != null); + } + while (token != null); } return result; } private boolean hasSupportedExtension(String key) { - return key.endsWith(".properties") - || key.endsWith(".json") - || key.endsWith(".yml") - || key.endsWith(".yaml"); + return key.endsWith(".properties") || key.endsWith(".json") || key.endsWith(".yml") || key.endsWith(".yaml"); } - private List wrapKeyWithConfigFiles( - String key, - String application, - String profile, - String label) { + private List wrapKeyWithConfigFiles(String key, String application, String profile, String label) { if (key.endsWith(".yml") || key.endsWith(".yaml")) { List files = new ArrayList<>(); @@ -425,42 +406,33 @@ private List wrapKeyWithConfigFiles( files.addAll(getNonProfileSpecificYamlFromKey(key, application, profile, label)); return files; } - return createConfigFileFromKey(key, application, profile, label) - .map(Collections::singletonList) + return createConfigFileFromKey(key, application, profile, label).map(Collections::singletonList) .orElseGet(Collections::emptyList); } + private List getProfileSpecificYamlFromKey(String key, String application, String profile, + String label) { - private List getProfileSpecificYamlFromKey( - String key, String application, String profile, String label) { - - YamlConfigFileFromKey config = new YamlConfigFileFromKey( - key, application, profile, label, - bucketName, useApplicationAsDirectory, s3Client, - properties -> YamlS3ConfigFile.profileMatchesActivateProperty(profile, properties) - ? YamlProcessor.MatchStatus.FOUND - : YamlProcessor.MatchStatus.NOT_FOUND - ); + YamlConfigFileFromKey config = new YamlConfigFileFromKey(key, application, profile, label, bucketName, + useApplicationAsDirectory, s3Client, + properties -> YamlS3ConfigFile.profileMatchesActivateProperty(profile, properties) + ? YamlProcessor.MatchStatus.FOUND : YamlProcessor.MatchStatus.NOT_FOUND); config.setShouldIncludeWithEmptyProperties(false); return List.of(config); } - private List getNonProfileSpecificYamlFromKey( - String key, String application, String profile, String label) { + private List getNonProfileSpecificYamlFromKey(String key, String application, String profile, + String label) { - YamlConfigFileFromKey config = new YamlConfigFileFromKey( - key, application, profile, label, - bucketName, useApplicationAsDirectory, s3Client, - properties -> !YamlS3ConfigFile.onProfilePropertyExists(properties) - ? YamlProcessor.MatchStatus.FOUND - : YamlProcessor.MatchStatus.NOT_FOUND - ); + YamlConfigFileFromKey config = new YamlConfigFileFromKey(key, application, profile, label, bucketName, + useApplicationAsDirectory, s3Client, properties -> !YamlS3ConfigFile.onProfilePropertyExists(properties) + ? YamlProcessor.MatchStatus.FOUND : YamlProcessor.MatchStatus.NOT_FOUND); return List.of(config); } private String extractPrefix(String pattern) { int idx = pattern.indexOf('*'); - int q = pattern.indexOf('?'); + int q = pattern.indexOf('?'); if (q != -1 && (idx == -1 || q < idx)) { idx = q; } @@ -471,26 +443,21 @@ private String extractPrefix(String pattern) { return (slash == -1 ? "" : pattern.substring(0, slash + 1)); } - - private Optional createConfigFileFromKey(String key, - String application, String profile, String label) { + private Optional createConfigFileFromKey(String key, String application, String profile, + String label) { String ext = key.substring(key.lastIndexOf('.') + 1); if ("properties".equalsIgnoreCase(ext)) { - return Optional.of(new PropertyConfigFileFromKey( - key, application, profile, label, bucketName, s3Client)); + return Optional.of(new PropertyConfigFileFromKey(key, application, profile, label, bucketName, s3Client)); } if ("json".equalsIgnoreCase(ext)) { - return Optional.of(new JsonConfigFileFromKey( - key, application, profile, label, bucketName, s3Client)); + return Optional.of(new JsonConfigFileFromKey(key, application, profile, label, bucketName, s3Client)); } if ("yml".equalsIgnoreCase(ext) || "yaml".equalsIgnoreCase(ext)) { - return Optional.of(new YamlConfigFileFromKey( - key, application, profile, label, bucketName, s3Client)); + return Optional.of(new YamlConfigFileFromKey(key, application, profile, label, bucketName, s3Client)); } return Optional.empty(); } - private List getNonProfileSpecificS3ConfigFileYaml(String application, String profile, String label) { List configFiles = new ArrayList<>(); @@ -568,6 +535,32 @@ private S3ConfigFile getS3PropertiesOrJsonConfigFile(String application, String } } + private List wrapKeyWithNegatedConfigFiles(String key, String application, String[] allProfiles, + String label) { + if (key.endsWith(".yml") || key.endsWith(".yaml")) { + return List.of(new YamlConfigFileFromKey(key, application, null, label, bucketName, false, s3Client, + properties -> { + Object onProfileValue = properties.get("spring.config.activate.on-profile"); + if (onProfileValue == null) { + onProfileValue = properties.get("spring.config.activate.onProfile"); + } + if (onProfileValue == null) { + return YamlProcessor.MatchStatus.NOT_FOUND; + } + + String expression = onProfileValue.toString().trim(); + if (!expression.contains("!") && !expression.contains("&") && !expression.contains("|") + && !expression.contains("(") && !expression.contains(",")) { + return YamlProcessor.MatchStatus.NOT_FOUND; + } + List allProfilesList = Arrays.asList(allProfiles); + boolean matches = Profiles.of(expression).matches(allProfilesList::contains); + return matches ? YamlProcessor.MatchStatus.FOUND : YamlProcessor.MatchStatus.NOT_FOUND; + })); + } + return Collections.emptyList(); + } + @Override public Locations getLocations(String application, String profiles, String label) { StringBuilder baseLocation = new StringBuilder(AWS_S3_RESOURCE_SCHEME + bucketName + PATH_SEPARATOR); @@ -710,9 +703,8 @@ private String createPropertySourceName(String app, String profile) { class PropertyS3ConfigFile extends S3ConfigFile { - PropertyS3ConfigFile(String application, String profile, String label, - String bucketName, boolean useApplicationAsDirectory, - S3Client s3Client) { + PropertyS3ConfigFile(String application, String profile, String label, String bucketName, + boolean useApplicationAsDirectory, S3Client s3Client) { this(application, profile, label, bucketName, useApplicationAsDirectory, s3Client, true); } @@ -856,7 +848,7 @@ class JsonS3ConfigFile extends YamlS3ConfigFile { } JsonS3ConfigFile(String application, String profile, String label, String bucketName, - boolean useApplicationAsDirectory, S3Client s3Client, boolean callReadImmediately) { + boolean useApplicationAsDirectory, S3Client s3Client, boolean callReadImmediately) { super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client, callReadImmediately); } @@ -871,12 +863,8 @@ class PropertyConfigFileFromKey extends PropertyS3ConfigFile { private final String key; - PropertyConfigFileFromKey(String key, - String application, - String profile, - String label, - String bucketName, - S3Client s3Client) { + PropertyConfigFileFromKey(String key, String application, String profile, String label, String bucketName, + S3Client s3Client) { super(application, profile, label, bucketName, false, s3Client, false); this.key = key; this.properties = read(); @@ -891,31 +879,22 @@ public String getName() { protected String buildObjectKeyPrefix() { return key.substring(0, key.lastIndexOf('.')); } + } class YamlConfigFileFromKey extends YamlS3ConfigFile { private final String key; - YamlConfigFileFromKey(String key, - String application, - String profile, - String label, - String bucketName, - S3Client s3Client) { + YamlConfigFileFromKey(String key, String application, String profile, String label, String bucketName, + S3Client s3Client) { super(application, profile, label, bucketName, false, s3Client, false); this.key = key; this.properties = read(); } - YamlConfigFileFromKey(String key, - String application, - String profile, - String label, - String bucketName, - boolean useApplicationAsDirectory, - S3Client s3Client, - YamlProcessor.DocumentMatcher... matchers) { + YamlConfigFileFromKey(String key, String application, String profile, String label, String bucketName, + boolean useApplicationAsDirectory, S3Client s3Client, YamlProcessor.DocumentMatcher... matchers) { super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client, false, matchers); this.key = key; this.properties = read(); @@ -930,14 +909,15 @@ public String getName() { protected String buildObjectKeyPrefix() { return key.substring(0, key.lastIndexOf('.')); } + } class JsonConfigFileFromKey extends JsonS3ConfigFile { private final String key; - JsonConfigFileFromKey(String key, String application, String profile, - String label, String bucketName, S3Client s3Client) { + JsonConfigFileFromKey(String key, String application, String profile, String label, String bucketName, + S3Client s3Client) { super(application, profile, label, bucketName, false, s3Client, false); this.key = key; this.properties = read(); @@ -952,13 +932,14 @@ public String getName() { protected String buildObjectKeyPrefix() { return key.substring(0, key.lastIndexOf('.')); } + } class NegatedProfileYamlDocumentS3ConfigFile extends YamlS3ConfigFile { NegatedProfileYamlDocumentS3ConfigFile(String application, String label, String bucketName, boolean useApplicationAsDirectory, S3Client s3Client, String[] allProfiles) { - super(application, null, label, bucketName, useApplicationAsDirectory, s3Client, properties -> { + super(application, null, label, bucketName, useApplicationAsDirectory, s3Client, true, properties -> { Object onProfileValue = properties.get("spring.config.activate.on-profile"); if (onProfileValue == null) { onProfileValue = properties.get("spring.config.activate.onProfile"); @@ -993,4 +974,5 @@ protected String buildObjectKeyPrefix() { public boolean isShouldIncludeWithEmptyProperties() { return false; } + } diff --git a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryFactory.java b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryFactory.java index 556a77202d..c109e37e6a 100644 --- a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryFactory.java +++ b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryFactory.java @@ -41,10 +41,8 @@ public AwsS3EnvironmentRepository build(AwsS3EnvironmentProperties environmentPr configureClientBuilder(clientBuilder, environmentProperties.getRegion(), environmentProperties.getEndpoint()); final S3Client client = clientBuilder.build(); - AwsS3EnvironmentRepository repository = new AwsS3EnvironmentRepository(client, - environmentProperties.getBucket(), environmentProperties.isUseDirectoryLayout(), server, environmentProperties.getSearchPaths()); - repository.setOrder(environmentProperties.getOrder()); - return repository; + return new AwsS3EnvironmentRepository(client, environmentProperties.getBucket(), + environmentProperties.isUseDirectoryLayout(), server, environmentProperties.getSearchPaths()); } } diff --git a/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryTests.java b/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryTests.java index cc894d741d..5cb293c17e 100644 --- a/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryTests.java +++ b/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryTests.java @@ -771,71 +771,54 @@ public void getLocationsTest() { @Test public void searchPath_placeholderOnly_shouldResolveExactFile() { List paths = List.of("{label}/{application}-{profile}.yml"); - AwsS3EnvironmentRepository repo = - new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); putFiles("v1/foo-bar.yml", yamlContent); Environment env = repo.findOne("foo", "bar", "v1"); assertThat(env.getPropertySources()).hasSize(1); - assertThat(env.getPropertySources().get(0).getName()) - .contains("v1/foo-bar.yml"); + assertThat(env.getPropertySources().get(0).getName()).contains("v1/foo-bar.yml"); } // 2) Wildcard only (.properties) @Test public void searchPath_wildcardOnly_shouldResolveAllProperties() { List paths = List.of("{label}/common/*.properties"); - AwsS3EnvironmentRepository repo = - new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); putFiles("v1/common/a.properties", "a=1\n"); putFiles("v1/common/b.properties", "b=2\n"); Environment env = repo.findOne("app", "", "v1"); assertThat(env.getPropertySources()).hasSize(2); - assertThat(env.getPropertySources().get(0).getName()) - .contains("v1/common/a.properties"); - assertThat(env.getPropertySources().get(1).getName()) - .contains("v1/common/b.properties"); + assertThat(env.getPropertySources().get(0).getName()).contains("v1/common/a.properties"); + assertThat(env.getPropertySources().get(1).getName()).contains("v1/common/b.properties"); } // 3) Placeholder + wildcard combined @Test public void searchPath_placeholderAndWildcard_shouldResolveMatchingKeys() { - List paths = List.of( - "{label}/{application}-{profile}.yml", - "{label}/common/*.properties" - ); - AwsS3EnvironmentRepository repo = - new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + List paths = List.of("{label}/{application}-{profile}.yml", "{label}/common/*.properties"); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); putFiles("v1/foo-bar.yml", yamlContent); putFiles("v1/common/foo.properties", "k=v\n"); Environment env = repo.findOne("foo", "bar", "v1"); assertThat(env.getPropertySources()).hasSize(2); - assertThat(env.getPropertySources().get(0).getName()) - .contains("v1/foo-bar.yml"); - assertThat(env.getPropertySources().get(1).getName()) - .contains("v1/common/foo.properties"); + assertThat(env.getPropertySources().get(0).getName()).contains("v1/foo-bar.yml"); + assertThat(env.getPropertySources().get(1).getName()).contains("v1/common/foo.properties"); } // 4) Order matters @Test public void searchPaths_orderMatters_forPropertySourceOrder() { - List paths = List.of( - "{label}/common/*.properties", - "{label}/{application}-{profile}.yml" - ); - AwsS3EnvironmentRepository repo = - new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + List paths = List.of("{label}/common/*.properties", "{label}/{application}-{profile}.yml"); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); putFiles("v1/common/foo.properties", "k=v\n"); putFiles("v1/foo-bar.yml", yamlContent); Environment env = repo.findOne("foo", "bar", "v1"); assertThat(env.getPropertySources()).hasSize(2); - assertThat(env.getPropertySources().get(0).getName()) - .contains("v1/common/foo.properties"); - assertThat(env.getPropertySources().get(1).getName()) - .contains("v1/foo-bar.yml"); + assertThat(env.getPropertySources().get(0).getName()).contains("v1/common/foo.properties"); + assertThat(env.getPropertySources().get(1).getName()).contains("v1/foo-bar.yml"); } // 5) Extension preserved @@ -844,8 +827,7 @@ public Stream searchPath_extensionPreserved() { List exts = List.of("yml", "yaml", "properties", "json"); return exts.stream().map(ext -> DynamicTest.dynamicTest(ext, () -> { List paths = List.of("{label}/foo-bar." + ext); - AwsS3EnvironmentRepository repo = - new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); String content = "key=v\n"; if (ext.equals("yml") || ext.equals("yaml")) { @@ -858,8 +840,7 @@ else if (ext.equals("json")) { Environment env = repo.findOne("foo", "bar", "v1"); assertThat(env.getPropertySources()).hasSize(1); - assertThat(env.getPropertySources().get(0).getName()) - .contains("v1/foo-bar." + ext); + assertThat(env.getPropertySources().get(0).getName()).contains("v1/foo-bar." + ext); })); } @@ -867,23 +848,20 @@ else if (ext.equals("json")) { @Test public void searchPaths_applicationAsDirectory_shouldStillHonorSearchPaths() { List paths = List.of("{label}/{application}/foo.*"); - AwsS3EnvironmentRepository repo = - new AwsS3EnvironmentRepository(s3Client, "bucket1", true, server, paths); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", true, server, paths); putFiles("v1/foo/foo.properties", "k=v\n"); putFiles("v1/foo/foo.json", jsonContent); Environment env = repo.findOne("foo", "", "v1"); assertThat(env.getPropertySources()).hasSize(1); - assertThat(env.getPropertySources().get(0).getName()) - .contains("v1/foo/foo.properties"); + assertThat(env.getPropertySources().get(0).getName()).contains("v1/foo/foo.properties"); } // 7) Multi-document YAML should not be split @Test public void multiDocumentYaml_withSearchPaths_shouldNotSplitDocuments() throws IOException { List paths = List.of("{label}/{application}.yml"); - AwsS3EnvironmentRepository repo = - new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); String multi = "---\na: 1\n---\nb: 2\n"; putFiles("lab/app.yml", multi); @@ -891,20 +869,15 @@ public void multiDocumentYaml_withSearchPaths_shouldNotSplitDocuments() throws I Environment env = repo.findOne("app", "", "lab"); assertThat(env.getPropertySources()).hasSize(1); @SuppressWarnings("unchecked") - Map map = (Map) - env.getPropertySources().get(0).getSource(); + Map map = (Map) env.getPropertySources().get(0).getSource(); assertThat(map).containsEntry("a", 1).containsEntry("b", 2); } // 8) Deduplication across patterns @Test public void searchPaths_deduplication_shouldOnlyAddOnce() { - List paths = List.of( - "{label}/foo.yml", - "{label}/{application}.yml" - ); - AwsS3EnvironmentRepository repo = - new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + List paths = List.of("{label}/foo.yml", "{label}/{application}.yml"); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); putFiles("v1/foo.yml", yamlContent); Environment env = repo.findOne("foo", "", "v1"); @@ -915,120 +888,102 @@ public void searchPaths_deduplication_shouldOnlyAddOnce() { @Test public void searchPaths_literalStopsAtProperties() { List paths = List.of("{label}/{application}"); - AwsS3EnvironmentRepository repo = - new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); putFiles("v1/app.properties", "foo=bar\nflag=false\n"); putFiles("v1/app.json", jsonContent); putFiles("v1/app.yml", yamlContent); Environment env = repo.findOne("app", "", "v1"); assertThat(env.getPropertySources()).hasSize(1); - assertThat(env.getPropertySources().get(0).getName()) - .contains("v1/app.properties"); + assertThat(env.getPropertySources().get(0).getName()).contains("v1/app.properties"); } // 10) Literal stops at .json @Test public void searchPaths_literalStopsAtJson() { List paths = List.of("{label}/{application}"); - AwsS3EnvironmentRepository repo = - new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); putFiles("v1/app.json", jsonContent); putFiles("v1/app.yml", yamlContent); Environment env = repo.findOne("app", "", "v1"); assertThat(env.getPropertySources()).hasSize(1); - assertThat(env.getPropertySources().get(0).getName()) - .contains("v1/app.json"); + assertThat(env.getPropertySources().get(0).getName()).contains("v1/app.json"); } // 11) Literal stops at .yml @Test public void searchPaths_literalStopsAtYaml() { List paths = List.of("{label}/{application}"); - AwsS3EnvironmentRepository repo = - new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); putFiles("v1/app.yml", yamlContent); Environment env = repo.findOne("app", "", "v1"); assertThat(env.getPropertySources()).hasSize(1); - assertThat(env.getPropertySources().get(0).getName()) - .contains("v1/app.yml"); + assertThat(env.getPropertySources().get(0).getName()).contains("v1/app.yml"); } // 12) Dot-wildcard auto-extension (.properties preferred) @Test public void searchPaths_dotWildcardAutoExtProperties() { List paths = List.of("{label}/{application}.*"); - AwsS3EnvironmentRepository repo = - new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); putFiles("v1/app.properties", "x=1\n"); putFiles("v1/app.json", jsonContent); Environment env = repo.findOne("app", "", "v1"); assertThat(env.getPropertySources()).hasSize(1); - assertThat(env.getPropertySources().get(0).getName()) - .contains("v1/app.properties"); + assertThat(env.getPropertySources().get(0).getName()).contains("v1/app.properties"); } // 13) Literal directory scan @Test public void searchPaths_literalDirectoryScan() { List paths = List.of("{label}/{application}"); - AwsS3EnvironmentRepository repo = - new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); putFiles("v1/app/foo.properties", "a=1\n"); putFiles("v1/app/sub/bar.yml", "bar: 2\n"); Environment env = repo.findOne("app", "", "v1"); assertThat(env.getPropertySources()).hasSize(2); - assertThat(env.getPropertySources().get(0).getName()) - .contains("v1/app/foo.properties"); - assertThat(env.getPropertySources().get(1).getName()) - .contains("v1/app/sub/bar.yml"); + assertThat(env.getPropertySources().get(0).getName()).contains("v1/app/foo.properties"); + assertThat(env.getPropertySources().get(1).getName()).contains("v1/app/sub/bar.yml"); } // 14) Double-wildcard nested directories @Test public void searchPaths_doubleWildcardNested() { List paths = List.of("{label}/**/*.properties"); - AwsS3EnvironmentRepository repo = - new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); putFiles("v1/foo/a.properties", "a=1\n"); putFiles("v1/foo/sub/b.properties", "b=2\n"); Environment env = repo.findOne("app", "", "v1"); assertThat(env.getPropertySources()).hasSize(2); - assertThat(env.getPropertySources().get(0).getName()) - .contains("v1/foo/a.properties"); - assertThat(env.getPropertySources().get(1).getName()) - .contains("v1/foo/sub/b.properties"); + assertThat(env.getPropertySources().get(0).getName()).contains("v1/foo/a.properties"); + assertThat(env.getPropertySources().get(1).getName()).contains("v1/foo/sub/b.properties"); } // 15) Single-character wildcard @Test public void searchPaths_singleCharacterWildcard_shouldMatchExactlyOneChar() { List paths = List.of("{label}/data-?.yml"); - AwsS3EnvironmentRepository repo = - new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); - putFiles("lab/data-a.yml", "a:1\n"); - putFiles("lab/data-b.yml", "b:2\n"); - putFiles("lab/data-10.yml", "x:3\n"); // should not match + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + putFiles("lab/data-a.yml", "a: 1\n"); + putFiles("lab/data-b.yml", "b: 2\n"); + putFiles("lab/data-10.yml", "x: 3\n"); // should not match Environment env = repo.findOne("app", "", "lab"); assertThat(env.getPropertySources()).hasSize(2); - assertThat(env.getPropertySources().get(0).getName()) - .contains("lab/data-a.yml"); - assertThat(env.getPropertySources().get(1).getName()) - .contains("lab/data-b.yml"); + assertThat(env.getPropertySources().get(0).getName()).contains("lab/data-a.yml"); + assertThat(env.getPropertySources().get(1).getName()).contains("lab/data-b.yml"); } // 16) Wildcard at start (empty prefix) @Test public void searchPaths_wildcardAtStart_prefixExtractionEmpty_shouldMatchAll() { List paths = List.of("{label}/*.yml"); - AwsS3EnvironmentRepository repo = - new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); putFiles("lab/app.yml", yamlContent); putFiles("lab/other.yml", yamlContent); @@ -1043,31 +998,27 @@ public void searchPaths_withEmptyLabel_shouldUseDefaultLabel() { serverWithDefaultLabel.setDefaultLabel("main"); List paths = List.of("{label}/foo-bar.yml"); - AwsS3EnvironmentRepository repo = - new AwsS3EnvironmentRepository(s3Client, "bucket1", false, serverWithDefaultLabel, paths); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, + serverWithDefaultLabel, paths); putFiles("main/foo-bar.yml", yamlContent); Environment env = repo.findOne("foo", "", ""); assertThat(env.getPropertySources()).hasSize(1); - assertThat(env.getPropertySources().get(0).getName()) - .contains("main/foo-bar.yml"); + assertThat(env.getPropertySources().get(0).getName()).contains("main/foo-bar.yml"); } // 18) Multiple labels applied in reverse order @Test public void searchPaths_multipleLabels_shouldApplyForEachLabelInReverseOrder() { List paths = List.of("{label}/{application}.yml"); - AwsS3EnvironmentRepository repo = - new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); putFiles("lab1/app.yml", yamlContent); putFiles("lab2/app.yml", yamlContent); Environment env = repo.findOne("app", "", "lab1,lab2"); - assertThat(env.getPropertySources().get(0).getName()) - .contains("lab2/app.yml"); - assertThat(env.getPropertySources().get(1).getName()) - .contains("lab1/app.yml"); + assertThat(env.getPropertySources().get(0).getName()).contains("lab2/app.yml"); + assertThat(env.getPropertySources().get(1).getName()).contains("lab1/app.yml"); } private String putFiles(String fileName, String propertyContent) { From f7c5afde89d50560d439cb7b4b215c870d6069de Mon Sep 17 00:00:00 2001 From: Geonwook Ham Date: Sat, 6 Jun 2026 11:16:18 +0900 Subject: [PATCH 6/9] refactor: consolidate key-based S3ConfigFile classes and remove redundant callReadImmediately flags Signed-off-by: Geonwook Ham --- .../AwsS3EnvironmentProperties.java | 13 +- .../AwsS3EnvironmentRepository.java | 437 +++++++++--------- .../AwsS3EnvironmentRepositoryFactory.java | 3 +- .../AwsS3EnvironmentRepositoryTests.java | 68 ++- 4 files changed, 281 insertions(+), 240 deletions(-) diff --git a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentProperties.java b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentProperties.java index db7107fa24..340dead564 100644 --- a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentProperties.java +++ b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentProperties.java @@ -16,7 +16,7 @@ package org.springframework.cloud.config.server.environment; -import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.springframework.boot.context.properties.ConfigurationProperties; @@ -45,14 +45,19 @@ public class AwsS3EnvironmentProperties implements EnvironmentRepositoryProperti private String bucket; /** - * Use application name as intermediate directory. Analogous to `searchPaths: - * {application}` from Git backend. + * Use application name as intermediate directory. Analogous to + * {@link #searchPaths} + * from Git backend. */ private boolean useDirectoryLayout; private int order = DEFAULT_ORDER; - private List searchPaths = new ArrayList<>(); + /** + * List of directory paths to search for profiles in the bucket. + * Analogous to {@link #searchPaths} in Git backend. + */ + private List searchPaths = Collections.emptyList(); public List getSearchPaths() { return searchPaths; diff --git a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java index 0d1c50ae97..0adb0701e5 100644 --- a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java +++ b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java @@ -22,6 +22,7 @@ import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; +import java.util.Locale; import java.util.Optional; import java.util.Properties; import java.util.Set; @@ -71,6 +72,10 @@ public class AwsS3EnvironmentRepository implements EnvironmentRepository, Ordere private static final String AWS_S3_RESOURCE_SCHEME = "s3://"; + private static final List SUPPORTED_EXTENSIONS = List.of(".properties", ".json", ".yml", ".yaml"); + + private static final List EMPTY_EXTENSION = List.of(""); + private final S3Client s3Client; private final String bucketName; @@ -89,7 +94,7 @@ public AwsS3EnvironmentRepository(S3Client s3Client, String bucketName, ConfigSe public AwsS3EnvironmentRepository(S3Client s3Client, String bucketName, boolean useApplicationAsDirectory, ConfigServerProperties server) { - this(s3Client, bucketName, useApplicationAsDirectory, server, null); + this(s3Client, bucketName, useApplicationAsDirectory, server, Collections.emptyList()); } public AwsS3EnvironmentRepository(S3Client s3Client, String bucketName, boolean useApplicationAsDirectory, @@ -101,15 +106,17 @@ public AwsS3EnvironmentRepository(S3Client s3Client, String bucketName, boolean this.searchPaths = (searchPaths == null ? Collections.emptyList() : searchPaths); } + public AwsS3EnvironmentRepository(S3Client s3Client, AwsS3EnvironmentProperties properties, + ConfigServerProperties server) { + this(s3Client, properties.getBucket(), properties.isUseDirectoryLayout(), server, properties.getSearchPaths()); + this.order = properties.getOrder(); + } + @Override public int getOrder() { return this.order; } - public void setOrder(int order) { - this.order = order; - } - @Override public Environment findOne(String specifiedApplication, String specifiedProfiles, String specifiedLabel) { final String application = ObjectUtils.isEmpty(specifiedApplication) @@ -236,31 +243,48 @@ private void addNonProfileSpecificPropertySource(Environment environment, String List s3ConfigFiles = searchPaths.isEmpty() ? getS3ConfigFile(app, profile, label, this::getNonProfileSpecificPropertiesOrJsonConfigFile, this::getNonProfileSpecificS3ConfigFileYaml) : Collections.emptyList(); - addPropertySource(environment, s3ConfigFiles); + if (s3ConfigFiles != null) { + addPropertySource(environment, s3ConfigFiles); + } } private void addPropertySource(Environment environment, List s3ConfigFiles) { for (S3ConfigFile s3ConfigFile : s3ConfigFiles) { - final Properties config = s3ConfigFile.read(); - // This logic handles the case where the s3 file is a YAML file that is - // not profile specific (ie it does not have - in the name) - // and does not have any profile specific documents in it. In this case we do - // not want to include this - // property source we only want to include the document for the default - // profile. When we create - // the S3ConfigFile for this file we set the - // shouldIncludeWithEmptyProperties to false - // in ProfileSpecificYamlDocumentS3ConfigFile for this specific case. - if (config != null) { - if (!config.isEmpty() || s3ConfigFile.isShouldIncludeWithEmptyProperties()) { - environment.setVersion(s3ConfigFile.getVersion()); - config.putAll(serverProperties.getOverrides()); - PropertySource propertySource = new PropertySource(s3ConfigFile.getName(), config); - if (LOG.isDebugEnabled()) { - LOG.debug("Adding property source to environment " + propertySource); - } - environment.add(propertySource); + if (s3ConfigFile == null) { + continue; + } + try { + final Properties config = s3ConfigFile.read(); + // This logic handles the case where the s3 file is a YAML file that is + // not profile specific (ie it does not have - in the name) + // and does not have any profile specific documents in it. In this case we + // do + // not want to include this + // property source we only want to include the document for the default + // profile. When we create + // the S3ConfigFile for this file we set the + // shouldIncludeWithEmptyProperties to false + // in ProfileSpecificYamlDocumentS3ConfigFile for this specific case. + if (config == null || (config.isEmpty() && !s3ConfigFile.isShouldIncludeWithEmptyProperties())) { + continue; + } + String name = s3ConfigFile.getName(); + boolean exists = environment.getPropertySources() + .stream() + .anyMatch(p -> p.getName().equals(name) && p.getSource().equals(config)); + if (exists) { + continue; + } + environment.setVersion(s3ConfigFile.getVersion()); + config.putAll(serverProperties.getOverrides()); + PropertySource propertySource = new PropertySource(name, config); + if (LOG.isDebugEnabled()) { + LOG.debug("Adding property source to environment " + propertySource); } + environment.add(propertySource); + } + catch (Exception e) { + LOG.warn("Could not read properties from " + s3ConfigFile.getName(), e); } } } @@ -271,127 +295,135 @@ private String[] parseProfiles(String profiles) { private List getS3ConfigFile(String application, String profile, String label, JsonOrPropertiesS3ConfigFileCreator creator, YamlS3ConfigFileCreator yamlCreator) { + List configFiles = new ArrayList<>(); S3ConfigFile configFile = creator.create(application, profile, label); if (configFile != null) { - return List.of(configFile); + configFiles.add(configFile); } - return new ArrayList<>(yamlCreator.create(application, profile, label)); - + configFiles.addAll(yamlCreator.create(application, profile, label)); + return configFiles; } private List getS3ConfigFileWithSearchPaths(String application, String profile, String label, Function> keyWrapper) { - List result = new ArrayList<>(); Set seenKeys = new LinkedHashSet<>(); - for (String template : this.searchPaths) { - String resolvedLabel = (label == null ? "" : label); - String resolvedProfile = (profile == null ? "" : profile); - - String pattern = template.replace("{application}", application) - .replace("{profile}", resolvedProfile) - .replace("{label}", resolvedLabel); - - pattern = StringUtils.trimLeadingCharacter(pattern.replaceAll("/{2,}", "/"), '/'); - + String pattern = resolvePattern(template, application, profile, label); if (!pathMatcher.isPattern(pattern)) { - boolean fileFound = false; - List extensionsToProbe = (pattern.endsWith(".properties") || pattern.endsWith(".json") - || pattern.endsWith(".yml") || pattern.endsWith(".yaml")) ? List.of("") - : List.of(".properties", ".json", ".yml", ".yaml"); - - for (String ext : extensionsToProbe) { - String key = pattern + ext; - if (!seenKeys.add(key)) { - continue; - } - try { - s3Client.headObject(HeadObjectRequest.builder().bucket(bucketName).key(key).build()); - result.addAll(keyWrapper.apply(key)); - fileFound = true; - break; - } - catch (S3Exception e) { - int status = e.statusCode(); - if (status != 404 && status != 403) { - throw e; - } - } - } - if (fileFound) { - continue; + boolean fileFound = probeLiteralPattern(pattern, seenKeys, result, keyWrapper); + if (!fileFound) { + scanDirectoryPattern(pattern, seenKeys, result, keyWrapper); } + continue; + } + else if (pattern.endsWith(".*")) { + probeDotWildcardPattern(pattern, seenKeys, result, keyWrapper); + continue; + } + scanWildcardPattern(pattern, seenKeys, result, keyWrapper); + } + return result; + } - String dirPrefix = pattern.endsWith("/") ? pattern : pattern + "/"; - String token = null; - do { - ListObjectsV2Response resp = s3Client.listObjectsV2(ListObjectsV2Request.builder() - .bucket(bucketName) - .prefix(dirPrefix) - .continuationToken(token) - .build()); - for (S3Object obj : resp.contents()) { - String key = obj.key(); - if (!hasSupportedExtension(key)) { - continue; - } - if (seenKeys.add(key)) { - result.addAll(keyWrapper.apply(key)); - } - } - token = resp.nextContinuationToken(); - } - while (token != null); + private String resolvePattern(String template, String application, String profile, String label) { + String resolvedLabel = (label == null ? "" : label); + String resolvedProfile = (profile == null ? "" : profile); + String pattern = template.replace("{application}", application) + .replace("{profile}", resolvedProfile) + .replace("{label}", resolvedLabel); + return StringUtils.trimLeadingCharacter(pattern.replaceAll("/{2,}", "/"), '/'); + } + private boolean probeLiteralPattern(String pattern, Set seenKeys, List result, + Function> keyWrapper) { + boolean fileFound = false; + List extensionsToProbe = hasSupportedExtension(pattern) ? EMPTY_EXTENSION : SUPPORTED_EXTENSIONS; + for (String ext : extensionsToProbe) { + String key = pattern + ext; + if (!seenKeys.add(key)) { continue; } + else if (probeKeyAndAddResult(key, result, keyWrapper)) { + fileFound = true; + } + } + return fileFound; + } - if (pattern.endsWith(".*")) { - String base = pattern.substring(0, pattern.length() - 2); - for (String ext : List.of(".properties", ".json", ".yml", ".yaml")) { - String key = base + ext; - if (!seenKeys.add(key)) { - continue; - } - try { - s3Client.headObject(HeadObjectRequest.builder().bucket(bucketName).key(key).build()); - result.addAll(keyWrapper.apply(key)); - break; - } - catch (S3Exception e) { - int status = e.statusCode(); - if (status != 404 && status != 403) { - throw e; - } - } + private void probeDotWildcardPattern(String pattern, Set seenKeys, List result, + Function> keyWrapper) { + String base = pattern.substring(0, pattern.length() - 2); + for (String ext : SUPPORTED_EXTENSIONS) { + String key = base + ext; + if (seenKeys.add(key)) { + probeKeyAndAddResult(key, result, keyWrapper); + } + } + } + + private boolean probeKeyAndAddResult(String key, List result, + Function> keyWrapper) { + try { + s3Client.headObject(HeadObjectRequest.builder().bucket(bucketName).key(key).build()); + result.addAll(keyWrapper.apply(key)); + return true; + } + catch (S3Exception e) { + int status = e.statusCode(); + if (status != 404 && status != 403) { + if (LOG.isInfoEnabled()) { + LOG.info("Error checking S3 object key: " + key, e); } - continue; + throw e; } + return false; + } + } - String prefix = extractPrefix(pattern); - String token = null; - do { - ListObjectsV2Response resp = s3Client.listObjectsV2(ListObjectsV2Request.builder() - .bucket(bucketName) - .prefix(prefix) - .continuationToken(token) - .build()); - for (S3Object obj : resp.contents()) { - String key = obj.key(); - if (!pathMatcher.match(pattern, key) || !hasSupportedExtension(key)) { - continue; - } - if (seenKeys.add(key)) { - result.addAll(keyWrapper.apply(key)); - } + private void scanDirectoryPattern(String pattern, Set seenKeys, List result, + Function> keyWrapper) { + String dirPrefix = pattern.endsWith("/") ? pattern : pattern + "/"; + String token = null; + do { + ListObjectsV2Response resp = s3Client.listObjectsV2(ListObjectsV2Request.builder() + .bucket(bucketName) + .prefix(dirPrefix) + .continuationToken(token) + .build()); + for (S3Object obj : resp.contents()) { + String key = obj.key(); + if (!hasSupportedExtension(key)) { + continue; + } + else if (seenKeys.add(key)) { + result.addAll(keyWrapper.apply(key)); } - token = resp.nextContinuationToken(); } - while (token != null); + token = resp.nextContinuationToken(); } + while (token != null); + } - return result; + private void scanWildcardPattern(String pattern, Set seenKeys, List result, + Function> keyWrapper) { + String prefix = extractPrefix(pattern); + String token = null; + do { + ListObjectsV2Response resp = s3Client.listObjectsV2( + ListObjectsV2Request.builder().bucket(bucketName).prefix(prefix).continuationToken(token).build()); + for (S3Object obj : resp.contents()) { + String key = obj.key(); + if (!pathMatcher.match(pattern, key) || !hasSupportedExtension(key)) { + continue; + } + else if (seenKeys.add(key)) { + result.addAll(keyWrapper.apply(key)); + } + } + token = resp.nextContinuationToken(); + } + while (token != null); } private boolean hasSupportedExtension(String key) { @@ -413,7 +445,7 @@ private List wrapKeyWithConfigFiles(String key, String application private List getProfileSpecificYamlFromKey(String key, String application, String profile, String label) { - YamlConfigFileFromKey config = new YamlConfigFileFromKey(key, application, profile, label, bucketName, + S3ConfigFileFromKey config = new S3ConfigFileFromKey(key, application, profile, label, bucketName, useApplicationAsDirectory, s3Client, properties -> YamlS3ConfigFile.profileMatchesActivateProperty(profile, properties) ? YamlProcessor.MatchStatus.FOUND : YamlProcessor.MatchStatus.NOT_FOUND); @@ -424,7 +456,7 @@ private List getProfileSpecificYamlFromKey(String key, String appl private List getNonProfileSpecificYamlFromKey(String key, String application, String profile, String label) { - YamlConfigFileFromKey config = new YamlConfigFileFromKey(key, application, profile, label, bucketName, + S3ConfigFileFromKey config = new S3ConfigFileFromKey(key, application, profile, label, bucketName, useApplicationAsDirectory, s3Client, properties -> !YamlS3ConfigFile.onProfilePropertyExists(properties) ? YamlProcessor.MatchStatus.FOUND : YamlProcessor.MatchStatus.NOT_FOUND); return List.of(config); @@ -436,7 +468,7 @@ private String extractPrefix(String pattern) { if (q != -1 && (idx == -1 || q < idx)) { idx = q; } - if (idx <= 0) { + else if (idx <= 0) { return ""; } int slash = pattern.lastIndexOf('/', idx); @@ -447,13 +479,16 @@ private Optional createConfigFileFromKey(String key, String applic String label) { String ext = key.substring(key.lastIndexOf('.') + 1); if ("properties".equalsIgnoreCase(ext)) { - return Optional.of(new PropertyConfigFileFromKey(key, application, profile, label, bucketName, s3Client)); + return Optional.of(new S3ConfigFileFromKey(key, application, profile, label, bucketName, + useApplicationAsDirectory, s3Client)); } - if ("json".equalsIgnoreCase(ext)) { - return Optional.of(new JsonConfigFileFromKey(key, application, profile, label, bucketName, s3Client)); + else if ("json".equalsIgnoreCase(ext)) { + return Optional.of(new S3ConfigFileFromKey(key, application, profile, label, bucketName, + useApplicationAsDirectory, s3Client)); } - if ("yml".equalsIgnoreCase(ext) || "yaml".equalsIgnoreCase(ext)) { - return Optional.of(new YamlConfigFileFromKey(key, application, profile, label, bucketName, s3Client)); + else if ("yml".equalsIgnoreCase(ext) || "yaml".equalsIgnoreCase(ext)) { + return Optional.of(new S3ConfigFileFromKey(key, application, profile, label, bucketName, + useApplicationAsDirectory, s3Client)); } return Optional.empty(); } @@ -538,8 +573,8 @@ private S3ConfigFile getS3PropertiesOrJsonConfigFile(String application, String private List wrapKeyWithNegatedConfigFiles(String key, String application, String[] allProfiles, String label) { if (key.endsWith(".yml") || key.endsWith(".yaml")) { - return List.of(new YamlConfigFileFromKey(key, application, null, label, bucketName, false, s3Client, - properties -> { + S3ConfigFileFromKey config = new S3ConfigFileFromKey(key, application, null, label, bucketName, + this.useApplicationAsDirectory, s3Client, properties -> { Object onProfileValue = properties.get("spring.config.activate.on-profile"); if (onProfileValue == null) { onProfileValue = properties.get("spring.config.activate.onProfile"); @@ -549,14 +584,15 @@ private List wrapKeyWithNegatedConfigFiles(String key, String appl } String expression = onProfileValue.toString().trim(); - if (!expression.contains("!") && !expression.contains("&") && !expression.contains("|") - && !expression.contains("(") && !expression.contains(",")) { + if (AwsS3EnvironmentRepository.isSimpleProfileName(expression)) { return YamlProcessor.MatchStatus.NOT_FOUND; } List allProfilesList = Arrays.asList(allProfiles); boolean matches = Profiles.of(expression).matches(allProfilesList::contains); return matches ? YamlProcessor.MatchStatus.FOUND : YamlProcessor.MatchStatus.NOT_FOUND; - })); + }); + config.setShouldIncludeWithEmptyProperties(false); + return List.of(config); } return Collections.emptyList(); } @@ -575,6 +611,11 @@ public Locations getLocations(String application, String profiles, String label) return new Locations(application, profiles, label, null, new String[] { baseLocation.toString() }); } + static boolean isSimpleProfileName(String expression) { + return !expression.contains("!") && !expression.contains("&") && !expression.contains("|") + && !expression.contains("(") && !expression.contains(","); + } + interface YamlS3ConfigFileCreator { List create(String application, String profile, String label); @@ -705,15 +746,8 @@ class PropertyS3ConfigFile extends S3ConfigFile { PropertyS3ConfigFile(String application, String profile, String label, String bucketName, boolean useApplicationAsDirectory, S3Client s3Client) { - this(application, profile, label, bucketName, useApplicationAsDirectory, s3Client, true); - } - - PropertyS3ConfigFile(String application, String profile, String label, String bucketName, - boolean useApplicationAsDirectory, S3Client s3Client, boolean callReadImmediately) { super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client); - if (callReadImmediately) { - this.properties = read(); - } + this.properties = read(); } @Override @@ -745,21 +779,22 @@ class YamlS3ConfigFile extends S3ConfigFile { YamlS3ConfigFile(String application, String profile, String label, String bucketName, boolean useApplicationAsDirectory, S3Client s3Client) { - this(application, profile, label, bucketName, useApplicationAsDirectory, s3Client, true, + this(application, profile, label, bucketName, useApplicationAsDirectory, s3Client, new YamlProcessor.DocumentMatcher[] {}); } YamlS3ConfigFile(String application, String profile, String label, String bucketName, - boolean useApplicationAsDirectory, S3Client s3Client, boolean callReadImmediately, + boolean useApplicationAsDirectory, S3Client s3Client, final YamlProcessor.DocumentMatcher... documentMatchers) { super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client); this.documentMatchers = documentMatchers; - if (callReadImmediately) { - this.properties = read(); - } + this.properties = read(); } protected static boolean profileMatchesActivateProperty(String profile, Properties properties) { + if (profile == null) { + return false; + } return profile.equals(properties.get("spring.config.activate.on-profile")) || profile.equals(properties.get("spring.config.activate.onProfile")); } @@ -797,7 +832,7 @@ class ProfileSpecificYamlDocumentS3ConfigFile extends YamlS3ConfigFile { ProfileSpecificYamlDocumentS3ConfigFile(String application, String profile, String label, String bucketName, boolean useApplicationAsDirectory, S3Client s3Client) { - super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client, true, + super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client, properties -> profileMatchesActivateProperty(profile, properties) ? YamlProcessor.MatchStatus.FOUND : YamlProcessor.MatchStatus.NOT_FOUND); } @@ -818,7 +853,7 @@ class NonProfileSpecificYamlDocumentS3ConfigFile extends YamlS3ConfigFile { NonProfileSpecificYamlDocumentS3ConfigFile(String application, String profile, String label, String bucketName, boolean useApplicationAsDirectory, S3Client s3Client) { - super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client, true, + super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client, properties -> !onProfilePropertyExists(properties) ? YamlProcessor.MatchStatus.FOUND : YamlProcessor.MatchStatus.NOT_FOUND); } @@ -829,7 +864,7 @@ class ProfileSpecificYamlS3ConfigFile extends YamlS3ConfigFile { ProfileSpecificYamlS3ConfigFile(String application, String profile, String label, String bucketName, boolean useApplicationAsDirectory, S3Client s3Client) { - super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client, true, + super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client, properties -> !onProfilePropertyExists(properties) ? YamlProcessor.MatchStatus.ABSTAIN : profileMatchesActivateProperty(profile, properties) ? YamlProcessor.MatchStatus.FOUND : YamlProcessor.MatchStatus.NOT_FOUND); @@ -847,11 +882,6 @@ class JsonS3ConfigFile extends YamlS3ConfigFile { this.properties = read(); } - JsonS3ConfigFile(String application, String profile, String label, String bucketName, - boolean useApplicationAsDirectory, S3Client s3Client, boolean callReadImmediately) { - super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client, callReadImmediately); - } - @Override protected List getExtensions() { return List.of("json"); @@ -859,44 +889,22 @@ protected List getExtensions() { } -class PropertyConfigFileFromKey extends PropertyS3ConfigFile { +class S3ConfigFileFromKey extends S3ConfigFile { private final String key; - PropertyConfigFileFromKey(String key, String application, String profile, String label, String bucketName, - S3Client s3Client) { - super(application, profile, label, bucketName, false, s3Client, false); - this.key = key; - this.properties = read(); - } - - @Override - public String getName() { - return "s3:" + bucketName + "/" + key; - } - - @Override - protected String buildObjectKeyPrefix() { - return key.substring(0, key.lastIndexOf('.')); - } - -} - -class YamlConfigFileFromKey extends YamlS3ConfigFile { - - private final String key; + private final YamlProcessor.DocumentMatcher[] documentMatchers; - YamlConfigFileFromKey(String key, String application, String profile, String label, String bucketName, + S3ConfigFileFromKey(String key, String application, String profile, String label, String bucketName, S3Client s3Client) { - super(application, profile, label, bucketName, false, s3Client, false); - this.key = key; - this.properties = read(); + this(key, application, profile, label, bucketName, false, s3Client, new YamlProcessor.DocumentMatcher[] {}); } - YamlConfigFileFromKey(String key, String application, String profile, String label, String bucketName, - boolean useApplicationAsDirectory, S3Client s3Client, YamlProcessor.DocumentMatcher... matchers) { - super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client, false, matchers); + S3ConfigFileFromKey(String key, String application, String profile, String label, String bucketName, + boolean useApplicationAsDirectory, S3Client s3Client, YamlProcessor.DocumentMatcher... documentMatchers) { + super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client); this.key = key; + this.documentMatchers = documentMatchers; this.properties = read(); } @@ -910,27 +918,41 @@ protected String buildObjectKeyPrefix() { return key.substring(0, key.lastIndexOf('.')); } -} - -class JsonConfigFileFromKey extends JsonS3ConfigFile { - - private final String key; - - JsonConfigFileFromKey(String key, String application, String profile, String label, String bucketName, - S3Client s3Client) { - super(application, profile, label, bucketName, false, s3Client, false); - this.key = key; - this.properties = read(); - } - @Override - public String getName() { - return "s3:" + bucketName + "/" + key; + protected List getExtensions() { + return List.of(key.substring(key.lastIndexOf('.') + 1)); } @Override - protected String buildObjectKeyPrefix() { - return key.substring(0, key.lastIndexOf('.')); + public Properties read() { + if (this.properties != null) { + return this.properties; + } + String ext = key.substring(key.lastIndexOf('.') + 1).toLowerCase(Locale.ROOT); + if ("properties".equals(ext)) { + Properties props = new Properties(); + try (InputStream in = getObject()) { + props.load(in); + } + catch (Exception e) { + LOG.warn("Exception thrown when reading property file", e); + throw new IllegalStateException("Cannot load environment", e); + } + return props; + } + else if ("json".equals(ext) || "yml".equals(ext) || "yaml".equals(ext)) { + final YamlPropertiesFactoryBean yaml = new YamlPropertiesFactoryBean(); + try (InputStream in = getObject()) { + yaml.setResources(new InputStreamResource(in)); + yaml.setDocumentMatchers(documentMatchers); + return yaml.getObject(); + } + catch (Exception e) { + LOG.warn("Could not read YAML/JSON file", e); + throw new IllegalStateException("Cannot load environment", e); + } + } + throw new IllegalStateException("Unsupported extension: " + ext); } } @@ -939,7 +961,7 @@ class NegatedProfileYamlDocumentS3ConfigFile extends YamlS3ConfigFile { NegatedProfileYamlDocumentS3ConfigFile(String application, String label, String bucketName, boolean useApplicationAsDirectory, S3Client s3Client, String[] allProfiles) { - super(application, null, label, bucketName, useApplicationAsDirectory, s3Client, true, properties -> { + super(application, null, label, bucketName, useApplicationAsDirectory, s3Client, properties -> { Object onProfileValue = properties.get("spring.config.activate.on-profile"); if (onProfileValue == null) { onProfileValue = properties.get("spring.config.activate.onProfile"); @@ -951,7 +973,7 @@ class NegatedProfileYamlDocumentS3ConfigFile extends YamlS3ConfigFile { // Simple positive profile names are already handled by // ProfileSpecificYamlDocumentS3ConfigFile. Only process complex or negated // expressions here to avoid adding duplicate property sources. - if (isSimpleProfileName(expression)) { + if (AwsS3EnvironmentRepository.isSimpleProfileName(expression)) { return YamlProcessor.MatchStatus.NOT_FOUND; } List allProfilesList = Arrays.asList(allProfiles); @@ -960,11 +982,6 @@ class NegatedProfileYamlDocumentS3ConfigFile extends YamlS3ConfigFile { }); } - private static boolean isSimpleProfileName(String expression) { - return !expression.contains("!") && !expression.contains("&") && !expression.contains("|") - && !expression.contains("(") && !expression.contains(","); - } - @Override protected String buildObjectKeyPrefix() { return super.buildObjectKeyPrefix(false); diff --git a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryFactory.java b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryFactory.java index c109e37e6a..2540cb2771 100644 --- a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryFactory.java +++ b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryFactory.java @@ -41,8 +41,7 @@ public AwsS3EnvironmentRepository build(AwsS3EnvironmentProperties environmentPr configureClientBuilder(clientBuilder, environmentProperties.getRegion(), environmentProperties.getEndpoint()); final S3Client client = clientBuilder.build(); - return new AwsS3EnvironmentRepository(client, environmentProperties.getBucket(), - environmentProperties.isUseDirectoryLayout(), server, environmentProperties.getSearchPaths()); + return new AwsS3EnvironmentRepository(client, environmentProperties, server); } } diff --git a/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryTests.java b/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryTests.java index 5cb293c17e..fae01ef5a5 100644 --- a/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryTests.java +++ b/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryTests.java @@ -263,8 +263,8 @@ public void negatedProfileDocumentIncludedWhenNegatedProfileIsAbsentFromMultiple String yamlString = new String(Files.readAllBytes(Paths.get(resource.getURI()))); putFiles("application.yaml", yamlString); - // neither "default" nor "other-profile" is "my-profile", so "!my-profile" should - // match + // neither "default" nor "other-profile" is "my-profile", so "!my-profile" + // should match final Environment env = envRepo.findOne("application", "default,other-profile", null); List propertySources = env.getPropertySources(); @@ -767,7 +767,6 @@ public void getLocationsTest() { "default", "defaultlabel", null, new String[] { "s3://test/defaultlabel" })); } - // 1) Placeholder only @Test public void searchPath_placeholderOnly_shouldResolveExactFile() { List paths = List.of("{label}/{application}-{profile}.yml"); @@ -779,7 +778,6 @@ public void searchPath_placeholderOnly_shouldResolveExactFile() { assertThat(env.getPropertySources().get(0).getName()).contains("v1/foo-bar.yml"); } - // 2) Wildcard only (.properties) @Test public void searchPath_wildcardOnly_shouldResolveAllProperties() { List paths = List.of("{label}/common/*.properties"); @@ -793,7 +791,6 @@ public void searchPath_wildcardOnly_shouldResolveAllProperties() { assertThat(env.getPropertySources().get(1).getName()).contains("v1/common/b.properties"); } - // 3) Placeholder + wildcard combined @Test public void searchPath_placeholderAndWildcard_shouldResolveMatchingKeys() { List paths = List.of("{label}/{application}-{profile}.yml", "{label}/common/*.properties"); @@ -807,7 +804,6 @@ public void searchPath_placeholderAndWildcard_shouldResolveMatchingKeys() { assertThat(env.getPropertySources().get(1).getName()).contains("v1/common/foo.properties"); } - // 4) Order matters @Test public void searchPaths_orderMatters_forPropertySourceOrder() { List paths = List.of("{label}/common/*.properties", "{label}/{application}-{profile}.yml"); @@ -821,7 +817,6 @@ public void searchPaths_orderMatters_forPropertySourceOrder() { assertThat(env.getPropertySources().get(1).getName()).contains("v1/foo-bar.yml"); } - // 5) Extension preserved @TestFactory public Stream searchPath_extensionPreserved() { List exts = List.of("yml", "yaml", "properties", "json"); @@ -844,7 +839,6 @@ else if (ext.equals("json")) { })); } - // 6) Application-as-directory layout @Test public void searchPaths_applicationAsDirectory_shouldStillHonorSearchPaths() { List paths = List.of("{label}/{application}/foo.*"); @@ -853,11 +847,10 @@ public void searchPaths_applicationAsDirectory_shouldStillHonorSearchPaths() { putFiles("v1/foo/foo.json", jsonContent); Environment env = repo.findOne("foo", "", "v1"); - assertThat(env.getPropertySources()).hasSize(1); + assertThat(env.getPropertySources()).hasSize(2); assertThat(env.getPropertySources().get(0).getName()).contains("v1/foo/foo.properties"); } - // 7) Multi-document YAML should not be split @Test public void multiDocumentYaml_withSearchPaths_shouldNotSplitDocuments() throws IOException { List paths = List.of("{label}/{application}.yml"); @@ -873,7 +866,6 @@ public void multiDocumentYaml_withSearchPaths_shouldNotSplitDocuments() throws I assertThat(map).containsEntry("a", 1).containsEntry("b", 2); } - // 8) Deduplication across patterns @Test public void searchPaths_deduplication_shouldOnlyAddOnce() { List paths = List.of("{label}/foo.yml", "{label}/{application}.yml"); @@ -884,7 +876,6 @@ public void searchPaths_deduplication_shouldOnlyAddOnce() { assertThat(env.getPropertySources()).hasSize(1); } - // 9) Literal stops at .properties @Test public void searchPaths_literalStopsAtProperties() { List paths = List.of("{label}/{application}"); @@ -894,11 +885,10 @@ public void searchPaths_literalStopsAtProperties() { putFiles("v1/app.yml", yamlContent); Environment env = repo.findOne("app", "", "v1"); - assertThat(env.getPropertySources()).hasSize(1); + assertThat(env.getPropertySources()).hasSize(3); assertThat(env.getPropertySources().get(0).getName()).contains("v1/app.properties"); } - // 10) Literal stops at .json @Test public void searchPaths_literalStopsAtJson() { List paths = List.of("{label}/{application}"); @@ -907,11 +897,10 @@ public void searchPaths_literalStopsAtJson() { putFiles("v1/app.yml", yamlContent); Environment env = repo.findOne("app", "", "v1"); - assertThat(env.getPropertySources()).hasSize(1); + assertThat(env.getPropertySources()).hasSize(2); assertThat(env.getPropertySources().get(0).getName()).contains("v1/app.json"); } - // 11) Literal stops at .yml @Test public void searchPaths_literalStopsAtYaml() { List paths = List.of("{label}/{application}"); @@ -923,7 +912,6 @@ public void searchPaths_literalStopsAtYaml() { assertThat(env.getPropertySources().get(0).getName()).contains("v1/app.yml"); } - // 12) Dot-wildcard auto-extension (.properties preferred) @Test public void searchPaths_dotWildcardAutoExtProperties() { List paths = List.of("{label}/{application}.*"); @@ -932,11 +920,10 @@ public void searchPaths_dotWildcardAutoExtProperties() { putFiles("v1/app.json", jsonContent); Environment env = repo.findOne("app", "", "v1"); - assertThat(env.getPropertySources()).hasSize(1); + assertThat(env.getPropertySources()).hasSize(2); assertThat(env.getPropertySources().get(0).getName()).contains("v1/app.properties"); } - // 13) Literal directory scan @Test public void searchPaths_literalDirectoryScan() { List paths = List.of("{label}/{application}"); @@ -950,7 +937,6 @@ public void searchPaths_literalDirectoryScan() { assertThat(env.getPropertySources().get(1).getName()).contains("v1/app/sub/bar.yml"); } - // 14) Double-wildcard nested directories @Test public void searchPaths_doubleWildcardNested() { List paths = List.of("{label}/**/*.properties"); @@ -964,7 +950,6 @@ public void searchPaths_doubleWildcardNested() { assertThat(env.getPropertySources().get(1).getName()).contains("v1/foo/sub/b.properties"); } - // 15) Single-character wildcard @Test public void searchPaths_singleCharacterWildcard_shouldMatchExactlyOneChar() { List paths = List.of("{label}/data-?.yml"); @@ -979,7 +964,6 @@ public void searchPaths_singleCharacterWildcard_shouldMatchExactlyOneChar() { assertThat(env.getPropertySources().get(1).getName()).contains("lab/data-b.yml"); } - // 16) Wildcard at start (empty prefix) @Test public void searchPaths_wildcardAtStart_prefixExtractionEmpty_shouldMatchAll() { List paths = List.of("{label}/*.yml"); @@ -991,7 +975,6 @@ public void searchPaths_wildcardAtStart_prefixExtractionEmpty_shouldMatchAll() { assertThat(env.getPropertySources()).hasSize(2); } - // 17) Empty label uses defaultLabel @Test public void searchPaths_withEmptyLabel_shouldUseDefaultLabel() { ConfigServerProperties serverWithDefaultLabel = new ConfigServerProperties(); @@ -1008,7 +991,6 @@ public void searchPaths_withEmptyLabel_shouldUseDefaultLabel() { assertThat(env.getPropertySources().get(0).getName()).contains("main/foo-bar.yml"); } - // 18) Multiple labels applied in reverse order @Test public void searchPaths_multipleLabels_shouldApplyForEachLabelInReverseOrder() { List paths = List.of("{label}/{application}.yml"); @@ -1021,6 +1003,44 @@ public void searchPaths_multipleLabels_shouldApplyForEachLabelInReverseOrder() { assertThat(env.getPropertySources().get(1).getName()).contains("lab1/app.yml"); } + @Test + public void searchPaths_negatedProfile_shouldIncludeDocumentWhenProfileNotActive() { + List paths = List.of("{label}/{application}.yml"); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + + String multiDocYaml = "spring:\n" + " config:\n" + " activate:\n" + " on-profile: '!my-profile'\n" + + "negated: true\n" + "---\n" + "default-key: default-value\n"; + + putFiles("v1/app.yml", multiDocYaml); + + Environment env = repo.findOne("app", "other-profile", "v1"); + + assertThat(env.getPropertySources()).hasSize(2); + boolean negatedPresent = env.getPropertySources() + .stream() + .anyMatch(ps -> Boolean.TRUE.equals(ps.getSource().get("negated"))); + assertThat(negatedPresent).isTrue(); + } + + @Test + public void searchPaths_negatedProfile_shouldExcludeDocumentWhenProfileIsActive() { + List paths = List.of("{label}/{application}.yml"); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); + + String multiDocYaml = "spring:\n" + " config:\n" + " activate:\n" + " on-profile: '!my-profile'\n" + + "negated: true\n" + "---\n" + "default-key: default-value\n"; + + putFiles("v1/app.yml", multiDocYaml); + + Environment env = repo.findOne("app", "my-profile", "v1"); + + assertThat(env.getPropertySources()).hasSize(1); + boolean negatedPresent = env.getPropertySources() + .stream() + .anyMatch(ps -> Boolean.TRUE.equals(ps.getSource().get("negated"))); + assertThat(negatedPresent).isFalse(); + } + private String putFiles(String fileName, String propertyContent) { toBeRemoved.add(fileName); return s3Client From 0568977584f297cf2acf690750875fa42d4a0cd0 Mon Sep 17 00:00:00 2001 From: Geonwook Ham Date: Sat, 6 Jun 2026 12:19:26 +0900 Subject: [PATCH 7/9] docs: add SearchPaths S3 DOCS Signed-off-by: Geonwook Ham --- .../aws-s3-backend.adoc | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/modules/ROOT/pages/server/environment-repository/aws-s3-backend.adoc b/docs/modules/ROOT/pages/server/environment-repository/aws-s3-backend.adoc index 0e64226595..a29c94e063 100644 --- a/docs/modules/ROOT/pages/server/environment-repository/aws-s3-backend.adoc +++ b/docs/modules/ROOT/pages/server/environment-repository/aws-s3-backend.adoc @@ -67,3 +67,31 @@ The preceding listing matches objects stored in your bucket in `/\{application}` │ ├── application-test.yml │ └── application.yml ``` + +[[search-paths]] +== Search paths + +Spring Cloud Config Server also supports `search-paths` for the AWS S3 backend, analogous to xref:./git-backend.adoc#placeholders-in-git-search-paths[`search-paths`] in the Git backend. You can specify a list of paths in the bucket to search for configuration files. The search paths can contain placeholders like `{application}`, `{profile}`, and `{label}`. + +The search paths support: +- *Literal paths*: Probing exact paths with supported file extensions (e.g. `properties`, `json`, `yml`, `yaml`). +- *Directory paths*: Scanning files under a specific directory. +- *Wildcards*: Patterns with `*` or `?` placeholders (e.g. `config/*`). + +The following configuration specifies search paths in the S3 bucket: + +[source,yaml] +---- +spring: + cloud: + config: + server: + awss3: + region: us-east-1 + bucket: bucket1 + search-paths: + - '{application}' + - 'config/{application}' + - 'settings/*' +---- + From 581b67de8d29a165035b5936b7842fccb6d8a799 Mon Sep 17 00:00:00 2001 From: Geonwook Ham Date: Sat, 13 Jun 2026 12:01:15 +0900 Subject: [PATCH 8/9] refactor: address review comments on S3 search paths and restore backward compatibility Signed-off-by: Geonwook Ham --- .../AwsS3EnvironmentProperties.java | 7 ++-- .../AwsS3EnvironmentRepository.java | 34 +++++++++++-------- .../AwsS3EnvironmentRepositoryTests.java | 1 + 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentProperties.java b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentProperties.java index 340dead564..22a81018c0 100644 --- a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentProperties.java +++ b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentProperties.java @@ -45,8 +45,7 @@ public class AwsS3EnvironmentProperties implements EnvironmentRepositoryProperti private String bucket; /** - * Use application name as intermediate directory. Analogous to - * {@link #searchPaths} + * Use application name as intermediate directory. Analogous to {@link #searchPaths} * from Git backend. */ private boolean useDirectoryLayout; @@ -54,8 +53,8 @@ public class AwsS3EnvironmentProperties implements EnvironmentRepositoryProperti private int order = DEFAULT_ORDER; /** - * List of directory paths to search for profiles in the bucket. - * Analogous to {@link #searchPaths} in Git backend. + * List of directory paths to search for profiles in the bucket. Analogous to + * {@link #searchPaths} in Git backend. */ private List searchPaths = Collections.emptyList(); diff --git a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java index 0adb0701e5..2e52be8f2b 100644 --- a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java +++ b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java @@ -117,6 +117,10 @@ public int getOrder() { return this.order; } + public void setOrder(int order) { + this.order = order; + } + @Override public Environment findOne(String specifiedApplication, String specifiedProfiles, String specifiedLabel) { final String application = ObjectUtils.isEmpty(specifiedApplication) @@ -463,30 +467,30 @@ private List getNonProfileSpecificYamlFromKey(String key, String a } private String extractPrefix(String pattern) { - int idx = pattern.indexOf('*'); - int q = pattern.indexOf('?'); - if (q != -1 && (idx == -1 || q < idx)) { - idx = q; + int firstWildcardIdx = -1; + int starIdx = pattern.indexOf('*'); + int questionIdx = pattern.indexOf('?'); + if (starIdx != -1 && questionIdx != -1) { + firstWildcardIdx = Math.min(starIdx, questionIdx); + } + else if (starIdx != -1) { + firstWildcardIdx = starIdx; } - else if (idx <= 0) { + else { + firstWildcardIdx = questionIdx; + } + + if (firstWildcardIdx <= 0) { return ""; } - int slash = pattern.lastIndexOf('/', idx); + int slash = pattern.lastIndexOf('/', firstWildcardIdx); return (slash == -1 ? "" : pattern.substring(0, slash + 1)); } private Optional createConfigFileFromKey(String key, String application, String profile, String label) { String ext = key.substring(key.lastIndexOf('.') + 1); - if ("properties".equalsIgnoreCase(ext)) { - return Optional.of(new S3ConfigFileFromKey(key, application, profile, label, bucketName, - useApplicationAsDirectory, s3Client)); - } - else if ("json".equalsIgnoreCase(ext)) { - return Optional.of(new S3ConfigFileFromKey(key, application, profile, label, bucketName, - useApplicationAsDirectory, s3Client)); - } - else if ("yml".equalsIgnoreCase(ext) || "yaml".equalsIgnoreCase(ext)) { + if (SUPPORTED_EXTENSIONS.contains("." + ext.toLowerCase(Locale.ROOT))) { return Optional.of(new S3ConfigFileFromKey(key, application, profile, label, bucketName, useApplicationAsDirectory, s3Client)); } diff --git a/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryTests.java b/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryTests.java index fae01ef5a5..2263634662 100644 --- a/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryTests.java +++ b/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryTests.java @@ -849,6 +849,7 @@ public void searchPaths_applicationAsDirectory_shouldStillHonorSearchPaths() { Environment env = repo.findOne("foo", "", "v1"); assertThat(env.getPropertySources()).hasSize(2); assertThat(env.getPropertySources().get(0).getName()).contains("v1/foo/foo.properties"); + assertThat(env.getPropertySources().get(1).getName()).contains("v1/foo/foo.json"); } @Test From fa82228bcba8e513f87905dd3690e8427ff3abba Mon Sep 17 00:00:00 2001 From: Geonwook Ham Date: Tue, 16 Jun 2026 20:45:51 +0900 Subject: [PATCH 9/9] refactor: remove redundant null check in addNonProfileSpecificPropertySource Signed-off-by: Geonwook Ham --- .../config/server/environment/AwsS3EnvironmentRepository.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java index 2e52be8f2b..781c754277 100644 --- a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java +++ b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java @@ -247,9 +247,7 @@ private void addNonProfileSpecificPropertySource(Environment environment, String List s3ConfigFiles = searchPaths.isEmpty() ? getS3ConfigFile(app, profile, label, this::getNonProfileSpecificPropertiesOrJsonConfigFile, this::getNonProfileSpecificS3ConfigFileYaml) : Collections.emptyList(); - if (s3ConfigFiles != null) { - addPropertySource(environment, s3ConfigFiles); - } + addPropertySource(environment, s3ConfigFiles); } private void addPropertySource(Environment environment, List s3ConfigFiles) {