diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMapping.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMapping.java index c8ecf2d0..8a1293eb 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMapping.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMapping.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2021 Expedia, Inc. + * Copyright (C) 2016-2026 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -90,4 +90,8 @@ void createDatabase(Database database) throws AlreadyExistsException, InvalidObjectException, MetaException, TException; long getLatency(); + + default boolean isGlueBackend() { + return false; + } } diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingDecorator.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingDecorator.java index 18dab355..e6dca3b2 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingDecorator.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingDecorator.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2023 Expedia, Inc. + * Copyright (C) 2016-2026 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -109,4 +109,9 @@ public long getLatency() { return metaStoreMapping.getLatency(); } + @Override + public boolean isGlueBackend() { + return metaStoreMapping.isGlueBackend(); + } + } diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImpl.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImpl.java index e7cdae8f..a9b02be1 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImpl.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImpl.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2025 Expedia, Inc. + * Copyright (C) 2016-2026 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -77,7 +77,7 @@ public MetaStoreMapping newInstance(AbstractMetaStore metaStore) { metaStore.getRemoteMetaStoreUris()); MetaStoreMapping metaStoreMapping = new MetaStoreMappingImpl(prefixNameFor(metaStore), metaStore.getName(), createClient(metaStore), accessControlHandlerFactory.newInstance(metaStore), metaStore.getConnectionType(), - metaStore.getLatency(), loadMetastoreFilterHook(metaStore)); + metaStore.getLatency(), loadMetastoreFilterHook(metaStore), metaStore.getGlueConfig() != null); if (waggleDanceConfiguration.getDatabaseResolution() == DatabaseResolution.PREFIXED) { return new DatabaseNameMapping(new PrefixMapping(metaStoreMapping), metaStore.getDatabaseNameBiMapping()); } else { diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImpl.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImpl.java index 8b0d683c..cd5412a9 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImpl.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImpl.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2025 Expedia, Inc. + * Copyright (C) 2016-2026 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -57,6 +57,7 @@ class MetaStoreMappingImpl implements MetaStoreMapping { private final ConnectionType connectionType; private final long latency; private final MetaStoreFilterHook metastoreFilter; + private final boolean glueBackend; private final ExecutorService executor = Executors.newSingleThreadExecutor(); MetaStoreMappingImpl( @@ -66,7 +67,8 @@ class MetaStoreMappingImpl implements MetaStoreMapping { AccessControlHandler accessControlHandler, ConnectionType connectionType, long latency, - MetaStoreFilterHook metastoreFilter) { + MetaStoreFilterHook metastoreFilter, + boolean glueBackend) { this.databasePrefix = databasePrefix; this.name = name; this.client = client; @@ -74,6 +76,7 @@ class MetaStoreMappingImpl implements MetaStoreMapping { this.connectionType = connectionType; this.latency = latency; this.metastoreFilter = metastoreFilter; + this.glueBackend = glueBackend; } @Override @@ -177,4 +180,9 @@ public long getLatency() { return latency; } + @Override + public boolean isGlueBackend() { + return glueBackend; + } + } diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/server/FederatedHMSHandler.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/server/FederatedHMSHandler.java index 1301e97b..5649b1c1 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/server/FederatedHMSHandler.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/server/FederatedHMSHandler.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2025 Expedia, Inc. + * Copyright (C) 2016-2026 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -364,8 +364,23 @@ public Database get_database(String name) throws NoSuchObjectException, MetaExce log.debug("Fetching database {}", name); DatabaseMapping mapping = databaseMappingService.databaseMapping(name); log.debug("Mapping is '{}'", mapping.getDatabasePrefix()); - Database result = mapping.getClient().get_database(mapping.transformInboundDatabaseName(name)); - return mapping.transformOutboundDatabase(mapping.getMetastoreFilter().filterDatabase(result)); + try { + Database result = mapping.getClient().get_database(mapping.transformInboundDatabaseName(name)); + return mapping.transformOutboundDatabase(mapping.getMetastoreFilter().filterDatabase(result)); + } catch (MetaException e) { + if (mapping.isGlueBackend() && isLakeFormationAccessDenied(e)) { + log.debug("Lake Formation returned AccessDeniedException for database '{}', translating to NoSuchObjectException", name, e); + throw new NoSuchObjectException(name); + } + throw e; + } + } + + private static boolean isLakeFormationAccessDenied(MetaException e) { + String message = e.getMessage(); + return message != null + && message.contains("AccessDeniedException") + && message.contains("Lake Formation permission"); } @Override @@ -523,8 +538,16 @@ public List get_all_tables(String db_name) throws MetaException, TExcept @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) public Table get_table(String dbname, String tbl_name) throws MetaException, NoSuchObjectException, TException { DatabaseMapping mapping = getDbMappingAndCheckTableAllowed(dbname, tbl_name); - Table table = mapping.getClient().get_table(mapping.transformInboundDatabaseName(dbname), tbl_name); - return mapping.transformOutboundTable(mapping.getMetastoreFilter().filterTable(table)); + try { + Table table = mapping.getClient().get_table(mapping.transformInboundDatabaseName(dbname), tbl_name); + return mapping.transformOutboundTable(mapping.getMetastoreFilter().filterTable(table)); + } catch (MetaException e) { + if (mapping.isGlueBackend() && isLakeFormationAccessDenied(e)) { + log.debug("Lake Formation returned AccessDeniedException for table '{}.{}', translating to NoSuchObjectException", dbname, tbl_name, e); + throw new NoSuchObjectException(dbname + "." + tbl_name); + } + throw e; + } } @Override @@ -2188,9 +2211,17 @@ public List get_materialized_views_for_rewriting(String dbName) throws M @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) public GetTableResult get_table_req(GetTableRequest req) throws MetaException, NoSuchObjectException, TException { DatabaseMapping mapping = getDbMappingAndCheckTableAllowed(req.getDbName(), req.getTblName()); - GetTableResult result = mapping.getClient().get_table_req(mapping.transformInboundGetTableRequest(req)); - result.setTable(mapping.getMetastoreFilter().filterTable(result.getTable())); - return mapping.transformOutboundGetTableResult(result); + try { + GetTableResult result = mapping.getClient().get_table_req(mapping.transformInboundGetTableRequest(req)); + result.setTable(mapping.getMetastoreFilter().filterTable(result.getTable())); + return mapping.transformOutboundGetTableResult(result); + } catch (MetaException e) { + if (mapping.isGlueBackend() && isLakeFormationAccessDenied(e)) { + log.debug("Lake Formation returned AccessDeniedException for table '{}.{}', translating to NoSuchObjectException", req.getDbName(), req.getTblName(), e); + throw new NoSuchObjectException(req.getDbName() + "." + req.getTblName()); + } + throw e; + } } @Override diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/ASTQueryMappingTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/ASTQueryMappingTest.java index 76876182..f0507f32 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/ASTQueryMappingTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/ASTQueryMappingTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2025 Expedia, Inc. + * Copyright (C) 2016-2026 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -39,7 +39,7 @@ public class ASTQueryMappingTest { @Before public void setUp() { metaStoreMapping = new PrefixMapping(new MetaStoreMappingImpl(PREFIX, "mapping", null, null, DIRECT, LATENCY, - new DefaultMetaStoreFilterHookImpl(new HiveConf()))); + new DefaultMetaStoreFilterHookImpl(new HiveConf()), false)); } @Test diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/DatabaseNameMappingTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/DatabaseNameMappingTest.java index 1076f146..19a5c20c 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/DatabaseNameMappingTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/DatabaseNameMappingTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2021 Expedia, Inc. + * Copyright (C) 2016-2026 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,7 +35,7 @@ public class DatabaseNameMappingTest { private final MetaStoreMapping metaStoreMapping = new PrefixMapping( new MetaStoreMappingImpl("pre_", "mapping", null, null, DIRECT, - 0L, new DefaultMetaStoreFilterHookImpl(new HiveConf()))); + 0L, new DefaultMetaStoreFilterHookImpl(new HiveConf()), false)); @Test public void mapNames() throws Exception { diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImplTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImplTest.java index dcb1d03b..c5936d0b 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImplTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImplTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2025 Expedia, Inc. + * Copyright (C) 2016-2026 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -61,9 +61,9 @@ public class MetaStoreMappingImplTest { @Before public void init() { metaStoreMapping = new MetaStoreMappingImpl(DATABASE_PREFIX, NAME, client, accessControlHandler, DIRECT, LATENCY, - new DefaultMetaStoreFilterHookImpl(new HiveConf())); + new DefaultMetaStoreFilterHookImpl(new HiveConf()), false); tunneledMetaStoreMapping = new MetaStoreMappingImpl(DATABASE_PREFIX, NAME, client, accessControlHandler, TUNNELED, - LATENCY, new DefaultMetaStoreFilterHookImpl(new HiveConf())); + LATENCY, new DefaultMetaStoreFilterHookImpl(new HiveConf()), false); } @Test diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/PrefixMappingTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/PrefixMappingTest.java index 61c10164..f96752e9 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/PrefixMappingTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/PrefixMappingTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2021 Expedia, Inc. + * Copyright (C) 2016-2026 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,7 +28,7 @@ public class PrefixMappingTest { private final MetaStoreMapping metaStoreMapping = new MetaStoreMappingImpl("prefix_", "mapping", null, null, DIRECT, - 0L, new DefaultMetaStoreFilterHookImpl(new HiveConf())); + 0L, new DefaultMetaStoreFilterHookImpl(new HiveConf()), false); @Test public void mapNames() throws Exception { diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/server/FederatedHMSHandlerTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/server/FederatedHMSHandlerTest.java index 74af2424..dcecdefe 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/server/FederatedHMSHandlerTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/server/FederatedHMSHandlerTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2025 Expedia, Inc. + * Copyright (C) 2016-2026 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -40,6 +40,7 @@ import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; +import static org.junit.Assert.fail; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -144,6 +145,7 @@ import org.apache.hadoop.hive.metastore.api.LockResponse; import org.apache.hadoop.hive.metastore.api.LockType; import org.apache.hadoop.hive.metastore.api.MapSchemaVersionToSerdeRequest; +import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.NotNullConstraintsRequest; import org.apache.hadoop.hive.metastore.api.NotNullConstraintsResponse; @@ -330,6 +332,32 @@ public void get_database() throws Exception { assertThat(result, is(outboundDB)); } + @Test(expected = NoSuchObjectException.class) + public void get_database_glueBackendLakeFormationAccessDenied_throwsNoSuchObjectException() throws Exception { + when(primaryMapping.isGlueBackend()).thenReturn(true); + when(primaryMapping.transformInboundDatabaseName(DB_P)).thenReturn("inbound"); + when(primaryClient.get_database("inbound")).thenThrow(new MetaException( + "An error occurred (AccessDeniedException) when calling the GetDatabase operation: " + + "Insufficient Lake Formation permission(s): Required Describe on " + DB_P)); + handler.get_database(DB_P); + } + + @Test + public void get_database_nonGlueBackend_lakeFormationLikeMessage_rethrowsMetaException() throws Exception { + MetaException expected = new MetaException( + "An error occurred (AccessDeniedException) when calling the GetDatabase operation: " + + "Insufficient Lake Formation permission(s): Required Describe on " + DB_P); + when(primaryMapping.isGlueBackend()).thenReturn(false); + when(primaryMapping.transformInboundDatabaseName(DB_P)).thenReturn("inbound"); + when(primaryClient.get_database("inbound")).thenThrow(expected); + try { + handler.get_database(DB_P); + fail("Expected MetaException"); + } catch (MetaException e) { + assertThat(e, is(sameInstance(expected))); + } + } + @Test public void drop_database() throws TException { when(primaryMapping.transformInboundDatabaseName(DB_P)).thenReturn("inbound"); @@ -486,6 +514,32 @@ public void get_table() throws TException { assertThat(result, is(outbound)); } + @Test(expected = NoSuchObjectException.class) + public void get_table_glueBackendLakeFormationAccessDenied_throwsNoSuchObjectException() throws Exception { + when(primaryMapping.isGlueBackend()).thenReturn(true); + when(primaryMapping.transformInboundDatabaseName(DB_P)).thenReturn("inbound"); + when(primaryClient.get_table("inbound", "table")).thenThrow(new MetaException( + "An error occurred (AccessDeniedException) when calling the GetTable operation: " + + "Insufficient Lake Formation permission(s): Required Describe on table")); + handler.get_table(DB_P, "table"); + } + + @Test + public void get_table_nonGlueBackend_lakeFormationLikeMessage_rethrowsMetaException() throws Exception { + MetaException expected = new MetaException( + "An error occurred (AccessDeniedException) when calling the GetTable operation: " + + "Insufficient Lake Formation permission(s): Required Describe on table"); + when(primaryMapping.isGlueBackend()).thenReturn(false); + when(primaryMapping.transformInboundDatabaseName(DB_P)).thenReturn("inbound"); + when(primaryClient.get_table("inbound", "table")).thenThrow(expected); + try { + handler.get_table(DB_P, "table"); + fail("Expected MetaException"); + } catch (MetaException e) { + assertThat(e, is(sameInstance(expected))); + } + } + @Test public void get_table_objects_by_name() throws TException { when(primaryMapping.transformInboundDatabaseName(DB_P)).thenReturn("inbound"); @@ -1001,6 +1055,40 @@ public void get_table_req() throws TException { assertThat(result.getTable().getTableName(), is("table")); } + @Test(expected = NoSuchObjectException.class) + public void get_table_req_glueBackendLakeFormationAccessDenied_throwsNoSuchObjectException() throws Exception { + when(primaryMapping.isGlueBackend()).thenReturn(true); + Table table = new Table(); + table.setDbName(DB_P); + table.setTableName("table"); + GetTableRequest request = new GetTableRequest(table.getDbName(), table.getTableName()); + when(primaryMapping.transformInboundGetTableRequest(request)).thenReturn(request); + when(primaryClient.get_table_req(request)).thenThrow(new MetaException( + "An error occurred (AccessDeniedException) when calling the GetTable operation: " + + "Insufficient Lake Formation permission(s): Required Describe on table")); + handler.get_table_req(request); + } + + @Test + public void get_table_req_nonGlueBackend_lakeFormationLikeMessage_rethrowsMetaException() throws Exception { + MetaException expected = new MetaException( + "An error occurred (AccessDeniedException) when calling the GetTable operation: " + + "Insufficient Lake Formation permission(s): Required Describe on table"); + when(primaryMapping.isGlueBackend()).thenReturn(false); + Table table = new Table(); + table.setDbName(DB_P); + table.setTableName("table"); + GetTableRequest request = new GetTableRequest(table.getDbName(), table.getTableName()); + when(primaryMapping.transformInboundGetTableRequest(request)).thenReturn(request); + when(primaryClient.get_table_req(request)).thenThrow(expected); + try { + handler.get_table_req(request); + fail("Expected MetaException"); + } catch (MetaException e) { + assertThat(e, is(sameInstance(expected))); + } + } + @Test public void get_table_objects_by_name_req() throws TException { Table table0 = new Table();