[ZEPPELIN-6171] Resolve nested LDAP groups on FreeIPA via memberOf

docs/setup/security/shiro_authentication.md

@@ -174,8 +174,22 @@ ldapRealm.groupSearchScope = subtree;
 ldapRealm.memberAttributeValueTemplate = cn={0},ou=people,dc=hadoop,dc=apache,dc=org
 ldapRealm.contextFactory.systemUsername = uid=guest,ou=people,dc=hadoop,dc=apache,dc=org
 ldapRealm.contextFactory.systemPassword = S{ALIAS=ldcSystemPassword}
-# enable support for nested groups using the LDAP_MATCHING_RULE_IN_CHAIN operator
+# enable support for nested groups using the LDAP_MATCHING_RULE_IN_CHAIN operator (Active Directory only)
 ldapRealm.groupSearchEnableMatchingRuleInChain = true
+# enable support for nested groups on directories that lack LDAP_MATCHING_RULE_IN_CHAIN
+# (e.g. FreeIPA / 389 Directory Server) by reading the user entry's own memberOf attribute,
+# which the MemberOf plugin pre-flattens to include direct and indirect group membership.
+# If both this and groupSearchEnableMatchingRuleInChain are enabled, the matching-rule-in-chain
+# path takes precedence and this setting is ignored.
+# Note: the LDAP bind used by ldapRealm.contextFactory must be authenticated (not anonymous) or
+# the directory may not return memberOf; if group members span multiple backends/replicas, the
+# directory's own server-side scope configuration must be set up for memberOf to be complete.
+# Only memberOf values under an explicitly configured groupSearchBase are treated as groups (other
+# entries such as HBAC/sudo rules or roles are ignored); without groupSearchBase no groups are resolved.
+# The leaf RDN type of each group DN must match groupIdAttribute, otherwise the value is skipped.
+ldapRealm.groupSearchEnableMemberOf = false
+# customize the attribute name read by groupSearchEnableMemberOf (defaults to memberOf)
+ldapRealm.memberOfAttribute = memberOf
 # optional mapping from physical groups to logical application roles
 ldapRealm.rolesByGroup = LDN_USERS: user_role, NYK_USERS: user_role, HKG_USERS: user_role, GLOBAL_ADMIN: admin_role
 # optional list of roles that are allowed to authenticate. Incase not present all groups are allowed to authenticate (login).

zeppelin-server/src/main/java/org/apache/zeppelin/realm/LdapRealm.java

@@ -33,6 +33,7 @@
 import java.util.regex.Pattern;
 import javax.naming.AuthenticationException;
 import javax.naming.Context;
+import javax.naming.InvalidNameException;
 import javax.naming.NamingEnumeration;
 import javax.naming.NamingException;
 import javax.naming.PartialResultException;
@@ -44,6 +45,7 @@
 import javax.naming.ldap.LdapContext;
 import javax.naming.ldap.LdapName;
 import javax.naming.ldap.PagedResultsControl;
+import javax.naming.ldap.Rdn;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.alias.CredentialProvider;
 import org.apache.hadoop.security.alias.CredentialProviderFactory;
@@ -165,6 +167,8 @@ public class LdapRealm extends DefaultLdapRealm {
   private String userSearchScope = "subtree";
   private String groupSearchScope = "subtree";
   private boolean groupSearchEnableMatchingRuleInChain;
+  private boolean groupSearchEnableMemberOf;
+  private String memberOfAttribute = "memberOf";
 
   private String groupSearchBase;
 
@@ -217,6 +221,15 @@ protected AuthenticationInfo doGetAuthenticationInfo(AuthenticationToken token)
   @Override
   protected void onInit() {
     super.onInit();
+    if (groupSearchEnableMatchingRuleInChain && groupSearchEnableMemberOf) {
+      LOGGER.warn("Both groupSearchEnableMatchingRuleInChain and groupSearchEnableMemberOf are "
+          + "enabled; groupSearchEnableMatchingRuleInChain takes precedence and "
+          + "groupSearchEnableMemberOf is ignored.");
+    }
+    if (groupSearchEnableMemberOf
+        && org.apache.commons.lang3.StringUtils.isEmpty(groupSearchBase)) {
+      LOGGER.warn("memberOf mode requires groupSearchBase; no groups will be resolved.");
+    }
     if (!org.apache.commons.lang3.StringUtils.isEmpty(this.hadoopSecurityCredentialPath)
         && getContextFactory() != null) {
       ((JndiLdapContextFactory) getContextFactory()).setSystemPassword(
@@ -356,62 +369,18 @@ protected Set<String> rolesFor(PrincipalCollection principals, String userNameIn
       // ldapsearch -h localhost -p 33389 -D
       // uid=guest,ou=people,dc=hadoop,dc=apache,dc=org -w guest-password
       // -b dc=hadoop,dc=apache,dc=org -s sub '(objectclass=*)'
-      NamingEnumeration<SearchResult> searchResultEnum = null;
       SearchControls searchControls = getGroupSearchControls();
       try {
         if (groupSearchEnableMatchingRuleInChain) {
-          searchResultEnum = ldapCtx.search(
-              getGroupSearchBase(),
-              String.format(
-                  MATCHING_RULE_IN_CHAIN_FORMAT,
-                  LdapFilterEncoder.escapeFilterValue(groupObjectClass),
-                  LdapFilterEncoder.escapeFilterValue(memberAttribute),
-                  LdapFilterEncoder.escapeFilterValue(userDn)),
-              searchControls);
-          while (searchResultEnum != null && searchResultEnum.hasMore()) {
-            // searchResults contains all the groups in search scope
-            numResults++;
-            final SearchResult group = searchResultEnum.next();
-
-            Attribute attribute = group.getAttributes().get(getGroupIdAttribute());
-            String groupName = attribute.get().toString();
-
-            String roleName = roleNameFor(groupName);
-            if (roleName != null) {
-              roleNames.add(roleName);
-            } else {
-              roleNames.add(groupName);
-            }
-          }
+          numResults += rolesForMatchingRuleInChain(userDn, ldapCtx, searchControls, roleNames);
+        } else if (groupSearchEnableMemberOf) {
+          numResults += rolesForMemberOf(userDn, ldapCtx, roleNames, groupNames);
         } else {
-          // Default group search filter
-          String searchFilter = String.format("(objectclass=%1$s)",
-              LdapFilterEncoder.escapeFilterValue(groupObjectClass));
-
-          // If group search filter is defined in Shiro config, then use it
-          if (groupSearchFilter != null) {
-            searchFilter = expandFilterTemplate(groupSearchFilter, userName);
-            //searchFilter = String.format("%1$s", groupSearchFilter);
-          }
-          LOGGER.debug("Group SearchBase|SearchFilter|GroupSearchScope: " + "{}|{}|{}",
-              getGroupSearchBase(), searchFilter, groupSearchScope);
-          searchResultEnum = ldapCtx.search(
-              getGroupSearchBase(),
-              searchFilter,
-              searchControls);
-          while (searchResultEnum != null && searchResultEnum.hasMore()) {
-            // searchResults contains all the groups in search scope
-            numResults++;
-            final SearchResult group = searchResultEnum.next();
-            addRoleIfMember(userDn, group, roleNames, groupNames, ldapContextFactory);
-          }
+          numResults += rolesForGroupMembership(userName, userDn, ldapCtx, searchControls,
+              ldapContextFactory, roleNames, groupNames);
         }
       } catch (PartialResultException e) {
         LOGGER.debug("Ignoring PartitalResultException");
-      } finally {
-        if (searchResultEnum != null) {
-          searchResultEnum.close();
-        }
       }
       // Re-activate paged results
       ldapCtx.setRequestControls(new Control[]{new PagedResultsControl(pageSize,
@@ -432,6 +401,177 @@ protected Set<String> rolesFor(PrincipalCollection principals, String userNameIn
     return roleNames;
   }
 
+  // AD-only path: LDAP_MATCHING_RULE_IN_CHAIN walks group ancestry server-side.
+  private int rolesForMatchingRuleInChain(String userDn, LdapContext ldapCtx,
+      SearchControls searchControls, Set<String> roleNames) throws NamingException {
+    int numResults = 0;
+    NamingEnumeration<SearchResult> searchResultEnum = null;
+    try {
+      searchResultEnum = ldapCtx.search(
+          getGroupSearchBase(),
+          String.format(
+              MATCHING_RULE_IN_CHAIN_FORMAT,
+              LdapFilterEncoder.escapeFilterValue(groupObjectClass),
+              LdapFilterEncoder.escapeFilterValue(memberAttribute),
+              LdapFilterEncoder.escapeFilterValue(userDn)),
+          searchControls);
+      while (searchResultEnum != null && searchResultEnum.hasMore()) {
+        // searchResults contains all the groups in search scope
+        numResults++;
+        final SearchResult group = searchResultEnum.next();
+
+        Attribute attribute = group.getAttributes().get(getGroupIdAttribute());
+        String groupName = attribute.get().toString();
+
+        String roleName = roleNameFor(groupName);
+        if (roleName != null) {
+          roleNames.add(roleName);
+        } else {
+          roleNames.add(groupName);
+        }
+      }
+    } finally {
+      if (searchResultEnum != null) {
+        searchResultEnum.close();
+      }
+    }
+    return numResults;
+  }
+
+  private int rolesForGroupMembership(String userName, String userDn, LdapContext ldapCtx,
+      SearchControls searchControls, LdapContextFactory ldapContextFactory,
+      Set<String> roleNames, Set<String> groupNames) throws NamingException {
+    int numResults = 0;
+    NamingEnumeration<SearchResult> searchResultEnum = null;
+    try {
+      // Default group search filter
+      String searchFilter = String.format("(objectclass=%1$s)",
+          LdapFilterEncoder.escapeFilterValue(groupObjectClass));
+
+      // If group search filter is defined in Shiro config, then use it
+      if (groupSearchFilter != null) {
+        searchFilter = expandFilterTemplate(groupSearchFilter, userName);
+        //searchFilter = String.format("%1$s", groupSearchFilter);
+      }
+      LOGGER.debug("Group SearchBase|SearchFilter|GroupSearchScope: " + "{}|{}|{}",
+          getGroupSearchBase(), searchFilter, groupSearchScope);
+      searchResultEnum = ldapCtx.search(
+          getGroupSearchBase(),
+          searchFilter,
+          searchControls);
+      while (searchResultEnum != null && searchResultEnum.hasMore()) {
+        // searchResults contains all the groups in search scope
+        numResults++;
+        final SearchResult group = searchResultEnum.next();
+        addRoleIfMember(userDn, group, roleNames, groupNames, ldapContextFactory);
+      }
+    } finally {
+      if (searchResultEnum != null) {
+        searchResultEnum.close();
+      }
+    }
+    return numResults;
+  }
+
+  /**
+   * FreeIPA/389 DS path: reads the user entry's {@code memberOf} attribute,
+   * which the directory pre-flattens with nested (indirect) group membership.
+   */
+  private int rolesForMemberOf(String userDn, LdapContext ldapCtx,
+      Set<String> roleNames, Set<String> groupNames) throws NamingException {
+    SearchControls memberOfControls = new SearchControls();
+    memberOfControls.setSearchScope(SearchControls.OBJECT_SCOPE);
+    memberOfControls.setReturningAttributes(new String[]{memberOfAttribute});
+
+    // Only an explicitly configured groupSearchBase bounds the group DNs; the
+    // searchBase fallback would accept every DN, including non-group entries.
+    final LdapName groupBase = org.apache.commons.lang3.StringUtils.isEmpty(groupSearchBase)
+        ? null : new LdapName(groupSearchBase);
+
+    int numResults = 0;
+    NamingEnumeration<SearchResult> searchResultEnum = null;
+    try {
+      searchResultEnum = ldapCtx.search(userDn, "(objectclass=*)", memberOfControls);
+      if (searchResultEnum != null && searchResultEnum.hasMore()) {
+        numResults++;
+        final SearchResult userEntry = searchResultEnum.next();
+        Attribute memberOf = userEntry.getAttributes().get(memberOfAttribute);
+        if (memberOf != null) {
+          NamingEnumeration<?> memberOfValues = memberOf.getAll();
+          try {
+            while (memberOfValues.hasMore()) {
+              String groupDn = memberOfValues.next().toString();
+              if (!isUnderGroupSearchBase(groupDn, groupBase)) {
+                LOGGER.debug("Skipping memberOf value '{}' outside groupSearchBase", groupDn);
+                continue;
+              }
+              String groupName = groupNameFromMemberOfDn(groupDn);
+              if (groupName != null) {
+                recordGroupRole(groupName, roleNames, groupNames);
+              }
+            }
+          } finally {
+            memberOfValues.close();
+          }
+        }
+      }
+    } finally {
+      if (searchResultEnum != null) {
+        searchResultEnum.close();
+      }
+    }
+    return numResults;
+  }
+
+  /**
+   * Extracts the group name from a memberOf DN value using its leaf RDN.
+   * Ancestor RDNs are not scanned: a container RDN on the path (e.g. FreeIPA's
+   * {@code cn=groups,cn=accounts}) shares the group's RDN type and would be
+   * mistaken for the group. Returns null for an unparseable DN or a leaf RDN whose
+   * type differs from groupIdAttribute so the caller skips it.
+   */
+  String groupNameFromMemberOfDn(String groupDn) {
+    try {
+      LdapName groupLdapName = new LdapName(groupDn);
+      List<Rdn> rdns = groupLdapName.getRdns();
+      if (rdns.isEmpty()) {
+        return null;
+      }
+      Rdn leafRdn = rdns.get(rdns.size() - 1);
+      if (!getGroupIdAttribute().equalsIgnoreCase(leafRdn.getType())) {
+        LOGGER.debug("Skipping memberOf value '{}': leaf RDN type '{}' does not match "
+            + "groupIdAttribute '{}'", groupDn, leafRdn.getType(), getGroupIdAttribute());
+        return null;
+      }
+      return leafRdn.getValue().toString();
+    } catch (InvalidNameException e) {
+      LOGGER.warn("Skipping malformed memberOf value '{}': {}", groupDn, e.getMessage());
+      return null;
+    }
+  }
+
+  private static boolean isUnderGroupSearchBase(String groupDn, LdapName groupBase) {
+    if (groupBase == null) {
+      return false;
+    }
+    try {
+      return new LdapName(groupDn).startsWith(groupBase);
+    } catch (InvalidNameException e) {
+      LOGGER.warn("Skipping malformed memberOf value '{}': {}", groupDn, e.getMessage());
+      return false;
+    }
+  }
+
+  private void recordGroupRole(String groupName, Set<String> roleNames, Set<String> groupNames) {
+    groupNames.add(groupName);
+    String roleName = roleNameFor(groupName);
+    if (roleName != null) {
+      roleNames.add(roleName);
+    } else {
+      roleNames.add(groupName);
+    }
+  }
+
   protected String getUserDnForSearch(String userName) {
     if (userSearchAttributeName == null || userSearchAttributeName.isEmpty()) {
       // memberAttributeValuePrefix and memberAttributeValueSuffix
@@ -821,6 +961,22 @@ public void setGroupSearchEnableMatchingRuleInChain(
     this.groupSearchEnableMatchingRuleInChain = groupSearchEnableMatchingRuleInChain;
   }
 
+  public boolean isGroupSearchEnableMemberOf() {
+    return groupSearchEnableMemberOf;
+  }
+
+  public void setGroupSearchEnableMemberOf(boolean groupSearchEnableMemberOf) {
+    this.groupSearchEnableMemberOf = groupSearchEnableMemberOf;
+  }
+
+  public String getMemberOfAttribute() {
+    return memberOfAttribute;
+  }
+
+  public void setMemberOfAttribute(String memberOfAttribute) {
+    this.memberOfAttribute = memberOfAttribute;
+  }
+
   private SearchControls getUserSearchControls() {
     SearchControls searchControls = SUBTREE_SCOPE;
     if ("onelevel".equalsIgnoreCase(userSearchScope)) {