SAMZA-1794: setting application acl in launch context
authorHai Lu <halu@linkedin.com>
Wed, 1 Aug 2018 22:20:10 +0000 (15:20 -0700)
committerJagadish <jvenkatraman@linkedin.com>
Wed, 1 Aug 2018 22:20:10 +0000 (15:20 -0700)
Currently we don't set application acl for container launch context. See https://hadoop.apache.org/docs/r2.6.4/api/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.html#setApplicationACLs(java.util.Map)

This could potentially cause problem if samza job is running on a secured YARN cluster. Say user A submits the job, then by default only user A can view the log and the status of the job. Even worse case is that user A submits the job through some proxy account, then even user A herself/himself couldn't access to logs/status of the application.

We need to make some changes for the YARN application submission to set application acls in launch context as configured.

Author: Hai Lu <halu@linkedin.com>

Reviewers: Jagadish<jagadish@apache.org>

Closes #592 from lhaiesp/master

docs/learn/documentation/versioned/jobs/configuration-table.html
samza-yarn/src/main/java/org/apache/samza/config/YarnConfig.java
samza-yarn/src/main/java/org/apache/samza/job/yarn/YarnClusterResourceManager.java
samza-yarn/src/main/scala/org/apache/samza/job/yarn/ClientHelper.scala

index 8557480..26b4661 100644 (file)
                 </tr>
 
                 <tr>
+                    <td class="property" id="yarn-job-view-acl">yarn.job.view.acl</td>
+                    <td class="default"></td>
+                    <td class="description">
+                        This is for secured YARN cluster only.
+                        The 'viewing' acl of the YARN application that controls who can view the application (e.g. application status, logs).
+                        See <a href="https://hadoop.apache.org/docs/r2.6.0/api/org/apache/hadoop/yarn/api/records/ApplicationAccessType.html">ApplicationAccessType</a> for more details.
+                    </td>
+                </tr>
+
+                <tr>
+                    <td class="property" id="yarn-job-modify-acl">yarn.job.modify.acl</td>
+                    <td class="default"></td>
+                    <td class="description">
+                        This is for secured YARN cluster only.
+                        The 'modify' acl of the YARN application that controls who can modify the application (e.g. killing the application).
+                        See <a href="https://hadoop.apache.org/docs/r2.6.0/api/org/apache/hadoop/yarn/api/records/ApplicationAccessType.html">ApplicationAccessType</a> for more details.
+                    </td>
+                </tr>
+
+                <tr>
                     <td class="property" id="yarn-token-renewal-interval-seconds">yarn.token.renewal.interval.seconds</td>
                     <td class="default"></td>
                     <td class="description">
index 466b8cf..58adf26 100644 (file)
 
 package org.apache.samza.config;
 
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.samza.SamzaException;
 
 public class YarnConfig extends MapConfig {
@@ -115,6 +119,22 @@ public class YarnConfig extends MapConfig {
    */
   public static final String YARN_JOB_STAGING_DIRECTORY = "yarn.job.staging.directory";
 
+  /**
+   * For secured YARN cluster only.
+   * The 'viewing' acl of the YARN application. This controls who can view the application,
+   * for example, application status, logs.
+   * {@link org.apache.hadoop.yarn.api.records.ApplicationAccessType} for more details
+   */
+  public static final String YARN_APPLICATION_VIEW_ACL = "yarn.job.view.acl";
+
+  /**
+   * For secured YARN cluster only.
+   * The 'modify' acl of the YARN application. This controls who can modify the application,
+   * for example, killing the job.
+   * {@link org.apache.hadoop.yarn.api.records.ApplicationAccessType} for more details
+   */
+  public static final String YARN_APPLICATION_MODIFY_ACL = "yarn.job.modify.acl";
+
   public YarnConfig(Config config) {
     super(config);
   }
@@ -190,4 +210,30 @@ public class YarnConfig extends MapConfig {
   public String getYarnJobStagingDirectory() {
     return get(YARN_JOB_STAGING_DIRECTORY, null);
   }
+
+  public String getYarnApplicationViewAcl() {
+    return get(YARN_APPLICATION_VIEW_ACL, null);
+  }
+
+  public String getYarnApplicationModifyAcl() {
+    return get(YARN_APPLICATION_MODIFY_ACL, null);
+  }
+
+  /**
+   * Helper function to get all application acls
+   * @return a map of {@link ApplicationAccessType} to {@link String} for all the acls defined
+   */
+  public Map<ApplicationAccessType, String> getYarnApplicationAcls() {
+    Map<ApplicationAccessType, String> acls = new HashMap<>();
+    String viewAcl = getYarnApplicationViewAcl();
+    String modifyAcl = getYarnApplicationModifyAcl();
+    if (viewAcl != null) {
+      acls.put(ApplicationAccessType.VIEW_APP, viewAcl);
+    }
+    if (modifyAcl != null) {
+      acls.put(ApplicationAccessType.MODIFY_APP, modifyAcl);
+    }
+    return acls;
+  }
+
 }
index 79a9083..6f175ea 100644 (file)
@@ -651,6 +651,13 @@ public class YarnClusterResourceManager extends ClusterResourceManager implement
     context.setCommands(new ArrayList<String>() {{add(cmd);}});
     context.setLocalResources(localResourceMap);
 
+    if (UserGroupInformation.isSecurityEnabled()) {
+      Map<ApplicationAccessType, String> acls = yarnConfig.getYarnApplicationAcls();
+      if (!acls.isEmpty()) {
+        context.setApplicationACLs(acls);
+      }
+    }
+
     log.debug("Setting localResourceMap to {}", localResourceMap);
     log.debug("Setting context to {}", context);
 
index d193ddb..6f65ec4 100644 (file)
@@ -191,12 +191,17 @@ class ClientHelper(conf: Configuration) extends Logging {
       }
     }
 
-    if (UserGroupInformation.isSecurityEnabled()) {
+    if (UserGroupInformation.isSecurityEnabled) {
       validateJobConfig(config)
 
       setupSecurityToken(fs, containerCtx)
       info("set security token for %s" format appId.get)
 
+      val acls = yarnConfig.getYarnApplicationAcls
+      if (!acls.isEmpty) {
+        containerCtx.setApplicationACLs(acls)
+      }
+
       val amLocalResources = setupAMLocalResources(fs, Option(yarnConfig.getYarnKerberosPrincipal), Option(yarnConfig.getYarnKerberosKeytab))
       localResources ++= amLocalResources