flink-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (FLINK-3929) Support for Kerberos Authentication with Keytab Credential
Date Wed, 27 Jul 2016 14:12:20 GMT

    [ https://issues.apache.org/jira/browse/FLINK-3929?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15395732#comment-15395732
] 

ASF GitHub Bot commented on FLINK-3929:
---------------------------------------

Github user mxm commented on a diff in the pull request:

    https://github.com/apache/flink/pull/2275#discussion_r72446082
  
    --- Diff: flink-streaming-connectors/flink-connector-filesystem/src/test/java/org/apache/flink/streaming/connectors/fs/RollingSinkSecuredITCase.java
---
    @@ -0,0 +1,195 @@
    +/**
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements.  See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership.  The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License.  You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +
    +package org.apache.flink.streaming.connectors.fs;
    +
    +import org.apache.flink.configuration.ConfigConstants;
    +import org.apache.flink.runtime.security.SecurityContext;
    +import org.apache.flink.streaming.util.TestStreamEnvironment;
    +import org.apache.flink.test.util.SecureTestEnvironment;
    +import org.apache.flink.test.util.TestingSecurityContext;
    +import org.apache.flink.test.util.TestBaseUtils;
    +import org.apache.flink.util.NetUtils;
    +import org.apache.hadoop.fs.FileUtil;
    +import org.apache.hadoop.fs.Path;
    +import org.apache.hadoop.hdfs.MiniDFSCluster;
    +import org.apache.hadoop.http.HttpConfig;
    +import org.apache.hadoop.security.SecurityUtil;
    +import org.apache.hadoop.security.UserGroupInformation;
    +import org.junit.AfterClass;
    +import org.junit.BeforeClass;
    +import org.slf4j.Logger;
    +import org.slf4j.LoggerFactory;
    +
    +import java.io.File;
    +import java.io.FileWriter;
    +import java.io.IOException;
    +import java.util.HashMap;
    +import java.util.Map;
    +
    +import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
    +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY;
    +
    +/**
    + * Tests for running {@link RollingSinkSecuredITCase} which is an extension of {@link
RollingSink} in secure environment
    + */
    +
    +public class RollingSinkSecuredITCase extends RollingSinkITCase {
    +
    +	protected static final Logger LOG = LoggerFactory.getLogger(RollingSinkSecuredITCase.class);
    +
    +	/*
    +	 * override super class static methods to avoid creating MiniDFS and MiniFlink with
wrong configurations
    +	 * and out-of-order sequence for secure cluster
    +	 */
    +	@BeforeClass
    +	public static void setup() throws Exception {}
    +
    +	@AfterClass
    +	public static void teardown() throws Exception {}
    +
    +	@BeforeClass
    +	public static void createHDFS() throws IOException {}
    +
    +	@AfterClass
    +	public static void destroyHDFS() {}
    +
    +	@BeforeClass
    +	public static void startSecureCluster() throws Exception {
    +
    +		LOG.info("starting secure cluster environment for testing");
    +
    +		dataDir = tempFolder.newFolder();
    +
    +		conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, dataDir.getAbsolutePath());
    +
    +		SecureTestEnvironment.prepare(tempFolder);
    +
    +		populateSecureConfigurations();
    +
    +		SecurityContext.SecurityConfiguration ctx = new SecurityContext.SecurityConfiguration();
    +		ctx.setCredentials(SecureTestEnvironment.getTestKeytab(), SecureTestEnvironment.getHadoopServicePrincipal());
    +		ctx.setHadoopConfiguration(conf);
    +		try {
    +			TestingSecurityContext.install(ctx, SecureTestEnvironment.getClientSecurityConfigurationMap());
    +		} catch(Exception e) {
    +			throw new RuntimeException("Exception occurred while setting up secure test context.
Reason: {}", e);
    +		}
    +
    +		File hdfsSiteXML = new File(dataDir.getAbsolutePath() + "/hdfs-site.xml");
    +
    +		FileWriter writer = new FileWriter(hdfsSiteXML);
    +		conf.writeXml(writer);
    +		writer.flush();
    +		writer.close();
    +
    +		Map<String, String> map = new HashMap<String, String>(System.getenv());
    +		map.put("HADOOP_CONF_DIR", hdfsSiteXML.getParentFile().getAbsolutePath());
    +		TestBaseUtils.setEnv(map);
    +
    +
    +		MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
    +		builder.checkDataNodeAddrConfig(true);
    +		builder.checkDataNodeHostConfig(true);
    +		hdfsCluster = builder.build();
    +
    +		dfs = hdfsCluster.getFileSystem();
    +
    +		hdfsURI = "hdfs://"
    +				+ NetUtils.hostAndPortToUrlString(hdfsCluster.getURI().getHost(), hdfsCluster.getNameNodePort())
    +				+ "/";
    +
    +		startSecureFlinkClusterWithRecoveryModeEnabled();
    +	}
    +
    +	@AfterClass
    +	public static void teardownSecureCluster() throws Exception {
    +		LOG.info("tearing down secure cluster environment");
    +
    +		TestStreamEnvironment.unsetAsContext();
    +		stopCluster(cluster, TestBaseUtils.DEFAULT_TIMEOUT);
    +
    +		hdfsCluster.shutdown();
    +
    +		SecureTestEnvironment.cleanup();
    +		FileUtil.fullyDelete(dataDir);
    --- End diff --
    
    Shouldn't be necessary because `TemporaryFolder` cleans up.


> Support for Kerberos Authentication with Keytab Credential
> ----------------------------------------------------------
>
>                 Key: FLINK-3929
>                 URL: https://issues.apache.org/jira/browse/FLINK-3929
>             Project: Flink
>          Issue Type: New Feature
>            Reporter: Eron Wright 
>            Assignee: Vijay Srinivasaraghavan
>              Labels: kerberos, security
>   Original Estimate: 672h
>  Remaining Estimate: 672h
>
> _This issue is part of a series of improvements detailed in the [Secure Data Access|https://docs.google.com/document/d/1-GQB6uVOyoaXGwtqwqLV8BHDxWiMO2WnVzBoJ8oPaAs/edit?usp=sharing]
design doc._
> Add support for a keytab credential to be associated with the Flink cluster, to facilitate:
> - Kerberos-authenticated data access for connectors
> - Kerberos-authenticated ZooKeeper access
> Support both the standalone and YARN deployment modes.
>  



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Mime
View raw message