From commits-return-3785-apmail-sentry-commits-archive=sentry.apache.org@sentry.incubator.apache.org Mon Oct 13 23:26:24 2014 Return-Path: X-Original-To: apmail-sentry-commits-archive@minotaur.apache.org Delivered-To: apmail-sentry-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id DE34817D56 for ; Mon, 13 Oct 2014 23:26:24 +0000 (UTC) Received: (qmail 13128 invoked by uid 500); 13 Oct 2014 23:26:24 -0000 Delivered-To: apmail-sentry-commits-archive@sentry.apache.org Received: (qmail 13080 invoked by uid 500); 13 Oct 2014 23:26:24 -0000 Mailing-List: contact commits-help@sentry.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@sentry.incubator.apache.org Delivered-To: mailing list commits@sentry.incubator.apache.org Received: (qmail 12996 invoked by uid 99); 13 Oct 2014 23:26:24 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 13 Oct 2014 23:26:24 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED,T_RP_MATCHES_RCVD X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO mail.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with SMTP; Mon, 13 Oct 2014 23:26:18 +0000 Received: (qmail 12073 invoked by uid 99); 13 Oct 2014 23:25:58 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 13 Oct 2014 23:25:58 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id EB27F915089; Mon, 13 Oct 2014 23:25:57 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: prasadm@apache.org To: commits@sentry.incubator.apache.org Date: Mon, 13 Oct 2014 23:26:10 -0000 Message-Id: <75309ddb9a444394b3ea4aad96644294@git.apache.org> In-Reply-To: <9a183453898e48e1803fb74c27a37ecb@git.apache.org> References: <9a183453898e48e1803fb74c27a37ecb@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [14/15] SENTRY-432: Synchronization of HDFS permissions with Sentry permissions. First refresh (Arun Suresh via Prasad Mujumdar) X-Virus-Checked: Checked by ClamAV on apache.org http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/78787d63/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceProcessor.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceProcessor.java b/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceProcessor.java deleted file mode 100644 index 1198619..0000000 --- a/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceProcessor.java +++ /dev/null @@ -1,100 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.hdfs; - -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -import org.apache.sentry.hdfs.service.thrift.SentryHDFSService; -import org.apache.sentry.hdfs.service.thrift.TAuthzUpdateResponse; -import org.apache.sentry.hdfs.service.thrift.TPathsUpdate; -import org.apache.sentry.hdfs.service.thrift.TPermissionsUpdate; -import org.apache.thrift.TException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class SentryHDFSServiceProcessor implements SentryHDFSService.Iface { - - private static final Logger LOGGER = LoggerFactory.getLogger(SentryHDFSServiceProcessor.class); - - public static final String SENTRY_HDFS_SERVICE_NAME = "SentryHDFSService"; - - @Override - public TAuthzUpdateResponse get_all_authz_updates_from(long permSeqNum, long pathSeqNum) - throws TException { - TAuthzUpdateResponse retVal = new TAuthzUpdateResponse(); - retVal.setAuthzPathUpdate(new LinkedList()); - retVal.setAuthzPermUpdate(new LinkedList()); - if (SentryPlugin.instance != null) { - List permUpdates = SentryPlugin.instance.getAllPermsUpdatesFrom(permSeqNum); - List pathUpdates = SentryPlugin.instance.getAllPathsUpdatesFrom(pathSeqNum); - try { - for (PathsUpdate update : pathUpdates) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("### Sending PATH preUpdate seq [" + update.getSeqNum() + "] ###"); - LOGGER.debug("### Sending PATH preUpdate [" + update.getThriftObject() + "] ###"); - } - retVal.getAuthzPathUpdate().add(update.getThriftObject()); - } - for (PermissionsUpdate update : permUpdates) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("### Sending PERM preUpdate seq [" + update.getSeqNum() + "] ###"); - LOGGER.debug("### Sending PERM preUpdate [" + update.getThriftObject() + "] ###"); - } - retVal.getAuthzPermUpdate().add(update.getThriftObject()); - } - } catch (Exception e) { - LOGGER.error("Error Sending updates to downstream Cache", e); - throw new TException(e); - } - } else { - LOGGER.error("SentryPlugin not initialized yet !!"); - } - - return retVal; - } - - @Override - public void handle_hms_notification(TPathsUpdate update) throws TException { - try { - PathsUpdate hmsUpdate = new PathsUpdate(update); - if (SentryPlugin.instance != null) { - SentryPlugin.instance.handlePathUpdateNotification(hmsUpdate); - LOGGER.info("Authz Paths update [" + hmsUpdate.getSeqNum() + "].."); - } else { - LOGGER.error("SentryPlugin not initialized yet !!"); - } - } catch (Exception e) { - LOGGER.error("Error handling notification from HMS", e); - throw new TException(e); - } - } - - /** - * Not implemented for the time being.. - */ - @Override - public Map> get_all_related_paths(String arg0, - boolean arg1) throws TException { - // TODO Auto-generated method stub - return null; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/78787d63/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceProcessorFactory.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceProcessorFactory.java b/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceProcessorFactory.java deleted file mode 100644 index bf64bbc..0000000 --- a/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceProcessorFactory.java +++ /dev/null @@ -1,86 +0,0 @@ -package org.apache.sentry.hdfs; - -import java.net.Socket; - -import org.apache.hadoop.conf.Configuration; -import org.apache.sentry.hdfs.service.thrift.SentryHDFSService; -import org.apache.sentry.hdfs.service.thrift.SentryHDFSService.Iface; -import org.apache.sentry.provider.db.log.util.CommandUtil; -import org.apache.sentry.service.thrift.ProcessorFactory; -import org.apache.thrift.TException; -import org.apache.thrift.TMultiplexedProcessor; -import org.apache.thrift.TProcessor; -import org.apache.thrift.protocol.TProtocol; -import org.apache.thrift.transport.TSaslClientTransport; -import org.apache.thrift.transport.TSaslServerTransport; -import org.apache.thrift.transport.TSocket; -import org.apache.thrift.transport.TTransport; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class SentryHDFSServiceProcessorFactory extends ProcessorFactory{ - - private static final Logger LOGGER = LoggerFactory.getLogger(SentryHDFSServiceProcessorFactory.class); - - static class ProcessorWrapper extends SentryHDFSService.Processor { - - public ProcessorWrapper(Iface iface) { - super(iface); - } - @Override - public boolean process(TProtocol in, TProtocol out) throws TException { - setIpAddress(in); - setImpersonator(in); - return super.process(in, out); - } - - private void setImpersonator(final TProtocol in) { - TTransport transport = in.getTransport(); - if (transport instanceof TSaslServerTransport) { - String impersonator = ((TSaslServerTransport) transport).getSaslServer().getAuthorizationID(); - CommandUtil.setImpersonator(impersonator); - } - } - - private void setIpAddress(final TProtocol in) { - TTransport transport = in.getTransport(); - TSocket tSocket = getUnderlyingSocketFromTransport(transport); - if (tSocket != null) { - setIpAddress(tSocket.getSocket()); - } else { - LOGGER.warn("Unknown Transport, cannot determine ipAddress"); - } - } - - private void setIpAddress(Socket socket) { - CommandUtil.setIpAddress(socket.getInetAddress().toString()); - } - - private TSocket getUnderlyingSocketFromTransport(TTransport transport) { - if (transport != null) { - if (transport instanceof TSaslServerTransport) { - transport = ((TSaslServerTransport) transport).getUnderlyingTransport(); - } else if (transport instanceof TSaslClientTransport) { - transport = ((TSaslClientTransport) transport).getUnderlyingTransport(); - } else if (transport instanceof TSocket) { - return (TSocket) transport; - } - } - return null; - } - } - - public SentryHDFSServiceProcessorFactory(Configuration conf) { - super(conf); - } - - - public boolean register(TMultiplexedProcessor multiplexedProcessor) throws Exception { - SentryHDFSServiceProcessor sentryServiceHandler = - new SentryHDFSServiceProcessor(); - TProcessor processor = new ProcessorWrapper(sentryServiceHandler); - multiplexedProcessor.registerProcessor( - SentryHDFSServiceProcessor.SENTRY_HDFS_SERVICE_NAME, processor); - return true; - } -} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/78787d63/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java b/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java deleted file mode 100644 index 262e893..0000000 --- a/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java +++ /dev/null @@ -1,238 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.hdfs; - -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.sentry.hdfs.UpdateForwarder.ExternalImageRetriever; -import org.apache.sentry.hdfs.service.thrift.TPathChanges; -import org.apache.sentry.hdfs.service.thrift.TPermissionsUpdate; -import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges; -import org.apache.sentry.hdfs.service.thrift.TRoleChanges; -import org.apache.sentry.provider.db.SentryPolicyStorePlugin; -import org.apache.sentry.provider.db.service.persistent.SentryStore; -import org.apache.sentry.provider.db.service.thrift.TAlterSentryRoleAddGroupsRequest; -import org.apache.sentry.provider.db.service.thrift.TAlterSentryRoleDeleteGroupsRequest; -import org.apache.sentry.provider.db.service.thrift.TAlterSentryRoleGrantPrivilegeRequest; -import org.apache.sentry.provider.db.service.thrift.TAlterSentryRoleRevokePrivilegeRequest; -import org.apache.sentry.provider.db.service.thrift.TDropSentryRoleRequest; -import org.apache.sentry.provider.db.service.thrift.TSentryGroup; -import org.apache.sentry.provider.db.service.thrift.TSentryPrivilege; -import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.Lists; - -public class SentryPlugin implements SentryPolicyStorePlugin { - - private static final Logger LOGGER = LoggerFactory.getLogger(SentryPlugin.class); - - public static volatile SentryPlugin instance; - - static class PermImageRetriever implements ExternalImageRetriever { - - private final SentryStore sentryStore; - - public PermImageRetriever(SentryStore sentryStore) { - this.sentryStore = sentryStore; - } - - @Override - public PermissionsUpdate retrieveFullImage(long currSeqNum) { - Map> privilegeImage = sentryStore.retrieveFullPrivilegeImage(); - Map> roleImage = sentryStore.retrieveFullRoleImage(); - - TPermissionsUpdate tPermUpdate = new TPermissionsUpdate(true, currSeqNum, - new HashMap(), - new HashMap()); - for (Map.Entry> privEnt : privilegeImage.entrySet()) { - String authzObj = privEnt.getKey(); - HashMap privs = privEnt.getValue(); - tPermUpdate.putToPrivilegeChanges(authzObj, new TPrivilegeChanges( - authzObj, privs, new HashMap())); - } - for (Map.Entry> privEnt : roleImage.entrySet()) { - String role = privEnt.getKey(); - LinkedList groups = privEnt.getValue(); - tPermUpdate.putToRoleChanges(role, new TRoleChanges(role, groups, new LinkedList())); - } - PermissionsUpdate permissionsUpdate = new PermissionsUpdate(tPermUpdate); - permissionsUpdate.setSeqNum(currSeqNum); - return permissionsUpdate; - } - - } - - private UpdateForwarder pathsUpdater; - private UpdateForwarder permsUpdater; - private final AtomicLong permSeqNum = new AtomicLong(5); - - @Override - public void initialize(Configuration conf, SentryStore sentryStore) throws SentryPluginException { - HiveConf hiveConf = new HiveConf(conf, Configuration.class); - final MetastoreClient hmsClient = new ExtendedMetastoreClient(hiveConf); - final String[] pathPrefixes = conf - .getStrings(ServerConfig.SENTRY_HDFS_INTEGRATION_PATH_PREFIXES, new String[]{"/"}); - pathsUpdater = new UpdateForwarder(new UpdateableAuthzPaths( - pathPrefixes), createHMSImageRetriever(pathPrefixes, hmsClient), 100); - PermImageRetriever permImageRetriever = new PermImageRetriever(sentryStore); - permsUpdater = new UpdateForwarder( - new UpdateablePermissions(permImageRetriever), permImageRetriever, 100); - instance = this; - } - - public List getAllPathsUpdatesFrom(long pathSeqNum) { - return pathsUpdater.getAllUpdatesFrom(pathSeqNum); - } - - public List getAllPermsUpdatesFrom(long permSeqNum) { - return permsUpdater.getAllUpdatesFrom(permSeqNum); - } - - public void handlePathUpdateNotification(PathsUpdate update) { - pathsUpdater.handleUpdateNotification(update); - LOGGER.info("Recieved Authz Path update [" + update.getSeqNum() + "].."); - } - - private ExternalImageRetriever createHMSImageRetriever( - final String[] pathPrefixes, final MetastoreClient hmsClient) { - return new ExternalImageRetriever() { - @Override - public PathsUpdate retrieveFullImage(long currSeqNum) { - PathsUpdate tempUpdate = new PathsUpdate(currSeqNum, false); - List allDatabases = hmsClient.getAllDatabases(); - for (Database db : allDatabases) { - tempUpdate.newPathChange(db.getName()).addToAddPaths( - PathsUpdate.cleanPath(db.getLocationUri())); - List allTables = hmsClient.getAllTablesOfDatabase(db); - for (Table tbl : allTables) { - TPathChanges tblPathChange = tempUpdate.newPathChange(tbl - .getDbName() + "." + tbl.getTableName()); - List tblParts = hmsClient.listAllPartitions(db, tbl); - tblPathChange.addToAddPaths(PathsUpdate.cleanPath(tbl.getSd() - .getLocation() == null ? db.getLocationUri() : tbl - .getSd().getLocation())); - for (Partition part : tblParts) { - tblPathChange.addToAddPaths(PathsUpdate.cleanPath(part.getSd() - .getLocation())); - } - } - } - UpdateableAuthzPaths tmpAuthzPaths = new UpdateableAuthzPaths( - pathPrefixes); - tmpAuthzPaths.updatePartial(Lists.newArrayList(tempUpdate), - new ReentrantReadWriteLock()); - PathsUpdate retUpdate = new PathsUpdate(currSeqNum, true); - retUpdate.getThriftObject().setPathsDump( - tmpAuthzPaths.getPathsDump().createPathsDump()); - return retUpdate; - } - }; - } - - @Override - public void onAlterSentryRoleAddGroups( - TAlterSentryRoleAddGroupsRequest request) throws SentryPluginException { - PermissionsUpdate update = new PermissionsUpdate(permSeqNum.incrementAndGet(), false); - TRoleChanges rUpdate = update.addRoleUpdate(request.getRoleName()); - for (TSentryGroup group : request.getGroups()) { - rUpdate.addToAddGroups(group.getGroupName()); - } - permsUpdater.handleUpdateNotification(update); - LOGGER.info("Authz Perm preUpdate [" + update.getSeqNum() + ", " + request.getRoleName() + "].."); - } - - @Override - public void onAlterSentryRoleDeleteGroups( - TAlterSentryRoleDeleteGroupsRequest request) - throws SentryPluginException { - PermissionsUpdate update = new PermissionsUpdate(permSeqNum.incrementAndGet(), false); - TRoleChanges rUpdate = update.addRoleUpdate(request.getRoleName()); - for (TSentryGroup group : request.getGroups()) { - rUpdate.addToDelGroups(group.getGroupName()); - } - permsUpdater.handleUpdateNotification(update); - LOGGER.info("Authz Perm preUpdate [" + update.getSeqNum() + ", " + request.getRoleName() + "].."); - } - - @Override - public void onAlterSentryRoleGrantPrivilege( - TAlterSentryRoleGrantPrivilegeRequest request) - throws SentryPluginException { - String authzObj = getAuthzObj(request.getPrivilege()); - if (authzObj != null) { - PermissionsUpdate update = new PermissionsUpdate(permSeqNum.incrementAndGet(), false); - update.addPrivilegeUpdate(authzObj).putToAddPrivileges( - request.getRoleName(), request.getPrivilege().getAction().toUpperCase()); - permsUpdater.handleUpdateNotification(update); - LOGGER.info("Authz Perm preUpdate [" + update.getSeqNum() + "].."); - } - } - - @Override - public void onAlterSentryRoleRevokePrivilege( - TAlterSentryRoleRevokePrivilegeRequest request) - throws SentryPluginException { - String authzObj = getAuthzObj(request.getPrivilege()); - if (authzObj != null) { - PermissionsUpdate update = new PermissionsUpdate(permSeqNum.incrementAndGet(), false); - update.addPrivilegeUpdate(authzObj).putToDelPrivileges( - request.getRoleName(), request.getPrivilege().getAction().toUpperCase()); - permsUpdater.handleUpdateNotification(update); - LOGGER.info("Authz Perm preUpdate [" + update.getSeqNum() + ", " + authzObj + "].."); - } - } - - @Override - public void onDropSentryRole(TDropSentryRoleRequest request) - throws SentryPluginException { - PermissionsUpdate update = new PermissionsUpdate(permSeqNum.incrementAndGet(), false); - update.addPrivilegeUpdate(PermissionsUpdate.ALL_AUTHZ_OBJ).putToDelPrivileges( - request.getRoleName(), PermissionsUpdate.ALL_AUTHZ_OBJ); - update.addRoleUpdate(request.getRoleName()).addToDelGroups(PermissionsUpdate.ALL_GROUPS); - permsUpdater.handleUpdateNotification(update); - LOGGER.info("Authz Perm preUpdate [" + update.getSeqNum() + ", " + request.getRoleName() + "].."); - } - - private String getAuthzObj(TSentryPrivilege privilege) { - String authzObj = null; - if (!SentryStore.isNULL(privilege.getDbName())) { - String dbName = privilege.getDbName(); - String tblName = privilege.getTableName(); - if (tblName == null) { - authzObj = dbName; - } else { - authzObj = dbName + "." + tblName; - } - } - return authzObj; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/78787d63/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java b/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java deleted file mode 100644 index b0fc5ed..0000000 --- a/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java +++ /dev/null @@ -1,227 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sentry.hdfs; - -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.Executor; -import java.util.concurrent.Executors; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import org.apache.sentry.hdfs.Updateable; - -import com.google.common.collect.Lists; - -public class UpdateForwarder implements - Updateable { - - public static interface ExternalImageRetriever { - - public K retrieveFullImage(long currSeqNum); - - } - - private final AtomicLong lastSeenSeqNum = new AtomicLong(0); - private final AtomicLong lastCommittedSeqNum = new AtomicLong(0); - // Updates should be handled in order - private final Executor updateHandler = Executors.newSingleThreadExecutor(); - - // Update log is used when propagate updates to a downstream cache. - // The preUpdate log stores all commits that were applied to this cache. - // When the update log is filled to capacity (updateLogSize), all - // entries are cleared and a compact image if the state of the cache is - // appended to the log. - // The first entry in an update log (consequently the first preUpdate a - // downstream cache sees) will be a full image. All subsequent entries are - // partial edits - private final LinkedList updateLog = new LinkedList(); - // UpdateLog is dissabled when updateLogSize = 0; - private final int updateLogSize; - - private final ExternalImageRetriever imageRetreiver; - - private volatile Updateable updateable; - - private final ReadWriteLock lock = new ReentrantReadWriteLock(); - private static final long INIT_SEQ_NUM = -2; - - public UpdateForwarder(Updateable updateable, - ExternalImageRetriever imageRetreiver, int updateLogSize) { - this.updateLogSize = updateLogSize; - this.imageRetreiver = imageRetreiver; - K fullImage = imageRetreiver.retrieveFullImage(INIT_SEQ_NUM); - appendToUpdateLog(fullImage); - this.updateable = updateable.updateFull(fullImage); - } - - /** - * Handle notifications from HMS plug-in or upstream Cache - * @param update - */ - public void handleUpdateNotification(final K update) { - // Correct the seqNums on the first update - if (lastCommittedSeqNum.get() == INIT_SEQ_NUM) { - K firstUpdate = updateLog.peek(); - long firstSeqNum = update.getSeqNum() - 1; - firstUpdate.setSeqNum(firstSeqNum); - lastCommittedSeqNum.set(firstSeqNum); - lastSeenSeqNum.set(firstSeqNum); - } - final boolean editNotMissed = - lastSeenSeqNum.incrementAndGet() == update.getSeqNum(); - if (!editNotMissed) { - lastSeenSeqNum.set(update.getSeqNum()); - } - Runnable task = new Runnable() { - @Override - public void run() { - K toUpdate = update; - if (update.hasFullImage()) { - updateable = updateable.updateFull(update); - } else { - if (editNotMissed) { - // apply partial preUpdate - updateable.updatePartial(Lists.newArrayList(update), lock); - } else { - // Retrieve full update from External Source and - toUpdate = imageRetreiver - .retrieveFullImage(update.getSeqNum()); - updateable = updateable.updateFull(toUpdate); - } - } - appendToUpdateLog(toUpdate); - } - }; - updateHandler.execute(task); - } - - private void appendToUpdateLog(K update) { - synchronized (updateLog) { - if (updateLogSize > 0) { - if (update.hasFullImage() || (updateLog.size() == updateLogSize)) { - // Essentially a log compaction - updateLog.clear(); - updateLog.add(update.hasFullImage() ? update - : createFullImageUpdate(update.getSeqNum())); - } else { - updateLog.add(update); - } - } - lastCommittedSeqNum.set(update.getSeqNum()); - } - } - - /** - * Return all updates from requested seqNum (inclusive) - * @param seqNum - * @return - */ - public List getAllUpdatesFrom(long seqNum) { - List retVal = new LinkedList(); - synchronized (updateLog) { - long currSeqNum = lastCommittedSeqNum.get(); - if (updateLogSize == 0) { - // no updatelog configured.. - return retVal; - } - K head = updateLog.peek(); - if (seqNum > currSeqNum + 1) { - // This process has probably restarted since downstream - // recieved last update - retVal.addAll(updateLog); - return retVal; - } - if (head.getSeqNum() > seqNum) { - // Caller has diverged greatly.. - if (head.hasFullImage()) { - // head is a refresh(full) image - // Send full image along with partial updates - for (K u : updateLog) { - retVal.add(u); - } - } else { - // Create a full image - // clear updateLog - // add fullImage to head of Log - // NOTE : This should ideally never happen - K fullImage = createFullImageUpdate(currSeqNum); - updateLog.clear(); - updateLog.add(fullImage); - retVal.add(fullImage); - } - } else { - // increment iterator to requested seqNum - Iterator iter = updateLog.iterator(); - K u = null; - while (iter.hasNext()) { - u = iter.next(); - if (u.getSeqNum() == seqNum) { - break; - } - } - // add all updates from requestedSeq - // to committedSeqNum - for (long seq = seqNum; seq <= currSeqNum; seq ++) { - retVal.add(u); - if (iter.hasNext()) { - u = iter.next(); - } else { - break; - } - } - } - } - return retVal; - } - - public boolean areAllUpdatesCommited() { - return lastCommittedSeqNum.get() == lastSeenSeqNum.get(); - } - - public long getLastCommitted() { - return lastCommittedSeqNum.get(); - } - - public long getLastSeen() { - return lastSeenSeqNum.get(); - } - - @Override - public Updateable updateFull(K update) { - return updateable.updateFull(update); - } - - @Override - public void updatePartial(Iterable updates, ReadWriteLock lock) { - updateable.updatePartial(updates, lock); - } - - @Override - public long getLastUpdatedSeqNum() { - return updateable.getLastUpdatedSeqNum(); - } - - @Override - public K createFullImageUpdate(long currSeqNum) { - return updateable.createFullImageUpdate(currSeqNum); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/78787d63/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateablePermissions.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateablePermissions.java b/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateablePermissions.java deleted file mode 100644 index 6b3e2e2..0000000 --- a/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateablePermissions.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sentry.hdfs; - -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.ReadWriteLock; - -import org.apache.sentry.hdfs.PermissionsUpdate; -import org.apache.sentry.hdfs.Updateable; -import org.apache.sentry.hdfs.UpdateForwarder.ExternalImageRetriever; - -public class UpdateablePermissions implements Updateable{ - - private AtomicLong seqNum = new AtomicLong(); - private final ExternalImageRetriever imageRetreiver; - - public UpdateablePermissions( - ExternalImageRetriever imageRetreiver) { - this.imageRetreiver = imageRetreiver; - } - - @Override - public PermissionsUpdate createFullImageUpdate(long currSeqNum) { - return imageRetreiver.retrieveFullImage(currSeqNum); - } - - @Override - public long getLastUpdatedSeqNum() { - return seqNum.get(); - } - - @Override - public void updatePartial(Iterable update, - ReadWriteLock lock) { - for (PermissionsUpdate permsUpdate : update) { - seqNum.set(permsUpdate.getSeqNum()); - } - } - - @Override - public Updateable updateFull(PermissionsUpdate update) { - UpdateablePermissions other = new UpdateablePermissions(imageRetreiver); - other.seqNum.set(update.getSeqNum()); - return other; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/78787d63/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java b/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java deleted file mode 100644 index d571df8..0000000 --- a/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java +++ /dev/null @@ -1,278 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.hdfs; - -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.locks.ReadWriteLock; - -import junit.framework.Assert; - -import org.apache.sentry.hdfs.UpdateForwarder; -import org.apache.sentry.hdfs.Updateable; -import org.apache.sentry.hdfs.UpdateForwarder.ExternalImageRetriever; -import org.apache.sentry.hdfs.Updateable.Update; -import org.junit.Test; - -import com.google.common.base.Joiner; -import com.google.common.collect.Lists; - -public class TestUpdateForwarder { - - static class DummyUpdate implements Update { - private long seqNum = 0; - private boolean hasFullUpdate = false; - private String stuff; - public DummyUpdate(long seqNum, boolean hasFullUpdate) { - this.seqNum = seqNum; - this.hasFullUpdate = hasFullUpdate; - } - public String getStuff() { - return stuff; - } - public DummyUpdate setStuff(String stuff) { - this.stuff = stuff; - return this; - } - @Override - public boolean hasFullImage() { - return hasFullUpdate; - } - @Override - public long getSeqNum() { - return seqNum; - } - @Override - public void setSeqNum(long seqNum) { - this.seqNum = seqNum; - } - } - - static class DummyUpdatable implements Updateable { - - private List state = new LinkedList(); - private long lastUpdatedSeqNum = 0; - - @Override - public void updatePartial(Iterable update, ReadWriteLock lock) { - for (DummyUpdate u : update) { - state.add(u.getStuff()); - lastUpdatedSeqNum = u.seqNum; - } - } - - @Override - public Updateable updateFull(DummyUpdate update) { - DummyUpdatable retVal = new DummyUpdatable(); - retVal.lastUpdatedSeqNum = update.seqNum; - retVal.state = Lists.newArrayList(update.stuff.split(",")); - return retVal; - } - - @Override - public long getLastUpdatedSeqNum() { - return lastUpdatedSeqNum; - } - - @Override - public DummyUpdate createFullImageUpdate(long currSeqNum) { - DummyUpdate retVal = new DummyUpdate(currSeqNum, true); - retVal.stuff = Joiner.on(",").join(state); - return retVal; - } - - public String getState() { - return Joiner.on(",").join(state); - } - } - - static class DummyImageRetreiver implements ExternalImageRetriever { - - private String state; - public void setState(String state) { - this.state = state; - } - @Override - public DummyUpdate retrieveFullImage(long currSeqNum) { - DummyUpdate retVal = new DummyUpdate(currSeqNum, true); - retVal.stuff = state; - return retVal; - } - } - - @Test - public void testInit() { - DummyImageRetreiver imageRetreiver = new DummyImageRetreiver(); - imageRetreiver.setState("a,b,c"); - UpdateForwarder updateForwarder = new UpdateForwarder( - new DummyUpdatable(), imageRetreiver, 10); - Assert.assertEquals(-2, updateForwarder.getLastUpdatedSeqNum()); - List allUpdates = updateForwarder.getAllUpdatesFrom(0); - Assert.assertTrue(allUpdates.size() == 1); - Assert.assertEquals("a,b,c", allUpdates.get(0).getStuff()); - - // If the current process has restarted the input seqNum will be > currSeq - allUpdates = updateForwarder.getAllUpdatesFrom(100); - Assert.assertTrue(allUpdates.size() == 1); - Assert.assertEquals("a,b,c", allUpdates.get(0).getStuff()); - Assert.assertEquals(-2, allUpdates.get(0).getSeqNum()); - allUpdates = updateForwarder.getAllUpdatesFrom(-1); - Assert.assertEquals(0, allUpdates.size()); - } - - @Test - public void testUpdateReceive() throws Exception { - DummyImageRetreiver imageRetreiver = new DummyImageRetreiver(); - imageRetreiver.setState("a,b,c"); - UpdateForwarder updateForwarder = new UpdateForwarder( - new DummyUpdatable(), imageRetreiver, 5); - updateForwarder.handleUpdateNotification(new DummyUpdate(5, false).setStuff("d")); - while(!updateForwarder.areAllUpdatesCommited()) { - Thread.sleep(100); - } - Assert.assertEquals(5, updateForwarder.getLastUpdatedSeqNum()); - List allUpdates = updateForwarder.getAllUpdatesFrom(0); - Assert.assertEquals(2, allUpdates.size()); - Assert.assertEquals("a,b,c", allUpdates.get(0).getStuff()); - Assert.assertEquals("d", allUpdates.get(1).getStuff()); - } - - @Test - public void testGetUpdates() throws Exception { - DummyImageRetreiver imageRetreiver = new DummyImageRetreiver(); - imageRetreiver.setState("a,b,c"); - UpdateForwarder updateForwarder = new UpdateForwarder( - new DummyUpdatable(), imageRetreiver, 5); - updateForwarder.handleUpdateNotification(new DummyUpdate(5, false).setStuff("d")); - while(!updateForwarder.areAllUpdatesCommited()) { - Thread.sleep(100); - } - Assert.assertEquals(5, updateForwarder.getLastUpdatedSeqNum()); - List allUpdates = updateForwarder.getAllUpdatesFrom(0); - Assert.assertEquals(2, allUpdates.size()); - - updateForwarder.handleUpdateNotification(new DummyUpdate(6, false).setStuff("e")); - updateForwarder.handleUpdateNotification(new DummyUpdate(7, false).setStuff("f")); - - while(!updateForwarder.areAllUpdatesCommited()) { - Thread.sleep(100); - } - Assert.assertEquals(7, updateForwarder.getLastUpdatedSeqNum()); - allUpdates = updateForwarder.getAllUpdatesFrom(0); - Assert.assertEquals(4, allUpdates.size()); - Assert.assertEquals("a,b,c", allUpdates.get(0).getStuff()); - Assert.assertEquals(4, allUpdates.get(0).getSeqNum()); - Assert.assertEquals("d", allUpdates.get(1).getStuff()); - Assert.assertEquals(5, allUpdates.get(1).getSeqNum()); - Assert.assertEquals("e", allUpdates.get(2).getStuff()); - Assert.assertEquals(6, allUpdates.get(2).getSeqNum()); - Assert.assertEquals("f", allUpdates.get(3).getStuff()); - Assert.assertEquals(7, allUpdates.get(3).getSeqNum()); - - updateForwarder.handleUpdateNotification(new DummyUpdate(8, false).setStuff("g")); - while(!updateForwarder.areAllUpdatesCommited()) { - Thread.sleep(100); - } - Assert.assertEquals(8, updateForwarder.getLastUpdatedSeqNum()); - allUpdates = updateForwarder.getAllUpdatesFrom(8); - Assert.assertEquals(1, allUpdates.size()); - Assert.assertEquals("g", allUpdates.get(0).getStuff()); - } - - @Test - public void testGetUpdatesAfterExternalEntityReset() throws Exception { - DummyImageRetreiver imageRetreiver = new DummyImageRetreiver(); - imageRetreiver.setState("a,b,c"); - UpdateForwarder updateForwarder = new UpdateForwarder( - new DummyUpdatable(), imageRetreiver, 5); - updateForwarder.handleUpdateNotification(new DummyUpdate(5, false).setStuff("d")); - while(!updateForwarder.areAllUpdatesCommited()) { - Thread.sleep(100); - } - - updateForwarder.handleUpdateNotification(new DummyUpdate(6, false).setStuff("e")); - updateForwarder.handleUpdateNotification(new DummyUpdate(7, false).setStuff("f")); - - while(!updateForwarder.areAllUpdatesCommited()) { - Thread.sleep(100); - } - Assert.assertEquals(7, updateForwarder.getLastUpdatedSeqNum()); - List allUpdates = updateForwarder.getAllUpdatesFrom(0); - Assert.assertEquals(4, allUpdates.size()); - Assert.assertEquals("f", allUpdates.get(3).getStuff()); - Assert.assertEquals(7, allUpdates.get(3).getSeqNum()); - - updateForwarder.handleUpdateNotification(new DummyUpdate(8, false).setStuff("g")); - while(!updateForwarder.areAllUpdatesCommited()) { - Thread.sleep(100); - } - Assert.assertEquals(8, updateForwarder.getLastUpdatedSeqNum()); - allUpdates = updateForwarder.getAllUpdatesFrom(8); - Assert.assertEquals(1, allUpdates.size()); - Assert.assertEquals("g", allUpdates.get(0).getStuff()); - - imageRetreiver.setState("a,b,c,d,e,f,g,h"); - - // New update comes with SeqNum = 1 - updateForwarder.handleUpdateNotification(new DummyUpdate(1, false).setStuff("h")); - while(!updateForwarder.areAllUpdatesCommited()) { - Thread.sleep(100); - } - // NN plugin asks for next update - allUpdates = updateForwarder.getAllUpdatesFrom(9); - Assert.assertEquals(1, allUpdates.size()); - Assert.assertEquals("a,b,c,d,e,f,g,h", allUpdates.get(0).getStuff()); - Assert.assertEquals(1, allUpdates.get(0).getSeqNum()); - } - - @Test - public void testUpdateLogCompression() throws Exception { - DummyImageRetreiver imageRetreiver = new DummyImageRetreiver(); - imageRetreiver.setState("a,b,c"); - UpdateForwarder updateForwarder = new UpdateForwarder( - new DummyUpdatable(), imageRetreiver, 5); - updateForwarder.handleUpdateNotification(new DummyUpdate(5, false).setStuff("d")); - while(!updateForwarder.areAllUpdatesCommited()) { - Thread.sleep(100); - } - Assert.assertEquals(5, updateForwarder.getLastUpdatedSeqNum()); - List allUpdates = updateForwarder.getAllUpdatesFrom(0); - Assert.assertEquals(2, allUpdates.size()); - - updateForwarder.handleUpdateNotification(new DummyUpdate(6, false).setStuff("e")); - updateForwarder.handleUpdateNotification(new DummyUpdate(7, false).setStuff("f")); - updateForwarder.handleUpdateNotification(new DummyUpdate(8, false).setStuff("g")); - updateForwarder.handleUpdateNotification(new DummyUpdate(9, false).setStuff("h")); - updateForwarder.handleUpdateNotification(new DummyUpdate(10, false).setStuff("i")); - updateForwarder.handleUpdateNotification(new DummyUpdate(11, false).setStuff("j")); - - while(!updateForwarder.areAllUpdatesCommited()) { - Thread.sleep(100); - } - Assert.assertEquals(11, updateForwarder.getLastUpdatedSeqNum()); - allUpdates = updateForwarder.getAllUpdatesFrom(0); - Assert.assertEquals(3, allUpdates.size()); - Assert.assertEquals("a,b,c,d,e,f,g,h", allUpdates.get(0).getStuff()); - Assert.assertEquals(9, allUpdates.get(0).getSeqNum()); - Assert.assertEquals("i", allUpdates.get(1).getStuff()); - Assert.assertEquals(10, allUpdates.get(1).getSeqNum()); - Assert.assertEquals("j", allUpdates.get(2).getStuff()); - Assert.assertEquals(11, allUpdates.get(2).getSeqNum()); - } -} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/78787d63/sentry-hdfs/bin/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-hdfs/bin/pom.xml b/sentry-hdfs/bin/pom.xml new file mode 100644 index 0000000..4c4691f --- /dev/null +++ b/sentry-hdfs/bin/pom.xml @@ -0,0 +1,38 @@ + + + + + 4.0.0 + + + org.apache.sentry + sentry + 1.5.0-incubating-SNAPSHOT + + + sentry-hdfs + Sentry HDFS + pom + + + sentry-hdfs-common + sentry-hdfs-service + sentry-hdfs-namenode-plugin + + + http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/78787d63/sentry-hdfs/bin/sentry-hdfs-common/.gitignore ---------------------------------------------------------------------- diff --git a/sentry-hdfs/bin/sentry-hdfs-common/.gitignore b/sentry-hdfs/bin/sentry-hdfs-common/.gitignore new file mode 100644 index 0000000..1a28cd6 --- /dev/null +++ b/sentry-hdfs/bin/sentry-hdfs-common/.gitignore @@ -0,0 +1 @@ +sentry-hdfs/src/test/java/org/apache/sentry/hdfs/DummyAdapter.java http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/78787d63/sentry-hdfs/bin/sentry-hdfs-common/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-hdfs/bin/sentry-hdfs-common/pom.xml b/sentry-hdfs/bin/sentry-hdfs-common/pom.xml new file mode 100644 index 0000000..511bc53 --- /dev/null +++ b/sentry-hdfs/bin/sentry-hdfs-common/pom.xml @@ -0,0 +1,148 @@ + + + + 4.0.0 + + org.apache.sentry + sentry-hdfs + 1.5.0-incubating-SNAPSHOT + + + sentry-hdfs-common + Sentry HDFS Common + + + + + org.apache.hadoop + hadoop-minicluster + 2.5.0 + provided + + + + junit + junit + test + + + com.google.guava + guava + + + org.apache.hive + hive-metastore + provided + + + org.apache.hadoop + hadoop-common + 2.5.0 + provided + + + + ${basedir}/src/main/java + ${basedir}/src/test/java + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-source + generate-sources + + add-source + + + + src/gen/thrift/gen-javabean + + + + + + + + + + thriftif + + + + org.apache.maven.plugins + maven-antrun-plugin + + + generate-thrift-sources + generate-sources + + + + + + + + + + + + + + + + + + + + + + run + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + enforce-property + + enforce + + + + + thrift.home + + + true + + + + + + + + + + http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/78787d63/sentry-hdfs/bin/sentry-hdfs-common/src/main/resources/sentry_hdfs_service.thrift ---------------------------------------------------------------------- diff --git a/sentry-hdfs/bin/sentry-hdfs-common/src/main/resources/sentry_hdfs_service.thrift b/sentry-hdfs/bin/sentry-hdfs-common/src/main/resources/sentry_hdfs_service.thrift new file mode 100644 index 0000000..9212b64 --- /dev/null +++ b/sentry-hdfs/bin/sentry-hdfs-common/src/main/resources/sentry_hdfs_service.thrift @@ -0,0 +1,87 @@ +#!/usr/local/bin/thrift -java + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# +# Thrift Service that the MetaStore is built on +# + +include "share/fb303/if/fb303.thrift" + +namespace java org.apache.sentry.hdfs.service.thrift +namespace php sentry.hdfs.thrift +namespace cpp Apache.Sentry.HDFS.Thrift + +struct TPathChanges { +1: required string authzObj; +2: required list> addPaths; +3: required list> delPaths; +} + +struct TPathEntry { +1: required byte type; +2: required string pathElement; +3: optional string authzObj; +4: required set children; +} + +struct TPathsDump { +1: required i32 rootId; +2: required map nodeMap; +} + +struct TPathsUpdate { +1: required bool hasFullImage; +2: optional TPathsDump pathsDump; +3: required i64 seqNum; +4: required list pathChanges; +} + +struct TPrivilegeChanges { +1: required string authzObj; +2: required map addPrivileges; +3: required map delPrivileges; +} + +struct TRoleChanges { +1: required string role; +2: required list addGroups; +3: required list delGroups; +} + +struct TPermissionsUpdate { +1: required bool hasfullImage; +2: required i64 seqNum; +3: required map privilegeChanges; +4: required map roleChanges; +} + +struct TAuthzUpdateResponse { +1: optional list authzPathUpdate, +2: optional list authzPermUpdate, +} + +service SentryHDFSService +{ + # HMS Path cache + void handle_hms_notification(1:TPathsUpdate pathsUpdate); + + TAuthzUpdateResponse get_all_authz_updates_from(1:i64 permSeqNum, 2:i64 pathSeqNum); + map> get_all_related_paths(1:string path, 2:bool exactMatch); +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/78787d63/sentry-hdfs/bin/sentry-hdfs-common/src/test/resources/hdfs-sentry.xml ---------------------------------------------------------------------- diff --git a/sentry-hdfs/bin/sentry-hdfs-common/src/test/resources/hdfs-sentry.xml b/sentry-hdfs/bin/sentry-hdfs-common/src/test/resources/hdfs-sentry.xml new file mode 100644 index 0000000..c23a431 --- /dev/null +++ b/sentry-hdfs/bin/sentry-hdfs-common/src/test/resources/hdfs-sentry.xml @@ -0,0 +1,22 @@ + + + + + + + http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/78787d63/sentry-hdfs/bin/sentry-hdfs-namenode-plugin/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-hdfs/bin/sentry-hdfs-namenode-plugin/pom.xml b/sentry-hdfs/bin/sentry-hdfs-namenode-plugin/pom.xml new file mode 100644 index 0000000..de1aabd --- /dev/null +++ b/sentry-hdfs/bin/sentry-hdfs-namenode-plugin/pom.xml @@ -0,0 +1,74 @@ + + + + 4.0.0 + + org.apache.sentry + sentry-hdfs + 1.5.0-incubating-SNAPSHOT + + + sentry-hdfs-namenode-plugin + Sentry HDFS Namenode Plugin + + + + + org.apache.sentry + sentry-hdfs-common + 1.5.0-incubating-SNAPSHOT + + + org.apache.sentry + sentry-service-client + 1.5.0-incubating-SNAPSHOT + + + org.apache.sentry + sentry-hdfs-service + 1.5.0-incubating-SNAPSHOT + + + + junit + junit + test + + + com.google.guava + guava + + + org.apache.hadoop + hadoop-common + provided + + + org.apache.hadoop + hadoop-hdfs + provided + + + org.apache.hadoop + hadoop-minicluster + test + + + + http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/78787d63/sentry-hdfs/bin/sentry-hdfs-namenode-plugin/src/test/resources/hdfs-sentry.xml ---------------------------------------------------------------------- diff --git a/sentry-hdfs/bin/sentry-hdfs-namenode-plugin/src/test/resources/hdfs-sentry.xml b/sentry-hdfs/bin/sentry-hdfs-namenode-plugin/src/test/resources/hdfs-sentry.xml new file mode 100644 index 0000000..511bfdd --- /dev/null +++ b/sentry-hdfs/bin/sentry-hdfs-namenode-plugin/src/test/resources/hdfs-sentry.xml @@ -0,0 +1,33 @@ + + + + + + + sentry.hdfs-plugin.path-prefixes + /user/hive/dw + + + sentry.hdfs-plugin.sentry-uri + thrift://localhost:1234 + + + sentry.hdfs-plugin.stale-threshold.ms + -1 + + http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/78787d63/sentry-hdfs/bin/sentry-hdfs-service/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-hdfs/bin/sentry-hdfs-service/pom.xml b/sentry-hdfs/bin/sentry-hdfs-service/pom.xml new file mode 100644 index 0000000..74c4f20 --- /dev/null +++ b/sentry-hdfs/bin/sentry-hdfs-service/pom.xml @@ -0,0 +1,108 @@ + + + + 4.0.0 + + org.apache.sentry + sentry-hdfs + 1.5.0-incubating-SNAPSHOT + + + sentry-hdfs-service + Sentry HDFS service + + + + org.apache.hadoop + hadoop-common + provided + + + junit + junit + test + + + log4j + log4j + + + org.apache.shiro + shiro-core + + + com.google.guava + guava + + + org.slf4j + slf4j-api + + + org.slf4j + slf4j-log4j12 + + + org.apache.sentry + sentry-hdfs-common + + + org.apache.sentry + sentry-provider-db + + + org.apache.sentry + sentry-service-client + + + org.apache.hive + hive-exec + ${hive.version} + + + org.apache.hive + hive-shims + ${hive.version} + + + org.apache.thrift + libfb303 + + + org.apache.thrift + libthrift + + + ant-contrib + ant-contrib + + + org.apache.hadoop + hadoop-minikdc + test + + + org.apache.hive + hive-metastore + ${hive.version} + + + + + http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/78787d63/sentry-hdfs/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-hdfs/pom.xml b/sentry-hdfs/pom.xml index 94c554f..4c4691f 100644 --- a/sentry-hdfs/pom.xml +++ b/sentry-hdfs/pom.xml @@ -1,4 +1,4 @@ - + - + + 4.0.0 + org.apache.sentry sentry 1.5.0-incubating-SNAPSHOT - .. sentry-hdfs - Sentry HDFS Integration - - - - - org.apache.hadoop - hadoop-minicluster - 2.5.0 - provided - + Sentry HDFS + pom - - junit - junit - test - - - com.google.guava - guava - - - org.apache.hive - hive-metastore - provided - - - org.apache.hadoop - hadoop-common - 2.5.0 - provided - - - - ${basedir}/src/main/java - ${basedir}/src/test/java - - - org.codehaus.mojo - build-helper-maven-plugin - - - add-source - generate-sources - - add-source - - - - src/gen/thrift/gen-javabean - - - - - - - - - - thriftif - - - - org.apache.maven.plugins - maven-antrun-plugin - - - generate-thrift-sources - generate-sources - - - - - - - - - - - - - - - - - - - - - - run - - - - - - org.apache.maven.plugins - maven-enforcer-plugin - - - enforce-property - - enforce - - - - - thrift.home - - - true - - - - - - - - + + sentry-hdfs-common + sentry-hdfs-service + sentry-hdfs-namenode-plugin + http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/78787d63/sentry-hdfs/sentry-hdfs-common/.gitignore ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/.gitignore b/sentry-hdfs/sentry-hdfs-common/.gitignore new file mode 100644 index 0000000..91ad75b --- /dev/null +++ b/sentry-hdfs/sentry-hdfs-common/.gitignore @@ -0,0 +1,18 @@ +*.class +target/ +.classpath +.project +.settings +.metadata +.idea/ +*.iml +derby.log +datanucleus.log +sentry-core/sentry-core-common/src/gen +**/TempStatsStore/ +# Package Files # +*.jar +*.war +*.ear +test-output/ +maven-repo/ http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/78787d63/sentry-hdfs/sentry-hdfs-common/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/pom.xml b/sentry-hdfs/sentry-hdfs-common/pom.xml new file mode 100644 index 0000000..511bc53 --- /dev/null +++ b/sentry-hdfs/sentry-hdfs-common/pom.xml @@ -0,0 +1,148 @@ + + + + 4.0.0 + + org.apache.sentry + sentry-hdfs + 1.5.0-incubating-SNAPSHOT + + + sentry-hdfs-common + Sentry HDFS Common + + + + + org.apache.hadoop + hadoop-minicluster + 2.5.0 + provided + + + + junit + junit + test + + + com.google.guava + guava + + + org.apache.hive + hive-metastore + provided + + + org.apache.hadoop + hadoop-common + 2.5.0 + provided + + + + ${basedir}/src/main/java + ${basedir}/src/test/java + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-source + generate-sources + + add-source + + + + src/gen/thrift/gen-javabean + + + + + + + + + + thriftif + + + + org.apache.maven.plugins + maven-antrun-plugin + + + generate-thrift-sources + generate-sources + + + + + + + + + + + + + + + + + + + + + + run + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + enforce-property + + enforce + + + + + thrift.home + + + true + + + + + + + + + +