knox-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lmc...@apache.org
Subject knox git commit: KNOX-1105 - Provide indication that topologies were generated from simple descriptors (Phil Zampino via lmccay)
Date Thu, 09 Nov 2017 00:04:55 GMT
Repository: knox
Updated Branches:
  refs/heads/master a068eb8da -> f52dc2015


KNOX-1105 - Provide indication that topologies were generated from simple descriptors (Phil
Zampino via lmccay)

Project: http://git-wip-us.apache.org/repos/asf/knox/repo
Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/f52dc201
Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/f52dc201
Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/f52dc201

Branch: refs/heads/master
Commit: f52dc20159ae1d8adcd6e9f9966b7bb6b42d14cf
Parents: a068eb8
Author: Larry McCay <lmccay@hortonworks.com>
Authored: Wed Nov 8 19:04:40 2017 -0500
Committer: Larry McCay <lmccay@hortonworks.com>
Committed: Wed Nov 8 19:04:40 2017 -0500

----------------------------------------------------------------------
 .../builder/BeanPropertyTopologyBuilder.java    | 11 +++++++++
 .../simple/SimpleDescriptorHandler.java         | 20 +++++++++++++++-
 .../xml/KnoxFormatXmlTopologyRules.java         |  2 ++
 .../src/main/resources/conf/topology-v1.xsd     |  1 +
 .../simple/SimpleDescriptorHandlerTest.java     |  8 +++++++
 .../service/admin/TopologiesResource.java       | 25 ++++++++++++++++----
 .../service/admin/beans/BeanConverter.java      |  2 ++
 .../gateway/service/admin/beans/Topology.java   | 11 +++++++++
 .../hadoop/gateway/i18n/GatewaySpiMessages.java |  3 +++
 .../hadoop/gateway/topology/Topology.java       |  9 +++++++
 10 files changed, 87 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/knox/blob/f52dc201/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/builder/BeanPropertyTopologyBuilder.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/builder/BeanPropertyTopologyBuilder.java
b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/builder/BeanPropertyTopologyBuilder.java
index 65278a1..5058508 100644
--- a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/builder/BeanPropertyTopologyBuilder.java
+++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/builder/BeanPropertyTopologyBuilder.java
@@ -28,6 +28,7 @@ public class BeanPropertyTopologyBuilder implements TopologyBuilder {
 
     private String name;
     private String defaultService;
+    private boolean isGenerated;
     private List<Provider> providers;
     private List<Service> services;
     private List<Application> applications;
@@ -47,6 +48,15 @@ public class BeanPropertyTopologyBuilder implements TopologyBuilder {
         return name;
     }
 
+    public BeanPropertyTopologyBuilder generated(String isGenerated) {
+        this.isGenerated = Boolean.valueOf(isGenerated);
+        return this;
+    }
+
+    public boolean isGenerated() {
+        return isGenerated;
+    }
+
     public BeanPropertyTopologyBuilder defaultService(String defaultService) {
       this.defaultService = defaultService;
       return this;
@@ -87,6 +97,7 @@ public class BeanPropertyTopologyBuilder implements TopologyBuilder {
         Topology topology = new Topology();
         topology.setName(name);
         topology.setDefaultServicePath(defaultService);
+        topology.setGenerated(isGenerated);
 
         for (Provider provider : providers) {
             topology.addProvider(provider);

http://git-wip-us.apache.org/repos/asf/knox/blob/f52dc201/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandler.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandler.java
b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandler.java
index 040eb86..089925d 100644
--- a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandler.java
+++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandler.java
@@ -77,7 +77,14 @@ public class SimpleDescriptorHandler {
         DefaultServiceDiscoveryConfig sdc = new DefaultServiceDiscoveryConfig(desc.getDiscoveryAddress());
         sdc.setUser(desc.getDiscoveryUser());
         sdc.setPasswordAlias(desc.getDiscoveryPasswordAlias());
-        ServiceDiscovery sd = ServiceDiscoveryFactory.get(desc.getDiscoveryType(), gatewayServices);
+
+        // Use the discovery type from the descriptor. If it's unspecified, employ the default
type.
+        String discoveryType = desc.getDiscoveryType();
+        if (discoveryType == null) {
+            discoveryType = "AMBARI";
+        }
+
+        ServiceDiscovery sd = ServiceDiscoveryFactory.get(discoveryType, gatewayServices);
         ServiceDiscovery.Cluster cluster = sd.discover(sdc, desc.getClusterName());
 
         List<String> validServiceNames = new ArrayList<>();
@@ -148,10 +155,20 @@ public class SimpleDescriptorHandler {
                 topologyFilename = desc.getClusterName();
             }
             topologyDescriptor = new File(destDirectory, topologyFilename + ".xml");
+
             fw = new BufferedWriter(new FileWriter(topologyDescriptor));
 
+            fw.write("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n");
+
+            fw.write("<!--==============================================-->\n");
+            fw.write("<!-- DO NOT EDIT. This is an auto-generated file. -->\n");
+            fw.write("<!--==============================================-->\n");
+
             fw.write("<topology>\n");
 
+            // KNOX-1105 Indicate that this topology was auto-generated
+            fw.write("    <generated>true</generated>\n");
+
             // Copy the externalized provider configuration content into the topology descriptor
in-line
             InputStreamReader policyReader = new InputStreamReader(new FileInputStream(providerConfig));
             char[] buffer = new char[1024];
@@ -243,6 +260,7 @@ public class SimpleDescriptorHandler {
         return result;
     }
 
+
     private static boolean validateURL(String serviceName, String url) {
         boolean result = false;
 

http://git-wip-us.apache.org/repos/asf/knox/blob/f52dc201/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/xml/KnoxFormatXmlTopologyRules.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/xml/KnoxFormatXmlTopologyRules.java
b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/xml/KnoxFormatXmlTopologyRules.java
index cb30769..7f9ec3e 100644
--- a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/xml/KnoxFormatXmlTopologyRules.java
+++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/xml/KnoxFormatXmlTopologyRules.java
@@ -33,6 +33,7 @@ public class KnoxFormatXmlTopologyRules extends AbstractRulesModule {
   private static final String NAME_TAG = "name";
   private static final String VERSION_TAG = "version";
   private static final String DEFAULT_SERVICE_TAG = "path";
+  private static final String GENERATED_TAG = "generated";
   private static final String APPLICATION_TAG = "application";
   private static final String SERVICE_TAG = "service";
   private static final String ROLE_TAG = "role";
@@ -50,6 +51,7 @@ public class KnoxFormatXmlTopologyRules extends AbstractRulesModule {
     forPattern( ROOT_TAG + "/" + NAME_TAG ).callMethod("name").usingElementBodyAsArgument();
     forPattern( ROOT_TAG + "/" + VERSION_TAG ).callMethod("version").usingElementBodyAsArgument();
     forPattern( ROOT_TAG + "/" + DEFAULT_SERVICE_TAG ).callMethod("defaultService").usingElementBodyAsArgument();
+    forPattern( ROOT_TAG + "/" + GENERATED_TAG ).callMethod("generated").usingElementBodyAsArgument();
 
     forPattern( ROOT_TAG + "/" + APPLICATION_TAG ).createObject().ofType( Application.class
).then().setNext( "addApplication" );
     forPattern( ROOT_TAG + "/" + APPLICATION_TAG + "/" + ROLE_TAG ).setBeanProperty();

http://git-wip-us.apache.org/repos/asf/knox/blob/f52dc201/gateway-server/src/main/resources/conf/topology-v1.xsd
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/resources/conf/topology-v1.xsd b/gateway-server/src/main/resources/conf/topology-v1.xsd
index 96c9ba2..936c701 100644
--- a/gateway-server/src/main/resources/conf/topology-v1.xsd
+++ b/gateway-server/src/main/resources/conf/topology-v1.xsd
@@ -22,6 +22,7 @@ limitations under the License.
 
                 <h:element name="name" minOccurs="0" maxOccurs="1"/>
                 <h:element name="path" minOccurs="0" maxOccurs="1"/>
+                <h:element name="generated" type="h:boolean" minOccurs="0"/>
 
                 <h:element name="gateway" minOccurs="0" maxOccurs="1">
                     <h:complexType>

http://git-wip-us.apache.org/repos/asf/knox/blob/f52dc201/gateway-server/src/test/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandlerTest.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/test/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandlerTest.java
b/gateway-server/src/test/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandlerTest.java
index a1fda1c..3a62980 100644
--- a/gateway-server/src/test/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandlerTest.java
+++ b/gateway-server/src/test/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandlerTest.java
@@ -44,9 +44,12 @@ import org.w3c.dom.Node;
 import org.w3c.dom.NodeList;
 import org.xml.sax.SAXException;
 
+import static org.hamcrest.Matchers.hasXPath;
+import static org.hamcrest.Matchers.is;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -231,6 +234,11 @@ public class SimpleDescriptorHandlerTest {
             // Parse the topology descriptor
             Document topologyXml = XmlUtils.readXml(topologyFile);
 
+            // KNOX-1105 Mark generated topology files
+            assertThat("Expected the \"generated\" marker element in the topology XML, with
value of \"true\".",
+                       topologyXml,
+                       hasXPath("/topology/generated", is("true")));
+
             // Validate the provider configuration
             Document extProviderConf = XmlUtils.readXml(new ByteArrayInputStream(TEST_PROVIDER_CONFIG.getBytes()));
             Node gatewayNode = (Node) xpath.compile("/topology/gateway").evaluate(topologyXml,
XPathConstants.NODE);

http://git-wip-us.apache.org/repos/asf/knox/blob/f52dc201/gateway-service-admin/src/main/java/org/apache/hadoop/gateway/service/admin/TopologiesResource.java
----------------------------------------------------------------------
diff --git a/gateway-service-admin/src/main/java/org/apache/hadoop/gateway/service/admin/TopologiesResource.java
b/gateway-service-admin/src/main/java/org/apache/hadoop/gateway/service/admin/TopologiesResource.java
index 445e85d..67a053b 100644
--- a/gateway-service-admin/src/main/java/org/apache/hadoop/gateway/service/admin/TopologiesResource.java
+++ b/gateway-service-admin/src/main/java/org/apache/hadoop/gateway/service/admin/TopologiesResource.java
@@ -135,16 +135,33 @@ public class TopologiesResource {
   @Consumes({APPLICATION_JSON, APPLICATION_XML})
   @Path(SINGLE_TOPOLOGY_API_PATH)
   public Topology uploadTopology(@PathParam("id") String id, Topology t) {
+    Topology result = null;
 
-    GatewayServices gs = (GatewayServices) request.getServletContext()
-        .getAttribute(GatewayServices.GATEWAY_SERVICES_ATTRIBUTE);
+    GatewayServices gs =
+                (GatewayServices) request.getServletContext().getAttribute(GatewayServices.GATEWAY_SERVICES_ATTRIBUTE);
 
     t.setName(id);
     TopologyService ts = gs.getService(GatewayServices.TOPOLOGY_SERVICE);
 
-    ts.deployTopology(BeanConverter.getTopology(t));
+    // Check for existing topology with the same name, to see if it had been generated
+    boolean existingGenerated = false;
+    for (org.apache.hadoop.gateway.topology.Topology existingTopology : ts.getTopologies())
{
+      if(existingTopology.getName().equals(id)) {
+        existingGenerated = existingTopology.isGenerated();
+        break;
+      }
+    }
 
-    return getTopology(id);
+    // If a topology with the same ID exists, which had been generated, then DO NOT overwrite
it because it will be
+    // out of sync with the source descriptor. Otherwise, deploy the updated version.
+    if (!existingGenerated) {
+      ts.deployTopology(BeanConverter.getTopology(t));
+      result = getTopology(id);
+    } else {
+      log.disallowedOverwritingGeneratedTopology(id);
+    }
+
+    return result;
   }
 
   @DELETE

http://git-wip-us.apache.org/repos/asf/knox/blob/f52dc201/gateway-service-admin/src/main/java/org/apache/hadoop/gateway/service/admin/beans/BeanConverter.java
----------------------------------------------------------------------
diff --git a/gateway-service-admin/src/main/java/org/apache/hadoop/gateway/service/admin/beans/BeanConverter.java
b/gateway-service-admin/src/main/java/org/apache/hadoop/gateway/service/admin/beans/BeanConverter.java
index f94dcad..b25f0de 100644
--- a/gateway-service-admin/src/main/java/org/apache/hadoop/gateway/service/admin/beans/BeanConverter.java
+++ b/gateway-service-admin/src/main/java/org/apache/hadoop/gateway/service/admin/beans/BeanConverter.java
@@ -29,6 +29,7 @@ public class BeanConverter {
     topologyResource.setTimestamp(topology.getTimestamp());
     topologyResource.setPath(topology.getDefaultServicePath());
     topologyResource.setUri(topology.getUri());
+    topologyResource.setGenerated(topology.isGenerated());
     for ( org.apache.hadoop.gateway.topology.Provider provider : topology.getProviders()
) {
       topologyResource.getProviders().add( getProvider(provider) );
     }
@@ -47,6 +48,7 @@ public class BeanConverter {
     deploymentTopology.setTimestamp(topology.getTimestamp());
     deploymentTopology.setDefaultServicePath(topology.getPath());
     deploymentTopology.setUri(topology.getUri());
+    deploymentTopology.setGenerated(topology.isGenerated());
     for ( Provider provider : topology.getProviders() ) {
       deploymentTopology.addProvider( getProvider(provider) );
     }

http://git-wip-us.apache.org/repos/asf/knox/blob/f52dc201/gateway-service-admin/src/main/java/org/apache/hadoop/gateway/service/admin/beans/Topology.java
----------------------------------------------------------------------
diff --git a/gateway-service-admin/src/main/java/org/apache/hadoop/gateway/service/admin/beans/Topology.java
b/gateway-service-admin/src/main/java/org/apache/hadoop/gateway/service/admin/beans/Topology.java
index a197c27..67ecc27 100644
--- a/gateway-service-admin/src/main/java/org/apache/hadoop/gateway/service/admin/beans/Topology.java
+++ b/gateway-service-admin/src/main/java/org/apache/hadoop/gateway/service/admin/beans/Topology.java
@@ -39,6 +39,9 @@ public class Topology {
   @XmlElement
   private long timestamp;
 
+  @XmlElement(name="generated")
+  private boolean isGenerated;
+
   @XmlElement(name="provider")
   @XmlElementWrapper(name="gateway")
   public List<Provider> providers;
@@ -84,6 +87,14 @@ public class Topology {
     this.timestamp = timestamp;
   }
 
+  public boolean isGenerated() {
+    return isGenerated;
+  }
+
+  public void setGenerated(boolean isGenerated) {
+    this.isGenerated = isGenerated;
+  }
+
   public List<Service> getServices() {
     if (services == null) {
       services = new ArrayList<>();

http://git-wip-us.apache.org/repos/asf/knox/blob/f52dc201/gateway-spi/src/main/java/org/apache/hadoop/gateway/i18n/GatewaySpiMessages.java
----------------------------------------------------------------------
diff --git a/gateway-spi/src/main/java/org/apache/hadoop/gateway/i18n/GatewaySpiMessages.java
b/gateway-spi/src/main/java/org/apache/hadoop/gateway/i18n/GatewaySpiMessages.java
index aad4d8a..4a5b926 100644
--- a/gateway-spi/src/main/java/org/apache/hadoop/gateway/i18n/GatewaySpiMessages.java
+++ b/gateway-spi/src/main/java/org/apache/hadoop/gateway/i18n/GatewaySpiMessages.java
@@ -88,4 +88,7 @@ public interface GatewaySpiMessages {
   @Message(level = MessageLevel.ERROR, text = "Invalid resource URI {0} : {1}")
   void invalidResourceURI(final String uri, final String reason, @StackTrace(level = MessageLevel.DEBUG)
Exception e );
 
+  @Message( level = MessageLevel.ERROR, text = "Topology {0} cannot be manually overwritten
because it was generated from a simple descriptor." )
+  void disallowedOverwritingGeneratedTopology(final String topologyName);
+
 }

http://git-wip-us.apache.org/repos/asf/knox/blob/f52dc201/gateway-spi/src/main/java/org/apache/hadoop/gateway/topology/Topology.java
----------------------------------------------------------------------
diff --git a/gateway-spi/src/main/java/org/apache/hadoop/gateway/topology/Topology.java b/gateway-spi/src/main/java/org/apache/hadoop/gateway/topology/Topology.java
index c366421..6415afe 100644
--- a/gateway-spi/src/main/java/org/apache/hadoop/gateway/topology/Topology.java
+++ b/gateway-spi/src/main/java/org/apache/hadoop/gateway/topology/Topology.java
@@ -33,6 +33,7 @@ public class Topology {
   private String name;
   private String defaultServicePath = null;
   private long timestamp;
+  private boolean isGenerated;
   public List<Provider> providerList = new ArrayList<Provider>();
   private Map<String,Map<String,Provider>> providerMap = new HashMap<>();
   public List<Service> services = new ArrayList<Service>();
@@ -76,6 +77,14 @@ public class Topology {
     defaultServicePath = servicePath;
   }
 
+  public void setGenerated(boolean isGenerated) {
+    this.isGenerated = isGenerated;
+  }
+
+  public boolean isGenerated() {
+    return isGenerated;
+  }
+
   public Collection<Service> getServices() {
     return services;
   }


Mime
View raw message