beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From reuven...@apache.org
Subject [2/3] beam git commit: Strip table decorators before creating tables.
Date Fri, 22 Sep 2017 03:23:20 GMT
Strip table decorators before creating tables.


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/0f50eb75
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/0f50eb75
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/0f50eb75

Branch: refs/heads/master
Commit: 0f50eb759dd3c810f0ef70d66a8077df227cb372
Parents: 0a073af
Author: Reuven Lax <relax@relax-macbookpro2.roam.corp.google.com>
Authored: Tue Sep 19 11:38:13 2017 -0700
Committer: Reuven Lax <relax@relax-macbookpro2.roam.corp.google.com>
Committed: Thu Sep 21 20:16:22 2017 -0700

----------------------------------------------------------------------
 .../apache/beam/sdk/io/gcp/bigquery/CreateTables.java    |  2 +-
 .../beam/sdk/io/gcp/bigquery/TableDestination.java       | 10 ++++++++++
 .../apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java  | 11 +++++++++++
 3 files changed, 22 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/0f50eb75/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CreateTables.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CreateTables.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CreateTables.java
index 7f83b83..aff5ff1 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CreateTables.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/CreateTables.java
@@ -113,7 +113,7 @@ public class CreateTables<DestinationT>
   private void possibleCreateTable(
       BigQueryOptions options, TableDestination tableDestination, TableSchema tableSchema)
       throws InterruptedException, IOException {
-    String tableSpec = tableDestination.getTableSpec();
+    String tableSpec = tableDestination.getStrippedTableSpec();
     TableReference tableReference = tableDestination.getTableReference();
     String tableDescription = tableDestination.getTableDescription();
     if (createDisposition != createDisposition.CREATE_NEVER && !createdTables.contains(tableSpec))
{

http://git-wip-us.apache.org/repos/asf/beam/blob/0f50eb75/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestination.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestination.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestination.java
index 79f1b22..4a4f66b 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestination.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableDestination.java
@@ -30,6 +30,7 @@ import javax.annotation.Nullable;
 public class TableDestination implements Serializable {
   private static final long serialVersionUID = 1L;
   private final String tableSpec;
+  @Nullable private String strippedTableSpec;
   @Nullable
   private final String tableDescription;
   @Nullable
@@ -59,6 +60,7 @@ public class TableDestination implements Serializable {
   public TableDestination(String tableSpec, @Nullable String tableDescription,
       @Nullable String jsonTimePartitioning) {
     this.tableSpec = tableSpec;
+    this.strippedTableSpec = null;
     this.tableDescription = tableDescription;
     this.jsonTimePartitioning = jsonTimePartitioning;
   }
@@ -68,6 +70,14 @@ public class TableDestination implements Serializable {
     return tableSpec;
   }
 
+  public String getStrippedTableSpec() {
+    if (strippedTableSpec == null) {
+      int index = tableSpec.lastIndexOf('$');
+      strippedTableSpec = (index  == -1) ? tableSpec : tableSpec.substring(0, index);
+    }
+    return strippedTableSpec;
+  }
+
   public TableReference getTableReference() {
     return BigQueryHelpers.parseTableSpec(tableSpec);
   }

http://git-wip-us.apache.org/repos/asf/beam/blob/0f50eb75/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
index 9120507..7927282 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
@@ -2343,4 +2343,15 @@ public class BigQueryIOTest implements Serializable {
     }
     return converted;
   }
+
+  @Test
+  public void testTableDecoratorStripping() {
+    TableDestination tableDestination = tableDestination = new TableDestination(
+        "project:dataset.table$decorator", "");
+    assertEquals("project:dataset.table", tableDestination.getStrippedTableSpec());
+
+    tableDestination = new TableDestination("project:dataset.table", "");
+    assertEquals("project:dataset.table", tableDestination.getStrippedTableSpec());
+
+  }
 }


Mime
View raw message