drill-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From GitBox <...@apache.org>
Subject [GitHub] [drill] dbw9580 commented on a change in pull request #2084: [WIP] DRILL-7745: Add storage plugin for IPFS
Date Mon, 22 Jun 2020 18:37:22 GMT

dbw9580 commented on a change in pull request #2084:
URL: https://github.com/apache/drill/pull/2084#discussion_r443752931



##########
File path: contrib/storage-ipfs/src/main/java/org/apache/drill/exec/store/ipfs/IPFSHelper.java
##########
@@ -0,0 +1,286 @@
+/*
+ * Copyright (c) 2018-2020 Bowen Ding, Yuedong Xu, Liang Wang
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.drill.exec.store.ipfs;
+
+import io.ipfs.api.IPFS;
+import io.ipfs.api.MerkleNode;
+import io.ipfs.multiaddr.MultiAddress;
+import io.ipfs.multihash.Multihash;
+import org.apache.drill.common.exceptions.UserException;
+import org.apache.drill.exec.store.ipfs.IPFSStoragePluginConfig.IPFSTimeOut;
+import org.bouncycastle.util.Strings;
+
+import java.io.IOException;
+import java.lang.ref.WeakReference;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
+
+
+
+public class IPFSHelper {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(IPFSHelper.class);
+
+  public static final String IPFS_NULL_OBJECT_HASH = "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n";
+  public static final Multihash IPFS_NULL_OBJECT = Multihash.fromBase58(IPFS_NULL_OBJECT_HASH);
+
+  private WeakReference<ExecutorService> executorService;
+  private static ExecutorService DEFAULT_EXECUTOR = Executors.newSingleThreadExecutor();
+  private IPFS client;
+  private IPFSPeer myself;
+  private int maxPeersPerLeaf;
+  private Map<IPFSTimeOut, Integer> timeouts;
+
+  class DefaultWeakReference<T> extends WeakReference<T> {
+    private T default_;
+    public DefaultWeakReference(T referent, T default_) {
+      super(referent);
+      this.default_ = default_;
+    }
+
+    @Override
+    public T get() {
+      T ret = super.get();
+      if (ret == null) {
+        return default_;
+      } else {
+        return ret;
+      }
+    }
+  }
+
+  public IPFSHelper(IPFS ipfs) {
+    executorService = new DefaultWeakReference<>(DEFAULT_EXECUTOR, DEFAULT_EXECUTOR);
+    this.client = ipfs;
+  }
+
+  public void setExecutorService(ExecutorService executorService) {
+    this.executorService = new DefaultWeakReference<>(executorService, DEFAULT_EXECUTOR);
+  }
+
+  public void setTimeouts(Map<IPFSTimeOut, Integer> timeouts) {
+    this.timeouts = timeouts;
+  }
+
+  public void setMyself(IPFSPeer myself) {
+    this.myself = myself;
+  }
+
+  public void setMaxPeersPerLeaf(int maxPeersPerLeaf) {
+    this.maxPeersPerLeaf = maxPeersPerLeaf;
+  }
+
+  public IPFS getClient() {
+    return client;
+  }
+
+  public List<Multihash> findprovsTimeout(Multihash id) throws IOException {
+    List<String> providers;
+    providers = client.dht.findprovsListTimeout(id, maxPeersPerLeaf, timeouts.get(IPFSTimeOut.FIND_PROV),
executorService.get());
+
+    List<Multihash> ret = providers.stream().map(str -> Multihash.fromBase58(str)).collect(Collectors.toList());
+    return ret;
+  }
+
+  public List<MultiAddress> findpeerTimeout(Multihash peerId) throws IOException {
+    // trying to resolve addresses of a node itself will always hang
+    // so we treat it specially
+    if(peerId.equals(myself.getId())) {
+      return myself.getMultiAddresses();
+    }
+
+    List<String> addrs;
+    addrs = client.dht.findpeerListTimeout(peerId, timeouts.get(IPFSTimeOut.FIND_PEER_INFO),
executorService.get());
+    List<MultiAddress>
+        ret = addrs
+        .stream()
+        .filter(addr -> !addr.equals(""))
+        .map(str -> new MultiAddress(str)).collect(Collectors.toList());
+    return ret;
+  }
+
+  @FunctionalInterface
+  public interface ThrowingFunction<T, R, E extends Exception>{
+    R apply(final T in) throws E;
+  }
+
+  @FunctionalInterface
+  public interface ThrowingSupplier<R, E extends Exception> {
+    R get() throws E;
+  }
+
+  /**
+   * Execute a time-critical operation op within time timeout. Throws TimeoutException, so
the
+   * caller has a chance to recover from a timeout.
+   * @param op a Function that represents the operation to perform
+   * @param in the parameter for op
+   * @param timeout consider the execution has timed out after this amount of time in seconds
+   * @param <T>
+   * @param <R>
+   * @param <E>
+   * @return R the result of the operation
+   * @throws TimeoutException
+   * @throws E
+   */
+  public <T, R, E extends Exception> R timed(ThrowingFunction<T, R, E> op, T
in, int timeout) throws TimeoutException, E {
+    Callable<R> task = () -> op.apply(in);
+    Future<R> res = executorService.get().submit(task);
+    try {
+      return res.get(timeout, TimeUnit.SECONDS);

Review comment:
       Removed in ca71f95.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



Mime
View raw message