libcloud-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From to...@apache.org
Subject [2/2] git commit: Add support for HTTP proxy to LibcloudHTTPConnection and LibcloudHTTPSConnection class.
Date Tue, 19 Aug 2014 16:24:03 GMT
Add support for HTTP proxy to LibcloudHTTPConnection and
LibcloudHTTPSConnection class.

User can specify which HTTP proxy to use using one of the following
approaches:

* by setting "proxy_url" environment variable (global / process wide)
* by passing "proxy_url"  argument to the Connection class constructor (per
  connection instance)
* by calling "set_http_proxy" method on the Connection class (per connection
  instance)


Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/f3f60002
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/f3f60002
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/f3f60002

Branch: refs/heads/trunk
Commit: f3f600028384bc19c0f7ff051306a401d57afd35
Parents: 5207038
Author: Tomaz Muraus <tomaz@apache.org>
Authored: Tue Aug 19 16:29:54 2014 +0200
Committer: Tomaz Muraus <tomaz@apache.org>
Committed: Tue Aug 19 18:22:18 2014 +0200

----------------------------------------------------------------------
 .../examples/http_proxy/constructor_argument.py |   6 +
 .../http_proxy/set_http_proxy_method.py         |  12 ++
 docs/other/using-http-proxy.rst                 |  52 ++++++++
 libcloud/common/base.py                         |  15 ++-
 libcloud/httplib_ssl.py                         | 129 ++++++++++++++++++-
 libcloud/test/test_connection.py                |  53 ++++++++
 6 files changed, 262 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/libcloud/blob/f3f60002/docs/examples/http_proxy/constructor_argument.py
----------------------------------------------------------------------
diff --git a/docs/examples/http_proxy/constructor_argument.py b/docs/examples/http_proxy/constructor_argument.py
new file mode 100644
index 0000000..ae54945
--- /dev/null
+++ b/docs/examples/http_proxy/constructor_argument.py
@@ -0,0 +1,6 @@
+from libcloud.compute.drivers.dreamhost import DreamhostConnection
+
+PROXY_URL = 'http://<proxy hostname>:<proxy port>'
+
+conn = DreamhostConnection(host='dreamhost.com', port=443,
+                           timeout=None, proxy_url=PROXY_URL)

http://git-wip-us.apache.org/repos/asf/libcloud/blob/f3f60002/docs/examples/http_proxy/set_http_proxy_method.py
----------------------------------------------------------------------
diff --git a/docs/examples/http_proxy/set_http_proxy_method.py b/docs/examples/http_proxy/set_http_proxy_method.py
new file mode 100644
index 0000000..d8c5e83
--- /dev/null
+++ b/docs/examples/http_proxy/set_http_proxy_method.py
@@ -0,0 +1,12 @@
+from pprint import pprint
+
+from libcloud.compute.types import Provider
+from libcloud.compute.providers import get_driver
+
+PROXY_URL = 'http://<proxy hostname>:<proxy port>'
+
+cls = get_driver(Provider.RACKSPACE)
+driver = cls('username', 'api key', region='ord')
+driver.set_http_proxy(proxy_url=PROXY_URL)
+
+pprint(driver.list_nodes())

http://git-wip-us.apache.org/repos/asf/libcloud/blob/f3f60002/docs/other/using-http-proxy.rst
----------------------------------------------------------------------
diff --git a/docs/other/using-http-proxy.rst b/docs/other/using-http-proxy.rst
new file mode 100644
index 0000000..8c12dc5
--- /dev/null
+++ b/docs/other/using-http-proxy.rst
@@ -0,0 +1,52 @@
+Using an HTTP proxy
+===================
+
+.. note::
+
+    Support for HTTP proxies is only available in Libcloud trunk and higher.
+
+Libcloud supports using an HTTP proxy for outgoing HTTP and HTTPS requests. At
+the moment, using a proxy is only supported if you are using Python 2.7 or
+above (it has been tested with 2.7, PyPy, 3.1, 3.3, 3.3, 3.4).
+
+You can specify which HTTP proxy to use using one of the approaches described
+bellow:
+
+* By setting ``http_proxy`` environment variable (this setting is system /
+  process wide)
+* By passing ``http_proxy`` argument to the
+  :class:`libcloud.common.base.LibcloudHTTPConnection` class constructor (this
+  setting is local to the connection instance)
+* By calling :meth:`libcloud.common.base.LibcloudHTTPConnection.set_http_proxy`
+  method (this setting is local to the connection instance)
+
+Known limitations
+-----------------
+
+* HTTP proxies which require authentication are not supported
+* Python 2.6 is not supported
+
+Examples
+--------
+
+This section includes some code examples which show how to use an HTTP proxy
+with Libcloud.
+
+1. Using http_proxy environment variable
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. sourcecode:: python
+
+    http_proxy=http://<proxy hostname>:<proxy port> python my_script.py
+
+2. Passing http_proxy argument to the connection class
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. literalinclude:: /examples/http_proxy/constructor_argument.py
+   :language: python
+
+3. Calling set_http_proxy method
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. literalinclude:: /examples/http_proxy/set_http_proxy_method.py
+   :language: python

http://git-wip-us.apache.org/repos/asf/libcloud/blob/f3f60002/libcloud/common/base.py
----------------------------------------------------------------------
diff --git a/libcloud/common/base.py b/libcloud/common/base.py
index 488f084..daf04d2 100644
--- a/libcloud/common/base.py
+++ b/libcloud/common/base.py
@@ -46,10 +46,9 @@ from libcloud.utils.misc import lowercase_keys
 from libcloud.utils.compression import decompress_data
 from libcloud.common.types import LibcloudError, MalformedResponseError
 
+from libcloud.httplib_ssl import LibcloudHTTPConnection
 from libcloud.httplib_ssl import LibcloudHTTPSConnection
 
-LibcloudHTTPConnection = httplib.HTTPConnection
-
 
 class HTTPResponse(httplib.HTTPResponse):
     # On python 2.6 some calls can hang because HEAD isn't quite properly
@@ -267,7 +266,9 @@ class LoggingConnection():
 
     :cvar log: file-like object that logs entries are written to.
     """
+
     log = None
+    http_proxy_used = False
 
     def _log_response(self, r):
         rv = "# -------- begin %d:%d response ----------\n" % (id(self), id(r))
@@ -341,7 +342,15 @@ class LoggingConnection():
         return (rr, rv)
 
     def _log_curl(self, method, url, body, headers):
-        cmd = ["curl", "-i"]
+        cmd = ["curl"]
+
+        if self.http_proxy_used:
+            proxy_url = 'http://%s:%s' % (self.proxy_host,
+                                          self.proxy_port)
+            proxy_url = pquote(proxy_url)
+            cmd.extend(['--proxy', proxy_url])
+
+        cmd.extend(['-i'])
 
         if method.lower() == 'head':
             # HEAD method need special handling

http://git-wip-us.apache.org/repos/asf/libcloud/blob/f3f60002/libcloud/httplib_ssl.py
----------------------------------------------------------------------
diff --git a/libcloud/httplib_ssl.py b/libcloud/httplib_ssl.py
index 29136ef..2024433 100644
--- a/libcloud/httplib_ssl.py
+++ b/libcloud/httplib_ssl.py
@@ -19,14 +19,126 @@ verification, depending on libcloud.security settings.
 import os
 import re
 import socket
+import sys
 import ssl
 import warnings
 
 import libcloud.security
 from libcloud.utils.py3 import httplib
+from libcloud.utils.py3 import urlparse
 
+__all__ = [
+    'LibcloudBaseConnection',
+    'LibcloudHTTPConnection',
+    'LibcloudHTTPSConnection'
+]
 
-class LibcloudHTTPSConnection(httplib.HTTPSConnection):
+HTTP_PROXY_ENV_VARIABLE_NAME = 'http_proxy'
+
+
+class LibcloudBaseConnection(object):
+    """
+    Base connection class to inherit from.
+
+    Note: This class should not be instantiated directly.
+    """
+
+    proxy_scheme = None
+    proxy_host = None
+    proxy_port = None
+    http_proxy_used = False
+
+    def set_http_proxy(self, proxy_url):
+        """
+        Set a HTTP proxy which will be used with this connection.
+
+        :param proxy_url: Proxy URL (e.g. http://hostname:3128)
+        :type proxy_url: ``str``
+        """
+        if sys.version_info[:2] == (2, 6):
+            raise Exception('HTTP proxy support requires Python 2.7 or higher')
+
+        scheme, host, port = self._parse_proxy_url(proxy_url=proxy_url)
+
+        self.proxy_scheme = scheme
+        self.proxy_host = host
+        self.proxy_port = port
+
+        self._setup_http_proxy()
+
+    def _parse_proxy_url(self, proxy_url):
+        """
+        Parse and validate a proxy URL.
+
+        :param proxy_url: Proxy URL (e.g. http://hostname:3128)
+        :type proxy_url: ``str``
+
+        :rtype: ``tuple`` (``scheme``, ``hostname``, ``port``)
+        """
+        parsed = urlparse.urlparse(proxy_url)
+
+        if parsed.scheme != 'http':
+            raise ValueError('Only http proxies are supported')
+
+        if not parsed.hostname or not parsed.port:
+            raise ValueError('proxy_url must be in the following format: '
+                             'http://<proxy host>:<proxy port>')
+
+        proxy_scheme = parsed.scheme
+        proxy_host, proxy_port = parsed.hostname, parsed.port
+
+        return (proxy_scheme, proxy_host, proxy_port)
+
+    def _setup_http_proxy(self):
+        """
+        Set up HTTP proxy.
+
+        :param proxy_url: Proxy URL (e.g. http://<host>:3128)
+        :type proxy_url: ``str``
+        """
+        self.set_tunnel(host=self.host, port=self.port)
+        self._set_hostport(host=self.proxy_host, port=self.proxy_port)
+
+    def _activate_http_proxy(self, sock):
+        self.sock = sock
+        self._tunnel()
+
+    def _set_hostport(self, host, port):
+        """
+        Backported from Python stdlib so Proxy support also works with
+        Python 3.4.
+        """
+        if port is None:
+            i = host.rfind(':')
+            j = host.rfind(']')         # ipv6 addresses have [...]
+            if i > j:
+                try:
+                    port = int(host[i+1:])
+                except ValueError:
+                    msg = "nonnumeric port: '%s'" % host[i+1:]
+                    raise httplib.InvalidURL(msg)
+                host = host[:i]
+            else:
+                port = self.default_port
+            if host and host[0] == '[' and host[-1] == ']':
+                host = host[1:-1]
+        self.host = host
+        self.port = port
+
+
+class LibcloudHTTPConnection(httplib.HTTPConnection, LibcloudBaseConnection):
+    def __init__(self, *args, **kwargs):
+        # Support for HTTP proxy
+        proxy_url_env = os.environ.get(HTTP_PROXY_ENV_VARIABLE_NAME, None)
+        proxy_url = kwargs.pop('proxy_url', proxy_url_env)
+
+        super(LibcloudHTTPConnection, self).__init__(*args, **kwargs)
+
+        if proxy_url:
+            self.set_http_proxy(proxy_url=proxy_url)
+
+
+class LibcloudHTTPSConnection(httplib.HTTPSConnection, LibcloudBaseConnection):
     """
     LibcloudHTTPSConnection
 
@@ -41,7 +153,15 @@ class LibcloudHTTPSConnection(httplib.HTTPSConnection):
         Constructor
         """
         self._setup_verify()
-        httplib.HTTPSConnection.__init__(self, *args, **kwargs)
+
+        # Support for HTTP proxy
+        proxy_url_env = os.environ.get(HTTP_PROXY_ENV_VARIABLE_NAME, None)
+        proxy_url = kwargs.pop('proxy_url', proxy_url_env)
+
+        super(LibcloudHTTPSConnection, self).__init__(*args, **kwargs)
+
+        if proxy_url:
+            self.set_http_proxy(proxy_url=proxy_url)
 
     def _setup_verify(self):
         """
@@ -97,6 +217,11 @@ class LibcloudHTTPSConnection(httplib.HTTPSConnection):
         else:
             sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
             sock.connect((self.host, self.port))
+
+        # Activate the HTTP proxy
+        if self.http_proxy_used:
+            self._activate_http_proxy(sock=sock)
+
         self.sock = ssl.wrap_socket(sock,
                                     self.key_file,
                                     self.cert_file,

http://git-wip-us.apache.org/repos/asf/libcloud/blob/f3f60002/libcloud/test/test_connection.py
----------------------------------------------------------------------
diff --git a/libcloud/test/test_connection.py b/libcloud/test/test_connection.py
index 5df7919..b5c2abb 100644
--- a/libcloud/test/test_connection.py
+++ b/libcloud/test/test_connection.py
@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import sys
 import ssl
 
@@ -22,6 +23,58 @@ from mock import Mock, call
 from libcloud.test import unittest
 from libcloud.common.base import Connection
 from libcloud.common.base import LoggingConnection
+from libcloud.httplib_ssl import LibcloudBaseConnection
+from libcloud.httplib_ssl import LibcloudHTTPConnection
+
+
+class BaseConnectionClassTestCase(unittest.TestCase):
+    def test_parse_proxy_url(self):
+        conn = LibcloudBaseConnection()
+
+        proxy_url = 'http://127.0.0.1:3128'
+        result = conn._parse_proxy_url(proxy_url=proxy_url)
+        self.assertEqual(result[0], 'http')
+        self.assertEqual(result[1], '127.0.0.1')
+        self.assertEqual(result[2], 3128)
+
+        proxy_url = 'https://127.0.0.1:3128'
+        expected_msg = 'Only http proxies are supported'
+        self.assertRaisesRegexp(ValueError, expected_msg,
+                                conn._parse_proxy_url,
+                                proxy_url=proxy_url)
+
+        proxy_url = 'http://127.0.0.1'
+        expected_msg = 'proxy_url must be in the following format'
+        self.assertRaisesRegexp(ValueError, expected_msg,
+                                conn._parse_proxy_url,
+                                proxy_url=proxy_url)
+
+    def test_constructor(self):
+        conn = LibcloudHTTPConnection(host='localhost', port=80)
+        self.assertEqual(conn.proxy_scheme, None)
+        self.assertEqual(conn.proxy_host, None)
+        self.assertEqual(conn.proxy_port, None)
+
+        proxy_url = 'http://127.0.0.3:3128'
+        conn.set_http_proxy(proxy_url=proxy_url)
+        self.assertEqual(conn.proxy_scheme, 'http')
+        self.assertEqual(conn.proxy_host, '127.0.0.3')
+        self.assertEqual(conn.proxy_port, 3128)
+
+        proxy_url = 'http://127.0.0.4:3128'
+        conn = LibcloudHTTPConnection(host='localhost', port=80,
+                                      proxy_url=proxy_url)
+        self.assertEqual(conn.proxy_scheme, 'http')
+        self.assertEqual(conn.proxy_host, '127.0.0.4')
+        self.assertEqual(conn.proxy_port, 3128)
+
+        os.environ['http_proxy'] = proxy_url
+        proxy_url = 'http://127.0.0.5:3128'
+        conn = LibcloudHTTPConnection(host='localhost', port=80,
+                                      proxy_url=proxy_url)
+        self.assertEqual(conn.proxy_scheme, 'http')
+        self.assertEqual(conn.proxy_host, '127.0.0.5')
+        self.assertEqual(conn.proxy_port, 3128)
 
 
 class ConnectionClassTestCase(unittest.TestCase):


Mime
View raw message