# Copyright 2014 Baidu, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file # except in compliance with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software distributed under the # License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, # either express or implied. See the License for the specific language governing permissions # and limitations under the License. # -*- coding: utf-8 -*- """ This module provide http request function for bce services. """ from future.utils import iteritems, iterkeys, itervalues from builtins import str, bytes import logging import http.client import sys import time import traceback import baidubce from baidubce import compat from baidubce import utils from baidubce.bce_response import BceResponse from baidubce.exception import BceHttpClientError from baidubce.exception import BceServerError from baidubce.exception import BceClientError from baidubce.http import http_headers try: from urllib.parse import urlparse except ImportError: from urlparse import urlparse _logger = logging.getLogger(__name__) def _get_connection(protocol, host, port, connection_timeout_in_millis, proxy_host=None, proxy_port=None): """ :param protocol :type protocol: baidubce.protocol.Protocol :param endpoint :type endpoint: str :param connection_timeout_in_millis :type connection_timeout_in_millis int """ host = compat.convert_to_string(host) if protocol.name == baidubce.protocol.HTTP.name: if proxy_host and proxy_port: _logger.debug('Using proxy host: %s, port: %d' % (proxy_host, proxy_port)) conn = http.client.HTTPConnection(host=proxy_host, port=proxy_port, timeout=connection_timeout_in_millis / 1000) conn.set_tunnel(host, port) return conn return http.client.HTTPConnection( host=host, port=port, timeout=connection_timeout_in_millis / 1000) elif protocol.name == baidubce.protocol.HTTPS.name: if proxy_host and proxy_port: _logger.debug('Using proxy host: %s, port: %d' % (host, port)) conn = http.client.HTTPSConnection(host=proxy_host, port=proxy_port, timeout=connection_timeout_in_millis / 1000) conn.set_tunnel(host, port) return conn return http.client.HTTPSConnection( host=host, port=port, timeout=connection_timeout_in_millis / 1000) else: raise ValueError( 'Invalid protocol: %s, either HTTP or HTTPS is expected.' % protocol) def _send_http_request(conn, http_method, uri, headers, body, send_buf_size): # putrequest() need that http_method and uri is Ascii on Py2 and unicode \ # on Py3 http_method = compat.convert_to_string(http_method) uri = compat.convert_to_string(uri) conn.putrequest(http_method, uri, skip_host=True, skip_accept_encoding=True) for k, v in iteritems(headers): k = utils.convert_to_standard_string(k) v = utils.convert_to_standard_string(v) conn.putheader(k, v) conn.endheaders() if body: if isinstance(body, (bytes, str)): conn.send(body) else: total = int(headers[http_headers.CONTENT_LENGTH]) sent = 0 while sent < total: size = total - sent if size > send_buf_size: size = send_buf_size buf = body.read(size) if not buf: raise BceClientError( 'Insufficient data, only %d bytes available while %s is %d' % ( sent, http_headers.CONTENT_LENGTH, total)) conn.send(buf) sent += len(buf) return conn.getresponse() def check_headers(headers): """ check value in headers, if \n in value, raise :param headers: :return: """ for k, v in iteritems(headers): if isinstance(v, (bytes, str)) and \ b'\n' in compat.convert_to_bytes(v): raise BceClientError(r'There should not be any "\n" in header[%s]:%s' % (k, v)) def send_request( config, sign_function, response_handler_functions, http_method, path, body, headers, params, use_backup_endpoint=False): """ Send request to BCE services. :param config :type config: baidubce.BceClientConfiguration :param sign_function: :param response_handler_functions: :type response_handler_functions: list :param request: :type request: baidubce.internal.InternalRequest :return: :rtype: baidubce.BceResponse """ _logger.debug(b'%s request start: %s %s, %s, %s', http_method, path, headers, params, body) headers = headers or {} user_agent = 'bce-sdk-python/%s/%s/%s' % ( compat.convert_to_string(baidubce.SDK_VERSION), sys.version, sys.platform) user_agent = user_agent.replace('\n', '') user_agent = compat.convert_to_bytes(user_agent) headers[http_headers.USER_AGENT] = user_agent should_get_new_date = False if http_headers.BCE_DATE not in headers: should_get_new_date = True request_endpoint = config.endpoint if use_backup_endpoint: request_endpoint = config.backup_endpoint headers[http_headers.HOST] = request_endpoint if isinstance(body, str): body = body.encode(baidubce.DEFAULT_ENCODING) if not body: headers[http_headers.CONTENT_LENGTH] = 0 elif isinstance(body, bytes): headers[http_headers.CONTENT_LENGTH] = len(body) elif http_headers.CONTENT_LENGTH not in headers: raise ValueError(b'No %s is specified.' % http_headers.CONTENT_LENGTH) # store the offset of fp body offset = None if hasattr(body, "tell") and hasattr(body, "seek"): offset = body.tell() protocol, host, port = utils.parse_host_port(request_endpoint, config.protocol) headers[http_headers.HOST] = host if port != config.protocol.default_port: headers[http_headers.HOST] += b':' + compat.convert_to_bytes(port) headers[http_headers.AUTHORIZATION] = sign_function( config.credentials, http_method, path, headers, params) encoded_params = utils.get_canonical_querystring(params, False) if len(encoded_params) > 0: uri = path + b'?' + encoded_params else: uri = path check_headers(headers) retries_attempted = 0 errors = [] while True: conn = None try: # restore the offset of fp body when retrying if should_get_new_date is True: headers[http_headers.BCE_DATE] = utils.get_canonical_time() headers[http_headers.AUTHORIZATION] = sign_function( config.credentials, http_method, path, headers, params) if retries_attempted > 0 and offset is not None: body.seek(offset) conn = _get_connection(protocol, host, port, config.connection_timeout_in_mills, config.proxy_host, config.proxy_port) _logger.debug('request args:method=%s, uri=%s, headers=%s,patams=%s, body=%s', http_method, uri, headers, params, body) http_response = _send_http_request( conn, http_method, uri, headers, body, config.send_buf_size) headers_list = http_response.getheaders() # on py3 ,values of headers_list is decoded with ios-8859-1 from # utf-8 binary bytes # headers_list[*][0] is lowercase on py2 # headers_list[*][0] is raw value py3 if compat.PY3 and isinstance(headers_list, list): temp_heads = [] for k, v in headers_list: k = k.encode('latin-1').decode('utf-8') v = v.encode('latin-1').decode('utf-8') k = k.lower() temp_heads.append((k, v)) headers_list = temp_heads _logger.debug( 'request return: status=%d, headers=%s' % (http_response.status, headers_list)) response = BceResponse() if config.under_line_headers: response.set_metadata_from_headers(dict(headers_list)) else: response.set_metadata_from_headers_no_underlined(dict(headers_list)) if config.auto_follow_redirect: if http_method == b'GET' and 300 <= http_response.status < 400: headers_map = {k: v for k, v in headers_list} if 'location' in headers_map: try: http_response.read() except Exception: _logger.info("ignore read response body error") try: http_response.close() except Exception: _logger.info("ignore close response connection error") location = headers_map.get('location') _logger.debug('request auto follow redirect location is %s', location) parsed_url = urlparse(location) redirect_conn = None if protocol.name == baidubce.protocol.HTTP.name: redirect_conn = http.client.HTTPConnection(parsed_url.netloc, timeout=config.connection_timeout_in_mills / 1000) elif protocol.name == baidubce.protocol.HTTPS.name: redirect_conn = http.client.HTTPSConnection(parsed_url.netloc, timeout=config.connection_timeout_in_mills / 1000) else: raise ValueError('Invalid protocol: %s, either HTTP or HTTPS is expected.' % protocol) redirect_conn.putrequest("GET", parsed_url.path + "?" + parsed_url.query, skip_host=True, skip_accept_encoding=True) redirect_conn.putheader("Host", parsed_url.netloc) redirect_conn.endheaders() http_response = redirect_conn.getresponse() for handler_function in response_handler_functions: if handler_function(http_response, response): break return response except Exception as e: if conn is not None: conn.close() # insert ">>>>" before all trace back lines and then save it errors.append('\n'.join('>>>>' + line for line in traceback.format_exc().splitlines())) if isinstance(e, BceServerError): request_id = e.request_id status_code = e.status_code code = e.code else: request_id = None status_code = None code = None if config.retry_policy.should_retry(e, retries_attempted): delay_in_millis = config.retry_policy.get_delay_before_next_retry_in_millis( e, retries_attempted) time.sleep(delay_in_millis / 1000.0) else: raise BceHttpClientError('Unable to execute HTTP request. Retried %d times. ' 'All trace backs:\n%s' % (retries_attempted, '\n'.join(errors)), e, status_code, code, request_id=request_id) retries_attempted += 1