1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 """Classes to encapsulate a single HTTP request.
16
17 The classes implement a command pattern, with every
18 object supporting an execute() method that does the
19 actuall HTTP request.
20 """
21 from __future__ import absolute_import
22 import six
23 from six.moves import http_client
24 from six.moves import range
25
26 __author__ = 'jcgregorio@google.com (Joe Gregorio)'
27
28 from six import BytesIO, StringIO
29 from six.moves.urllib.parse import urlparse, urlunparse, quote, unquote
30
31 import base64
32 import copy
33 import gzip
34 import httplib2
35 import json
36 import logging
37 import mimetypes
38 import os
39 import random
40 import socket
41 import sys
42 import time
43 import uuid
44
45
46 try:
47 import ssl
48 except ImportError:
49 _ssl_SSLError = object()
50 else:
51 _ssl_SSLError = ssl.SSLError
52
53 from email.generator import Generator
54 from email.mime.multipart import MIMEMultipart
55 from email.mime.nonmultipart import MIMENonMultipart
56 from email.parser import FeedParser
57
58
59
60 try:
61 from oauth2client import util
62 except ImportError:
63 from oauth2client import _helpers as util
64
65 from googleapiclient import mimeparse
66 from googleapiclient.errors import BatchError
67 from googleapiclient.errors import HttpError
68 from googleapiclient.errors import InvalidChunkSizeError
69 from googleapiclient.errors import ResumableUploadError
70 from googleapiclient.errors import UnexpectedBodyError
71 from googleapiclient.errors import UnexpectedMethodError
72 from googleapiclient.model import JsonModel
73
74
75 LOGGER = logging.getLogger(__name__)
76
77 DEFAULT_CHUNK_SIZE = 512*1024
78
79 MAX_URI_LENGTH = 2048
80
81 _TOO_MANY_REQUESTS = 429
85 """Determines whether a response should be retried.
86
87 Args:
88 resp_status: The response status received.
89 content: The response content body.
90
91 Returns:
92 True if the response should be retried, otherwise False.
93 """
94
95 if resp_status >= 500:
96 return True
97
98
99 if resp_status == _TOO_MANY_REQUESTS:
100 return True
101
102
103
104 if resp_status == six.moves.http_client.FORBIDDEN:
105
106 if not content:
107 return False
108
109
110 try:
111 data = json.loads(content.decode('utf-8'))
112 reason = data['error']['errors'][0]['reason']
113 except (UnicodeDecodeError, ValueError, KeyError):
114 LOGGER.warning('Invalid JSON content from response: %s', content)
115 return False
116
117 LOGGER.warning('Encountered 403 Forbidden with reason "%s"', reason)
118
119
120 if reason in ('userRateLimitExceeded', 'rateLimitExceeded', ):
121 return True
122
123
124 return False
125
126
127 -def _retry_request(http, num_retries, req_type, sleep, rand, uri, method, *args,
128 **kwargs):
129 """Retries an HTTP request multiple times while handling errors.
130
131 If after all retries the request still fails, last error is either returned as
132 return value (for HTTP 5xx errors) or thrown (for ssl.SSLError).
133
134 Args:
135 http: Http object to be used to execute request.
136 num_retries: Maximum number of retries.
137 req_type: Type of the request (used for logging retries).
138 sleep, rand: Functions to sleep for random time between retries.
139 uri: URI to be requested.
140 method: HTTP method to be used.
141 args, kwargs: Additional arguments passed to http.request.
142
143 Returns:
144 resp, content - Response from the http request (may be HTTP 5xx).
145 """
146 resp = None
147 content = None
148 for retry_num in range(num_retries + 1):
149 if retry_num > 0:
150
151 sleep_time = rand() * 2 ** retry_num
152 LOGGER.warning(
153 'Sleeping %.2f seconds before retry %d of %d for %s: %s %s, after %s',
154 sleep_time, retry_num, num_retries, req_type, method, uri,
155 resp.status if resp else exception)
156 sleep(sleep_time)
157
158 try:
159 exception = None
160 resp, content = http.request(uri, method, *args, **kwargs)
161
162 except _ssl_SSLError as ssl_error:
163 exception = ssl_error
164 except socket.error as socket_error:
165
166 if socket.errno.errorcode.get(socket_error.errno) not in (
167 'WSAETIMEDOUT', 'ETIMEDOUT', 'EPIPE', 'ECONNABORTED', ):
168 raise
169 exception = socket_error
170
171 if exception:
172 if retry_num == num_retries:
173 raise exception
174 else:
175 continue
176
177 if not _should_retry_response(resp.status, content):
178 break
179
180 return resp, content
181
208
234
377
502
569
598
690
693 """Truncated stream.
694
695 Takes a stream and presents a stream that is a slice of the original stream.
696 This is used when uploading media in chunks. In later versions of Python a
697 stream can be passed to httplib in place of the string of data to send. The
698 problem is that httplib just blindly reads to the end of the stream. This
699 wrapper presents a virtual stream that only reads to the end of the chunk.
700 """
701
702 - def __init__(self, stream, begin, chunksize):
703 """Constructor.
704
705 Args:
706 stream: (io.Base, file object), the stream to wrap.
707 begin: int, the seek position the chunk begins at.
708 chunksize: int, the size of the chunk.
709 """
710 self._stream = stream
711 self._begin = begin
712 self._chunksize = chunksize
713 self._stream.seek(begin)
714
715 - def read(self, n=-1):
716 """Read n bytes.
717
718 Args:
719 n, int, the number of bytes to read.
720
721 Returns:
722 A string of length 'n', or less if EOF is reached.
723 """
724
725 cur = self._stream.tell()
726 end = self._begin + self._chunksize
727 if n == -1 or cur + n > end:
728 n = end - cur
729 return self._stream.read(n)
730
733 """Encapsulates a single HTTP request."""
734
735 @util.positional(4)
736 - def __init__(self, http, postproc, uri,
737 method='GET',
738 body=None,
739 headers=None,
740 methodId=None,
741 resumable=None):
742 """Constructor for an HttpRequest.
743
744 Args:
745 http: httplib2.Http, the transport object to use to make a request
746 postproc: callable, called on the HTTP response and content to transform
747 it into a data object before returning, or raising an exception
748 on an error.
749 uri: string, the absolute URI to send the request to
750 method: string, the HTTP method to use
751 body: string, the request body of the HTTP request,
752 headers: dict, the HTTP request headers
753 methodId: string, a unique identifier for the API method being called.
754 resumable: MediaUpload, None if this is not a resumbale request.
755 """
756 self.uri = uri
757 self.method = method
758 self.body = body
759 self.headers = headers or {}
760 self.methodId = methodId
761 self.http = http
762 self.postproc = postproc
763 self.resumable = resumable
764 self.response_callbacks = []
765 self._in_error_state = False
766
767
768 major, minor, params = mimeparse.parse_mime_type(
769 self.headers.get('content-type', 'application/json'))
770
771
772 self.body_size = len(self.body or '')
773
774
775 self.resumable_uri = None
776
777
778 self.resumable_progress = 0
779
780
781 self._rand = random.random
782 self._sleep = time.sleep
783
784 @util.positional(1)
785 - def execute(self, http=None, num_retries=0):
786 """Execute the request.
787
788 Args:
789 http: httplib2.Http, an http object to be used in place of the
790 one the HttpRequest request object was constructed with.
791 num_retries: Integer, number of times to retry with randomized
792 exponential backoff. If all retries fail, the raised HttpError
793 represents the last request. If zero (default), we attempt the
794 request only once.
795
796 Returns:
797 A deserialized object model of the response body as determined
798 by the postproc.
799
800 Raises:
801 googleapiclient.errors.HttpError if the response was not a 2xx.
802 httplib2.HttpLib2Error if a transport error has occured.
803 """
804 if http is None:
805 http = self.http
806
807 if self.resumable:
808 body = None
809 while body is None:
810 _, body = self.next_chunk(http=http, num_retries=num_retries)
811 return body
812
813
814
815 if 'content-length' not in self.headers:
816 self.headers['content-length'] = str(self.body_size)
817
818 if len(self.uri) > MAX_URI_LENGTH and self.method == 'GET':
819 self.method = 'POST'
820 self.headers['x-http-method-override'] = 'GET'
821 self.headers['content-type'] = 'application/x-www-form-urlencoded'
822 parsed = urlparse(self.uri)
823 self.uri = urlunparse(
824 (parsed.scheme, parsed.netloc, parsed.path, parsed.params, None,
825 None)
826 )
827 self.body = parsed.query
828 self.headers['content-length'] = str(len(self.body))
829
830
831 resp, content = _retry_request(
832 http, num_retries, 'request', self._sleep, self._rand, str(self.uri),
833 method=str(self.method), body=self.body, headers=self.headers)
834
835 for callback in self.response_callbacks:
836 callback(resp)
837 if resp.status >= 300:
838 raise HttpError(resp, content, uri=self.uri)
839 return self.postproc(resp, content)
840
841 @util.positional(2)
843 """add_response_headers_callback
844
845 Args:
846 cb: Callback to be called on receiving the response headers, of signature:
847
848 def cb(resp):
849 # Where resp is an instance of httplib2.Response
850 """
851 self.response_callbacks.append(cb)
852
853 @util.positional(1)
855 """Execute the next step of a resumable upload.
856
857 Can only be used if the method being executed supports media uploads and
858 the MediaUpload object passed in was flagged as using resumable upload.
859
860 Example:
861
862 media = MediaFileUpload('cow.png', mimetype='image/png',
863 chunksize=1000, resumable=True)
864 request = farm.animals().insert(
865 id='cow',
866 name='cow.png',
867 media_body=media)
868
869 response = None
870 while response is None:
871 status, response = request.next_chunk()
872 if status:
873 print "Upload %d%% complete." % int(status.progress() * 100)
874
875
876 Args:
877 http: httplib2.Http, an http object to be used in place of the
878 one the HttpRequest request object was constructed with.
879 num_retries: Integer, number of times to retry with randomized
880 exponential backoff. If all retries fail, the raised HttpError
881 represents the last request. If zero (default), we attempt the
882 request only once.
883
884 Returns:
885 (status, body): (ResumableMediaStatus, object)
886 The body will be None until the resumable media is fully uploaded.
887
888 Raises:
889 googleapiclient.errors.HttpError if the response was not a 2xx.
890 httplib2.HttpLib2Error if a transport error has occured.
891 """
892 if http is None:
893 http = self.http
894
895 if self.resumable.size() is None:
896 size = '*'
897 else:
898 size = str(self.resumable.size())
899
900 if self.resumable_uri is None:
901 start_headers = copy.copy(self.headers)
902 start_headers['X-Upload-Content-Type'] = self.resumable.mimetype()
903 if size != '*':
904 start_headers['X-Upload-Content-Length'] = size
905 start_headers['content-length'] = str(self.body_size)
906
907 resp, content = _retry_request(
908 http, num_retries, 'resumable URI request', self._sleep, self._rand,
909 self.uri, method=self.method, body=self.body, headers=start_headers)
910
911 if resp.status == 200 and 'location' in resp:
912 self.resumable_uri = resp['location']
913 else:
914 raise ResumableUploadError(resp, content)
915 elif self._in_error_state:
916
917
918
919 headers = {
920 'Content-Range': 'bytes */%s' % size,
921 'content-length': '0'
922 }
923 resp, content = http.request(self.resumable_uri, 'PUT',
924 headers=headers)
925 status, body = self._process_response(resp, content)
926 if body:
927
928 return (status, body)
929
930 if self.resumable.has_stream():
931 data = self.resumable.stream()
932 if self.resumable.chunksize() == -1:
933 data.seek(self.resumable_progress)
934 chunk_end = self.resumable.size() - self.resumable_progress - 1
935 else:
936
937 data = _StreamSlice(data, self.resumable_progress,
938 self.resumable.chunksize())
939 chunk_end = min(
940 self.resumable_progress + self.resumable.chunksize() - 1,
941 self.resumable.size() - 1)
942 else:
943 data = self.resumable.getbytes(
944 self.resumable_progress, self.resumable.chunksize())
945
946
947 if len(data) < self.resumable.chunksize():
948 size = str(self.resumable_progress + len(data))
949
950 chunk_end = self.resumable_progress + len(data) - 1
951
952 headers = {
953 'Content-Range': 'bytes %d-%d/%s' % (
954 self.resumable_progress, chunk_end, size),
955
956
957 'Content-Length': str(chunk_end - self.resumable_progress + 1)
958 }
959
960 for retry_num in range(num_retries + 1):
961 if retry_num > 0:
962 self._sleep(self._rand() * 2**retry_num)
963 LOGGER.warning(
964 'Retry #%d for media upload: %s %s, following status: %d'
965 % (retry_num, self.method, self.uri, resp.status))
966
967 try:
968 resp, content = http.request(self.resumable_uri, method='PUT',
969 body=data,
970 headers=headers)
971 except:
972 self._in_error_state = True
973 raise
974 if not _should_retry_response(resp.status, content):
975 break
976
977 return self._process_response(resp, content)
978
980 """Process the response from a single chunk upload.
981
982 Args:
983 resp: httplib2.Response, the response object.
984 content: string, the content of the response.
985
986 Returns:
987 (status, body): (ResumableMediaStatus, object)
988 The body will be None until the resumable media is fully uploaded.
989
990 Raises:
991 googleapiclient.errors.HttpError if the response was not a 2xx or a 308.
992 """
993 if resp.status in [200, 201]:
994 self._in_error_state = False
995 return None, self.postproc(resp, content)
996 elif resp.status == 308:
997 self._in_error_state = False
998
999 self.resumable_progress = int(resp['range'].split('-')[1]) + 1
1000 if 'location' in resp:
1001 self.resumable_uri = resp['location']
1002 else:
1003 self._in_error_state = True
1004 raise HttpError(resp, content, uri=self.uri)
1005
1006 return (MediaUploadProgress(self.resumable_progress, self.resumable.size()),
1007 None)
1008
1010 """Returns a JSON representation of the HttpRequest."""
1011 d = copy.copy(self.__dict__)
1012 if d['resumable'] is not None:
1013 d['resumable'] = self.resumable.to_json()
1014 del d['http']
1015 del d['postproc']
1016 del d['_sleep']
1017 del d['_rand']
1018
1019 return json.dumps(d)
1020
1021 @staticmethod
1023 """Returns an HttpRequest populated with info from a JSON object."""
1024 d = json.loads(s)
1025 if d['resumable'] is not None:
1026 d['resumable'] = MediaUpload.new_from_json(d['resumable'])
1027 return HttpRequest(
1028 http,
1029 postproc,
1030 uri=d['uri'],
1031 method=d['method'],
1032 body=d['body'],
1033 headers=d['headers'],
1034 methodId=d['methodId'],
1035 resumable=d['resumable'])
1036
1039 """Batches multiple HttpRequest objects into a single HTTP request.
1040
1041 Example:
1042 from googleapiclient.http import BatchHttpRequest
1043
1044 def list_animals(request_id, response, exception):
1045 \"\"\"Do something with the animals list response.\"\"\"
1046 if exception is not None:
1047 # Do something with the exception.
1048 pass
1049 else:
1050 # Do something with the response.
1051 pass
1052
1053 def list_farmers(request_id, response, exception):
1054 \"\"\"Do something with the farmers list response.\"\"\"
1055 if exception is not None:
1056 # Do something with the exception.
1057 pass
1058 else:
1059 # Do something with the response.
1060 pass
1061
1062 service = build('farm', 'v2')
1063
1064 batch = BatchHttpRequest()
1065
1066 batch.add(service.animals().list(), list_animals)
1067 batch.add(service.farmers().list(), list_farmers)
1068 batch.execute(http=http)
1069 """
1070
1071 @util.positional(1)
1072 - def __init__(self, callback=None, batch_uri=None):
1073 """Constructor for a BatchHttpRequest.
1074
1075 Args:
1076 callback: callable, A callback to be called for each response, of the
1077 form callback(id, response, exception). The first parameter is the
1078 request id, and the second is the deserialized response object. The
1079 third is an googleapiclient.errors.HttpError exception object if an HTTP error
1080 occurred while processing the request, or None if no error occurred.
1081 batch_uri: string, URI to send batch requests to.
1082 """
1083 if batch_uri is None:
1084 batch_uri = 'https://www.googleapis.com/batch'
1085 self._batch_uri = batch_uri
1086
1087
1088 self._callback = callback
1089
1090
1091 self._requests = {}
1092
1093
1094 self._callbacks = {}
1095
1096
1097 self._order = []
1098
1099
1100 self._last_auto_id = 0
1101
1102
1103 self._base_id = None
1104
1105
1106 self._responses = {}
1107
1108
1109 self._refreshed_credentials = {}
1110
1112 """Refresh the credentials and apply to the request.
1113
1114 Args:
1115 request: HttpRequest, the request.
1116 http: httplib2.Http, the global http object for the batch.
1117 """
1118
1119
1120
1121 creds = None
1122 if request.http is not None and hasattr(request.http.request,
1123 'credentials'):
1124 creds = request.http.request.credentials
1125 elif http is not None and hasattr(http.request, 'credentials'):
1126 creds = http.request.credentials
1127 if creds is not None:
1128 if id(creds) not in self._refreshed_credentials:
1129 creds.refresh(http)
1130 self._refreshed_credentials[id(creds)] = 1
1131
1132
1133
1134 if request.http is None or not hasattr(request.http.request,
1135 'credentials'):
1136 creds.apply(request.headers)
1137
1139 """Convert an id to a Content-ID header value.
1140
1141 Args:
1142 id_: string, identifier of individual request.
1143
1144 Returns:
1145 A Content-ID header with the id_ encoded into it. A UUID is prepended to
1146 the value because Content-ID headers are supposed to be universally
1147 unique.
1148 """
1149 if self._base_id is None:
1150 self._base_id = uuid.uuid4()
1151
1152 return '<%s+%s>' % (self._base_id, quote(id_))
1153
1155 """Convert a Content-ID header value to an id.
1156
1157 Presumes the Content-ID header conforms to the format that _id_to_header()
1158 returns.
1159
1160 Args:
1161 header: string, Content-ID header value.
1162
1163 Returns:
1164 The extracted id value.
1165
1166 Raises:
1167 BatchError if the header is not in the expected format.
1168 """
1169 if header[0] != '<' or header[-1] != '>':
1170 raise BatchError("Invalid value for Content-ID: %s" % header)
1171 if '+' not in header:
1172 raise BatchError("Invalid value for Content-ID: %s" % header)
1173 base, id_ = header[1:-1].rsplit('+', 1)
1174
1175 return unquote(id_)
1176
1178 """Convert an HttpRequest object into a string.
1179
1180 Args:
1181 request: HttpRequest, the request to serialize.
1182
1183 Returns:
1184 The request as a string in application/http format.
1185 """
1186
1187 parsed = urlparse(request.uri)
1188 request_line = urlunparse(
1189 ('', '', parsed.path, parsed.params, parsed.query, '')
1190 )
1191 status_line = request.method + ' ' + request_line + ' HTTP/1.1\n'
1192 major, minor = request.headers.get('content-type', 'application/json').split('/')
1193 msg = MIMENonMultipart(major, minor)
1194 headers = request.headers.copy()
1195
1196 if request.http is not None and hasattr(request.http.request,
1197 'credentials'):
1198 request.http.request.credentials.apply(headers)
1199
1200
1201 if 'content-type' in headers:
1202 del headers['content-type']
1203
1204 for key, value in six.iteritems(headers):
1205 msg[key] = value
1206 msg['Host'] = parsed.netloc
1207 msg.set_unixfrom(None)
1208
1209 if request.body is not None:
1210 msg.set_payload(request.body)
1211 msg['content-length'] = str(len(request.body))
1212
1213
1214 fp = StringIO()
1215
1216 g = Generator(fp, maxheaderlen=0)
1217 g.flatten(msg, unixfrom=False)
1218 body = fp.getvalue()
1219
1220 return status_line + body
1221
1223 """Convert string into httplib2 response and content.
1224
1225 Args:
1226 payload: string, headers and body as a string.
1227
1228 Returns:
1229 A pair (resp, content), such as would be returned from httplib2.request.
1230 """
1231
1232 status_line, payload = payload.split('\n', 1)
1233 protocol, status, reason = status_line.split(' ', 2)
1234
1235
1236 parser = FeedParser()
1237 parser.feed(payload)
1238 msg = parser.close()
1239 msg['status'] = status
1240
1241
1242 resp = httplib2.Response(msg)
1243 resp.reason = reason
1244 resp.version = int(protocol.split('/', 1)[1].replace('.', ''))
1245
1246 content = payload.split('\r\n\r\n', 1)[1]
1247
1248 return resp, content
1249
1251 """Create a new id.
1252
1253 Auto incrementing number that avoids conflicts with ids already used.
1254
1255 Returns:
1256 string, a new unique id.
1257 """
1258 self._last_auto_id += 1
1259 while str(self._last_auto_id) in self._requests:
1260 self._last_auto_id += 1
1261 return str(self._last_auto_id)
1262
1263 @util.positional(2)
1264 - def add(self, request, callback=None, request_id=None):
1265 """Add a new request.
1266
1267 Every callback added will be paired with a unique id, the request_id. That
1268 unique id will be passed back to the callback when the response comes back
1269 from the server. The default behavior is to have the library generate it's
1270 own unique id. If the caller passes in a request_id then they must ensure
1271 uniqueness for each request_id, and if they are not an exception is
1272 raised. Callers should either supply all request_ids or nevery supply a
1273 request id, to avoid such an error.
1274
1275 Args:
1276 request: HttpRequest, Request to add to the batch.
1277 callback: callable, A callback to be called for this response, of the
1278 form callback(id, response, exception). The first parameter is the
1279 request id, and the second is the deserialized response object. The
1280 third is an googleapiclient.errors.HttpError exception object if an HTTP error
1281 occurred while processing the request, or None if no errors occurred.
1282 request_id: string, A unique id for the request. The id will be passed to
1283 the callback with the response.
1284
1285 Returns:
1286 None
1287
1288 Raises:
1289 BatchError if a media request is added to a batch.
1290 KeyError is the request_id is not unique.
1291 """
1292 if request_id is None:
1293 request_id = self._new_id()
1294 if request.resumable is not None:
1295 raise BatchError("Media requests cannot be used in a batch request.")
1296 if request_id in self._requests:
1297 raise KeyError("A request with this ID already exists: %s" % request_id)
1298 self._requests[request_id] = request
1299 self._callbacks[request_id] = callback
1300 self._order.append(request_id)
1301
1302 - def _execute(self, http, order, requests):
1303 """Serialize batch request, send to server, process response.
1304
1305 Args:
1306 http: httplib2.Http, an http object to be used to make the request with.
1307 order: list, list of request ids in the order they were added to the
1308 batch.
1309 request: list, list of request objects to send.
1310
1311 Raises:
1312 httplib2.HttpLib2Error if a transport error has occured.
1313 googleapiclient.errors.BatchError if the response is the wrong format.
1314 """
1315 message = MIMEMultipart('mixed')
1316
1317 setattr(message, '_write_headers', lambda self: None)
1318
1319
1320 for request_id in order:
1321 request = requests[request_id]
1322
1323 msg = MIMENonMultipart('application', 'http')
1324 msg['Content-Transfer-Encoding'] = 'binary'
1325 msg['Content-ID'] = self._id_to_header(request_id)
1326
1327 body = self._serialize_request(request)
1328 msg.set_payload(body)
1329 message.attach(msg)
1330
1331
1332
1333 fp = StringIO()
1334 g = Generator(fp, mangle_from_=False)
1335 g.flatten(message, unixfrom=False)
1336 body = fp.getvalue()
1337
1338 headers = {}
1339 headers['content-type'] = ('multipart/mixed; '
1340 'boundary="%s"') % message.get_boundary()
1341
1342 resp, content = http.request(self._batch_uri, method='POST', body=body,
1343 headers=headers)
1344
1345 if resp.status >= 300:
1346 raise HttpError(resp, content, uri=self._batch_uri)
1347
1348
1349 header = 'content-type: %s\r\n\r\n' % resp['content-type']
1350
1351
1352 if six.PY3:
1353 content = content.decode('utf-8')
1354 for_parser = header + content
1355
1356 parser = FeedParser()
1357 parser.feed(for_parser)
1358 mime_response = parser.close()
1359
1360 if not mime_response.is_multipart():
1361 raise BatchError("Response not in multipart/mixed format.", resp=resp,
1362 content=content)
1363
1364 for part in mime_response.get_payload():
1365 request_id = self._header_to_id(part['Content-ID'])
1366 response, content = self._deserialize_response(part.get_payload())
1367
1368 if isinstance(content, six.text_type):
1369 content = content.encode('utf-8')
1370 self._responses[request_id] = (response, content)
1371
1372 @util.positional(1)
1374 """Execute all the requests as a single batched HTTP request.
1375
1376 Args:
1377 http: httplib2.Http, an http object to be used in place of the one the
1378 HttpRequest request object was constructed with. If one isn't supplied
1379 then use a http object from the requests in this batch.
1380
1381 Returns:
1382 None
1383
1384 Raises:
1385 httplib2.HttpLib2Error if a transport error has occured.
1386 googleapiclient.errors.BatchError if the response is the wrong format.
1387 """
1388
1389 if len(self._order) == 0:
1390 return None
1391
1392
1393 if http is None:
1394 for request_id in self._order:
1395 request = self._requests[request_id]
1396 if request is not None:
1397 http = request.http
1398 break
1399
1400 if http is None:
1401 raise ValueError("Missing a valid http object.")
1402
1403
1404
1405 if getattr(http.request, 'credentials', None) is not None:
1406 creds = http.request.credentials
1407 if not getattr(creds, 'access_token', None):
1408 LOGGER.info('Attempting refresh to obtain initial access_token')
1409 creds.refresh(http)
1410
1411 self._execute(http, self._order, self._requests)
1412
1413
1414
1415 redo_requests = {}
1416 redo_order = []
1417
1418 for request_id in self._order:
1419 resp, content = self._responses[request_id]
1420 if resp['status'] == '401':
1421 redo_order.append(request_id)
1422 request = self._requests[request_id]
1423 self._refresh_and_apply_credentials(request, http)
1424 redo_requests[request_id] = request
1425
1426 if redo_requests:
1427 self._execute(http, redo_order, redo_requests)
1428
1429
1430
1431
1432
1433 for request_id in self._order:
1434 resp, content = self._responses[request_id]
1435
1436 request = self._requests[request_id]
1437 callback = self._callbacks[request_id]
1438
1439 response = None
1440 exception = None
1441 try:
1442 if resp.status >= 300:
1443 raise HttpError(resp, content, uri=request.uri)
1444 response = request.postproc(resp, content)
1445 except HttpError as e:
1446 exception = e
1447
1448 if callback is not None:
1449 callback(request_id, response, exception)
1450 if self._callback is not None:
1451 self._callback(request_id, response, exception)
1452
1455 """Mock of HttpRequest.
1456
1457 Do not construct directly, instead use RequestMockBuilder.
1458 """
1459
1460 - def __init__(self, resp, content, postproc):
1461 """Constructor for HttpRequestMock
1462
1463 Args:
1464 resp: httplib2.Response, the response to emulate coming from the request
1465 content: string, the response body
1466 postproc: callable, the post processing function usually supplied by
1467 the model class. See model.JsonModel.response() as an example.
1468 """
1469 self.resp = resp
1470 self.content = content
1471 self.postproc = postproc
1472 if resp is None:
1473 self.resp = httplib2.Response({'status': 200, 'reason': 'OK'})
1474 if 'reason' in self.resp:
1475 self.resp.reason = self.resp['reason']
1476
1478 """Execute the request.
1479
1480 Same behavior as HttpRequest.execute(), but the response is
1481 mocked and not really from an HTTP request/response.
1482 """
1483 return self.postproc(self.resp, self.content)
1484
1487 """A simple mock of HttpRequest
1488
1489 Pass in a dictionary to the constructor that maps request methodIds to
1490 tuples of (httplib2.Response, content, opt_expected_body) that should be
1491 returned when that method is called. None may also be passed in for the
1492 httplib2.Response, in which case a 200 OK response will be generated.
1493 If an opt_expected_body (str or dict) is provided, it will be compared to
1494 the body and UnexpectedBodyError will be raised on inequality.
1495
1496 Example:
1497 response = '{"data": {"id": "tag:google.c...'
1498 requestBuilder = RequestMockBuilder(
1499 {
1500 'plus.activities.get': (None, response),
1501 }
1502 )
1503 googleapiclient.discovery.build("plus", "v1", requestBuilder=requestBuilder)
1504
1505 Methods that you do not supply a response for will return a
1506 200 OK with an empty string as the response content or raise an excpetion
1507 if check_unexpected is set to True. The methodId is taken from the rpcName
1508 in the discovery document.
1509
1510 For more details see the project wiki.
1511 """
1512
1513 - def __init__(self, responses, check_unexpected=False):
1514 """Constructor for RequestMockBuilder
1515
1516 The constructed object should be a callable object
1517 that can replace the class HttpResponse.
1518
1519 responses - A dictionary that maps methodIds into tuples
1520 of (httplib2.Response, content). The methodId
1521 comes from the 'rpcName' field in the discovery
1522 document.
1523 check_unexpected - A boolean setting whether or not UnexpectedMethodError
1524 should be raised on unsupplied method.
1525 """
1526 self.responses = responses
1527 self.check_unexpected = check_unexpected
1528
1529 - def __call__(self, http, postproc, uri, method='GET', body=None,
1530 headers=None, methodId=None, resumable=None):
1531 """Implements the callable interface that discovery.build() expects
1532 of requestBuilder, which is to build an object compatible with
1533 HttpRequest.execute(). See that method for the description of the
1534 parameters and the expected response.
1535 """
1536 if methodId in self.responses:
1537 response = self.responses[methodId]
1538 resp, content = response[:2]
1539 if len(response) > 2:
1540
1541 expected_body = response[2]
1542 if bool(expected_body) != bool(body):
1543
1544
1545 raise UnexpectedBodyError(expected_body, body)
1546 if isinstance(expected_body, str):
1547 expected_body = json.loads(expected_body)
1548 body = json.loads(body)
1549 if body != expected_body:
1550 raise UnexpectedBodyError(expected_body, body)
1551 return HttpRequestMock(resp, content, postproc)
1552 elif self.check_unexpected:
1553 raise UnexpectedMethodError(methodId=methodId)
1554 else:
1555 model = JsonModel(False)
1556 return HttpRequestMock(None, '{}', model.response)
1557
1560 """Mock of httplib2.Http"""
1561
1562 - def __init__(self, filename=None, headers=None):
1563 """
1564 Args:
1565 filename: string, absolute filename to read response from
1566 headers: dict, header to return with response
1567 """
1568 if headers is None:
1569 headers = {'status': '200'}
1570 if filename:
1571 f = open(filename, 'rb')
1572 self.data = f.read()
1573 f.close()
1574 else:
1575 self.data = None
1576 self.response_headers = headers
1577 self.headers = None
1578 self.uri = None
1579 self.method = None
1580 self.body = None
1581 self.headers = None
1582
1583
1584 - def request(self, uri,
1585 method='GET',
1586 body=None,
1587 headers=None,
1588 redirections=1,
1589 connection_type=None):
1590 self.uri = uri
1591 self.method = method
1592 self.body = body
1593 self.headers = headers
1594 return httplib2.Response(self.response_headers), self.data
1595
1598 """Mock of httplib2.Http
1599
1600 Mocks a sequence of calls to request returning different responses for each
1601 call. Create an instance initialized with the desired response headers
1602 and content and then use as if an httplib2.Http instance.
1603
1604 http = HttpMockSequence([
1605 ({'status': '401'}, ''),
1606 ({'status': '200'}, '{"access_token":"1/3w","expires_in":3600}'),
1607 ({'status': '200'}, 'echo_request_headers'),
1608 ])
1609 resp, content = http.request("http://examples.com")
1610
1611 There are special values you can pass in for content to trigger
1612 behavours that are helpful in testing.
1613
1614 'echo_request_headers' means return the request headers in the response body
1615 'echo_request_headers_as_json' means return the request headers in
1616 the response body
1617 'echo_request_body' means return the request body in the response body
1618 'echo_request_uri' means return the request uri in the response body
1619 """
1620
1622 """
1623 Args:
1624 iterable: iterable, a sequence of pairs of (headers, body)
1625 """
1626 self._iterable = iterable
1627 self.follow_redirects = True
1628
1629 - def request(self, uri,
1630 method='GET',
1631 body=None,
1632 headers=None,
1633 redirections=1,
1634 connection_type=None):
1635 resp, content = self._iterable.pop(0)
1636 if content == 'echo_request_headers':
1637 content = headers
1638 elif content == 'echo_request_headers_as_json':
1639 content = json.dumps(headers)
1640 elif content == 'echo_request_body':
1641 if hasattr(body, 'read'):
1642 content = body.read()
1643 else:
1644 content = body
1645 elif content == 'echo_request_uri':
1646 content = uri
1647 if isinstance(content, six.text_type):
1648 content = content.encode('utf-8')
1649 return httplib2.Response(resp), content
1650
1653 """Set the user-agent on every request.
1654
1655 Args:
1656 http - An instance of httplib2.Http
1657 or something that acts like it.
1658 user_agent: string, the value for the user-agent header.
1659
1660 Returns:
1661 A modified instance of http that was passed in.
1662
1663 Example:
1664
1665 h = httplib2.Http()
1666 h = set_user_agent(h, "my-app-name/6.0")
1667
1668 Most of the time the user-agent will be set doing auth, this is for the rare
1669 cases where you are accessing an unauthenticated endpoint.
1670 """
1671 request_orig = http.request
1672
1673
1674 def new_request(uri, method='GET', body=None, headers=None,
1675 redirections=httplib2.DEFAULT_MAX_REDIRECTS,
1676 connection_type=None):
1677 """Modify the request headers to add the user-agent."""
1678 if headers is None:
1679 headers = {}
1680 if 'user-agent' in headers:
1681 headers['user-agent'] = user_agent + ' ' + headers['user-agent']
1682 else:
1683 headers['user-agent'] = user_agent
1684 resp, content = request_orig(uri, method, body, headers,
1685 redirections, connection_type)
1686 return resp, content
1687
1688 http.request = new_request
1689 return http
1690
1693 """Tunnel PATCH requests over POST.
1694 Args:
1695 http - An instance of httplib2.Http
1696 or something that acts like it.
1697
1698 Returns:
1699 A modified instance of http that was passed in.
1700
1701 Example:
1702
1703 h = httplib2.Http()
1704 h = tunnel_patch(h, "my-app-name/6.0")
1705
1706 Useful if you are running on a platform that doesn't support PATCH.
1707 Apply this last if you are using OAuth 1.0, as changing the method
1708 will result in a different signature.
1709 """
1710 request_orig = http.request
1711
1712
1713 def new_request(uri, method='GET', body=None, headers=None,
1714 redirections=httplib2.DEFAULT_MAX_REDIRECTS,
1715 connection_type=None):
1716 """Modify the request headers to add the user-agent."""
1717 if headers is None:
1718 headers = {}
1719 if method == 'PATCH':
1720 if 'oauth_token' in headers.get('authorization', ''):
1721 LOGGER.warning(
1722 'OAuth 1.0 request made with Credentials after tunnel_patch.')
1723 headers['x-http-method-override'] = "PATCH"
1724 method = 'POST'
1725 resp, content = request_orig(uri, method, body, headers,
1726 redirections, connection_type)
1727 return resp, content
1728
1729 http.request = new_request
1730 return http
1731