You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
Describe the problem or feature request clearly here.
When I try to invoke my newly created endoint, I get a 104 error.
Is this a timeout issue? It is a fairly large network
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, **httplib_request_kw)
348 # urllib3.request. It also calls makefile (recv) on the socket.
--> 349 conn.request(method, url, **httplib_request_kw)
350
~/anaconda3/envs/tensorflow_p36/lib/python3.6/http/client.py in request(self, method, url, body, headers, encode_chunked)
1238 """Send a complete request to the server."""
-> 1239 self._send_request(method, url, body, headers, encode_chunked)
1240
~/anaconda3/envs/tensorflow_p36/lib/python3.6/ssl.py in sendall(self, data, flags)
971 while count < amount:
--> 972 v = self.send(byte_view[count:])
973 count += v
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/vendored/requests/packages/urllib3/util/retry.py in increment(self, method, url, response, error, _pool, _stacktrace)
244 if read is False:
--> 245 raise six.reraise(type(error), error, _stacktrace)
246 elif read is not None:
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/vendored/requests/packages/urllib3/packages/six.py in reraise(tp, value, tb)
308 if value.traceback is not tb:
--> 309 raise value.with_traceback(tb)
310 raise value
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, **httplib_request_kw)
348 # urllib3.request. It also calls makefile (recv) on the socket.
--> 349 conn.request(method, url, **httplib_request_kw)
350
~/anaconda3/envs/tensorflow_p36/lib/python3.6/http/client.py in request(self, method, url, body, headers, encode_chunked)
1238 """Send a complete request to the server."""
-> 1239 self._send_request(method, url, body, headers, encode_chunked)
1240
~/anaconda3/envs/tensorflow_p36/lib/python3.6/ssl.py in sendall(self, data, flags)
971 while count < amount:
--> 972 v = self.send(byte_view[count:])
973 count += v
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/client.py in _api_call(self, *args, **kwargs)
312 "%s() only accepts keyword arguments." % py_operation_name)
313 # The "self" in this scope is referring to the BaseClient.
--> 314 return self._make_api_call(operation_name, kwargs)
315
316 _api_call.name = str(py_operation_name)
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/endpoint.py in make_request(self, operation_model, request_dict)
146 logger.debug("Making request for %s (verify_ssl=%s) with params: %s",
147 operation_model, self.verify, request_dict)
--> 148 return self._send_request(request_dict, operation_model)
149
150 def create_request(self, params, operation_model=None):
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/endpoint.py in _send_request(self, request_dict, operation_model)
175 request, operation_model, attempts)
176 while self._needs_retry(attempts, operation_model, request_dict,
--> 177 success_response, exception):
178 attempts += 1
179 # If there is a stream associated with the request, we need
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/endpoint.py in _needs_retry(self, attempts, operation_model, request_dict, response, caught_exception)
271 event_name, response=response, endpoint=self,
272 operation=operation_model, attempts=attempts,
--> 273 caught_exception=caught_exception, request_dict=request_dict)
274 handler_response = first_non_none_response(responses)
275 if handler_response is None:
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/hooks.py in _emit(self, event_name, kwargs, stop_on_response)
208 for handler in handlers_to_call:
209 logger.debug('Event %s: calling handler %s', event_name, handler)
--> 210 response = handler(**kwargs)
211 responses.append((handler, response))
212 if stop_on_response and response is not None:
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/retryhandler.py in call(self, attempts, response, caught_exception, **kwargs)
181
182 """
--> 183 if self._checker(attempts, response, caught_exception):
184 result = self._action(attempts=attempts)
185 logger.debug("Retry needed, action of: %s", result)
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/retryhandler.py in call(self, attempt_number, response, caught_exception)
249 def call(self, attempt_number, response, caught_exception):
250 should_retry = self._should_retry(attempt_number, response,
--> 251 caught_exception)
252 if should_retry:
253 if attempt_number >= self._max_attempts:
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/retryhandler.py in _should_retry(self, attempt_number, response, caught_exception)
275 # If we've exceeded the max attempts we just let the exception
276 # propogate if one has occurred.
--> 277 return self._checker(attempt_number, response, caught_exception)
278
279
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/retryhandler.py in call(self, attempt_number, response, caught_exception)
315 for checker in self._checkers:
316 checker_response = checker(attempt_number, response,
--> 317 caught_exception)
318 if checker_response:
319 return checker_response
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/retryhandler.py in call(self, attempt_number, response, caught_exception)
221 elif caught_exception is not None:
222 return self._check_caught_exception(
--> 223 attempt_number, caught_exception)
224 else:
225 raise ValueError("Both response and caught_exception are None.")
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/retryhandler.py in _check_caught_exception(self, attempt_number, caught_exception)
357 # the MaxAttemptsDecorator is not interested in retrying the exception
358 # then this exception just propogates out past the retry code.
--> 359 raise caught_exception
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/endpoint.py in _get_response(self, request, operation_model, attempts)
220 request, verify=self.verify,
221 stream=streaming,
--> 222 proxies=self.proxies, timeout=self.timeout)
223 except ConnectionError as e:
224 # For a connection error, if it looks like it's a DNS
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/vendored/requests/sessions.py in send(self, request, **kwargs)
571
572 # Send the request
--> 573 r = adapter.send(request, **kwargs)
574
575 # Total elapsed time of the request (approximately)
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/vendored/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
413
414 except (ProtocolError, socket.error) as err:
--> 415 raise ConnectionError(err, request=request)
416
417 except MaxRetryError as e:
ConnectionError: ('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer'))`
Exact command to reproduce:
endpoint = my_estimator.deploy(initial_instance_count=1, instance_type="ml.m4.xlarge")
endpoint.predict(test_dict)
The text was updated successfully, but these errors were encountered:
I'm closing this ticket because it looks like a duplicate of aws/amazon-sagemaker-examples#324. If you are still having trouble, feel free to reopen this.
Please fill out the form below.
System Information
Describe the problem
Describe the problem or feature request clearly here.
When I try to invoke my newly created endoint, I get a 104 error.
Is this a timeout issue? It is a fairly large network
Minimal repro / logs
`---------------------------------------------------------------------------
ConnectionResetError Traceback (most recent call last)
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, **response_kw)
543 timeout=timeout_obj,
--> 544 body=body, headers=headers)
545
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, **httplib_request_kw)
348 # urllib3.request. It also calls makefile (recv) on the socket.
--> 349 conn.request(method, url, **httplib_request_kw)
350
~/anaconda3/envs/tensorflow_p36/lib/python3.6/http/client.py in request(self, method, url, body, headers, encode_chunked)
1238 """Send a complete request to the server."""
-> 1239 self._send_request(method, url, body, headers, encode_chunked)
1240
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/awsrequest.py in _send_request(self, method, url, body, headers, *args, **kwargs)
129 rval = HTTPConnection._send_request(
--> 130 self, method, url, body, headers, *args, **kwargs)
131 self._expect_header_set = False
~/anaconda3/envs/tensorflow_p36/lib/python3.6/http/client.py in _send_request(self, method, url, body, headers, encode_chunked)
1284 body = _encode(body, 'body')
-> 1285 self.endheaders(body, encode_chunked=encode_chunked)
1286
~/anaconda3/envs/tensorflow_p36/lib/python3.6/http/client.py in endheaders(self, message_body, encode_chunked)
1233 raise CannotSendHeader()
-> 1234 self._send_output(message_body, encode_chunked=encode_chunked)
1235
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/awsrequest.py in _send_output(self, message_body, *args, **kwargs)
156 message_body = None
--> 157 self.send(msg)
158 if self._expect_header_set:
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/awsrequest.py in send(self, str)
241 return
--> 242 return HTTPConnection.send(self, str)
243
~/anaconda3/envs/tensorflow_p36/lib/python3.6/http/client.py in send(self, data)
985 try:
--> 986 self.sock.sendall(data)
987 except TypeError:
~/anaconda3/envs/tensorflow_p36/lib/python3.6/ssl.py in sendall(self, data, flags)
971 while count < amount:
--> 972 v = self.send(byte_view[count:])
973 count += v
~/anaconda3/envs/tensorflow_p36/lib/python3.6/ssl.py in send(self, data, flags)
940 self.class)
--> 941 return self._sslobj.write(data)
942 else:
~/anaconda3/envs/tensorflow_p36/lib/python3.6/ssl.py in write(self, data)
641 """
--> 642 return self._sslobj.write(data)
643
ConnectionResetError: [Errno 104] Connection reset by peer
During handling of the above exception, another exception occurred:
ProtocolError Traceback (most recent call last)
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/vendored/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
369 retries=self.max_retries,
--> 370 timeout=timeout
371 )
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, **response_kw)
596 retries = retries.increment(method, url, error=e, _pool=self,
--> 597 _stacktrace=sys.exc_info()[2])
598 retries.sleep()
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/vendored/requests/packages/urllib3/util/retry.py in increment(self, method, url, response, error, _pool, _stacktrace)
244 if read is False:
--> 245 raise six.reraise(type(error), error, _stacktrace)
246 elif read is not None:
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/vendored/requests/packages/urllib3/packages/six.py in reraise(tp, value, tb)
308 if value.traceback is not tb:
--> 309 raise value.with_traceback(tb)
310 raise value
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, **response_kw)
543 timeout=timeout_obj,
--> 544 body=body, headers=headers)
545
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, **httplib_request_kw)
348 # urllib3.request. It also calls makefile (recv) on the socket.
--> 349 conn.request(method, url, **httplib_request_kw)
350
~/anaconda3/envs/tensorflow_p36/lib/python3.6/http/client.py in request(self, method, url, body, headers, encode_chunked)
1238 """Send a complete request to the server."""
-> 1239 self._send_request(method, url, body, headers, encode_chunked)
1240
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/awsrequest.py in _send_request(self, method, url, body, headers, *args, **kwargs)
129 rval = HTTPConnection._send_request(
--> 130 self, method, url, body, headers, *args, **kwargs)
131 self._expect_header_set = False
~/anaconda3/envs/tensorflow_p36/lib/python3.6/http/client.py in _send_request(self, method, url, body, headers, encode_chunked)
1284 body = _encode(body, 'body')
-> 1285 self.endheaders(body, encode_chunked=encode_chunked)
1286
~/anaconda3/envs/tensorflow_p36/lib/python3.6/http/client.py in endheaders(self, message_body, encode_chunked)
1233 raise CannotSendHeader()
-> 1234 self._send_output(message_body, encode_chunked=encode_chunked)
1235
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/awsrequest.py in _send_output(self, message_body, *args, **kwargs)
156 message_body = None
--> 157 self.send(msg)
158 if self._expect_header_set:
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/awsrequest.py in send(self, str)
241 return
--> 242 return HTTPConnection.send(self, str)
243
~/anaconda3/envs/tensorflow_p36/lib/python3.6/http/client.py in send(self, data)
985 try:
--> 986 self.sock.sendall(data)
987 except TypeError:
~/anaconda3/envs/tensorflow_p36/lib/python3.6/ssl.py in sendall(self, data, flags)
971 while count < amount:
--> 972 v = self.send(byte_view[count:])
973 count += v
~/anaconda3/envs/tensorflow_p36/lib/python3.6/ssl.py in send(self, data, flags)
940 self.class)
--> 941 return self._sslobj.write(data)
942 else:
~/anaconda3/envs/tensorflow_p36/lib/python3.6/ssl.py in write(self, data)
641 """
--> 642 return self._sslobj.write(data)
643
ProtocolError: ('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer'))
During handling of the above exception, another exception occurred:
ConnectionError Traceback (most recent call last)
in ()
----> 1 endpoint.predict(json.dumps(test_dict))
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/sagemaker/predictor.py in predict(self, data)
85 request_args['Accept'] = self.accept
86
---> 87 response = self.sagemaker_session.sagemaker_runtime_client.invoke_endpoint(**request_args)
88
89 response_body = response['Body']
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/client.py in _api_call(self, *args, **kwargs)
312 "%s() only accepts keyword arguments." % py_operation_name)
313 # The "self" in this scope is referring to the BaseClient.
--> 314 return self._make_api_call(operation_name, kwargs)
315
316 _api_call.name = str(py_operation_name)
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/client.py in _make_api_call(self, operation_name, api_params)
597 else:
598 http, parsed_response = self._endpoint.make_request(
--> 599 operation_model, request_dict)
600
601 self.meta.events.emit(
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/endpoint.py in make_request(self, operation_model, request_dict)
146 logger.debug("Making request for %s (verify_ssl=%s) with params: %s",
147 operation_model, self.verify, request_dict)
--> 148 return self._send_request(request_dict, operation_model)
149
150 def create_request(self, params, operation_model=None):
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/endpoint.py in _send_request(self, request_dict, operation_model)
175 request, operation_model, attempts)
176 while self._needs_retry(attempts, operation_model, request_dict,
--> 177 success_response, exception):
178 attempts += 1
179 # If there is a stream associated with the request, we need
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/endpoint.py in _needs_retry(self, attempts, operation_model, request_dict, response, caught_exception)
271 event_name, response=response, endpoint=self,
272 operation=operation_model, attempts=attempts,
--> 273 caught_exception=caught_exception, request_dict=request_dict)
274 handler_response = first_non_none_response(responses)
275 if handler_response is None:
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/hooks.py in emit(self, event_name, **kwargs)
225 handlers.
226 """
--> 227 return self._emit(event_name, kwargs)
228
229 def emit_until_response(self, event_name, **kwargs):
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/hooks.py in _emit(self, event_name, kwargs, stop_on_response)
358 aliased_event_name = self._alias_event_name(event_name)
359 return super(AliasedEventEmitter, self)._emit(
--> 360 aliased_event_name, kwargs, stop_on_response
361 )
362
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/hooks.py in _emit(self, event_name, kwargs, stop_on_response)
208 for handler in handlers_to_call:
209 logger.debug('Event %s: calling handler %s', event_name, handler)
--> 210 response = handler(**kwargs)
211 responses.append((handler, response))
212 if stop_on_response and response is not None:
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/retryhandler.py in call(self, attempts, response, caught_exception, **kwargs)
181
182 """
--> 183 if self._checker(attempts, response, caught_exception):
184 result = self._action(attempts=attempts)
185 logger.debug("Retry needed, action of: %s", result)
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/retryhandler.py in call(self, attempt_number, response, caught_exception)
249 def call(self, attempt_number, response, caught_exception):
250 should_retry = self._should_retry(attempt_number, response,
--> 251 caught_exception)
252 if should_retry:
253 if attempt_number >= self._max_attempts:
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/retryhandler.py in _should_retry(self, attempt_number, response, caught_exception)
275 # If we've exceeded the max attempts we just let the exception
276 # propogate if one has occurred.
--> 277 return self._checker(attempt_number, response, caught_exception)
278
279
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/retryhandler.py in call(self, attempt_number, response, caught_exception)
315 for checker in self._checkers:
316 checker_response = checker(attempt_number, response,
--> 317 caught_exception)
318 if checker_response:
319 return checker_response
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/retryhandler.py in call(self, attempt_number, response, caught_exception)
221 elif caught_exception is not None:
222 return self._check_caught_exception(
--> 223 attempt_number, caught_exception)
224 else:
225 raise ValueError("Both response and caught_exception are None.")
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/retryhandler.py in _check_caught_exception(self, attempt_number, caught_exception)
357 # the MaxAttemptsDecorator is not interested in retrying the exception
358 # then this exception just propogates out past the retry code.
--> 359 raise caught_exception
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/endpoint.py in _get_response(self, request, operation_model, attempts)
220 request, verify=self.verify,
221 stream=streaming,
--> 222 proxies=self.proxies, timeout=self.timeout)
223 except ConnectionError as e:
224 # For a connection error, if it looks like it's a DNS
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/vendored/requests/sessions.py in send(self, request, **kwargs)
571
572 # Send the request
--> 573 r = adapter.send(request, **kwargs)
574
575 # Total elapsed time of the request (approximately)
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/vendored/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
413
414 except (ProtocolError, socket.error) as err:
--> 415 raise ConnectionError(err, request=request)
416
417 except MaxRetryError as e:
ConnectionError: ('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer'))`
endpoint = my_estimator.deploy(initial_instance_count=1, instance_type="ml.m4.xlarge")
endpoint.predict(test_dict)
The text was updated successfully, but these errors were encountered: