Question:
So I’m trying to figure out Sagemaker and can get an endpoint for a tensorflow model up and running. Strangely enough, when I send small images to it, up to 220×220 (40kb), it runs just fine! However I get a connection closed error when I try anything larger, say 225×225.
I’ve tried from different networks and computers and get the same error. Also, I don’t think I’m anywhere near the 5MB limit for sagemaker requests so I must admit I’m stumped on this one. Anyone happen to know what might be the cause of this? Thanks for your help.
I’m calling my model via boto as follows:
|
1 2 3 4 5 6 7 8 9 |
config = Config(connect_timeout=999999, read_timeout=9999999) client = boto3.client("runtime.sagemaker", config=config) data = {"instances": [{"x":x, "x2":x2}]} response = client.invoke_endpoint(EndpointName='dcscn', Body=json.dumps(data), ContentType="application/json") response_body = response['Body'] |
The error I get is:
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 |
Traceback (most recent call last): File "/Users/bhhj/anaconda/lib/python3.6/site-packages/urllib3/connectionpool.py", line 600, in urlopen chunked=chunked) File "/Users/bhhj/anaconda/lib/python3.6/site-packages/urllib3/connectionpool.py", line 354, in _make_request conn.request(method, url, **httplib_request_kw) File "/Users/bhhj/anaconda/lib/python3.6/http/client.py", line 1239, in request self._send_request(method, url, body, headers, encode_chunked) File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/awsrequest.py", line 125, in _send_request method, url, body, headers, *args, **kwargs) File "/Users/bhhj/anaconda/lib/python3.6/http/client.py", line 1285, in _send_request self.endheaders(body, encode_chunked=encode_chunked) File "/Users/bhhj/anaconda/lib/python3.6/http/client.py", line 1234, in endheaders self._send_output(message_body, encode_chunked=encode_chunked) File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/awsrequest.py", line 152, in _send_output self.send(msg) File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/awsrequest.py", line 236, in send return super(AWSConnection, self).send(str) File "/Users/bhhj/anaconda/lib/python3.6/http/client.py", line 986, in send self.sock.sendall(data) File "/Users/bhhj/anaconda/lib/python3.6/ssl.py", line 975, in sendall v = self.send(byte_view[count:]) File "/Users/bhhj/anaconda/lib/python3.6/ssl.py", line 944, in send return self._sslobj.write(data) File "/Users/bhhj/anaconda/lib/python3.6/ssl.py", line 642, in write return self._sslobj.write(data) BrokenPipeError: [Errno 32] Broken pipe During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/httpsession.py", line 258, in send decode_content=False, File "/Users/bhhj/anaconda/lib/python3.6/site-packages/urllib3/connectionpool.py", line 638, in urlopen _stacktrace=sys.exc_info()[2]) File "/Users/bhhj/anaconda/lib/python3.6/site-packages/urllib3/util/retry.py", line 343, in increment raise six.reraise(type(error), error, _stacktrace) File "/Users/bhhj/anaconda/lib/python3.6/site-packages/urllib3/packages/six.py", line 685, in reraise raise value.with_traceback(tb) File "/Users/bhhj/anaconda/lib/python3.6/site-packages/urllib3/connectionpool.py", line 600, in urlopen chunked=chunked) File "/Users/bhhj/anaconda/lib/python3.6/site-packages/urllib3/connectionpool.py", line 354, in _make_request conn.request(method, url, **httplib_request_kw) File "/Users/bhhj/anaconda/lib/python3.6/http/client.py", line 1239, in request self._send_request(method, url, body, headers, encode_chunked) File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/awsrequest.py", line 125, in _send_request method, url, body, headers, *args, **kwargs) File "/Users/bhhj/anaconda/lib/python3.6/http/client.py", line 1285, in _send_request self.endheaders(body, encode_chunked=encode_chunked) File "/Users/bhhj/anaconda/lib/python3.6/http/client.py", line 1234, in endheaders self._send_output(message_body, encode_chunked=encode_chunked) File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/awsrequest.py", line 152, in _send_output self.send(msg) File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/awsrequest.py", line 236, in send return super(AWSConnection, self).send(str) File "/Users/bhhj/anaconda/lib/python3.6/http/client.py", line 986, in send self.sock.sendall(data) File "/Users/bhhj/anaconda/lib/python3.6/ssl.py", line 975, in sendall v = self.send(byte_view[count:]) File "/Users/bhhj/anaconda/lib/python3.6/ssl.py", line 944, in send return self._sslobj.write(data) File "/Users/bhhj/anaconda/lib/python3.6/ssl.py", line 642, in write return self._sslobj.write(data) urllib3.exceptions.ProtocolError: ('Connection aborted.', BrokenPipeError(32, 'Broken pipe')) During handling of the above exception, another exception occurred: Traceback (most recent call last): File "ping_endpoint.py", line 100, in ContentType="application/json") File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/client.py", line 357, in _api_call return self._make_api_call(operation_name, kwargs) File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/client.py", line 648, in _make_api_call operation_model, request_dict, request_context) File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/client.py", line 667, in _make_request return self._endpoint.make_request(operation_model, request_dict) File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/endpoint.py", line 102, in make_request return self._send_request(request_dict, operation_model) File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/endpoint.py", line 137, in _send_request success_response, exception): File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/endpoint.py", line 231, in _needs_retry caught_exception=caught_exception, request_dict=request_dict) File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/hooks.py", line 356, in emit return self._emitter.emit(aliased_event_name, **kwargs) File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/hooks.py", line 228, in emit return self._emit(event_name, kwargs) File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/hooks.py", line 211, in _emit response = handler(**kwargs) File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/retryhandler.py", line 183, in __call__ if self._checker(attempts, response, caught_exception): File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/retryhandler.py", line 251, in __call__ caught_exception) File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/retryhandler.py", line 277, in _should_retry return self._checker(attempt_number, response, caught_exception) File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/retryhandler.py", line 317, in __call__ caught_exception) File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/retryhandler.py", line 223, in __call__ attempt_number, caught_exception) File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/retryhandler.py", line 359, in _check_caught_exception raise caught_exception File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/endpoint.py", line 200, in _do_get_response http_response = self._send(request) File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/endpoint.py", line 244, in _send return self.http_session.send(request) File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/httpsession.py", line 289, in send endpoint_url=request.url botocore.exceptions.ConnectionClosedError: Connection was closed before we received a valid response from endpoint URL: |
Answer:
It can be a network error.
If you are connected to a VPN , try after disconnecting it
try with better internet connection
my case, closing the VPN worked