I am getting a http.client.RemoteDisconnected: Remote end closed connection without response from this function call:
driver.save_screenshot(img_path)
the full error stack trace is:
[ERROR] RemoteDisconnected: Remote end closed connection without response
Traceback (most recent call last):
File "/var/task/rssdiff.py", line 531, in lambda_function
main()
File "/var/task/rssdiff.py", line 573, in main
raise(e)
File "/var/task/rssdiff.py", line 567, in main
results = rss.parse_rss()
File "/var/task/rssdiff.py", line 513, in parse_rss
results = self.loop_entries(r.entries)
File "/var/task/rssdiff.py", line 501, in loop_entries
raise(e)
File "/var/task/rssdiff.py", line 493, in loop_entries
result = self.store_data(article_dict)
File "/var/task/rssdiff.py", line 467, in store_data
if self.show_diff(row['title']['S'], data['title'], img_path):
File "/var/task/rssdiff.py", line 270, in show_diff
driver.save_screenshot(img_path)
File "/var/task/selenium/webdriver/remote/webdriver.py", line 798, in get_screenshot_as_file
png = self.get_screenshot_as_png()
File "/var/task/selenium/webdriver/remote/webdriver.py", line 817, in get_screenshot_as_png
return base64.b64decode(self.get_screenshot_as_base64().encode('ascii'))
File "/var/task/selenium/webdriver/remote/webdriver.py", line 827, in get_screenshot_as_base64
return self.execute(Command.SCREENSHOT)['value']
File "/var/task/selenium/webdriver/remote/webdriver.py", line 234, in execute
response = self.command_executor.execute(driver_command, params)
File "/var/task/selenium/webdriver/remote/remote_connection.py", line 401, in execute
return self._request(command_info[0], url, body=data)
File "/var/task/selenium/webdriver/remote/remote_connection.py", line 471, in _request
resp = opener.open(request, timeout=self._timeout)
File "/var/lang/lib/python3.7/urllib/request.py", line 525, in open
response = self._open(req, data)
File "/var/lang/lib/python3.7/urllib/request.py", line 543, in _open
'_open', req)
File "/var/lang/lib/python3.7/urllib/request.py", line 503, in _call_chain
result = func(*args)
File "/var/lang/lib/python3.7/urllib/request.py", line 1378, in http_open
return self.do_open(http.client.HTTPConnection, req)
File "/var/lang/lib/python3.7/urllib/request.py", line 1353, in do_open
r = h.getresponse()
File "/var/lang/lib/python3.7/http/client.py", line 1373, in getresponse
response.begin()
File "/var/lang/lib/python3.7/http/client.py", line 319, in begin
version, status, reason = self._read_status()
File "/var/lang/lib/python3.7/http/client.py", line 288, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
[ERROR] RemoteDisconnected: Remote end closed connection without response Traceback (most recent call last): File "/var/task/rssdiff.py", line 531, in lambda_function main() File "/var/task/rssdiff.py", line 573, in main raise(e) File "/var/task/rssdiff.py", line 567, in main results = rss.parse_rss() File "/var/task/rssdiff.py", line 513, in parse_rss results = self.loop_entries(r.entries) File "/var/task/rssdiff.py", line 501, in loop_entries raise(e) File "/var/task/rssdiff.py", line 493, in loop_entries result = self.store_data(article_dict) File "/var/task/rssdiff.py", line 467, in store_data if self.show_diff(row['title']['S'], data['title'], img_path): File "/var/task/rssdiff.py", line 270, in show_diff driver.save_screenshot(img_path) File "/var/task/selenium/webdriver/remote/webdriver.py", line 798, in get_screenshot_as_file png = self.get_screenshot_as_png() File "/var/task/selenium/webdriver/remote/webdriver.py", line 817, in get_screenshot_as_png return base64.b64decode(self.get_screenshot_as_base64().encode('ascii')) File "/var/task/selenium/webdriver/remote/webdriver.py", line 827, in get_screenshot_as_base64 return self.execute(Command.SCREENSHOT)['value'] File "/var/task/selenium/webdriver/remote/webdriver.py", line 234, in execute response = self.command_executor.execute(driver_command, params) File "/var/task/selenium/webdriver/remote/remote_connection.py", line 401, in execute return self._request(command_info[0], url, body=data) File "/var/task/selenium/webdriver/remote/remote_connection.py", line 471, in _request resp = opener.open(request, timeout=self._timeout) File "/var/lang/lib/python3.7/urllib/request.py", line 525, in open response = self._open(req, data) File "/var/lang/lib/python3.7/urllib/request.py", line 543, in _open '_open', req) File "/var/lang/lib/python3.7/urllib/request.py", line 503, in _call_chain result = func(*args) File "/var/lang/lib/python3.7/urllib/request.py", line 1378, in http_open return self.do_open(http.client.HTTPConnection, req) File "/var/lang/lib/python3.7/urllib/request.py", line 1353, in do_open r = h.getresponse() File "/var/lang/lib/python3.7/http/client.py", line 1373, in getresponse response.begin() File "/var/lang/lib/python3.7/http/client.py", line 319, in begin version, status, reason = self._read_status() File "/var/lang/lib/python3.7/http/client.py", line 288, in _read_status raise RemoteDisconnected("Remote end closed connection without"
all the relevant selenium code is here:
driver = webdriver.PhantomJS(
executable_path=PHANTOMJS_PATH + '/phantomjs',
service_log_path='/tmp/ghostdriver.log')
driver.get(tmp_path)
driver.save_screenshot(img_path)
The interesting thing is I am running some python code in an AWS lambda, and
- I can't reproduce the error locally, even though I'm running the same packages
- This only happens on successive calls to the driver (the first one succeeds)
I looked this error up online, and I found Python HTTP Server/Client: Remote end closed connection without response error, so it's possible the version of selenium I'm using (2.53.6) isn't doing that? Since it only happens on successive calls to the driver, I also tried adding a driver.quit() but that did not work.
the relevant bits in the requirements.txt are:
requests==2.11.1
requests-oauthlib==0.7.0
selenium==2.53.6
UPDATE: I just double checked and tested locally, and it works fine locally, which I think means that theres some configuration I need to figure out between AWS lambda and selenium :/. I wonder if it's one of these flags: https://robertorocha.info/setting-up-a-selenium-web-scraper-on-aws-lambda-with-python/
