Hi guys, I am testing it with datahub docker quick...
# ingestion
b
Hi guys, I am testing it with datahub docker quickstart.(i am using aws ec2) I am trying to ingest the glue dataset, but I get a connection refused error. What should I check?
Copy code
RUN_INGEST - {'errors': [],
 'exec_id': '0d20d560-1942-4d49-bc09-9a1f11cbf44a',
 'infos': ['2022-06-08 02:28:40.227718 [exec_id=0d20d560-1942-4d49-bc09-9a1f11cbf44a] INFO: Starting execution for task with name=RUN_INGEST',
           '2022-06-08 02:29:17.394482 [exec_id=0d20d560-1942-4d49-bc09-9a1f11cbf44a] INFO: stdout=Requirement already satisfied: pip in '
           '[2022-06-08 02:29:04,730] INFO     {datahub.cli.ingest_cli:96} - DataHub CLI version: 0.8.35\n'
           '[2022-06-08 02:29:04,733] WARNING  {urllib3.connectionpool:810} - Retrying (Retry(total=2, connect=None, read=None, redirect=None, '
           "status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7fb3f56213a0>: Failed to "
           "establish a new connection: [Errno 111] Connection refused')': /config\n"
           '[2022-06-08 02:29:08,735] WARNING  {urllib3.connectionpool:810} - Retrying (Retry(total=1, connect=None, read=None, redirect=None, '
           "status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7fb3f56215b0>: Failed to "
           "establish a new connection: [Errno 111] Connection refused')': /config\n"
           '[2022-06-08 02:29:16,744] WARNING  {urllib3.connectionpool:810} - Retrying (Retry(total=0, connect=None, read=None, redirect=None, '
           "status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7fb3f56217c0>: Failed to "
           "establish a new connection: [Errno 111] Connection refused')': /config\n"
           '[2022-06-08 02:29:17,146] ERROR    {datahub.entrypoints:167} - File '
           '"/tmp/datahub/ingest/venv-0d20d560-1942-4d49-bc09-9a1f11cbf44a/lib/python3.9/site-packages/urllib3/connection.py", line 174, in '
           '_new_conn\n'
           '    161  def _new_conn(self):\n'
           ' (...)\n'
           '    170      if self.socket_options:\n'
           '    171          extra_kw["socket_options"] = self.socket_options\n'
           '    172  \n'
           '    173      try:\n'
           '--> 174          conn = connection.create_connection(\n'
           '    175              (self._dns_host, self.port), self.timeout, **extra_kw\n'
           '\n'
           'File "/tmp/datahub/ingest/venv-0d20d560-1942-4d49-bc09-9a1f11cbf44a/lib/python3.9/site-packages/urllib3/util/connection.py", line 95, in '
           'create_connection\n'
           '    37   def create_connection(\n'
           '    38       address,\n'
           '    39       timeout=socket._GLOBAL_DEFAULT_TIMEOUT,\n'
           '    40       source_address=None,\n'
           '    41       socket_options=None,\n'
           '    42   ):\n'
           ' (...)\n'
           '    91                   sock.close()\n'
           '    92                   sock = None\n'
           '    93   \n'
           '    94       if err is not None:\n'
           '--> 95           raise err\n'
           '    96   \n'
           '\n'
           'File "/tmp/datahub/ingest/venv-0d20d560-1942-4d49-bc09-9a1f11cbf44a/lib/python3.9/site-packages/urllib3/util/connection.py", line 85, in '
           'create_connection\n'
           '    37   def create_connection(\n'
           '    38       address,\n'
           '    39       timeout=socket._GLOBAL_DEFAULT_TIMEOUT,\n'
           '    40       source_address=None,\n'
           '    41       socket_options=None,\n'
           '    42   ):\n'
           ' (...)\n'
           '    81               if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:\n'
           '    82                   sock.settimeout(timeout)\n'
           '    83               if source_address:\n'
           '    84                   sock.bind(source_address)\n'
           '--> 85               sock.connect(sa)\n'
           '    86               return sock\n'
           '\n'
           'ConnectionRefusedError: [Errno 111] Connection refused\n'
           '\n'
           'While handling the above exception, another exception occurred:\n'
           '\n'
           'File "/tmp/datahub/ingest/venv-0d20d560-1942-4d49-bc09-9a1f11cbf44a/lib/python3.9/site-packages/urllib3/connectionpool.py", line 703, in '
           'urlopen\n'
           '    522  def urlopen(\n'
           '    523      self,\n'
           '    524      method,\n'
           '    525      url,\n'
           '    526      body=None,\n'
           '    527      headers=None,\n'
           '    528      retries=None,\n'
           '    529      redirect=True,\n'
           '    530      assert_same_host=True,\n'
           '    531      timeout=_Default,\n'
           '    532      pool_timeout=None,\n'
           '    533      release_conn=None,\n'
           '    534      chunked=False,\n'
           '    535      body_pos=None,\n'
           '    536      **response_kw\n'
           '    537  ):\n'
           ' (...)\n'
           '    699          if is_new_proxy_conn and http_tunnel_required:\n'
           '    700              self._prepare_proxy(conn)\n'
           '    701  \n'
           '    702          # Make the request on the httplib connection object.\n'
           '--> 703          httplib_response = self._make_request(\n'
           '    704              conn,\n'
           '\n'
           'File "/tmp/datahub/ingest/venv-0d20d560-1942-4d49-bc09-9a1f11cbf44a/lib/python3.9/site-packages/urllib3/connectionpool.py", line 398, in '
           '_make_request\n'
           '    361  def _make_request(\n'
           '    362      self, conn, method, url, timeout=_Default, chunked=False, **httplib_request_kw\n'
           '    363  ):\n'
           ' (...)\n'
           '    394      try:\n'
           '    395          if chunked:\n'
           '    396              conn.request_chunked(method, url, **httplib_request_kw)\n'
           '    397          else:\n'
           '--> 398              conn.request(method, url, **httplib_request_kw)\n'
           '    399  \n'
           '\n'
           'File "/tmp/datahub/ingest/venv-0d20d560-1942-4d49-bc09-9a1f11cbf44a/lib/python3.9/site-packages/urllib3/connection.py", line 239, in '
           'request\n'
           '    231  def request(self, method, url, body=None, headers=None):\n'
           ' (...)\n'
           '    235          # Avoid modifying the headers passed into .request()\n'
           '    236          headers = headers.copy()\n'
           '    237      if "user-agent" not in (six.ensure_str(k.lower()) for k in headers):\n'
           '    238          headers["User-Agent"] = _get_default_user_agent()\n'
           '--> 239      super(HTTPConnection, self).request(method, url, body=body, headers=headers)\n'
           '\n'
           'File "/usr/local/lib/python3.9/http/client.py", line 1285, in request\n'
           '    1282  def request(self, method, url, body=None, headers={}, *,\n'
           '    1283              encode_chunked=False):\n'
           '    1284      """Send a complete request to the server."""\n'
           '--> 1285      self._send_request(method, url, body, headers, encode_chunked)\n'
           '\n'
           'File "/usr/local/lib/python3.9/http/client.py", line 1331, in _send_request\n'
           '    1287  def _send_request(self, method, url, body, headers, encode_chunked):\n'
           ' (...)\n'
           '    1327      if isinstance(body, str):\n'
           '    1328          # RFC 2616 Section 3.7.1 says that text default has a\n'
           '    1329          # default charset of iso-8859-1.\n'
           "    1330          body = _encode(body, 'body')\n"
           '--> 1331      self.endheaders(body, encode_chunked=encode_chunked)\n'
           '\n'
           'File "/usr/local/lib/python3.9/http/client.py", line 1280, in endheaders\n'
           '    1269  def endheaders(self, message_body=None, *, encode_chunked=False):\n'
           ' (...)\n'
           '    1276      if self.__state == _CS_REQ_STARTED:\n'
           '    1277          self.__state = _CS_REQ_SENT\n'
           '    1278      else:\n'
           '    1279          raise CannotSendHeader()\n'
           '--> 1280      self._send_output(message_body, encode_chunked=encode_chunked)\n'
           '\n'
           'File "/usr/local/lib/python3.9/http/client.py", line 1040, in _send_output\n'
           '    1031  def _send_output(self, message_body=None, encode_chunked=False):\n'
           ' (...)\n'
           '    1036      """\n'
           '    1037      self._buffer.extend((b"", b""))\n'
           '    1038      msg = b"\\r\\n".join(self._buffer)\n'
           '    1039      del self._buffer[:]\n'
           '--> 1040      self.send(msg)\n'
           '    1041  \n'
           '\n'
           'File "/usr/local/lib/python3.9/http/client.py", line 980, in send\n'
           '    972  def send(self, data):\n'
           ' (...)\n'
           '    976      """\n'
           '    977  \n'
           '    978      if self.sock is None:\n'
           '    979          if self.auto_open:\n'
           '--> 980              self.connect()\n'
           '    981          else:\n'
           '\n'
           'File "/tmp/datahub/ingest/venv-0d20d560-1942-4d49-bc09-9a1f11cbf44a/lib/python3.9/site-packages/urllib3/connection.py", line 205, in '
           'connect\n'
           '    204  def connect(self):\n'
           '--> 205      conn = self._new_conn()\n'
           '    206      self._prepare_conn(conn)\n'
           '\n'
           'File "/tmp/datahub/ingest/venv-0d20d560-1942-4d49-bc09-9a1f11cbf44a/lib/python3.9/site-packages/urllib3/connection.py", line 186, in '
           '_new_conn\n'
           '    161  def _new_conn(self):\n'
           ' (...)\n'
           '    182              % (self.host, self.timeout),\n'
           '    183          )\n'
           '    184  \n'
           '    185      except SocketError as e:\n'
           '--> 186          raise NewConnectionError(\n'
           '    187              self, "Failed to establish a new connection: %s" % e\n'
           '\n'
c
Have you opened port 8080 on your EC2 instance?
b
Yes When I run it from the datahub UI, an error occurs, When I run it with the datahub ingest command, it works fine.
c
Are you trying datahub ingest from same EC2 cmd line?
b
Yes I ran it on the same ec2