My Truenas scale began to freeze everytime I go to apps tab. All system is slow and I can’t even access dashboard. All CPUs at almost 100%. And system very slow. I can’t find what is the issue. I had no hardware change recently.
OS Version:25.10.0.1
Product:Precision T3600
Model:Intel(R) Xeon(R) CPU E5-1620 0 @ 3.60GHz
Memory:63 GiB
LSI 9300 16i
pool for SMB with 10 HDDs and 3 SSDs for special vdev
pool for apps triple mirror data ssd
boot pool sata ssd
Error Name: EINVAL
Error Code: 22
Reason: UnixHTTPConnectionPool(host=‘localhost’, port=None): Read timed out. (read timeout=60)
Error Class: ReadTimeout
Trace: Traceback (most recent call last):
File “/usr/lib/python3/dist-packages/urllib3/connectionpool.py”, line 451, in _make_request
six.raise_from(e, None)
File “”, line 3, in raise_from
File “/usr/lib/python3/dist-packages/urllib3/connectionpool.py”, line 446, in _make_request
httplib_response = conn.getresponse()
^^^^^^^^^^^^^^^^^^
File “/usr/lib/python3.11/http/client.py”, line 1395, in getresponse
response.begin()
File “/usr/lib/python3.11/http/client.py”, line 325, in begin
version, status, reason = self._read_status()
^^^^^^^^^^^^^^^^^^^
File “/usr/lib/python3.11/http/client.py”, line 286, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), “iso-8859-1”)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/usr/lib/python3.11/socket.py”, line 706, in readinto
return self._sock.recv_into(b)
^^^^^^^^^^^^^^^^^^^^^^^
TimeoutError: timed outDuring handling of the above exception, another exception occurred:
Traceback (most recent call last):
File “/usr/lib/python3/dist-packages/requests/adapters.py”, line 489, in send
resp = conn.urlopen(
^^^^^^^^^^^^^
File “/usr/lib/python3/dist-packages/urllib3/connectionpool.py”, line 789, in urlopen
retries = retries.increment(
^^^^^^^^^^^^^^^^^^
File “/usr/lib/python3/dist-packages/urllib3/util/retry.py”, line 552, in increment
raise six.reraise(type(error), error, _stacktrace)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/usr/lib/python3/dist-packages/six.py”, line 719, in reraise
raise value
File “/usr/lib/python3/dist-packages/urllib3/connectionpool.py”, line 705, in urlopen
httplib_response = self._make_request(
^^^^^^^^^^^^^^^^^^^
File “/usr/lib/python3/dist-packages/urllib3/connectionpool.py”, line 453, in _make_request
self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
File “/usr/lib/python3/dist-packages/urllib3/connectionpool.py”, line 342, in _raise_timeout
raise ReadTimeoutError(
urllib3.exceptions.ReadTimeoutError: UnixHTTPConnectionPool(host=‘localhost’, port=None): Read timed out. (read timeout=60)During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File “/usr/lib/python3/dist-packages/middlewared/api/base/server/ws_handler/rpc.py”, line 360, in process_method_call
result = await method.call(app, id_, params)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/usr/lib/python3/dist-packages/middlewared/api/base/server/method.py”, line 57, in call
result = await self.middleware.call_with_audit(self.name, self.serviceobj, methodobj, params, app,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/usr/lib/python3/dist-packages/middlewared/main.py”, line 954, in call_with_audit
result = await self._call(method, serviceobj, methodobj, params, app=app,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/usr/lib/python3/dist-packages/middlewared/main.py”, line 771, in _call
return await methodobj(*prepared_call.args)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/usr/lib/python3/dist-packages/middlewared/api/base/decorator.py”, line 108, in wrapped
result = await func(*args)
^^^^^^^^^^^^^^^^^
File “/usr/lib/python3/dist-packages/middlewared/plugins/catalog/apps.py”, line 23, in latest
await self.middleware.call(
File “/usr/lib/python3/dist-packages/middlewared/main.py”, line 1043, in call
return await self._call(
^^^^^^^^^^^^^^^^^
File “/usr/lib/python3/dist-packages/middlewared/main.py”, line 782, in _call
return await self.run_in_executor(prepared_call.executor, methodobj, *prepared_call.args)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/usr/lib/python3/dist-packages/middlewared/main.py”, line 665, in run_in_executor
return await loop.run_in_executor(pool, functools.partial(method, *args, **kwargs))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/usr/lib/python3.11/concurrent/futures/thread.py”, line 58, in run
result = self.fn(*self.args, **self.kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/usr/lib/python3/dist-packages/middlewared/api/base/decorator.py”, line 116, in wrapped
result = func(*args)
^^^^^^^^^^^
File “/usr/lib/python3/dist-packages/middlewared/plugins/catalog/apps.py”, line 42, in available
for app in self.middleware.call_sync(‘app.query’)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/usr/lib/python3/dist-packages/middlewared/main.py”, line 1076, in call_sync
return methodobj(*prepared_call.args)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/usr/lib/python3/dist-packages/middlewared/api/base/decorator.py”, line 116, in wrapped
result = func(*args)
^^^^^^^^^^^
File “/usr/lib/python3/dist-packages/middlewared/plugins/apps/crud.py”, line 72, in query
apps = list_apps(available_apps_mapping, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/usr/lib/python3/dist-packages/middlewared/plugins/apps/ix_apps/query.py”, line 86, in list_apps
for app_name, app_resources in list_resources_by_project(
^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/usr/lib/python3/dist-packages/middlewared/plugins/apps/ix_apps/docker/query.py”, line 13, in list_resources_by_project
return list_resources_by_project_internal(project_name)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/usr/lib/python3/dist-packages/middlewared/plugins/apps/ix_apps/docker/query.py”, line 23, in list_resources_by_project_internal
containers = client.containers.list(all=True, filters=label_filter, sparse=False)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/usr/lib/python3/dist-packages/docker/models/containers.py”, line 1018, in list
containers.append(self.get(r[‘Id’]))
^^^^^^^^^^^^^^^^^
File “/usr/lib/python3/dist-packages/docker/models/containers.py”, line 954, in get
resp = self.client.api.inspect_container(container_id)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/usr/lib/python3/dist-packages/docker/utils/decorators.py”, line 19, in wrapped
return f(self, resource_id, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/usr/lib/python3/dist-packages/docker/api/container.py”, line 794, in inspect_container
self._get(self._url(“/containers/{0}/json”, container)), True
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/usr/lib/python3/dist-packages/docker/utils/decorators.py”, line 44, in inner
return f(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
File “/usr/lib/python3/dist-packages/docker/api/client.py”, line 246, in _get
return self.get(url, **self._set_request_timeout(kwargs))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/usr/lib/python3/dist-packages/requests/sessions.py”, line 600, in get
return self.request(“GET”, url, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/usr/lib/python3/dist-packages/requests/sessions.py”, line 587, in request
resp = self.send(prep, **send_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/usr/lib/python3/dist-packages/requests/sessions.py”, line 701, in send
r = adapter.send(request, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/usr/lib/python3/dist-packages/requests/adapters.py”, line 578, in send
raise ReadTimeout(e, request=request)
requests.exceptions.ReadTimeout: UnixHTTPConnectionPool(host=‘localhost’, port=None): Read timed out. (read timeout=60)