I have a replication task for a Dataset setup from one truenas server to another truenas server. It’s a “Full Filesystem Replication”.
After I added a new zvol to the dataset the replication task fails:
Replication "pool - pool" failed: Last full ZFS replication failed to transfer all the children of the snapshot pool/vm-zvols@auto-2024-12-12_00-00. The snapshot pool/vm-zvols/ios_paths_processor@auto-2024-12-12_00-00 was not transferred. Please run `zfs destroy -r pool/vm-zvols@auto-2024-12-12_00-00` on the target system and run replication again..
This issue seems to have been previously discussed in the old forum with the title: " Replication failing b/c of new dataset?"
And a bugreport submitted and presumably fixed NAS-109476
However this seems to have only fixed the bug that intermediate snapshots are not transfered.
My bug is that a newly created zvol in the datasets leads to a failing replication task.
Here’s the full replication.json from the debug:
{
"key": "replication_query",
"output": [
{
"id": 1,
"target_dataset": "pool/vm-zvols",
"recursive": true,
"compression": "LZ4",
"speed_limit": null,
"enabled": true,
"direction": "PUSH",
"transport": "SSH",
"sudo": true,
"netcat_active_side": null,
"netcat_active_side_port_min": null,
"netcat_active_side_port_max": null,
"source_datasets": [
"pool/vm-zvols"
],
"exclude": [],
"naming_schema": [],
"name_regex": null,
"auto": true,
"only_matching_schedule": false,
"readonly": "IGNORE",
"allow_from_scratch": true,
"hold_pending_snapshots": false,
"retention_policy": "SOURCE",
"lifetime_unit": null,
"lifetime_value": null,
"lifetimes": [],
"large_block": true,
"embed": false,
"compressed": true,
"retries": 5,
"netcat_active_side_listen_address": null,
"netcat_passive_side_connect_address": null,
"logging_level": null,
"name": "pool - pool",
"state": {
"state": "ERROR",
"datetime": {
"$date": 1734303601000
},
"error": "Last full ZFS replication failed to transfer all the children of the snapshot pool/vm-zvols@auto-2024-12-12_00-00. The snapshot pool/vm-zvols/ios_paths_processor@auto-2024-12-12_00-00 was not transferred. Please run `zfs destroy -r pool/vm-zvols@auto-2024-12-12_00-00` on the target system and run replication again.",
"last_snapshot": "pool/vm-zvols@auto-2024-12-12_00-00"
},
"properties": true,
"properties_exclude": [],
"properties_override": {},
"replicate": true,
"encryption": false,
"encryption_inherit": null,
"encryption_key": null,
"encryption_key_format": null,
"encryption_key_location": null,
"periodic_snapshot_tasks": [
{
"id": 2,
"dataset": "pool/vm-zvols",
"recursive": true,
"lifetime_value": 2,
"lifetime_unit": "WEEK",
"enabled": true,
"exclude": [],
"naming_schema": "auto-%Y-%m-%d_%H-%M",
"allow_empty": true,
"schedule": {
"minute": "0",
"hour": "0",
"dom": "*",
"month": "*",
"dow": "*",
"begin": "00:00",
"end": "23:59"
}
}
],
"also_include_naming_schema": [],
"schedule": null,
"restrict_schedule": null,
"job": {
"id": 205601,
"method": "replication.run",
"arguments": [
1,
false
],
"transient": false,
"description": null,
"abortable": false,
"logs_path": "/var/log/jobs/205601.log",
"logs_excerpt": "[2024/12/16 00:00:00] INFO [Thread-723] [zettarepl.paramiko.replication_task__task_1] Connected (version 2.0, client OpenSSH_9.2p1)\n[2024/12/16 00:00:00] INFO [Thread-723] [zettarepl.paramiko.replication_task__task_1] Authentication (publickey) successful!\n[2024/12/16 00:00:01] INFO [replication_task__task_1] [zettarepl.retention.calculate] Not destroying 'auto-2024-12-12_00-00' as it is the only snapshot left for naming schema 'auto-%Y-%m-%d_%H-%M'\n[2024/12/16 00:00:01] INFO [replication_task__task_1] [zettarepl.replication.pre_retention] Pre-retention destroying snapshots: []\n[2024/12/16 00:00:01] ERROR [replication_task__task_1] [zettarepl.replication.run] For task 'task_1' non-recoverable replication error ReplicationError('Last full ZFS replication failed to transfer all the children of the snapshot pool/vm-zvols@auto-2024-12-12_00-00. The snapshot pool/vm-zvols/ios_paths_processor@auto-2024-12-12_00-00 was not transferred. Please run `zfs destroy -r pool/vm-zvols@auto-2024-12-12_00-00` on the target system and run replication again.')\n",
"progress": {
"percent": 0,
"description": "",
"extra": null
},
"result": null,
"error": "[EFAULT] Last full ZFS replication failed to transfer all the children of the snapshot pool/vm-zvols@auto-2024-12-12_00-00. The snapshot pool/vm-zvols/ios_paths_processor@auto-2024-12-12_00-00 was not transferred. Please run `zfs destroy -r pool/vm-zvols@auto-2024-12-12_00-00` on the target system and run replication again.",
"exception": "Traceback (most recent call last):\n File \"/usr/lib/python3/dist-packages/middlewared/job.py\", line 427, in run\n await self.future\n File \"/usr/lib/python3/dist-packages/middlewared/job.py\", line 465, in __run_body\n rv = await self.method(*([self] + args))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/lib/python3/dist-packages/middlewared/schema/processor.py\", line 177, in nf\n return await func(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/lib/python3/dist-packages/middlewared/plugins/replication.py\", line 484, in run\n await self.middleware.call(\"zettarepl.run_replication_task\", id, really_run, job)\n File \"/usr/lib/python3/dist-packages/middlewared/main.py\", line 1399, in call\n return await self._call(\n ^^^^^^^^^^^^^^^^^\n File \"/usr/lib/python3/dist-packages/middlewared/main.py\", line 1353, in _call\n return await self.run_in_executor(prepared_call.executor, methodobj, *prepared_call.args)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/lib/python3/dist-packages/middlewared/main.py\", line 1251, in run_in_executor\n return await loop.run_in_executor(pool, functools.partial(method, *args, **kwargs))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/lib/python3.11/concurrent/futures/thread.py\", line 58, in run\n result = self.fn(*self.args, **self.kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/lib/python3/dist-packages/middlewared/plugins/zettarepl.py\", line 401, in run_replication_task\n self._run_replication_task_job(f\"task_{id}\", job)\n File \"/usr/lib/python3/dist-packages/middlewared/plugins/zettarepl.py\", line 461, in _run_replication_task_job\n raise CallError(make_sentence(message.error))\nmiddlewared.service_exception.CallError: [EFAULT] Last full ZFS replication failed to transfer all the children of the snapshot pool/vm-zvols@auto-2024-12-12_00-00. The snapshot pool/vm-zvols/ios_paths_processor@auto-2024-12-12_00-00 was not transferred. Please run `zfs destroy -r pool/vm-zvols@auto-2024-12-12_00-00` on the target system and run replication again.\n",
"exc_info": {
"repr": "CallError('Last full ZFS replication failed to transfer all the children of the snapshot pool/vm-zvols@auto-2024-12-12_00-00. The snapshot pool/vm-zvols/ios_paths_processor@auto-2024-12-12_00-00 was not transferred. Please run `zfs destroy -r pool/vm-zvols@auto-2024-12-12_00-00` on the target system and run replication again.')",
"type": "CallError",
"extra": null
},
"state": "FAILED",
"time_started": {
"$date": 1734303600000
},
"time_finished": {
"$date": 1734303601000
},
"credentials": null
}
},
{
"id": 2,
"target_dataset": "pool/vm-zvols",
"recursive": true,
"compression": null,
"speed_limit": null,
"enabled": true,
"direction": "PUSH",
"transport": "SSH",
"sudo": false,
"netcat_active_side": null,
"netcat_active_side_port_min": null,
"netcat_active_side_port_max": null,
"source_datasets": [
"pool/vm-zvols"
],
"exclude": [],
"naming_schema": [],
"name_regex": ".*",
"auto": false,
"only_matching_schedule": false,
"readonly": "IGNORE",
"allow_from_scratch": false,
"hold_pending_snapshots": false,
"retention_policy": "SOURCE",
"lifetime_unit": null,
"lifetime_value": null,
"lifetimes": [],
"large_block": true,
"embed": false,
"compressed": true,
"retries": 5,
"netcat_active_side_listen_address": null,
"netcat_passive_side_connect_address": null,
"logging_level": null,
"name": "Replicate Now",
"state": {
"state": "FINISHED",
"datetime": {
"$date": 1731408806000
},
"warnings": [],
"last_snapshot": "pool/vm-zvols@test manuell"
},
"properties": true,
"properties_exclude": [],
"properties_override": {},
"replicate": true,
"encryption": false,
"encryption_inherit": null,
"encryption_key": null,
"encryption_key_format": null,
"encryption_key_location": null,
"periodic_snapshot_tasks": [],
"also_include_naming_schema": [],
"schedule": null,
"restrict_schedule": null,
"job": {
"id": 106413,
"method": "replication.run",
"arguments": [
2
],
"transient": false,
"description": null,
"abortable": false,
"logs_path": "/var/log/jobs/106413.log",
"logs_excerpt": "[2024/11/12 11:53:26] INFO [Thread-381] [zettarepl.paramiko.replication_task__task_2] Connected (version 2.0, client OpenSSH_9.2p1)\n[2024/11/12 11:53:26] INFO [Thread-381] [zettarepl.paramiko.replication_task__task_2] Authentication (publickey) successful!\n[2024/11/12 11:53:26] INFO [replication_task__task_2] [zettarepl.replication.pre_retention] Pre-retention destroying snapshots: []\n[2024/11/12 11:53:26] INFO [replication_task__task_2] [zettarepl.replication.run] No snapshots to send for replication task 'task_2' on dataset 'pool/vm-zvols'\n",
"progress": {
"percent": 100,
"description": "",
"extra": null
},
"result": null,
"error": null,
"exception": null,
"exc_info": null,
"state": "SUCCESS",
"time_started": {
"$date": 1731408805000
},
"time_finished": {
"$date": 1731408806000
},
"credentials": {
"type": "LOGIN_PASSWORD",
"data": {
"username": "admin"
}
}
}
}
]
}
PS: The forum does not allow me to post links. There should atleast be an execption for other truenas/ixsystem sites.