Unable to start VMs in Scale 24.10.1

I just noticed that after upgrading to 24.10.1, that none of my VMs start. When I try to start them, I get a message " [EFAULT] Unable to define domain for axtools: ‘type’ " with an error log looking like this:

 Error: Traceback (most recent call last):
  File "/usr/lib/python3/dist-packages/middlewared/plugins/vm/vm_supervisor.py", line 45, in _check_add_domain
    self._add(self._vm_from_name(vm_name)['id'])
  File "/usr/lib/python3/dist-packages/middlewared/plugins/vm/vm_supervisor.py", line 16, in _add
    self._add_with_vm_data(vm)
  File "/usr/lib/python3/dist-packages/middlewared/plugins/vm/vm_supervisor.py", line 19, in _add_with_vm_data
    self.vms[vm['name']] = VMSupervisor(vm, self.middleware)
                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3/dist-packages/middlewared/plugins/vm/supervisor/supervisor.py", line 30, in __init__
    self.update_domain()
  File "/usr/lib/python3/dist-packages/middlewared/plugins/vm/supervisor/supervisor.py", line 66, in update_domain
    self.__define_domain()
  File "/usr/lib/python3/dist-packages/middlewared/plugins/vm/supervisor/supervisor.py", line 98, in __define_domain
    vm_xml = etree.tostring(self.construct_xml()).decode()
                            ^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3/dist-packages/middlewared/plugins/vm/supervisor/supervisor.py", line 272, in construct_xml
    'domain', type='kvm', id=str(self.vm_data['id']), attribute_dict={'children': self.get_domain_children()}
                                                                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3/dist-packages/middlewared/plugins/vm/supervisor/supervisor.py", line 268, in get_domain_children
    return domain_children(self.vm_data, context)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3/dist-packages/middlewared/plugins/vm/supervisor/domain_xml.py", line 31, in domain_children
    devices_xml(vm_data, context),
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3/dist-packages/middlewared/plugins/vm/supervisor/domain_xml.py", line 145, in devices_xml
    device_xml = device.xml(disk_number=disk_no, boot_number=boot_no())
                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3/dist-packages/middlewared/plugins/vm/devices/storage_devices.py", line 28, in xml
    virtio = self.data['attributes']['type'] == 'VIRTIO'
             ~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^
KeyError: 'type'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/lib/python3/dist-packages/middlewared/main.py", line 211, in call_method
    result = await self.middleware.call_with_audit(message['method'], serviceobj, methodobj, params, self)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3/dist-packages/middlewared/main.py", line 1529, in call_with_audit
    result = await self._call(method, serviceobj, methodobj, params, app=app,
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3/dist-packages/middlewared/main.py", line 1460, in _call
    return await methodobj(*prepared_call.args)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3/dist-packages/middlewared/schema/processor.py", line 179, in nf
    return await func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3/dist-packages/middlewared/schema/processor.py", line 49, in nf
    res = await f(*args, **kwargs)
          ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3/dist-packages/middlewared/plugins/vm/vm_lifecycle.py", line 58, in start
    await self.middleware.run_in_thread(self._start, vm['name'])
  File "/usr/lib/python3/dist-packages/middlewared/main.py", line 1367, in run_in_thread
    return await self.run_in_executor(io_thread_pool_executor, method, *args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3/dist-packages/middlewared/main.py", line 1364, in run_in_executor
    return await loop.run_in_executor(pool, functools.partial(method, *args, **kwargs))
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3.11/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3/dist-packages/middlewared/plugins/vm/vm_supervisor.py", line 67, in _start
    self._check_add_domain(vm_name)
  File "/usr/lib/python3/dist-packages/middlewared/plugins/vm/vm_supervisor.py", line 47, in _check_add_domain
    raise CallError(f'Unable to define domain for {vm_name}: {e}')
middlewared.service_exception.CallError: [EFAULT] Unable to define domain for axtools: 'type'

since I see “VIRTIO” mentioned, I checked the disks for each machine and i can’t get it to populate anything, but my guess is maybe syntax changed in the xml or something in new version? anyone else have this issue? When I try to “Edit” a disk, it shows “Options cannot be loaded” under zvol, and nothing else is visible except the device order #.

Edit, ok while waiting, I found that i can use midclt to update the device, but im having an issue:

midclt call vm.device.update 110 '{ "dtype": "DISK", "attributes": { "type": "VIRTIO", "iotype": "THREADS", "path": "/dev/zvol/sirius/vmstorage/saltmaster-z1nte", "physical_sectorsize": null, "logical_sectorsize": null }, "vm": 19 }'

throws the following error:

'zvol_name'
Traceback (most recent call last):
  File "/usr/lib/python3/dist-packages/middlewared/main.py", line 211, in call_method
    result = await self.middleware.call_with_audit(message['method'], serviceobj, methodobj, params, self)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3/dist-packages/middlewared/main.py", line 1529, in call_with_audit
    result = await self._call(method, serviceobj, methodobj, params, app=app,
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3/dist-packages/middlewared/main.py", line 1460, in _call
    return await methodobj(*prepared_call.args)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3/dist-packages/middlewared/service/crud_service.py", line 240, in update
    return await self.middleware._call(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3/dist-packages/middlewared/main.py", line 1460, in _call
    return await methodobj(*prepared_call.args)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3/dist-packages/middlewared/service/crud_service.py", line 261, in nf
    rv = await func(*args, **kwargs)
         ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3/dist-packages/middlewared/schema/processor.py", line 179, in nf
    return await func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3/dist-packages/middlewared/schema/processor.py", line 49, in nf
    res = await f(*args, **kwargs)
          ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3/dist-packages/middlewared/plugins/vm/vm_devices.py", line 181, in do_update
    new = await self.validate_device(new, device)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3/dist-packages/middlewared/plugins/vm/vm_devices.py", line 292, in validate_device
    await self.middleware.run_in_thread(device_obj.validate, device, old, vm_instance, update)
  File "/usr/lib/python3/dist-packages/middlewared/main.py", line 1367, in run_in_thread
    return await self.run_in_executor(io_thread_pool_executor, method, *args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3/dist-packages/middlewared/main.py", line 1364, in run_in_executor
    return await loop.run_in_executor(pool, functools.partial(method, *args, **kwargs))
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3.11/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3/dist-packages/middlewared/plugins/vm/devices/device.py", line 44, in validate
    self._validate(device, verrors, old, vm_instance, update)
  File "/usr/lib/python3/dist-packages/middlewared/plugins/vm/devices/storage_devices.py", line 177, in _validate
    super()._validate(device, verrors, old, vm_instance, update)
  File "/usr/lib/python3/dist-packages/middlewared/plugins/vm/devices/storage_devices.py", line 56, in _validate
    if not self.middleware.call_sync('vm.device.disk_uniqueness_integrity_check', device, vm_instance):
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3/dist-packages/middlewared/main.py", line 1654, in call_sync
    return self.run_coroutine(methodobj(*prepared_call.args))
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3/dist-packages/middlewared/main.py", line 1694, in run_coroutine
    return fut.result()
           ^^^^^^^^^^^^
  File "/usr/lib/python3.11/concurrent/futures/_base.py", line 449, in result
    return self.__get_result()
           ^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3.11/concurrent/futures/_base.py", line 401, in __get_result
    raise self._exception
  File "/usr/lib/python3/dist-packages/middlewared/plugins/vm/vm_devices.py", line 264, in disk_uniqueness_integrity_check
    disks = [
            ^
  File "/usr/lib/python3/dist-packages/middlewared/plugins/vm/vm_devices.py", line 266, in <listcomp>
    if d['dtype'] in ('DISK', 'RAW', 'CDROM') and translate_device(d) == translate_device(device)
                                                  ^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3/dist-packages/middlewared/plugins/vm/vm_devices.py", line 262, in translate_device
    return dev['attributes'].get('path') or zvol_name_to_path(dev['attributes']['zvol_name'])
                                                              ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^
KeyError: 'zvol_name'

I was able to set the Nic data, but i can’t set the disk data anymore. Anyone know how to update the disk data?

So I just ended up restoring a backup and it fixed my VM issues, so I’m not sure how it got corrupted, but its fixed now