TrueNas disk partitions overwritten

TrueNAS General
,
21

also i’ve switched to kUBUNTU (not vanilla ubuntu as that was going to take hours to download) to see if that makes things easier in the future.

okay, much longer this time:

------------------------------------
LABEL 0 
------------------------------------
    version: 5000
    name: 'sapphire'
    state: 0
    txg: 6510548
    pool_guid: 4129880466361988471
    errata: 0
    hostid: 3938147066
    hostname: ''
    top_guid: 8818973256395834896
    guid: 2858572398724299414
    vdev_children: 1
    vdev_tree:
        type: 'raidz'
        id: 0
        guid: 8818973256395834896
        nparity: 2
        metaslab_array: 135
        metaslab_shift: 34
        ashift: 12
        asize: 39997054648320
        is_log: 0
        create_txg: 4
        children[0]:
            type: 'disk'
            id: 0
            guid: 2858572398724299414
            path: '/dev/gptid/40eb2a5f-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5060
            create_txg: 4
        children[1]:
            type: 'disk'
            id: 1
            guid: 1194783631545835994
            path: '/dev/gptid/40de4124-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5059
            create_txg: 4
        children[2]:
            type: 'disk'
            id: 2
            guid: 8569878839378827450
            path: '/dev/gptid/40f43471-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5058
            create_txg: 4
        children[3]:
            type: 'disk'
            id: 3
            guid: 6839407397638309827
            path: '/dev/gptid/40ffa4dd-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5057
            create_txg: 4
        children[4]:
            type: 'disk'
            id: 4
            guid: 6751170395373396232
            path: '/dev/gptid/4109a6d0-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5056
            create_txg: 4
    features_for_read:
        com.delphix:hole_birth
        com.delphix:embedded_data
    labels = 0 1 2 3 
    Uberblock[0]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833472
	guid_sum = 2269178237999389052
	timestamp = 1733096802 UTC = Sun Dec  1 23:46:42 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[1]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833473
	guid_sum = 2269178237999389052
	timestamp = 1733096807 UTC = Sun Dec  1 23:46:47 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[2]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833474
	guid_sum = 2269178237999389052
	timestamp = 1733096813 UTC = Sun Dec  1 23:46:53 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[3]
	magic = 0000000000bab10c
	version = 5000
	txg = 6842275
	guid_sum = 2269178237999389052
	timestamp = 1733140952 UTC = Mon Dec  2 12:02:32 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[4]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833476
	guid_sum = 2269178237999389052
	timestamp = 1733096823 UTC = Sun Dec  1 23:47:03 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[5]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833477
	guid_sum = 2269178237999389052
	timestamp = 1733096828 UTC = Sun Dec  1 23:47:08 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[6]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833478
	guid_sum = 2269178237999389052
	timestamp = 1733096833 UTC = Sun Dec  1 23:47:13 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[7]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833479
	guid_sum = 2269178237999389052
	timestamp = 1733096838 UTC = Sun Dec  1 23:47:18 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[8]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833480
	guid_sum = 2269178237999389052
	timestamp = 1733096843 UTC = Sun Dec  1 23:47:23 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[9]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833513
	guid_sum = 2269178237999389052
	timestamp = 1733097009 UTC = Sun Dec  1 23:50:09 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[10]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833482
	guid_sum = 2269178237999389052
	timestamp = 1733096853 UTC = Sun Dec  1 23:47:33 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[11]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833483
	guid_sum = 2269178237999389052
	timestamp = 1733096858 UTC = Sun Dec  1 23:47:38 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[12]
	magic = 0000000000bab10c
	version = 5000
	txg = 6842316
	guid_sum = 2269178237999389052
	timestamp = 1733141157 UTC = Mon Dec  2 12:05:57 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[13]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833485
	guid_sum = 2269178237999389052
	timestamp = 1733096868 UTC = Sun Dec  1 23:47:48 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[14]
	magic = 0000000000bab10c
	version = 5000
	txg = 6842254
	guid_sum = 2269178237999389052
	timestamp = 1733140846 UTC = Mon Dec  2 12:00:46 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[15]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833519
	guid_sum = 2269178237999389052
	timestamp = 1733097039 UTC = Sun Dec  1 23:50:39 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[16]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833552
	guid_sum = 2269178237999389052
	timestamp = 1733097205 UTC = Sun Dec  1 23:53:25 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[17]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833457
	guid_sum = 2269178237999389052
	timestamp = 1733096727 UTC = Sun Dec  1 23:45:27 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[18]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833458
	guid_sum = 2269178237999389052
	timestamp = 1733096732 UTC = Sun Dec  1 23:45:32 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[19]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833523
	guid_sum = 2269178237999389052
	timestamp = 1733097059 UTC = Sun Dec  1 23:50:59 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[20]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833460
	guid_sum = 2269178237999389052
	timestamp = 1733096742 UTC = Sun Dec  1 23:45:42 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[21]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833461
	guid_sum = 2269178237999389052
	timestamp = 1733096747 UTC = Sun Dec  1 23:45:47 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[22]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833462
	guid_sum = 2269178237999389052
	timestamp = 1733096752 UTC = Sun Dec  1 23:45:52 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[23]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833463
	guid_sum = 2269178237999389052
	timestamp = 1733096757 UTC = Sun Dec  1 23:45:57 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[24]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833464
	guid_sum = 2269178237999389052
	timestamp = 1733096762 UTC = Sun Dec  1 23:46:02 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[25]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833465
	guid_sum = 2269178237999389052
	timestamp = 1733096767 UTC = Sun Dec  1 23:46:07 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[26]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833466
	guid_sum = 2269178237999389052
	timestamp = 1733096772 UTC = Sun Dec  1 23:46:12 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[27]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833499
	guid_sum = 2269178237999389052
	timestamp = 1733096939 UTC = Sun Dec  1 23:48:59 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[28]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833564
	guid_sum = 2269178237999389052
	timestamp = 1733097265 UTC = Sun Dec  1 23:54:25 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[29]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833565
	guid_sum = 2269178237999389052
	timestamp = 1733097270 UTC = Sun Dec  1 23:54:30 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[30]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833566
	guid_sum = 2269178237999389052
	timestamp = 1733097275 UTC = Sun Dec  1 23:54:35 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[31]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833567
	guid_sum = 2269178237999389052
	timestamp = 1733097280 UTC = Sun Dec  1 23:54:40 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3

cautiously optimistic is good :slight_smile:

thought i’d show this log before doing anything further.

what’s the best way to generate UUIDs? a quick google suggests ‘uuidgen’. Is that good enough.
do i just use the same UUIDs you’ve given and change a digit instead? not sure how specific it is, or how truenas will handle it all. probably a silly thing to ask, and to just use uuidgen as that’s what it’s for right

'juggled with “-d” '? is that something that will also be done on the command line to import. The GUI not a good fit for this sort of thing?
Happy to use the command line to do this given your knowledge and input, but just curious is all.

Given that this disk has worked, I would expect the others to also work. That being said, if something does go wrong, the data is already gone. it cant really be more gone… right.

does the above output look good enough to try another disk?

1 Like
22

uuidgen should be fine here. The actual ZFS guid value for the child devices in the vdev is stored on-disk - we just need to have differing values for each of the partition tables so that Linux can tell them apart.

I’m uncertain. The path values in your labels will be invalid, unless we take a few extra laps of restoring a placeholder partition table with a generated UUID, doing a zdb query against the disk, and then matching the on-disk GUID with the UUID in the path and then re-writing a partition table with the correct UUID again.

For example, the drive you’ve got in there, we threw a partition ID of 5F5AFE87-D1D1-11EF-8370-000C29ED785D on there, and then queried the label to find out it’s ZFS GUID 2858572398724299414 which we can then map in the vdev_tree section to children[0] and understand that ZFS expects it to be partition 40eb2a5f-3012-11ee-9ec2-b9204f9364fa … so you could then update the partition table with that corrected UUID and do the sfdisk command again.

But that’s a lot of extra steps. zpool import -d tells it to search the devices, and we’re likely to be passing extra parameters to tell it to do things like “search a few transactions back” and “mount the pool read-only” as well - so the CLI may be how we do the first few safety checks, and then we’ll do a final import, export, and reboot.

It’s important to not have UUID collisions anywhere, but the UUIDs themselves shouldn’t be important enough to matter here.

dding /dev/random over the entire drive would make it “more gone” but I don’t think we want to do that. :wink:

Yes. I think we’re good enough to try another disk here. So take the sudo sfdisk -d descriptive output of the drive as you did up in the previous post and save that somewhere for safekeeping. I expect it to be identical (save for the partitions) and then use the shot-in-the-dark.txt file as a template, replacing the lines for device: and the /dev/sdb1 with the correct device name (unless you do one disk at a time) and the uuid values. Don’t touch the type, start, or size values.

More than happy to validate your file or generate it myself for you if you’d prefer.

1 Like
23

i was going to just go through the process with each disk and get them done…
redid the first disk (non clone) and that went exactly the same as the clone. one down.

shutdown, popped in my second disk
second ‘shot in the dark.txt’ looks like this

label: gpt
label-id: 4056AF31-3012-11EE-9EC2-B9204F9364FA
device: /dev/sda
unit: sectors
first-lba: 40
last-lba: 15628053134
sector-size: 512

/dev/sda1 : start=         128, size=     4194304, type=516E7CB5-6ECF-11D6-8FF8-00022D09712B, uuid=720da5fc-df08-11ef-ad55-64006a1a4fce
/dev/sda2 : start=     4194432, size= 15623858703, type=516E7CBA-6ECF-11D6-8FF8-00022D09712B, uuid=72c27590-df08-11ef-ad55-64006a1a4fce

only the UUIDs and label-id have changed.
not sure if the lowercase uuid makes any difference. this is how they were generated on kubuntu.

disk01 (non-clone) zdb -ul output:

------------------------------------
LABEL 0 
------------------------------------
    version: 5000
    name: 'sapphire'
    state: 0
    txg: 6510548
    pool_guid: 4129880466361988471
    errata: 0
    hostid: 3938147066
    hostname: ''
    top_guid: 8818973256395834896
    guid: 2858572398724299414
    vdev_children: 1
    vdev_tree:
        type: 'raidz'
        id: 0
        guid: 8818973256395834896
        nparity: 2
        metaslab_array: 135
        metaslab_shift: 34
        ashift: 12
        asize: 39997054648320
        is_log: 0
        create_txg: 4
        children[0]:
            type: 'disk'
            id: 0
            guid: 2858572398724299414
            path: '/dev/gptid/40eb2a5f-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5060
            create_txg: 4
        children[1]:
            type: 'disk'
            id: 1
            guid: 1194783631545835994
            path: '/dev/gptid/40de4124-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5059
            create_txg: 4
        children[2]:
            type: 'disk'
            id: 2
            guid: 8569878839378827450
            path: '/dev/gptid/40f43471-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5058
            create_txg: 4
        children[3]:
            type: 'disk'
            id: 3
            guid: 6839407397638309827
            path: '/dev/gptid/40ffa4dd-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5057
            create_txg: 4
        children[4]:
            type: 'disk'
            id: 4
            guid: 6751170395373396232
            path: '/dev/gptid/4109a6d0-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5056
            create_txg: 4
    features_for_read:
        com.delphix:hole_birth
        com.delphix:embedded_data
    labels = 0 1 2 3 
    Uberblock[0]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833472
	guid_sum = 2269178237999389052
	timestamp = 1733096802 UTC = Sun Dec  1 23:46:42 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[1]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833473
	guid_sum = 2269178237999389052
	timestamp = 1733096807 UTC = Sun Dec  1 23:46:47 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[2]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833474
	guid_sum = 2269178237999389052
	timestamp = 1733096813 UTC = Sun Dec  1 23:46:53 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[3]
	magic = 0000000000bab10c
	version = 5000
	txg = 6842275
	guid_sum = 2269178237999389052
	timestamp = 1733140952 UTC = Mon Dec  2 12:02:32 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[4]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833476
	guid_sum = 2269178237999389052
	timestamp = 1733096823 UTC = Sun Dec  1 23:47:03 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[5]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833477
	guid_sum = 2269178237999389052
	timestamp = 1733096828 UTC = Sun Dec  1 23:47:08 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[6]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833478
	guid_sum = 2269178237999389052
	timestamp = 1733096833 UTC = Sun Dec  1 23:47:13 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[7]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833479
	guid_sum = 2269178237999389052
	timestamp = 1733096838 UTC = Sun Dec  1 23:47:18 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[8]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833480
	guid_sum = 2269178237999389052
	timestamp = 1733096843 UTC = Sun Dec  1 23:47:23 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[9]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833513
	guid_sum = 2269178237999389052
	timestamp = 1733097009 UTC = Sun Dec  1 23:50:09 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[10]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833482
	guid_sum = 2269178237999389052
	timestamp = 1733096853 UTC = Sun Dec  1 23:47:33 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[11]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833483
	guid_sum = 2269178237999389052
	timestamp = 1733096858 UTC = Sun Dec  1 23:47:38 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[12]
	magic = 0000000000bab10c
	version = 5000
	txg = 6842316
	guid_sum = 2269178237999389052
	timestamp = 1733141157 UTC = Mon Dec  2 12:05:57 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[13]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833485
	guid_sum = 2269178237999389052
	timestamp = 1733096868 UTC = Sun Dec  1 23:47:48 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[14]
	magic = 0000000000bab10c
	version = 5000
	txg = 6842254
	guid_sum = 2269178237999389052
	timestamp = 1733140846 UTC = Mon Dec  2 12:00:46 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[15]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833519
	guid_sum = 2269178237999389052
	timestamp = 1733097039 UTC = Sun Dec  1 23:50:39 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[16]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833552
	guid_sum = 2269178237999389052
	timestamp = 1733097205 UTC = Sun Dec  1 23:53:25 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[17]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833457
	guid_sum = 2269178237999389052
	timestamp = 1733096727 UTC = Sun Dec  1 23:45:27 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[18]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833458
	guid_sum = 2269178237999389052
	timestamp = 1733096732 UTC = Sun Dec  1 23:45:32 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[19]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833523
	guid_sum = 2269178237999389052
	timestamp = 1733097059 UTC = Sun Dec  1 23:50:59 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[20]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833460
	guid_sum = 2269178237999389052
	timestamp = 1733096742 UTC = Sun Dec  1 23:45:42 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[21]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833461
	guid_sum = 2269178237999389052
	timestamp = 1733096747 UTC = Sun Dec  1 23:45:47 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[22]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833462
	guid_sum = 2269178237999389052
	timestamp = 1733096752 UTC = Sun Dec  1 23:45:52 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[23]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833463
	guid_sum = 2269178237999389052
	timestamp = 1733096757 UTC = Sun Dec  1 23:45:57 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[24]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833464
	guid_sum = 2269178237999389052
	timestamp = 1733096762 UTC = Sun Dec  1 23:46:02 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[25]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833465
	guid_sum = 2269178237999389052
	timestamp = 1733096767 UTC = Sun Dec  1 23:46:07 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[26]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833466
	guid_sum = 2269178237999389052
	timestamp = 1733096772 UTC = Sun Dec  1 23:46:12 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[27]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833499
	guid_sum = 2269178237999389052
	timestamp = 1733096939 UTC = Sun Dec  1 23:48:59 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[28]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833564
	guid_sum = 2269178237999389052
	timestamp = 1733097265 UTC = Sun Dec  1 23:54:25 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[29]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833565
	guid_sum = 2269178237999389052
	timestamp = 1733097270 UTC = Sun Dec  1 23:54:30 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[30]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833566
	guid_sum = 2269178237999389052
	timestamp = 1733097275 UTC = Sun Dec  1 23:54:35 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[31]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833567
	guid_sum = 2269178237999389052
	timestamp = 1733097280 UTC = Sun Dec  1 23:54:40 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3

disk02 zdb -ul output:

------------------------------------
LABEL 0 
------------------------------------
    version: 5000
    name: 'sapphire'
    state: 0
    txg: 6510548
    pool_guid: 4129880466361988471
    errata: 0
    hostid: 3938147066
    hostname: ''
    top_guid: 8818973256395834896
    guid: 2858572398724299414
    vdev_children: 1
    vdev_tree:
        type: 'raidz'
        id: 0
        guid: 8818973256395834896
        nparity: 2
        metaslab_array: 135
        metaslab_shift: 34
        ashift: 12
        asize: 39997054648320
        is_log: 0
        create_txg: 4
        children[0]:
            type: 'disk'
            id: 0
            guid: 2858572398724299414
            path: '/dev/gptid/40eb2a5f-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5060
            create_txg: 4
        children[1]:
            type: 'disk'
            id: 1
            guid: 1194783631545835994
            path: '/dev/gptid/40de4124-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5059
            create_txg: 4
        children[2]:
            type: 'disk'
            id: 2
            guid: 8569878839378827450
            path: '/dev/gptid/40f43471-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5058
            create_txg: 4
        children[3]:
            type: 'disk'
            id: 3
            guid: 6839407397638309827
            path: '/dev/gptid/40ffa4dd-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5057
            create_txg: 4
        children[4]:
            type: 'disk'
            id: 4
            guid: 6751170395373396232
            path: '/dev/gptid/4109a6d0-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5056
            create_txg: 4
    features_for_read:
        com.delphix:hole_birth
        com.delphix:embedded_data
    labels = 0 1 
    Uberblock[0]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833472
	guid_sum = 2269178237999389052
	timestamp = 1733096802 UTC = Sun Dec  1 23:46:42 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[1]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833473
	guid_sum = 2269178237999389052
	timestamp = 1733096807 UTC = Sun Dec  1 23:46:47 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[2]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833474
	guid_sum = 2269178237999389052
	timestamp = 1733096813 UTC = Sun Dec  1 23:46:53 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[3]
	magic = 0000000000bab10c
	version = 5000
	txg = 6842275
	guid_sum = 2269178237999389052
	timestamp = 1733140952 UTC = Mon Dec  2 12:02:32 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[4]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833476
	guid_sum = 2269178237999389052
	timestamp = 1733096823 UTC = Sun Dec  1 23:47:03 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[5]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833477
	guid_sum = 2269178237999389052
	timestamp = 1733096828 UTC = Sun Dec  1 23:47:08 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[6]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833478
	guid_sum = 2269178237999389052
	timestamp = 1733096833 UTC = Sun Dec  1 23:47:13 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[7]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833479
	guid_sum = 2269178237999389052
	timestamp = 1733096838 UTC = Sun Dec  1 23:47:18 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[8]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833480
	guid_sum = 2269178237999389052
	timestamp = 1733096843 UTC = Sun Dec  1 23:47:23 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[9]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833513
	guid_sum = 2269178237999389052
	timestamp = 1733097009 UTC = Sun Dec  1 23:50:09 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[10]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833482
	guid_sum = 2269178237999389052
	timestamp = 1733096853 UTC = Sun Dec  1 23:47:33 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[11]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833483
	guid_sum = 2269178237999389052
	timestamp = 1733096858 UTC = Sun Dec  1 23:47:38 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[12]
	magic = 0000000000bab10c
	version = 5000
	txg = 6842316
	guid_sum = 2269178237999389052
	timestamp = 1733141157 UTC = Mon Dec  2 12:05:57 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[13]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833485
	guid_sum = 2269178237999389052
	timestamp = 1733096868 UTC = Sun Dec  1 23:47:48 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[14]
	magic = 0000000000bab10c
	version = 5000
	txg = 6842254
	guid_sum = 2269178237999389052
	timestamp = 1733140846 UTC = Mon Dec  2 12:00:46 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[15]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833519
	guid_sum = 2269178237999389052
	timestamp = 1733097039 UTC = Sun Dec  1 23:50:39 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[16]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833552
	guid_sum = 2269178237999389052
	timestamp = 1733097205 UTC = Sun Dec  1 23:53:25 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[17]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833457
	guid_sum = 2269178237999389052
	timestamp = 1733096727 UTC = Sun Dec  1 23:45:27 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[18]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833458
	guid_sum = 2269178237999389052
	timestamp = 1733096732 UTC = Sun Dec  1 23:45:32 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[19]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833523
	guid_sum = 2269178237999389052
	timestamp = 1733097059 UTC = Sun Dec  1 23:50:59 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[20]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833460
	guid_sum = 2269178237999389052
	timestamp = 1733096742 UTC = Sun Dec  1 23:45:42 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[21]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833461
	guid_sum = 2269178237999389052
	timestamp = 1733096747 UTC = Sun Dec  1 23:45:47 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[22]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833462
	guid_sum = 2269178237999389052
	timestamp = 1733096752 UTC = Sun Dec  1 23:45:52 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[23]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833463
	guid_sum = 2269178237999389052
	timestamp = 1733096757 UTC = Sun Dec  1 23:45:57 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[24]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833464
	guid_sum = 2269178237999389052
	timestamp = 1733096762 UTC = Sun Dec  1 23:46:02 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[25]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833465
	guid_sum = 2269178237999389052
	timestamp = 1733096767 UTC = Sun Dec  1 23:46:07 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[26]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833466
	guid_sum = 2269178237999389052
	timestamp = 1733096772 UTC = Sun Dec  1 23:46:12 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[27]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833499
	guid_sum = 2269178237999389052
	timestamp = 1733096939 UTC = Sun Dec  1 23:48:59 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[28]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833564
	guid_sum = 2269178237999389052
	timestamp = 1733097265 UTC = Sun Dec  1 23:54:25 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[29]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833565
	guid_sum = 2269178237999389052
	timestamp = 1733097270 UTC = Sun Dec  1 23:54:30 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[30]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833566
	guid_sum = 2269178237999389052
	timestamp = 1733097275 UTC = Sun Dec  1 23:54:35 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[31]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833567
	guid_sum = 2269178237999389052
	timestamp = 1733097280 UTC = Sun Dec  1 23:54:40 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
------------------------------------
LABEL 2 (Bad label cksum)
------------------------------------
    version: 5000
    name: 'sapphire'
    state: 0
    txg: 6510548
    pool_guid: 4129880466361988471
    errata: 0
    hostid: 3938147066
    hostname: ''
    top_guid: 8818973256395834896
    guid: 6751170395373396232
    vdev_children: 1
    vdev_tree:
        type: 'raidz'
        id: 0
        guid: 8818973256395834896
        nparity: 2
        metaslab_array: 135
        metaslab_shift: 34
        ashift: 12
        asize: 39997054648320
        is_log: 0
        create_txg: 4
        children[0]:
            type: 'disk'
            id: 0
            guid: 2858572398724299414
            path: '/dev/gptid/40eb2a5f-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5060
            create_txg: 4
        children[1]:
            type: 'disk'
            id: 1
            guid: 1194783631545835994
            path: '/dev/gptid/40de4124-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5059
            create_txg: 4
        children[2]:
            type: 'disk'
            id: 2
            guid: 8569878839378827450
            path: '/dev/gptid/40f43471-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5058
            create_txg: 4
        children[3]:
            type: 'disk'
            id: 3
            guid: 6839407397638309827
            path: '/dev/gptid/40ffa4dd-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5057
            create_txg: 4
        children[4]:
            type: 'disk'
            id: 4
            guid: 6751170395373396232
            path: '/dev/gptid/4109a6d0-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5056
            create_txg: 4
    features_for_read:
        com.delphix:hole_birth
        com.delphix:embedded_data
    labels = 2 3

the issue here, is im noting a new section that shows “LABEL 2 (Bad label cksum)” that is stopping me from doing anything further before asking
I’m hoping this is just a part of our broken recovery process :sweat_smile: and not something i’ve done that’s going to stop everything in its tracks…

haha. i dont intend to do something this silly… i do want the data back. i swear. :joy:

24

When you did your disk02 did you export the existing label with sfdisk -d first?

You want to use the existing (exported) label values for the first section that defines the disk itself.

Notably though - if the values for first-lba and last-lba are not identical to your other disk, I’ll need to math out the correct/expected partition sizes again. So let’s see a regular plain output of sfdisk -d /dev/whatever-sd-disk02-is - perhaps we should be referring to them by the last couple digits of their serial numbers for clarity.

label: gpt
label-id: 4056AF31-3012-11EE-9EC2-B9204F9364FA
device: /dev/sda
unit: sectors
first-lba: 40
last-lba: 15628053134
sector-size: 512

Right now it looks like the second shot-in-the-dark file used the same label-id line #2 as the first one.

So it’s

  1. export the existing label with sfdisk -d to a file and make a backup copy
  2. modify only the last two lines of that exported file, adjusting the uuid there
  3. rewrite that label back with the sfdisk < disk02.modified.txt

I don’t believe so but you can re-dump the label with sfdisk -d and see if it shows them in uppercase. It’s hex so ideally it shouldn’t be case-sensitive.

I expected this to be possible. What it likely means is that the Xpenology installer clobbered one of the four redundant labels that ZFS writes to the drive (or the drive is very slightly larger/smaller and we need to twiddle the partition size values)

Ideally ZFS just uses one of the other valid labels to import the pool - worst case you appear to have made a Z2 so you can survive a disk being lost.

25

i did export the sfdisk -d, however, i didnt save the output :
i made notes of what to check, but didnt check/modify the LBA sizes… :sweat_smile:
only changing the label-id, and the two uuid’s

oh, i have a sticky note on the disks to remind myself, but thats probably a decent idea also.

it shouldnt be.
disk 1: 4071A906-3012-11EE-9EC2-B9204F9364FA
disk 2: 4056AF31-3012-11EE-9EC2-B9204F9364FA
but i can see at a glance it might look that way

so yeah, i should be doing what you’re saying.

disk01

label: gpt
label-id: 4071A906-3012-11EE-9EC2-B9204F9364FA
device: /dev/sda
unit: sectors
first-lba: 40
last-lba: 15628053134
sector-size: 512

/dev/sda1 : start=         128, size=     4194304, type=516E7CB5-6ECF-11D6-8FF8-00022D09712B, uuid=5F56F119-D1D1-11EF-8370-000C29ED785D
/dev/sda2 : start=     4194432, size= 15623858703, type=516E7CBA-6ECF-11D6-8FF8-00022D09712B, uuid=5F5AFE87-D1D1-11EF-8370-000C29ED785D

disk02

label: gpt
label-id: 4056AF31-3012-11EE-9EC2-B9204F9364FA
device: /dev/sda
unit: sectors
first-lba: 40
last-lba: 15628053134
sector-size: 512

/dev/sda1 : start=         128, size=     4194304, type=516E7CB5-6ECF-11D6-8FF8-00022D09712B, uuid=720da5fc-df08-11ef-ad55-64006a1a4fce
/dev/sda2 : start=     4194432, size= 15623858703, type=516E7CBA-6ECF-11D6-8FF8-00022D09712B, uuid=72c27590-df08-11ef-ad55-64006a1a4fce

and again, while i didnt change the LBA numbers, i believe the rest should be correct.

i’ll redump the second disk and see what it gives. if not, ill specifically write uppercase string again

uh oh. presumably thats what the LBA stuff was meant to be for…
ill have to compare the rest of the disks before i modify anything further.

oh, that would be weird. i’m sure it was 4 disks and 1 redundant (ie z1). but then again, my brain could be playing tricks on me…

26

If they’re all identical models then they should have identical LBA geometry. Unless one is formatted to 4K sectors in which case it’s just gonna be all out of whack.

From your label dump:

    vdev_tree:
        type: 'raidz'
        ...
        nparity: 2

According to this, you’ve got a RAIDZ2 :slight_smile:

27

Screenshot 2025-01-31 060446
been scouring my photos as i thought maybe i sent a screengrab to someone. this is the closest thing i can find.
5 disks, 20TB available, 8% used. works out to about 3 disks and 2 redundant…
so you’re most likely right on the money here.

I dont see why they should be the same for the LBA stuff. havent had time to check yet, but i would assume truenas would format them to something usable. and all the same thing.

i’ll check the rest of the disks for LBA, and if they’re all the same, i’ll go through and update the partition tables with each of the modified 'shot-in-the-dark’s

what would be the next step after that? maybe doing a dry run of zpool import -d perhaps? but i’ll leave that up to the professional, and just wait to see what your next bit of info is.
cheers

28

okay, checked the remaining three disks

raw sfdisk 3

label: gpt
label-id: 40411EB0-3012-11EE-9EC2-B9204F9364FA
device: /dev/sda
unit: sectors
first-lba: 40
last-lba: 15628053134
sector-size: 512

/dev/sda1 : start=        8192, size=    16777216, type=A19D880F-05FC-4D3B-A006-743F0F84911E, uuid=27FD1BD7-93F0-4FEB-ADAE-484D82B80C8A
/dev/sda2 : start=    16785408, size=     4194304, type=A19D880F-05FC-4D3B-A006-743F0F84911E, uuid=37B0A24B-4141-4690-92F4-51F47C6364FF

raw sfdisk4

label: gpt
label-id: 4060CFDC-3012-11EE-9EC2-B9204F9364FA
device: /dev/sda
unit: sectors
first-lba: 40
last-lba: 15628053134
sector-size: 512

/dev/sda1 : start=        8192, size=    16777216, type=A19D880F-05FC-4D3B-A006-743F0F84911E, uuid=12114953-8AB9-4F09-8B16-A0185CEFF7CB
/dev/sda2 : start=    16785408, size=     4194304, type=A19D880F-05FC-4D3B-A006-743F0F84911E, uuid=72D8F4D0-6826-4F81-9DE0-FD064429C79F

raw sfdisk5

label: gpt
label-id: 404BB92A-3012-11EE-9EC2-B9204F9364FA
device: /dev/sdb
unit: sectors
first-lba: 40
last-lba: 15628053134
sector-size: 512

/dev/sdb1 : start=        8192, size=    16777216, type=A19D880F-05FC-4D3B-A006-743F0F84911E, uuid=0EC7C280-DCF5-47C5-94E0-12FF1E0A4490
/dev/sdb2 : start=    16785408, size=     4194304, type=A19D880F-05FC-4D3B-A006-743F0F84911E, uuid=84DF0CEF-D9DD-43E2-83C2-E00BB81E03C1

all the sector sizes and LBAs match, so looks like we’re good to go.

the shot-in-the-dark-03

label: gpt
label-id: 40411EB0-3012-11EE-9EC2-B9204F9364FA
device: /dev/sda
unit: sectors
first-lba: 40
last-lba: 15628053134
sector-size: 512

/dev/sda1 : start=         128, size=     4194304, type=516E7CB5-6ECF-11D6-8FF8-00022D09712B, uuid=2c7835b6-dfdf-11ef-b663-64006a1a4fce
/dev/sda2 : start=     4194432, size= 15623858703, type=516E7CBA-6ECF-11D6-8FF8-00022D09712B, uuid=2cd4fada-dfdf-11ef-b663-64006a1a4fce

the shot-in-the-dark-04

label: gpt
label-id: 4060CFDC-3012-11EE-9EC2-B9204F9364FA
device: /dev/sda
unit: sectors
first-lba: 40
last-lba: 15628053134
sector-size: 512

/dev/sda1 : start=         128, size=     4194304, type=516E7CB5-6ECF-11D6-8FF8-00022D09712B, uuid=74cd848a-dfdd-11ef-9c48-65e8d1055310
/dev/sda2 : start=     4194432, size= 15623858703, type=516E7CBA-6ECF-11D6-8FF8-00022D09712B, uuid=752afdcc-dfdd-11ef-9c48-65e8d1055310

the shot-in-the-dark-05

label: gpt
label-id: 404BB92A-3012-11EE-9EC2-B9204F9364FA
device: /dev/sdb
unit: sectors
first-lba: 40
last-lba: 15628053134
sector-size: 512

/dev/sdb1 : start=         128, size=     4194304, type=516E7CB5-6ECF-11D6-8FF8-00022D09712B, uuid=d2dd3cbe-dfdd-11ef-9c48-65e8d1055310
/dev/sdb2 : start=     4194432, size= 15623858703, type=516E7CBA-6ECF-11D6-8FF8-00022D09712B, uuid=d3305976-dfdd-11ef-9c48-65e8d1055310

*note did 4 and 5 at the same time, so 5 is the only one that uses sdb in these tests. should be fine. just saying.

i double checked disk 2 with the lower case UUID and it was showing in uppercase so we good.

finally the outputs for zdb -ul

disk3

------------------------------------
LABEL 0 
------------------------------------
    version: 5000
    name: 'sapphire'
    state: 0
    txg: 6510548
    pool_guid: 4129880466361988471
    errata: 0
    hostid: 3938147066
    hostname: ''
    top_guid: 8818973256395834896
    guid: 2858572398724299414
    vdev_children: 1
    vdev_tree:
        type: 'raidz'
        id: 0
        guid: 8818973256395834896
        nparity: 2
        metaslab_array: 135
        metaslab_shift: 34
        ashift: 12
        asize: 39997054648320
        is_log: 0
        create_txg: 4
        children[0]:
            type: 'disk'
            id: 0
            guid: 2858572398724299414
            path: '/dev/gptid/40eb2a5f-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5060
            create_txg: 4
        children[1]:
            type: 'disk'
            id: 1
            guid: 1194783631545835994
            path: '/dev/gptid/40de4124-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5059
            create_txg: 4
        children[2]:
            type: 'disk'
            id: 2
            guid: 8569878839378827450
            path: '/dev/gptid/40f43471-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5058
            create_txg: 4
        children[3]:
            type: 'disk'
            id: 3
            guid: 6839407397638309827
            path: '/dev/gptid/40ffa4dd-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5057
            create_txg: 4
        children[4]:
            type: 'disk'
            id: 4
            guid: 6751170395373396232
            path: '/dev/gptid/4109a6d0-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5056
            create_txg: 4
    features_for_read:
        com.delphix:hole_birth
        com.delphix:embedded_data
    labels = 0 1 
    Uberblock[0]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833472
	guid_sum = 2269178237999389052
	timestamp = 1733096802 UTC = Sun Dec  1 23:46:42 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[1]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833473
	guid_sum = 2269178237999389052
	timestamp = 1733096807 UTC = Sun Dec  1 23:46:47 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[2]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833474
	guid_sum = 2269178237999389052
	timestamp = 1733096813 UTC = Sun Dec  1 23:46:53 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[3]
	magic = 0000000000bab10c
	version = 5000
	txg = 6842275
	guid_sum = 2269178237999389052
	timestamp = 1733140952 UTC = Mon Dec  2 12:02:32 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[4]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833476
	guid_sum = 2269178237999389052
	timestamp = 1733096823 UTC = Sun Dec  1 23:47:03 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[5]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833477
	guid_sum = 2269178237999389052
	timestamp = 1733096828 UTC = Sun Dec  1 23:47:08 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[6]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833478
	guid_sum = 2269178237999389052
	timestamp = 1733096833 UTC = Sun Dec  1 23:47:13 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[7]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833479
	guid_sum = 2269178237999389052
	timestamp = 1733096838 UTC = Sun Dec  1 23:47:18 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[8]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833480
	guid_sum = 2269178237999389052
	timestamp = 1733096843 UTC = Sun Dec  1 23:47:23 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[9]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833513
	guid_sum = 2269178237999389052
	timestamp = 1733097009 UTC = Sun Dec  1 23:50:09 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[10]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833482
	guid_sum = 2269178237999389052
	timestamp = 1733096853 UTC = Sun Dec  1 23:47:33 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[11]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833483
	guid_sum = 2269178237999389052
	timestamp = 1733096858 UTC = Sun Dec  1 23:47:38 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[12]
	magic = 0000000000bab10c
	version = 5000
	txg = 6842316
	guid_sum = 2269178237999389052
	timestamp = 1733141157 UTC = Mon Dec  2 12:05:57 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[13]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833485
	guid_sum = 2269178237999389052
	timestamp = 1733096868 UTC = Sun Dec  1 23:47:48 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[14]
	magic = 0000000000bab10c
	version = 5000
	txg = 6842254
	guid_sum = 2269178237999389052
	timestamp = 1733140846 UTC = Mon Dec  2 12:00:46 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[15]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833519
	guid_sum = 2269178237999389052
	timestamp = 1733097039 UTC = Sun Dec  1 23:50:39 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[16]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833552
	guid_sum = 2269178237999389052
	timestamp = 1733097205 UTC = Sun Dec  1 23:53:25 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[17]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833457
	guid_sum = 2269178237999389052
	timestamp = 1733096727 UTC = Sun Dec  1 23:45:27 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[18]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833458
	guid_sum = 2269178237999389052
	timestamp = 1733096732 UTC = Sun Dec  1 23:45:32 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[19]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833523
	guid_sum = 2269178237999389052
	timestamp = 1733097059 UTC = Sun Dec  1 23:50:59 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[20]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833460
	guid_sum = 2269178237999389052
	timestamp = 1733096742 UTC = Sun Dec  1 23:45:42 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[21]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833461
	guid_sum = 2269178237999389052
	timestamp = 1733096747 UTC = Sun Dec  1 23:45:47 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[22]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833462
	guid_sum = 2269178237999389052
	timestamp = 1733096752 UTC = Sun Dec  1 23:45:52 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[23]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833463
	guid_sum = 2269178237999389052
	timestamp = 1733096757 UTC = Sun Dec  1 23:45:57 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[24]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833464
	guid_sum = 2269178237999389052
	timestamp = 1733096762 UTC = Sun Dec  1 23:46:02 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[25]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833465
	guid_sum = 2269178237999389052
	timestamp = 1733096767 UTC = Sun Dec  1 23:46:07 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[26]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833466
	guid_sum = 2269178237999389052
	timestamp = 1733096772 UTC = Sun Dec  1 23:46:12 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[27]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833499
	guid_sum = 2269178237999389052
	timestamp = 1733096939 UTC = Sun Dec  1 23:48:59 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[28]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833564
	guid_sum = 2269178237999389052
	timestamp = 1733097265 UTC = Sun Dec  1 23:54:25 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[29]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833565
	guid_sum = 2269178237999389052
	timestamp = 1733097270 UTC = Sun Dec  1 23:54:30 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[30]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833566
	guid_sum = 2269178237999389052
	timestamp = 1733097275 UTC = Sun Dec  1 23:54:35 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[31]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833567
	guid_sum = 2269178237999389052
	timestamp = 1733097280 UTC = Sun Dec  1 23:54:40 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
------------------------------------
LABEL 2 (Bad label cksum)
------------------------------------
    version: 5000
    name: 'sapphire'
    state: 0
    txg: 6510548
    pool_guid: 4129880466361988471
    errata: 0
    hostid: 3938147066
    hostname: ''
    top_guid: 8818973256395834896
    guid: 6839407397638309827
    vdev_children: 1
    vdev_tree:
        type: 'raidz'
        id: 0
        guid: 8818973256395834896
        nparity: 2
        metaslab_array: 135
        metaslab_shift: 34
        ashift: 12
        asize: 39997054648320
        is_log: 0
        create_txg: 4
        children[0]:
            type: 'disk'
            id: 0
            guid: 2858572398724299414
            path: '/dev/gptid/40eb2a5f-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5060
            create_txg: 4
        children[1]:
            type: 'disk'
            id: 1
            guid: 1194783631545835994
            path: '/dev/gptid/40de4124-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5059
            create_txg: 4
        children[2]:
            type: 'disk'
            id: 2
            guid: 8569878839378827450
            path: '/dev/gptid/40f43471-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5058
            create_txg: 4
        children[3]:
            type: 'disk'
            id: 3
            guid: 6839407397638309827
            path: '/dev/gptid/40ffa4dd-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5057
            create_txg: 4
        children[4]:
            type: 'disk'
            id: 4
            guid: 6751170395373396232
            path: '/dev/gptid/4109a6d0-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5056
            create_txg: 4
    features_for_read:
        com.delphix:hole_birth
        com.delphix:embedded_data
    labels = 2 3

zdb 4

------------------------------------
LABEL 0 
------------------------------------
    version: 5000
    name: 'sapphire'
    state: 0
    txg: 6510548
    pool_guid: 4129880466361988471
    errata: 0
    hostid: 3938147066
    hostname: ''
    top_guid: 8818973256395834896
    guid: 2858572398724299414
    vdev_children: 1
    vdev_tree:
        type: 'raidz'
        id: 0
        guid: 8818973256395834896
        nparity: 2
        metaslab_array: 135
        metaslab_shift: 34
        ashift: 12
        asize: 39997054648320
        is_log: 0
        create_txg: 4
        children[0]:
            type: 'disk'
            id: 0
            guid: 2858572398724299414
            path: '/dev/gptid/40eb2a5f-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5060
            create_txg: 4
        children[1]:
            type: 'disk'
            id: 1
            guid: 1194783631545835994
            path: '/dev/gptid/40de4124-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5059
            create_txg: 4
        children[2]:
            type: 'disk'
            id: 2
            guid: 8569878839378827450
            path: '/dev/gptid/40f43471-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5058
            create_txg: 4
        children[3]:
            type: 'disk'
            id: 3
            guid: 6839407397638309827
            path: '/dev/gptid/40ffa4dd-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5057
            create_txg: 4
        children[4]:
            type: 'disk'
            id: 4
            guid: 6751170395373396232
            path: '/dev/gptid/4109a6d0-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5056
            create_txg: 4
    features_for_read:
        com.delphix:hole_birth
        com.delphix:embedded_data
    labels = 0 1 
    Uberblock[0]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833472
	guid_sum = 2269178237999389052
	timestamp = 1733096802 UTC = Sun Dec  1 23:46:42 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[1]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833473
	guid_sum = 2269178237999389052
	timestamp = 1733096807 UTC = Sun Dec  1 23:46:47 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[2]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833474
	guid_sum = 2269178237999389052
	timestamp = 1733096813 UTC = Sun Dec  1 23:46:53 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[3]
	magic = 0000000000bab10c
	version = 5000
	txg = 6842275
	guid_sum = 2269178237999389052
	timestamp = 1733140952 UTC = Mon Dec  2 12:02:32 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[4]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833476
	guid_sum = 2269178237999389052
	timestamp = 1733096823 UTC = Sun Dec  1 23:47:03 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[5]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833477
	guid_sum = 2269178237999389052
	timestamp = 1733096828 UTC = Sun Dec  1 23:47:08 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[6]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833478
	guid_sum = 2269178237999389052
	timestamp = 1733096833 UTC = Sun Dec  1 23:47:13 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[7]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833479
	guid_sum = 2269178237999389052
	timestamp = 1733096838 UTC = Sun Dec  1 23:47:18 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[8]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833480
	guid_sum = 2269178237999389052
	timestamp = 1733096843 UTC = Sun Dec  1 23:47:23 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[9]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833513
	guid_sum = 2269178237999389052
	timestamp = 1733097009 UTC = Sun Dec  1 23:50:09 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[10]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833482
	guid_sum = 2269178237999389052
	timestamp = 1733096853 UTC = Sun Dec  1 23:47:33 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[11]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833483
	guid_sum = 2269178237999389052
	timestamp = 1733096858 UTC = Sun Dec  1 23:47:38 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[12]
	magic = 0000000000bab10c
	version = 5000
	txg = 6842316
	guid_sum = 2269178237999389052
	timestamp = 1733141157 UTC = Mon Dec  2 12:05:57 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[13]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833485
	guid_sum = 2269178237999389052
	timestamp = 1733096868 UTC = Sun Dec  1 23:47:48 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[14]
	magic = 0000000000bab10c
	version = 5000
	txg = 6842254
	guid_sum = 2269178237999389052
	timestamp = 1733140846 UTC = Mon Dec  2 12:00:46 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[15]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833519
	guid_sum = 2269178237999389052
	timestamp = 1733097039 UTC = Sun Dec  1 23:50:39 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[16]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833552
	guid_sum = 2269178237999389052
	timestamp = 1733097205 UTC = Sun Dec  1 23:53:25 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[17]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833457
	guid_sum = 2269178237999389052
	timestamp = 1733096727 UTC = Sun Dec  1 23:45:27 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[18]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833458
	guid_sum = 2269178237999389052
	timestamp = 1733096732 UTC = Sun Dec  1 23:45:32 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[19]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833523
	guid_sum = 2269178237999389052
	timestamp = 1733097059 UTC = Sun Dec  1 23:50:59 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[20]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833460
	guid_sum = 2269178237999389052
	timestamp = 1733096742 UTC = Sun Dec  1 23:45:42 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[21]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833461
	guid_sum = 2269178237999389052
	timestamp = 1733096747 UTC = Sun Dec  1 23:45:47 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[22]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833462
	guid_sum = 2269178237999389052
	timestamp = 1733096752 UTC = Sun Dec  1 23:45:52 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[23]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833463
	guid_sum = 2269178237999389052
	timestamp = 1733096757 UTC = Sun Dec  1 23:45:57 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[24]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833464
	guid_sum = 2269178237999389052
	timestamp = 1733096762 UTC = Sun Dec  1 23:46:02 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[25]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833465
	guid_sum = 2269178237999389052
	timestamp = 1733096767 UTC = Sun Dec  1 23:46:07 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[26]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833466
	guid_sum = 2269178237999389052
	timestamp = 1733096772 UTC = Sun Dec  1 23:46:12 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[27]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833499
	guid_sum = 2269178237999389052
	timestamp = 1733096939 UTC = Sun Dec  1 23:48:59 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[28]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833564
	guid_sum = 2269178237999389052
	timestamp = 1733097265 UTC = Sun Dec  1 23:54:25 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[29]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833565
	guid_sum = 2269178237999389052
	timestamp = 1733097270 UTC = Sun Dec  1 23:54:30 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[30]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833566
	guid_sum = 2269178237999389052
	timestamp = 1733097275 UTC = Sun Dec  1 23:54:35 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[31]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833567
	guid_sum = 2269178237999389052
	timestamp = 1733097280 UTC = Sun Dec  1 23:54:40 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
------------------------------------
LABEL 2 (Bad label cksum)
------------------------------------
    version: 5000
    name: 'sapphire'
    state: 0
    txg: 6510548
    pool_guid: 4129880466361988471
    errata: 0
    hostid: 3938147066
    hostname: ''
    top_guid: 8818973256395834896
    guid: 8569878839378827450
    vdev_children: 1
    vdev_tree:
        type: 'raidz'
        id: 0
        guid: 8818973256395834896
        nparity: 2
        metaslab_array: 135
        metaslab_shift: 34
        ashift: 12
        asize: 39997054648320
        is_log: 0
        create_txg: 4
        children[0]:
            type: 'disk'
            id: 0
            guid: 2858572398724299414
            path: '/dev/gptid/40eb2a5f-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5060
            create_txg: 4
        children[1]:
            type: 'disk'
            id: 1
            guid: 1194783631545835994
            path: '/dev/gptid/40de4124-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5059
            create_txg: 4
        children[2]:
            type: 'disk'
            id: 2
            guid: 8569878839378827450
            path: '/dev/gptid/40f43471-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5058
            create_txg: 4
        children[3]:
            type: 'disk'
            id: 3
            guid: 6839407397638309827
            path: '/dev/gptid/40ffa4dd-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5057
            create_txg: 4
        children[4]:
            type: 'disk'
            id: 4
            guid: 6751170395373396232
            path: '/dev/gptid/4109a6d0-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5056
            create_txg: 4
    features_for_read:
        com.delphix:hole_birth
        com.delphix:embedded_data
    labels = 2 3

zdb 5

------------------------------------
LABEL 0 
------------------------------------
    version: 5000
    name: 'sapphire'
    state: 0
    txg: 6510548
    pool_guid: 4129880466361988471
    errata: 0
    hostid: 3938147066
    hostname: ''
    top_guid: 8818973256395834896
    guid: 2858572398724299414
    vdev_children: 1
    vdev_tree:
        type: 'raidz'
        id: 0
        guid: 8818973256395834896
        nparity: 2
        metaslab_array: 135
        metaslab_shift: 34
        ashift: 12
        asize: 39997054648320
        is_log: 0
        create_txg: 4
        children[0]:
            type: 'disk'
            id: 0
            guid: 2858572398724299414
            path: '/dev/gptid/40eb2a5f-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5060
            create_txg: 4
        children[1]:
            type: 'disk'
            id: 1
            guid: 1194783631545835994
            path: '/dev/gptid/40de4124-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5059
            create_txg: 4
        children[2]:
            type: 'disk'
            id: 2
            guid: 8569878839378827450
            path: '/dev/gptid/40f43471-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5058
            create_txg: 4
        children[3]:
            type: 'disk'
            id: 3
            guid: 6839407397638309827
            path: '/dev/gptid/40ffa4dd-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5057
            create_txg: 4
        children[4]:
            type: 'disk'
            id: 4
            guid: 6751170395373396232
            path: '/dev/gptid/4109a6d0-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5056
            create_txg: 4
    features_for_read:
        com.delphix:hole_birth
        com.delphix:embedded_data
    labels = 0 1 
    Uberblock[0]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833472
	guid_sum = 2269178237999389052
	timestamp = 1733096802 UTC = Sun Dec  1 23:46:42 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[1]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833473
	guid_sum = 2269178237999389052
	timestamp = 1733096807 UTC = Sun Dec  1 23:46:47 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[2]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833474
	guid_sum = 2269178237999389052
	timestamp = 1733096813 UTC = Sun Dec  1 23:46:53 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[3]
	magic = 0000000000bab10c
	version = 5000
	txg = 6842275
	guid_sum = 2269178237999389052
	timestamp = 1733140952 UTC = Mon Dec  2 12:02:32 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[4]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833476
	guid_sum = 2269178237999389052
	timestamp = 1733096823 UTC = Sun Dec  1 23:47:03 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[5]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833477
	guid_sum = 2269178237999389052
	timestamp = 1733096828 UTC = Sun Dec  1 23:47:08 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[6]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833478
	guid_sum = 2269178237999389052
	timestamp = 1733096833 UTC = Sun Dec  1 23:47:13 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[7]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833479
	guid_sum = 2269178237999389052
	timestamp = 1733096838 UTC = Sun Dec  1 23:47:18 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[8]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833480
	guid_sum = 2269178237999389052
	timestamp = 1733096843 UTC = Sun Dec  1 23:47:23 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[9]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833513
	guid_sum = 2269178237999389052
	timestamp = 1733097009 UTC = Sun Dec  1 23:50:09 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[10]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833482
	guid_sum = 2269178237999389052
	timestamp = 1733096853 UTC = Sun Dec  1 23:47:33 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[11]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833483
	guid_sum = 2269178237999389052
	timestamp = 1733096858 UTC = Sun Dec  1 23:47:38 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[12]
	magic = 0000000000bab10c
	version = 5000
	txg = 6842316
	guid_sum = 2269178237999389052
	timestamp = 1733141157 UTC = Mon Dec  2 12:05:57 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[13]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833485
	guid_sum = 2269178237999389052
	timestamp = 1733096868 UTC = Sun Dec  1 23:47:48 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[14]
	magic = 0000000000bab10c
	version = 5000
	txg = 6842254
	guid_sum = 2269178237999389052
	timestamp = 1733140846 UTC = Mon Dec  2 12:00:46 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[15]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833519
	guid_sum = 2269178237999389052
	timestamp = 1733097039 UTC = Sun Dec  1 23:50:39 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[16]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833552
	guid_sum = 2269178237999389052
	timestamp = 1733097205 UTC = Sun Dec  1 23:53:25 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[17]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833457
	guid_sum = 2269178237999389052
	timestamp = 1733096727 UTC = Sun Dec  1 23:45:27 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[18]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833458
	guid_sum = 2269178237999389052
	timestamp = 1733096732 UTC = Sun Dec  1 23:45:32 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[19]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833523
	guid_sum = 2269178237999389052
	timestamp = 1733097059 UTC = Sun Dec  1 23:50:59 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[20]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833460
	guid_sum = 2269178237999389052
	timestamp = 1733096742 UTC = Sun Dec  1 23:45:42 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[21]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833461
	guid_sum = 2269178237999389052
	timestamp = 1733096747 UTC = Sun Dec  1 23:45:47 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[22]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833462
	guid_sum = 2269178237999389052
	timestamp = 1733096752 UTC = Sun Dec  1 23:45:52 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[23]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833463
	guid_sum = 2269178237999389052
	timestamp = 1733096757 UTC = Sun Dec  1 23:45:57 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[24]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833464
	guid_sum = 2269178237999389052
	timestamp = 1733096762 UTC = Sun Dec  1 23:46:02 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[25]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833465
	guid_sum = 2269178237999389052
	timestamp = 1733096767 UTC = Sun Dec  1 23:46:07 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[26]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833466
	guid_sum = 2269178237999389052
	timestamp = 1733096772 UTC = Sun Dec  1 23:46:12 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[27]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833499
	guid_sum = 2269178237999389052
	timestamp = 1733096939 UTC = Sun Dec  1 23:48:59 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[28]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833564
	guid_sum = 2269178237999389052
	timestamp = 1733097265 UTC = Sun Dec  1 23:54:25 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[29]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833565
	guid_sum = 2269178237999389052
	timestamp = 1733097270 UTC = Sun Dec  1 23:54:30 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[30]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833566
	guid_sum = 2269178237999389052
	timestamp = 1733097275 UTC = Sun Dec  1 23:54:35 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
    Uberblock[31]
	magic = 0000000000bab10c
	version = 5000
	txg = 6833567
	guid_sum = 2269178237999389052
	timestamp = 1733097280 UTC = Sun Dec  1 23:54:40 2024
	mmp_magic = 00000000a11cea11
	mmp_delay = 0
	mmp_valid = 0
	checkpoint_txg = 0
        labels = 0 1 2 3 
------------------------------------
LABEL 2 (Bad label cksum)
------------------------------------
    version: 5000
    name: 'sapphire'
    state: 0
    txg: 6510548
    pool_guid: 4129880466361988471
    errata: 0
    hostid: 3938147066
    hostname: ''
    top_guid: 8818973256395834896
    guid: 1194783631545835994
    vdev_children: 1
    vdev_tree:
        type: 'raidz'
        id: 0
        guid: 8818973256395834896
        nparity: 2
        metaslab_array: 135
        metaslab_shift: 34
        ashift: 12
        asize: 39997054648320
        is_log: 0
        create_txg: 4
        children[0]:
            type: 'disk'
            id: 0
            guid: 2858572398724299414
            path: '/dev/gptid/40eb2a5f-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5060
            create_txg: 4
        children[1]:
            type: 'disk'
            id: 1
            guid: 1194783631545835994
            path: '/dev/gptid/40de4124-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5059
            create_txg: 4
        children[2]:
            type: 'disk'
            id: 2
            guid: 8569878839378827450
            path: '/dev/gptid/40f43471-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5058
            create_txg: 4
        children[3]:
            type: 'disk'
            id: 3
            guid: 6839407397638309827
            path: '/dev/gptid/40ffa4dd-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5057
            create_txg: 4
        children[4]:
            type: 'disk'
            id: 4
            guid: 6751170395373396232
            path: '/dev/gptid/4109a6d0-3012-11ee-9ec2-b9204f9364fa'
            DTL: 5056
            create_txg: 4
    features_for_read:
        com.delphix:hole_birth
        com.delphix:embedded_data
    labels = 2 3

hopefully this all looks good (otherwise i’m kinda screwed aren’t I :joy:)

what’s the next step oh wise sage.

29

Unique labels and partition UUIDs all around … the next step is to connect them all back to a TrueNAS installation and try importing the pool.

First port of call would be to check the UI and see if it sees it as a candidate under Storage → Import Pool - if it does, take a stab at it there.

If not, then we’ll have to run back to the CLI and see what the results of just zpool import with no parameters are, to see if we can see the pool at a raw level.

30

connected everything to the instance and showed this.

Screenshot 2025-02-01 031240
Screenshot 2025-02-01 0312401189×682 117 KB

checking importing pools and it cannot see anything initially.
Screenshot 2025-02-01 031427
Screenshot 2025-02-01 031427704×479 28.8 KB

which doesnt look good.
Screenshot 2025-02-01 031623
Screenshot 2025-02-01 031623691×311 49.2 KB

which also doesnt look good… :sweat_smile:
i mean, it’s there, so maybe thats something.
Screenshot 2025-02-01 031743
Screenshot 2025-02-01 031743934×570 32.9 KB

the disks are definitely all being seen.

31

Well, it at least sees that there’s a ZFS pool with a partition on each disk, but it seems pretty confident that it’s borked.

I don’t know how much the Xpenology installer would have overwritten, but I find it interesting that one of the disks shows as “ONLINE” but all of the other four are “FAULTED” even before an import - yet we’ve got seemingly valid ZFS labels.

If you try a zpool import -Ff sapphire from the command-line, does it immediately reply with “I/O Error - Destroy and restore” or similar?

32

dang, not looking good hey?

it doesnt say that, no.

Screenshot 2025-02-05 102648
Screenshot 2025-02-05 102648732×379 57.3 KB

is this better or worse you think?

33

We’re still very much in an “all or nothing” scenario here.

It was able to pick up the one gptid off of ada0 that was manually assigned by the first shot-in-the-dark file - and it’s even tagging the partition as ONLINE - but the others still seem to be searching for it by disk label which is odd to me. It shouldn’t matter but I can’t shake the feeling that something here does matter.

We might try putting the right gptid values onto the partition to see if it will at least attempt it.

What does gpart list in TrueNAS CORE show you? It’ll be a long text file, so use the codeblocks or DM it to me as a text file if you have to.

1 Like
34

this is what i got

PS C:\Users\USER> ssh USER@IP
USER@IP's password:
FreeBSD 13.1-RELEASE-p7 n245428-4dfb91682c1 TRUENAS

        TrueNAS (c) 2009-2023, iXsystems, Inc.
        All rights reserved.
        TrueNAS code is released under the modified BSD license with some
        files copyrighted by (c) iXsystems, Inc.

        For more information, documentation, help or support, go here:
        http://truenas.com
Welcome to TrueNAS
Could not chdir to home directory /nonexistent: No such file or directory
truenas% gpart list
Geom name: da0
modified: false
state: OK
fwheads: 255
fwsectors: 63
last: 67108823
first: 40
entries: 128
scheme: GPT
Providers:
1. Name: da0p1
   Mediasize: 524288 (512K)
   Sectorsize: 512
   Stripesize: 4096
   Stripeoffset: 0
   Mode: r0w0e0
   efimedia: HD(1,GPT,c12923ce-2897-11ee-adfb-7d36ee228720,0x28,0x400)
   rawuuid: c12923ce-2897-11ee-adfb-7d36ee228720
   rawtype: 83bd6b9d-7f41-11dc-be0b-001560b84f0f
   label: (null)
   length: 524288
   offset: 20480
   type: freebsd-boot
   index: 1
   end: 1063
   start: 40
2. Name: da0p2
   Mediasize: 34342961152 (32G)
   Sectorsize: 512
   Stripesize: 4096
   Stripeoffset: 0
   Mode: r1w1e1
   efimedia: HD(2,GPT,c130ec61-2897-11ee-adfb-7d36ee228720,0x428,0x3ff8000)
   rawuuid: c130ec61-2897-11ee-adfb-7d36ee228720
   rawtype: 516e7cba-6ecf-11d6-8ff8-00022d09712b
   label: (null)
   length: 34342961152
   offset: 544768
   type: freebsd-zfs
   index: 2
   end: 67077159
   start: 1064
Consumers:
1. Name: da0
   Mediasize: 34359738368 (32G)
   Sectorsize: 512
   Stripesize: 4096
   Stripeoffset: 0
   Mode: r1w1e2

Geom name: ada0
modified: false
state: OK
fwheads: 16
fwsectors: 63
last: 15628053134
first: 40
entries: 128
scheme: GPT
Providers:
1. Name: ada0p1
   Mediasize: 2147483648 (2.0G)
   Sectorsize: 512
   Stripesize: 4096
   Stripeoffset: 0
   Mode: r0w0e0
   efimedia: HD(1,GPT,d2dd3cbe-dfdd-11ef-9c48-65e8d1055310,0x80,0x400000)
   rawuuid: d2dd3cbe-dfdd-11ef-9c48-65e8d1055310
   rawtype: 516e7cb5-6ecf-11d6-8ff8-00022d09712b
   label: (null)
   length: 2147483648
   offset: 65536
   type: freebsd-swap
   index: 1
   end: 4194431
   start: 128
2. Name: ada0p2
   Mediasize: 7999415655936 (7.3T)
   Sectorsize: 512
   Stripesize: 4096
   Stripeoffset: 0
   Mode: r0w0e0
   efimedia: HD(2,GPT,d3305976-dfdd-11ef-9c48-65e8d1055310,0x400080,0x3a3412a0f)
   rawuuid: d3305976-dfdd-11ef-9c48-65e8d1055310
   rawtype: 516e7cba-6ecf-11d6-8ff8-00022d09712b
   label: (null)
   length: 7999415655936
   offset: 2147549184
   type: freebsd-zfs
   index: 2
   end: 15628053134
   start: 4194432
Consumers:
1. Name: ada0
   Mediasize: 8001563222016 (7.3T)
   Sectorsize: 512
   Stripesize: 4096
   Stripeoffset: 0
   Mode: r0w0e0

Geom name: ada1
modified: false
state: OK
fwheads: 16
fwsectors: 63
last: 15628053134
first: 40
entries: 128
scheme: GPT
Providers:
1. Name: ada1p1
   Mediasize: 2147483648 (2.0G)
   Sectorsize: 512
   Stripesize: 4096
   Stripeoffset: 0
   Mode: r0w0e0
   efimedia: HD(1,GPT,74cd848a-dfdd-11ef-9c48-65e8d1055310,0x80,0x400000)
   rawuuid: 74cd848a-dfdd-11ef-9c48-65e8d1055310
   rawtype: 516e7cb5-6ecf-11d6-8ff8-00022d09712b
   label: (null)
   length: 2147483648
   offset: 65536
   type: freebsd-swap
   index: 1
   end: 4194431
   start: 128
2. Name: ada1p2
   Mediasize: 7999415655936 (7.3T)
   Sectorsize: 512
   Stripesize: 4096
   Stripeoffset: 0
   Mode: r0w0e0
   efimedia: HD(2,GPT,752afdcc-dfdd-11ef-9c48-65e8d1055310,0x400080,0x3a3412a0f)
   rawuuid: 752afdcc-dfdd-11ef-9c48-65e8d1055310
   rawtype: 516e7cba-6ecf-11d6-8ff8-00022d09712b
   label: (null)
   length: 7999415655936
   offset: 2147549184
   type: freebsd-zfs
   index: 2
   end: 15628053134
   start: 4194432
Consumers:
1. Name: ada1
   Mediasize: 8001563222016 (7.3T)
   Sectorsize: 512
   Stripesize: 4096
   Stripeoffset: 0
   Mode: r0w0e0

Geom name: ada2
modified: false
state: OK
fwheads: 16
fwsectors: 63
last: 15628053134
first: 40
entries: 128
scheme: GPT
Providers:
1. Name: ada2p1
   Mediasize: 2147483648 (2.0G)
   Sectorsize: 512
   Stripesize: 4096
   Stripeoffset: 0
   Mode: r0w0e0
   efimedia: HD(1,GPT,2c7835b6-dfdf-11ef-b663-64006a1a4fce,0x80,0x400000)
   rawuuid: 2c7835b6-dfdf-11ef-b663-64006a1a4fce
   rawtype: 516e7cb5-6ecf-11d6-8ff8-00022d09712b
   label: (null)
   length: 2147483648
   offset: 65536
   type: freebsd-swap
   index: 1
   end: 4194431
   start: 128
2. Name: ada2p2
   Mediasize: 7999415655936 (7.3T)
   Sectorsize: 512
   Stripesize: 4096
   Stripeoffset: 0
   Mode: r0w0e0
   efimedia: HD(2,GPT,2cd4fada-dfdf-11ef-b663-64006a1a4fce,0x400080,0x3a3412a0f)
   rawuuid: 2cd4fada-dfdf-11ef-b663-64006a1a4fce
   rawtype: 516e7cba-6ecf-11d6-8ff8-00022d09712b
   label: (null)
   length: 7999415655936
   offset: 2147549184
   type: freebsd-zfs
   index: 2
   end: 15628053134
   start: 4194432
Consumers:
1. Name: ada2
   Mediasize: 8001563222016 (7.3T)
   Sectorsize: 512
   Stripesize: 4096
   Stripeoffset: 0
   Mode: r0w0e0

Geom name: ada3
modified: false
state: OK
fwheads: 16
fwsectors: 63
last: 15628053134
first: 40
entries: 128
scheme: GPT
Providers:
1. Name: ada3p1
   Mediasize: 2147483648 (2.0G)
   Sectorsize: 512
   Stripesize: 4096
   Stripeoffset: 0
   Mode: r0w0e0
   efimedia: HD(1,GPT,720da5fc-df08-11ef-ad55-64006a1a4fce,0x80,0x400000)
   rawuuid: 720da5fc-df08-11ef-ad55-64006a1a4fce
   rawtype: 516e7cb5-6ecf-11d6-8ff8-00022d09712b
   label: (null)
   length: 2147483648
   offset: 65536
   type: freebsd-swap
   index: 1
   end: 4194431
   start: 128
2. Name: ada3p2
   Mediasize: 7999415655936 (7.3T)
   Sectorsize: 512
   Stripesize: 4096
   Stripeoffset: 0
   Mode: r0w0e0
   efimedia: HD(2,GPT,72c27590-df08-11ef-ad55-64006a1a4fce,0x400080,0x3a3412a0f)
   rawuuid: 72c27590-df08-11ef-ad55-64006a1a4fce
   rawtype: 516e7cba-6ecf-11d6-8ff8-00022d09712b
   label: (null)
   length: 7999415655936
   offset: 2147549184
   type: freebsd-zfs
   index: 2
   end: 15628053134
   start: 4194432
Consumers:
1. Name: ada3
   Mediasize: 8001563222016 (7.3T)
   Sectorsize: 512
   Stripesize: 4096
   Stripeoffset: 0
   Mode: r0w0e0

Geom name: ada4
modified: false
state: OK
fwheads: 16
fwsectors: 63
last: 15628053134
first: 40
entries: 128
scheme: GPT
Providers:
1. Name: ada4p1
   Mediasize: 2147483648 (2.0G)
   Sectorsize: 512
   Stripesize: 4096
   Stripeoffset: 0
   Mode: r0w0e0
   efimedia: HD(1,GPT,5f56f119-d1d1-11ef-8370-000c29ed785d,0x80,0x400000)
   rawuuid: 5f56f119-d1d1-11ef-8370-000c29ed785d
   rawtype: 516e7cb5-6ecf-11d6-8ff8-00022d09712b
   label: (null)
   length: 2147483648
   offset: 65536
   type: freebsd-swap
   index: 1
   end: 4194431
   start: 128
2. Name: ada4p2
   Mediasize: 7999415655936 (7.3T)
   Sectorsize: 512
   Stripesize: 4096
   Stripeoffset: 0
   Mode: r0w0e0
   efimedia: HD(2,GPT,5f5afe87-d1d1-11ef-8370-000c29ed785d,0x400080,0x3a3412a0f)
   rawuuid: 5f5afe87-d1d1-11ef-8370-000c29ed785d
   rawtype: 516e7cba-6ecf-11d6-8ff8-00022d09712b
   label: (null)
   length: 7999415655936
   offset: 2147549184
   type: freebsd-zfs
   index: 2
   end: 15628053134
   start: 4194432
Consumers:
1. Name: ada4
   Mediasize: 8001563222016 (7.3T)
   Sectorsize: 512
   Stripesize: 4096
   Stripeoffset: 0
   Mode: r0w0e0

truenas%

I’m happy for it to be here, unless there’s something personal in here that i should not be showing.
if we ever get somewhere with all this, maybe this could be a piece of the puzzle for someone else to solve their issue next time :slight_smile:

35

any word from HQ?