(alter flatbert auf loudhoward hardware) == Alter Zustand == flatbert hatte sda mit 80 GB und sdb mit 250 GB, softraid1 md0 (1GB) für boot und md1 (ca. 65GB) für / (ROOT) filesystem, da die lxc container nicht aufs md1 passten wurde ein zfs singlepool storage für die container erstellt (zfsonlinux 0.6.3 / (spl/zfs kernelmodul git version) gedacht waren regelmäßige backup-snapshots auf freenas(freebert) ... hatte aber keiner eingerichtet ... == Was passiert ist == nach 51 Tagen uptime, (bis 15.02.2015) ata fehler, zpool konnte nicht mehr import werden == bisheriger Vorgang == 250 GB platte als Image mit ddrescue (auf zaubert) gesichert: Fehler: 8 bereiche ca 1.5 MB Log: root@zaubert:/# cat flatbert.log # Rescue Logfile. Created by GNU ddrescue version 1.19 # Command line: ddrescue -n /dev/sda /flatbert_backup_16.02.2015.img flatbert.log # Start time: 2015-02-16 01:09:36 # Current time: 2015-02-16 03:54:17 # Finished # current_pos current_status 0x17E0A4E000 + # pos size status 0x00000000 0x4D20F000 + 0x4D20F000 0x00000200 - 0x4D20F200 0x00000C00 / 0x4D20FE00 0x00000200 - 0x4D210000 0x0012F000 + 0x4D33F000 0x00000200 - 0x4D33F200 0x00000C00 / 0x4D33FE00 0x00000200 - 0x4D340000 0x038A5000 + 0x50BE5000 0x00000200 - 0x50BE5200 0x006FFE00 + 0x512E5000 0x00000200 - 0x512E5200 0x00000C00 / 0x512E5E00 0x00000200 - 0x512E6000 0x38F252000 + 0x3E0538000 0x00000200 - 0x3E0538200 0x00000C00 / 0x3E0538E00 0x00000200 - 0x3E0539000 0x648203000 + 0xA2873C000 0x00000200 - 0xA2873C200 0x00000C00 / 0xA2873CE00 0x00000200 - 0xA2873D000 0xD866A9000 + 0x17AEDE6000 0x00000200 - 0x17AEDE6200 0x00000C00 / 0x17AEDE6E00 0x00000200 - 0x17AEDE7000 0x31C66000 + 0x17E0A4D000 0x00000200 - 0x17E0A4D200 0x00000C00 / 0x17E0A4DE00 0x00000200 - 0x17E0A4E000 0x22580E0000 + IMAGE liegt derzeit auf Zaubert unter: root@zaubert:/ROOT/default# ls -all | grep flatbert -rw-r--r-- 1 root root 250059350016 Feb 16 03:54 flatbert_backup_16.02.2015.img -rw-r--r-- 1 root root 1135 Feb 16 03:54 flatbert.log == knusbert backup == als hardware wurde knusbert genommen ein zfs snapshot backup von dem freebsd für cryptostorage, exolastic, rsyslog, asterisk liegt auf freenas zroot/BACKUP/knusbert 4.90G 192G 140K /mnt/BACKUP/knusbert zroot/BACKUP/knusbert-cpool 180G 192G 140K /mnt/BACKUP/knusbert-cpool zroot/BACKUP/knusbert-cpool/cpool 180G 192G 140K /mnt/BACKUP/knusbert-cpool/cpool zroot/BACKUP/knusbert-cpool/cpool/BACKUP 86.9G 192G 61.4G /mnt/BACKUP/knusbert-cpool/cpool/BACKUP zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot 25.5G 192G 140K /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail 25.5G 192G 174K /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail-admin 140K 192G 140K /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail-admin zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/avahi.hq.c3d2.de 427M 192G 427M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/avahi.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/basejail 1.14G 192G 1.14G /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/basejail zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/beastbert.hq.c3d2.de 211M 192G 211M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/beastbert.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/bitcoin.hq.c3d2.de 168M 192G 168M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/bitcoin.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/couchdb1.hq.c3d2.de 1.32G 192G 1.32G /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/couchdb1.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/couchdb2.hq.c3d2.de 613M 192G 613M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/couchdb2.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/couchdb3.hq.c3d2.de 608M 192G 608M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/couchdb3.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/dhcp.hq.c3d2.de 941M 192G 941M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/dhcp.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/dilbert.hq.c3d2.de 544M 192G 544M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/dilbert.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/distcc1.hq.c3d2.de 428M 192G 428M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/distcc1.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/distcc2.hq.c3d2.de 428M 192G 428M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/distcc2.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/distcc3.hq.c3d2.de 428M 192G 428M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/distcc3.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/distcc4.hq.c3d2.de 428M 192G 428M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/distcc4.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/dn42.hq.c3d2.de 188M 192G 188M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/dn42.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/dnscache.hq.c3d2.de 209M 192G 209M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/dnscache.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/dnstunnel.hq.c3d2.de 134M 192G 134M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/dnstunnel.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/downpressor.hq.c3d2.de 568M 192G 568M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/downpressor.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/fulljail 140K 192G 140K /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/fulljail zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/gitbert.hq.c3d2.de 185M 192G 185M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/gitbert.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/haproxy.hq.c3d2.de 93.4M 192G 93.4M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/haproxy.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/ipredator.hq.c3d2.de 190M 192G 190M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/ipredator.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/listbert1.hq.c3d2.de 2.95G 192G 2.95G /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/listbert1.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/listbert2.hq.c3d2.de 185M 192G 185M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/listbert2.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/music.hq.c3d2.de 1.47G 192G 1.47G /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/music.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/newjail 2.90M 192G 2.90M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/newjail zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/pentabot.hq.c3d2.de 499M 192G 499M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/pentabot.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/printer.hq.c3d2.de 58.5M 192G 58.5M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/printer.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/privoxy.hq.c3d2.de 419M 192G 419M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/privoxy.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/public-ip.hq.c3d2.de 166M 192G 166M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/public-ip.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/reverseproxy1.hq.c3d2.de 242M 192G 242M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/reverseproxy1.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/reverseproxy2.hq.c3d2.de 224M 192G 224M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/reverseproxy2.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/rippen.hq.c3d2.de 1010M 192G 1010M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/rippen.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/saugbert.hq.c3d2.de 1.70G 192G 1.70G /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/saugbert.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/sniffer.hq.c3d2.de 321M 192G 321M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/sniffer.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/squid.hq.c3d2.de 2.10G 192G 2.10G /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/squid.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/storage.hq.c3d2.de 2.79G 192G 2.71G /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/storage.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/storage.hq.c3d2.de/samba4db 79.8M 192G 79.8M - zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/storage.hq.c3d2.de/zimport 174K 192G 174K /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/storage.hq.c3d2.de/zimport zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/tor.hq.c3d2.de 216M 192G 216M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/tor.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/vert.hq.c3d2.de 358M 192G 358M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/vert.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/watchbert.hq.c3d2.de 1.31G 192G 1.31G /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/watchbert.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/wire.hq.c3d2.de 528M 192G 528M /mnt/BACKUP/knusbert-cpool/cpool/BACKUP/zroot/ezjail/wire.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/ezjail 65.6G 192G 186K /mnt/BACKUP/knusbert-cpool/cpool/ezjail zroot/BACKUP/knusbert-cpool/cpool/ezjail/asterisk.hq.c3d2.de 1.11G 192G 1.11G /mnt/BACKUP/knusbert-cpool/cpool/ezjail/asterisk.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/ezjail/basejail 1.10G 192G 1.10G /mnt/BACKUP/knusbert-cpool/cpool/ezjail/basejail zroot/BACKUP/knusbert-cpool/cpool/ezjail/cryptostorage.hq.c3d2.de 58.2G 192G 589M /mnt/BACKUP/knusbert-cpool/cpool/ezjail/cryptostorage.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/ezjail/cryptostorage.hq.c3d2.de/samba4db 15.4M 192G 15.4M - zroot/BACKUP/knusbert-cpool/cpool/ezjail/cryptostorage.hq.c3d2.de/storage 57.6G 192G 57.6G /mnt/BACKUP/knusbert-cpool/cpool/ezjail/cryptostorage.hq.c3d2.de/storage zroot/BACKUP/knusbert-cpool/cpool/ezjail/dn42-freeland.hq.c3d2.de 385M 192G 385M /mnt/BACKUP/knusbert-cpool/cpool/ezjail/dn42-freeland.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/ezjail/exolastic.hq.c3d2.de 4.09G 192G 4.09G /mnt/BACKUP/knusbert-cpool/cpool/ezjail/exolastic.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/ezjail/freebot.hq.c3d2.de 526M 192G 526M /mnt/BACKUP/knusbert-cpool/cpool/ezjail/freebot.hq.c3d2.de zroot/BACKUP/knusbert-cpool/cpool/ezjail/newjail 3.12M 192G 3.12M /mnt/BACKUP/knusbert-cpool/cpool/ezjail/newjail zroot/BACKUP/knusbert-cpool/cpool/ezjail/syslog.hq.c3d2.de 240M 192G 240M /mnt/BACKUP/knusbert-cpool/cpool/ezjail/syslog.hq.c3d2.de zroot/BACKUP/knusbert/singlestorage 140K 192G 140K /mnt/BACKUP/knusbert/singlestorage zroot/BACKUP/knusbert/zroot 4.90G 192G 151K /mnt/BACKUP/knusbert/zroot zroot/BACKUP/knusbert/zroot/ROOT 1.04G 192G 140K /mnt/BACKUP/knusbert/zroot/ROOT zroot/BACKUP/knusbert/zroot/ROOT/default 1.04G 192G 1.04G /mnt/BACKUP/knusbert/zroot/ROOT/default zroot/BACKUP/knusbert/zroot/admin 186K 192G 186K /mnt/BACKUP/knusbert/zroot/admin zroot/BACKUP/knusbert/zroot/tmp 192K 192G 192K /mnt/BACKUP/knusbert/zroot/tmp zroot/BACKUP/knusbert/zroot/usr 3.86G 192G 140K /mnt/BACKUP/knusbert/zroot/usr zroot/BACKUP/knusbert/zroot/usr/home 198K 192G 198K /mnt/BACKUP/knusbert/zroot/usr/home zroot/BACKUP/knusbert/zroot/usr/obj 1.69G 192G 1.69G /mnt/BACKUP/knusbert/zroot/usr/obj zroot/BACKUP/knusbert/zroot/usr/ports 931M 192G 931M /mnt/BACKUP/knusbert/zroot/usr/ports zroot/BACKUP/knusbert/zroot/usr/src 1.26G 192G 1.26G /mnt/BACKUP/knusbert/zroot/usr/src zroot/BACKUP/knusbert/zroot/var 942K 192G 140K /mnt/BACKUP/knusbert/zroot/var zroot/BACKUP/knusbert/zroot/var/crash 140K 192G 140K /mnt/BACKUP/knusbert/zroot/var/crash zroot/BACKUP/knusbert/zroot/var/log 279K 192G 279K /mnt/BACKUP/knusbert/zroot/var/log zroot/BACKUP/knusbert/zroot/var/mail 151K 192G 151K /mnt/BACKUP/knusbert/zroot/var/mail zroot/BACKUP/knusbert/zroot/var/tmp 140K 192G 140K /mnt/BACKUP/knusbert/zroot/var/tmp == neuer flatbert == knusbert hardware: sda - IDE (200 GB) sdb - SATA (1 TB) sdc - SATA (1 TB) sdd - SATA (2 TB) das flatbert IMAGE wurde mit netcat auf /dev/sdb geschoben, alle alten kernel (in einer live usb-stick chroot umgebung) entfernt, kernel 3.16 (jessie) installiert und softraid1 für md0/md1 mit /dev/sdb und /dev/sdc erstellt die zfs singlepool partition liegt derzeit auf /dev/sdb kompiliert wurde zfsonlinux version 0.6.3-26 (spl 17/zfs 26) === restore phase 1 === der zpool scheint kaputt gegangen zu sein: [root@flatbert:~]# zpool status -v pool: singlepool state: ONLINE status: One or more devices has experienced an error resulting in data corruption. Applications may be affected. action: Restore the file in question if possible. Otherwise restore the entire pool from backup. see: http://zfsonlinux.org/msg/ZFS-8000-8A scan: none requested config: NAME STATE READ WRITE CKSUM singlepool ONLINE 0 0 29 sdb4 ONLINE 0 0 116 errors: Permanent errors have been detected in the following files: singlepool/rpool/disk6:/var/log/daemon.log singlepool/rpool/disk6:/var/log/syslog.1 singlepool/rpool/disk12:/var/spool/exim4/input/1YMghG-0001t5-E8-H singlepool/rpool/disk19:/home/admin/.ghc/x86_64-linux-7.8.4/package.conf.d/package.cache singlepool/rpool/disk19:<0xf84c> singlepool/rpool/disk19:<0xf84d> singlepool/rpool/disk19:/home/admin/.cabal/logs singlepool/rpool/disk19:/broken_dir singlepool/rpool/disk24:<0x0> [root@flatbert:~]# aktuelle datasets: [root@flatbert:~]# zfs list cannot iterate filesystems: I/O error NAME USED AVAIL REFER MOUNTPOINT singlepool 77,4G 78,1G 25K /singlepool singlepool/rpool 77,4G 78,1G 3,06G /rpool singlepool/rpool/disk10 1,67G 78,1G 1,61G /rpool/disk10 singlepool/rpool/disk11 341M 78,1G 341M /rpool/disk11 singlepool/rpool/disk13 457M 78,1G 457M /rpool/disk13 singlepool/rpool/disk14 1005M 78,1G 960M /rpool/disk14 singlepool/rpool/disk17 422M 78,1G 422M /rpool/disk17 singlepool/rpool/disk19 1,70G 78,1G 1,16G /rpool/disk19 singlepool/rpool/disk2 1,72G 78,1G 1,67G /rpool/disk2 singlepool/rpool/disk21 1,23G 78,1G 1,23G /rpool/disk21 singlepool/rpool/disk22 148M 78,1G 148M /rpool/disk22 singlepool/rpool/disk23 685M 78,1G 664M /rpool/disk23 singlepool/rpool/disk25 3,20G 78,1G 3,20G /rpool/disk25 singlepool/rpool/disk26 402M 78,1G 331M /rpool/disk26 singlepool/rpool/disk28 57K 78,1G 25K /rpool/disk28 singlepool/rpool/disk29 57K 78,1G 25K /rpool/disk29 singlepool/rpool/disk3 624M 78,1G 368M /rpool/disk3 singlepool/rpool/disk30 57K 78,1G 25K /rpool/disk30 singlepool/rpool/disk4 761M 78,1G 761M /rpool/disk4 singlepool/rpool/disk7 398M 78,1G 364M /rpool/disk7 singlepool/rpool/disk8 1,44G 78,1G 1,18G /rpool/disk8 singlepool/rpool/disk9 1,06G 78,1G 1,05G /rpool/disk9 singlepool/rpool/gitlab 744M 78,1G 741M /rpool/gitlab singlepool/rpool/gitolite 185M 78,1G 183M /rpool/gitolite [root@flatbert:~]# ein backup wurde erstmal vom alten snapshot stand _BACKUP_24.12.2014 auf freenas geschoben eigentliche lxc-mount-struktur: [root@flatbert:/lxc-container]# ls -al insgesamt 8 drwxr-xr-x 2 root root 4096 Okt 6 14:50 . drwxr-xr-x 37 root root 4096 Feb 16 08:18 .. lrwxrwxrwx 1 root root 12 Mär 31 2014 astrom -> /rpool/disk1 lrwxrwxrwx 1 root root 12 Mär 31 2014 astron -> /rpool/disk2 lrwxrwxrwx 1 root root 12 Mär 31 2014 blackhole -> /rpool/disk3 lrwxrwxrwx 1 root root 12 Mär 31 2014 cloudybay -> /rpool/disk4 lrwxrwxrwx 1 root root 12 Mär 31 2014 debcache -> /rpool/disk5 lrwxrwxrwx 1 root root 12 Mär 31 2014 dhcp -> /rpool/disk7 lrwxrwxrwx 1 root root 14 Jun 3 2014 distcc5 -> /rpool/distcc5 lrwxrwxrwx 1 root root 14 Jun 7 2014 distcc6 -> /rpool/distcc6 lrwxrwxrwx 1 root root 12 Mär 31 2014 dn42 -> /rpool/disk6 lrwxrwxrwx 1 root root 12 Mär 31 2014 drucker -> /rpool/disk8 lrwxrwxrwx 1 root root 12 Mär 31 2014 feile -> /rpool/disk9 lrwxrwxrwx 1 root root 13 Mär 31 2014 fernandopoo -> /rpool/disk10 lrwxrwxrwx 1 root root 13 Mär 31 2014 flatbert-extra-backups -> /rpool/disk11 lrwxrwxrwx 1 root root 13 Mär 31 2014 git -> /rpool/disk12 lrwxrwxrwx 1 root root 13 Okt 6 14:44 gitlab -> /rpool/gitlab lrwxrwxrwx 1 root root 15 Okt 6 14:44 gitolite -> /rpool/gitolite lrwxrwxrwx 1 root root 13 Mär 31 2014 global -> /rpool/disk13 lrwxrwxrwx 1 root root 13 Mär 31 2014 jabber1 -> /rpool/disk14 lrwxrwxrwx 1 root root 13 Mär 31 2014 jabber2 -> /rpool/disk15 lrwxrwxrwx 1 root root 13 Mär 31 2014 knot -> /rpool/disk16 lrwxrwxrwx 1 root root 13 Mär 31 2014 leviathan -> /rpool/disk17 lrwxrwxrwx 1 root root 13 Mär 31 2014 lxc-cache -> /rpool/disk18 lrwxrwxrwx 1 root root 13 Mär 31 2014 matemat -> /rpool/disk19 lrwxrwxrwx 1 root root 13 Mär 31 2014 semanta -> /rpool/disk20 lrwxrwxrwx 1 root root 14 Okt 1 00:18 sharing -> /rpool/sharing lrwxrwxrwx 1 root root 13 Okt 1 00:18 sharing.old -> /rpool/disk21 lrwxrwxrwx 1 root root 13 Mär 31 2014 thron -> /rpool/disk23 lrwxrwxrwx 1 root root 13 Mär 31 2014 thron2 -> /rpool/disk22 lrwxrwxrwx 1 root root 13 Mär 31 2014 wiefelspuetz -> /rpool/disk24 lrwxrwxrwx 1 root root 13 Mär 31 2014 wolke7 -> /rpool/disk25 lrwxrwxrwx 1 root root 13 Mär 31 2014 wormhole -> /rpool/disk26 lrwxrwxrwx 1 root root 13 Mär 31 2014 www1 -> /rpool/disk27 === restore phase 2 === astrom -> /rpool/disk1 ### FEHLT !!! ### astron -> /rpool/disk2 blackhole -> /rpool/disk3 cloudybay -> /rpool/disk4 debcache -> /rpool/disk5 ### FEHLT !!! ### dhcp -> /rpool/disk7 distcc5 -> /rpool/distcc5 distcc6 -> /rpool/distcc6 dn42 -> /rpool/disk6 ### FEHLT !!! ### drucker -> /rpool/disk8 feile -> /rpool/disk9 fernandopoo -> /rpool/disk10 flatbert-extra-backups -> /rpool/disk11 git -> /rpool/disk12 ### FEHLT !!! ### gitlab -> /rpool/gitlab gitolite -> /rpool/gitolite global -> /rpool/disk13 jabber1 -> /rpool/disk14 jabber2 -> /rpool/disk15 ### FEHLT !!! ### knot -> /rpool/disk16 ### FEHLT !!! ### leviathan -> /rpool/disk17 lxc-cache -> /rpool/disk18 ### FEHLT !!! ### matemat -> /rpool/disk19 semanta -> /rpool/disk20 ### FEHLT !!! ### sharing -> /rpool/sharing sharing.old -> /rpool/disk21 thron -> /rpool/disk23 thron2 -> /rpool/disk22 wiefelspuetz -> /rpool/disk24 ### FEHLT !!! ### wolke7 -> /rpool/disk25 wormhole -> /rpool/disk26 www1 -> /rpool/disk27 ### FEHLT !!! ### ... in arbeit ... ==== zfs debug ==== http://docs.oracle.com/cd/E18752_01/html/819-5461/gbbwl.html [root@flatbert:~]# zdb singlepool: version: 5000 name: 'singlepool' state: 0 txg: 2268294 pool_guid: 11752217384613672320 errata: 0 hostid: 380373859 hostname: 'flatbert' vdev_children: 1 vdev_tree: type: 'root' id: 0 guid: 11752217384613672320 children[0]: type: 'disk' id: 0 guid: 311963733056151185 path: '/dev/sdb4' whole_disk: 0 metaslab_array: 34 metaslab_shift: 30 ashift: 9 asize: 169995141120 is_log: 0 create_txg: 4 features_for_read: com.delphix:hole_birth com.delphix:embedded_data [root@flatbert:~]# zpool history History for 'singlepool': 2014-10-06.12:22:26 zpool create singlepool /dev/sda4 2014-10-06.12:23:15 zfs set checksum=fletcher4 singlepool 2014-10-06.12:23:24 zfs set compression=lz4 singlepool 2014-10-06.12:24:16 zfs create -o checksum=fletcher4 -o compression=lz4 -o mountpoint=/singlepool/rpool singlepool/rpool 2014-10-06.12:24:56 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/disk1 2014-10-06.12:24:57 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/disk2 2014-10-06.12:24:58 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/disk3 2014-10-06.12:24:59 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/disk4 2014-10-06.12:25:00 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/disk5 2014-10-06.12:25:04 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/disk6 2014-10-06.12:25:06 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/disk7 2014-10-06.12:25:07 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/disk8 2014-10-06.12:25:10 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/disk9 2014-10-06.12:25:11 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/disk10 2014-10-06.12:25:13 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/disk11 2014-10-06.12:25:14 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/disk12 2014-10-06.12:25:15 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/disk13 2014-10-06.12:25:17 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/disk14 2014-10-06.12:25:18 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/disk15 2014-10-06.12:25:19 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/disk16 2014-10-06.12:25:20 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/disk17 2014-10-06.12:25:22 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/disk18 2014-10-06.12:25:24 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/disk19 2014-10-06.12:25:25 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/disk20 2014-10-06.12:25:26 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/disk21 2014-10-06.12:25:27 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/disk22 2014-10-06.12:25:28 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/disk23 2014-10-06.12:25:30 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/disk24 2014-10-06.12:25:31 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/disk25 2014-10-06.12:25:32 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/disk26 2014-10-06.12:25:33 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/disk27 2014-10-06.12:25:35 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/disk28 2014-10-06.12:25:37 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/disk29 2014-10-06.12:25:43 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/disk30 2014-10-06.12:26:45 zfs snapshot -r singlepool@_0000_clean_06.10.2014 2014-10-06.14:34:26 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/gitlab 2014-10-06.14:34:48 zfs create -o checksum=fletcher4 -o compression=lz4 singlepool/rpool/gitolite 2014-10-06.14:47:10 zfs snapshot -r singlepool@_0001_RECOVERY_06.10.2014 2014-10-06.14:49:27 zfs set mountpoint=/rpool singlepool/rpool 2014-10-06.15:01:18 zpool export singlepool 2014-10-12.15:19:56 zfs snapshot -r singlepool@_0002_RUN_12.10.2014 2014-12-24.12:17:03 zfs snapshot -r singlepool@_BACKUP_24.12.2014 2014-12-26.04:26:21 zfs snapshot -r singlepool@_RUN_26.12.2014 In der zpool history fehlt massiv viel [root@flatbert:~]# zpool status -v singlepool pool: singlepool state: ONLINE status: One or more devices has experienced an error resulting in data corruption. Applications may be affected. action: Restore the file in question if possible. Otherwise restore the entire pool from backup. see: http://zfsonlinux.org/msg/ZFS-8000-8A scan: none requested config: NAME STATE READ WRITE CKSUM singlepool ONLINE 0 0 40 sdb4 ONLINE 0 0 160 errors: Permanent errors have been detected in the following files: singlepool/rpool/disk6:/var/log/daemon.log singlepool/rpool/disk6:/var/log/syslog.1 singlepool/rpool/disk12:/var/spool/exim4/input/1YMghG-0001t5-E8-H singlepool/rpool/disk19:/home/admin/.ghc/x86_64-linux-7.8.4/package.conf.d/package.cache singlepool/rpool/disk19:<0xf84c> singlepool/rpool/disk19:<0xf84d> singlepool/rpool/disk19:/home/admin/.cabal/logs singlepool/rpool/disk19:/broken_dir singlepool/rpool/disk24:<0x0> Die Pool Metadaten sind nicht defekt und die korrupten datasets werden nicht aufgelistet !!! === löschen von Corrupted File or Directory === zfs mount singlepool/rpool/disk6 ... hat funktioniert! zpool scrub singlepool [root@flatbert:~]# zpool status -v pool: singlepool state: ONLINE status: One or more devices has experienced an error resulting in data corruption. Applications may be affected. action: Restore the file in question if possible. Otherwise restore the entire pool from backup. see: http://zfsonlinux.org/msg/ZFS-8000-8A scan: scrub repaired 16K in 0h25m with 42 errors on Mon Feb 16 20:05:10 2015 config: NAME STATE READ WRITE CKSUM singlepool ONLINE 0 0 204 sdb4 ONLINE 0 0 806 errors: Permanent errors have been detected in the following files: singlepool/rpool/disk6:<0x7391> /rpool/disk6/var/log/syslog.1 singlepool/rpool/disk12:/var/spool/exim4/msglog/1YMghG-0001t5-E8 singlepool/rpool/disk12:/var/spool/exim4/input/1YMghG-0001t5-E8-H singlepool/rpool/disk19:<0x0> singlepool/rpool/disk19:/home/admin/.ghc/x86_64-linux-7.8.4/package.conf.d/package.cache singlepool/rpool/disk19:<0xe11f> singlepool/rpool/disk19:<0xe135> singlepool/rpool/disk19:<0xf84c> singlepool/rpool/disk19:/home/admin/.cabal/logs singlepool/rpool/disk19:<0xe1a7> singlepool/rpool/disk19:<0xe1a8> singlepool/rpool/disk19:/broken_dir singlepool/rpool/disk24:<0x0> [root@flatbert:~]# z.B. zfs mount singlepool/rpool/disk19 rm -rfv /rpool/disk19/broken_dir zfs umount -a zpool scrub singlepool === notfall frickelei === zpool mit einem alten txg Uberblock Stand importieren zdb -l /dev/sdb4 zpool import -N -o readonly=on -f -R /mnt/broken -F -T 2272146 zdb -c -e -t 2272100 -p /dev/sdb4 rpool === manuelles remote snapshot backupen der funktionierenden datasets === for i in $(zfs list -t snapshot | grep "_BACKUP_24.12.2014" | awk '{print $1}'); do zfs send $i | ssh root@172.22.99.10 zfs recv zroot/BACKUP/flatbert-old/$(echo $i | sed 's/@_BACKUP_24.12.2014//g'); done === manuelles remote snapshot backupen der invisible datasets === zfs send singlepool/rpool/disk1@_BACKUP_24.12.2014 | ssh root@172.22.99.10 zfs recv zroot/BACKUP/flatbert-old/singlepool/rpool/disk1 zfs send singlepool/rpool/disk5@_BACKUP_24.12.2014 | ssh root@172.22.99.10 zfs recv zroot/BACKUP/flatbert-old/singlepool/rpool/disk5 zfs send singlepool/rpool/disk6@_BACKUP_24.12.2014 | ssh root@172.22.99.10 zfs recv zroot/BACKUP/flatbert-old/singlepool/rpool/disk6 zfs send singlepool/rpool/disk12@_BACKUP_24.12.2014 | ssh root@172.22.99.10 zfs recv zroot/BACKUP/flatbert-old/singlepool/rpool/disk12 zfs send singlepool/rpool/disk15@_BACKUP_24.12.2014 | ssh root@172.22.99.10 zfs recv zroot/BACKUP/flatbert-old/singlepool/rpool/disk15 zfs send singlepool/rpool/disk16@_BACKUP_24.12.2014 | ssh root@172.22.99.10 zfs recv zroot/BACKUP/flatbert-old/singlepool/rpool/disk16 zfs send singlepool/rpool/disk18@_BACKUP_24.12.2014 | ssh root@172.22.99.10 zfs recv zroot/BACKUP/flatbert-old/singlepool/rpool/disk18 zfs send singlepool/rpool/disk20@_BACKUP_24.12.2014 | ssh root@172.22.99.10 zfs recv zroot/BACKUP/flatbert-old/singlepool/rpool/disk20 zfs send singlepool/rpool/disk24@_BACKUP_24.12.2014 | ssh root@172.22.99.10 zfs recv zroot/BACKUP/flatbert-old/singlepool/rpool/disk24 zfs send singlepool/rpool/disk27@_BACKUP_24.12.2014 | ssh root@172.22.99.10 zfs recv zroot/BACKUP/flatbert-old/singlepool/rpool/disk27 cannot open 'singlepool/rpool/disk24': I/O error === aktuelle snapshots sichern === zpool clear -F singlepool zfs snapshot singlepool/rpool/disk1@_BROKEN_16.02.2015 ... bis ... zfs snapshot singlepool/rpool/disk30@_BROKEN_16.02.2015 zfs send singlepool/rpool/disk1@_BROKEN_16.02.2015 | ssh root@172.22.99.10 zfs recv zroot/BACKUP/flatbert-broken/disk1 == ZFS BACKUP Festplatte (sdd) == parted /dev/sdd mklabel gpt mkpart backup zfs 0% 50% zpool create -o ashift=12 backup-pool /dev/sdd1 zfs set checksum=sha256 backup-pool zfs set compression=lz4 backup-pool zfs set copies=2 backup-pool zfs set reservation=1G backup-pool zfs create -o checksum=sha256 -o compression=lz4 -o copies=2 -o reservation=1G -o mountpoint=/backup-pool/broken backup-pool/broken === RESTORE auf ZFS backup-pool === zfs send singlepool/rpool/disk1@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/disk1 zfs send singlepool/rpool/disk2@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/disk2 zfs send singlepool/rpool/disk3@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/disk3 zfs send singlepool/rpool/disk4@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/disk4 zfs send singlepool/rpool/disk5@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/disk5 zfs send singlepool/rpool/disk6@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/disk6 internal error: Ungültiger Austausch cannot receive new filesystem stream: invalid backup stream [1] 27085 abort zfs send singlepool/rpool/disk6@_BROKEN_16.02.2015 | 27086 exit 1 zfs recv backup-pool/broken/disk6 zfs send singlepool/rpool/disk6@_RUN_26.12.2014 | zfs recv backup-pool/broken/disk6 ### !!! ALT !!! ### zfs send singlepool/rpool/disk7@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/disk7 zfs send singlepool/rpool/disk8@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/disk8 zfs send singlepool/rpool/disk9@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/disk9 zfs send singlepool/rpool/disk10@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/disk10 zfs send singlepool/rpool/disk11@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/disk11 zfs send singlepool/rpool/disk12@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/disk12 zfs send singlepool/rpool/disk13@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/disk13 zfs send singlepool/rpool/disk14@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/disk14 zfs send singlepool/rpool/disk15@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/disk15 zfs send singlepool/rpool/disk16@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/disk16 zfs send singlepool/rpool/disk17@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/disk17 zfs send singlepool/rpool/disk18@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/disk18 zfs send singlepool/rpool/disk19@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/disk19 warning: cannot send 'singlepool/rpool/disk19@_BROKEN_16.02.2015': Eingabe-/Ausgabefehler cannot receive new filesystem stream: invalid backup stream zfs send singlepool/rpool/disk19@_RUN_26.12.2014 | zfs recv backup-pool/broken/disk19 ### !!! ALT !!! ### zfs send singlepool/rpool/disk20@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/disk20 zfs send singlepool/rpool/disk21@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/disk21 zfs send singlepool/rpool/disk22@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/disk22 zfs send singlepool/rpool/disk23@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/disk23 zfs send singlepool/rpool/disk24@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/disk24 cannot open 'singlepool/rpool/disk24': I/O error ### !!! TOTAL KAPUTT !!! ### cannot receive: failed to read from stream ### !!! TOTAL KAPUTT !!! ### zfs send singlepool/rpool/disk24@_RUN_26.12.2014 | zfs recv backup-pool/broken/disk24 ### !!! TOTAL KAPUTT !!! ### cannot open 'singlepool/rpool/disk24': I/O error ### !!! TOTAL KAPUTT !!! ### cannot receive: failed to read from stream ### !!! TOTAL KAPUTT !!! ### zfs send singlepool/rpool/disk24@_BACKUP_24.12.2014 | zfs recv backup-pool/broken/disk24 ### !!! TOTAL KAPUTT !!! ### cannot open 'singlepool/rpool/disk24': I/O error ### !!! TOTAL KAPUTT !!! ### cannot receive: failed to read from stream ### !!! TOTAL KAPUTT !!! ### zfs send singlepool/rpool/disk24@_0001_RECOVERY_06.10.2014 | zfs recv backup-pool/broken/disk24 ### !!! TOTAL KAPUTT !!! ### cannot open 'singlepool/rpool/disk24': I/O error ### !!! TOTAL KAPUTT !!! ### cannot receive: failed to read from stream ### !!! TOTAL KAPUTT !!! ### zfs send singlepool/rpool/disk25@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/disk25 zfs send singlepool/rpool/disk26@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/disk26 zfs send singlepool/rpool/disk27@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/disk27 zfs send singlepool/rpool/disk28@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/disk28 zfs send singlepool/rpool/disk29@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/disk29 zfs send singlepool/rpool/disk30@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/disk30 zfs send singlepool/rpool/gitlab@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/gitlab zfs send singlepool/rpool/gitolite@_BROKEN_16.02.2015 | zfs recv backup-pool/broken/gitolite === current snapshot rsync === # dn42 zfs mount singlepool/rpool/disk6 zfs create -o checksum=sha256 -o compression=lz4 -o copies=2 -o mountpoint=/backup-pool/broken/disk6_last backup-pool/broken/disk6last rsync -aAXv --progress /rpool/disk6/ /backup-pool/broken/disk6_last/ ### ### ### sending incremental file list sent 615,627 bytes received 1,952 bytes 1,235,158.00 bytes/sec total size is 829,978,916 speedup is 1,343.92 ### ### ### # matemat zfs mount singlepool/rpool/disk19 zfs create -o checksum=sha256 -o compression=lz4 -o copies=2 -o mountpoint=/backup-pool/broken/disk19_last backup-pool/broken/disk19last rsync -aAXv --progress /rpool/disk19/ /backup-pool/broken/disk19_last/ ### ### ### [root@flatbert:~]# rsync -aAXv --progress /rpool/disk19/ /backup-pool/broken/disk19_last/ sending incremental file list rsync: readdir("/rpool/disk19/broken_dir"): Invalid exchange (52) rsync: readdir("/rpool/disk19/home/admin/.cabal/logs"): Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/bin/elfedit") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/bin/nm") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/bin/ld.gold") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/bin/cpp-4.9") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/bin/c++filt") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/bin/strip") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/bin/x86_64-linux-gnu-cpp-4.9") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/bin/gcc-4.9") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/bin/gcov-4.9") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/bin/ld.bfd") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/bin/gcc-nm-4.9") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/bin/strings") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/bin/gcc-ranlib-4.9") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/bin/ar") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/bin/objdump") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/bin/gprof") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/bin/gcc-ar-4.9") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/argz.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/sgtty.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/argp.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/stdlib.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/tgmath.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/sched.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/aliases.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/stab.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/ucontext.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/syslog.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/semaphore.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/aio.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/termio.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/thread_db.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/setjmp.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/stdc-predef.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/stdint.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/ttyent.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/syscall.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/search.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/signal.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/string.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/spawn.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/_G_config.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/uchar.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/stdio_ext.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/stdio.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/termios.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/sysexits.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/alloca.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/ar.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/tar.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/strings.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/time.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/shadow.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/stropts.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/arpa/ftp.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/arpa/telnet.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/scsi/sg.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/scsi/scsi_ioctl.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/include/scsi/scsi.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/libbfd-2.24.90-system.20141023.so") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/libopcodes-2.24.90-system.20141023.so") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/liblsan.so") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/libquadmath.so") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/lto-wrapper") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/libtsan.so") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/include-fixed") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/libgcc_s.so") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/libasan.so") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/crtbeginT.o") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/crtend.o") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/libsanitizer.spec") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/libatomic.so") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/libcilkrts.so") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/crtprec32.o") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/libubsan.so") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/liblto_plugin.so") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/libitm.so") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/libasan_preinit.o") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/include") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/libcilkrts.spec") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/liblto_plugin.so.0.0.0") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/cc1") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/libquadmath.a") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/lto1") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/liblsan.a") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/libgcc_eh.a") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/libgomp.spec") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/libcilkrts.a") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/collect2") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/crtendS.o") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/libitm.spec") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/libtsan.a") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/libgomp.so") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/libitm.a") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/libbacktrace.a") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/crtprec80.o") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/liblto_plugin.so.0") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/crtbegin.o") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/libasan.a") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/gcc/x86_64-linux-gnu/4.9/libssp_nonshared.a") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/base-4.7.0.2/Foreign") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/base-4.7.0.2/Debug") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/base-4.7.0.2/Data") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/base-4.7.0.2/GHC") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/base-4.7.0.2/Text") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/base-4.7.0.2/System") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/base-4.7.0.2/Control") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/base-4.7.0.2/include/OldTypeable.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/base-4.7.0.2/include/Typeable.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/base-4.7.0.2/include/consUtils.h") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString.hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString.dyn_hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString/Char8.hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString/Short.dyn_hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString/Builder.dyn_hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString/Char8.dyn_hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString/Short.hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString/Lazy.hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString/Unsafe.hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString/Internal.dyn_hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString/Lazy.dyn_hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString/Builder.hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString/Unsafe.dyn_hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString/Internal.hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString/Builder/Extra.hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString/Builder/Prim.hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString/Builder/Extra.dyn_hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString/Builder/Internal.dyn_hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString/Builder/Prim.dyn_hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString/Builder/ASCII.hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString/Builder/ASCII.dyn_hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString/Builder/Internal.hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString/Builder/Prim/Internal.hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString/Builder/Prim/Binary.dyn_hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString/Builder/Prim/Binary.hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString/Builder/Prim/ASCII.hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString/Builder/Prim/Internal.dyn_hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString/Builder/Prim/ASCII.dyn_hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString/Builder/Prim/Internal/UncheckedShifts.dyn_hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString/Builder/Prim/Internal/Base16.dyn_hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString/Builder/Prim/Internal/Floating.dyn_hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/bytestring-0.10.4.0/Data/ByteString/Builder/Prim/Internal/Base16.hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/time-1.4.2/Data/Time/LocalTime") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/time-1.4.2/Data/Time/Clock") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/time-1.4.2/Data/Time/Calendar/Gregorian.dyn_hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/time-1.4.2/Data/Time/Calendar/Private.dyn_hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/time-1.4.2/Data/Time/Calendar/OrdinalDate.dyn_hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/time-1.4.2/Data/Time/Calendar/JulianYearDay.dyn_hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/time-1.4.2/Data/Time/Calendar/Gregorian.hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/time-1.4.2/Data/Time/Calendar/Days.dyn_hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/time-1.4.2/Data/Time/Calendar/Easter.dyn_hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/time-1.4.2/Data/Time/Calendar/JulianYearDay.hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/time-1.4.2/Data/Time/Calendar/Julian.dyn_hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/time-1.4.2/Data/Time/Calendar/Days.hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/time-1.4.2/Data/Time/Calendar/MonthDay.dyn_hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ghc/time-1.4.2/Data/Time/Calendar/WeekDate.dyn_hi") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_k1om.xu") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_l1om.xw") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_k1om.x") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf32_x86_64.xdw") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_l1om.xdc") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf32_x86_64.xdc") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_l1om.xdw") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf32_x86_64.xsc") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_l1om.xsw") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_l1om.xsc") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_x86_64.xs") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf32_x86_64.xu") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_k1om.xr") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/i386linux.xr") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_x86_64.xc") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/i386pep.xr") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/i386pe.xbn") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/i386pep.xu") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_l1om.xs") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/i386pep.xa") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_i386.x") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/i386pep.xbn") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_x86_64.xd") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf32_x86_64.xbn") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_l1om.xbn") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_k1om.xsc") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_x86_64.xr") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_k1om.xs") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_i386.xbn") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_l1om.x") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_k1om.xdc") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_k1om.xdw") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_x86_64.xsc") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/i386linux.xbn") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/i386linux.xn") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_x86_64.xdc") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_i386.xsw") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_i386.xsc") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_i386.xd") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_i386.xn") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_x86_64.x") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf32_x86_64.xc") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf32_x86_64.xw") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_k1om.xbn") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_i386.xdw") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_l1om.xu") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_k1om.xw") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf32_x86_64.xd") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf32_x86_64.xn") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/ldscripts/elf_i386.xc") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libnss_nis.so") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libg.a") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libnss_hesiod.so") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/librt.a") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libm.so") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libm.a") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libpthread.a") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/crti.o") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libieee.a") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libBrokenLocale.a") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libisl.so.10.2.2") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libdl.so") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/gcrt1.o") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libpthread.so") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libcloog-isl.so.4") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libcidn.so") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libmpfr.so.4.1.2") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libmpfr.so.4") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libutil.a") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/Mcrt1.o") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libresolv.a") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libnss_files.so") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libmpc.so.3.0.0") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libisl.so.10") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libnsl.a") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libmpc.so.3") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libcrypt.so") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libc_nonshared.a") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libpthread_nonshared.a") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/librt.so") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libnss_compat.so") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libcloog-isl.so.4.0.0") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/Scrt1.o") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libcrypt.a") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/crtn.o") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libnsl.so") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libdl.a") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libnss_nisplus.so") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libc.so") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libutil.so") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/librpcsvc.a") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/crt1.o") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libthread_db.so") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libnss_dns.so") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libresolv.so") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libmcheck.a") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libc.a") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/lib/x86_64-linux-gnu/libanl.a") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/cpp-4.9") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/libgcc-4.9-dev") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/binutils/gas/ChangeLog.gz") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/gcc-4.9-base/test-summary.gz") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/gcc-4.9-base/sanitizer") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/gcc-4.9-base/itm") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/gcc-4.9-base/gomp") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/gcc-4.9-base/README.Bugs") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/gcc-4.9-base/NEWS.html") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/gcc-4.9-base/README.ssp") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/gcc-4.9-base/NEWS.gz") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/gcc-4.9-base/changelog.gz") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/gcc-4.9-base/quadmath") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/gcc-4.9-base/gcc") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/gcc-4.9-base/cilkrts") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/libcloog-isl4/copyright") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/libcloog-isl4/changelog.Debian.gz") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/libcloog-isl4/changelog.Debian.amd64.gz") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/libisl10/changelog.Debian.gz") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/libisl10/copyright") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/libisl10/changelog.gz") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/libmpc3/copyright") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/libmpc3/changelog.Debian.gz") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/libmpfr4/NEWS.gz") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/libmpfr4/BUGS") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/libmpfr4/copyright") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/libmpfr4/TODO.gz") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/libmpfr4/changelog.Debian.gz") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/libmpfr4/AUTHORS") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/libmpfr4/README") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/libmpfr4/README.Debian") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/doc/libmpfr4/changelog.gz") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/lintian/overrides/binutils") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/man/man6") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/man/man2") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/man/sv") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/man/zh_TW") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/man/pl") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/man/gl") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/man/man1/gcc-ar-4.9.1.gz") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/man/man1/gcc-ranlib-4.9.1.gz") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/man/ru/man1") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/usr/share/man/ru/man5") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/var/lib/dpkg/info/libcloog-isl4:amd64.list") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/var/lib/dpkg/info/cpp.md5sums") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/var/lib/dpkg/info/libcloog-isl4:amd64.postinst") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/var/lib/dpkg/info/libmpc3:amd64.postinst") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/var/lib/dpkg/info/libcloog-isl4:amd64.postrm") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/var/lib/dpkg/info/libcloog-isl4:amd64.shlibs") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/var/lib/dpkg/info/libmpfr4:amd64.postinst") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/var/lib/dpkg/info/libcloog-isl4:amd64.symbols") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/var/lib/dpkg/info/libmpc3:amd64.shlibs") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/var/lib/dpkg/info/libmpc3:amd64.postrm") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/var/lib/dpkg/info/libmpfr4:amd64.shlibs") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/var/lib/dpkg/info/libisl10:amd64.list") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/var/lib/dpkg/info/cpp-4.9.list") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/var/lib/dpkg/info/libcloog-isl4:amd64.md5sums") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/var/lib/dpkg/info/libmpfr4:amd64.postrm") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/var/lib/dpkg/info/cpp-4.9.md5sums") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/var/lib/dpkg/info/gcc-4.9.md5sums") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/var/lib/dpkg/info/libgcc-4.9-dev:amd64.list") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/var/lib/dpkg/info/libmpfr4:amd64.md5sums") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/var/lib/dpkg/info/libmpc3:amd64.md5sums") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/var/lib/dpkg/info/libisl10:amd64.symbols") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/var/lib/dpkg/info/libmpc3:amd64.list") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/var/lib/dpkg/info/libisl10:amd64.shlibs") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/var/lib/dpkg/info/libmpfr4:amd64.list") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/var/lib/dpkg/triggers/Unincorp") failed: Invalid exchange (52) rsync: readlink_stat("/rpool/disk19/var/lib/ghc/package.conf.d/package.cache") failed: Invalid exchange (52) sent 951,513 bytes received 2,519 bytes 82,959.30 bytes/sec total size is 2,039,360,175 speedup is 2,137.62 rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1183) [sender=3.1.1] ### ### ### # wiefelspuetz / disk24 - leider irreparabel kaputt :( == singlepool entsorgen & neuer datenpool == zfs umount -a zpool destroy singlepool zpool export backup-pool [root@flatbert:~]# fdisk /dev/sdb (sowie sdc) Command (m for help): p Disk /dev/sdb: 931,5 GiB, 1000204886016 bytes, 1953525168 sectors Units: sectors of 1 * 512 = 512 bytes Sector size (logical/physical): 512 bytes / 512 bytes I/O size (minimum/optimal): 512 bytes / 512 bytes Disklabel type: dos Disk identifier: 0x000a8556 Device Boot Start End Sectors Size Id Type /dev/sdb1 * 2048 1953791 1951744 953M fd Linux raid autodetect /dev/sdb2 1953792 17577983 15624192 7,5G 82 Linux swap / Solaris /dev/sdb3 17577984 154296319 136718336 65,2G fd Linux raid autodetect /dev/sdb4 * 154296320 486327739 332031420 158,3G bf Solaris Command (m for help): d Partition number (1-4, default 4): 4 Partition 4 has been deleted. Command (m for help): p Disk /dev/sdb: 931,5 GiB, 1000204886016 bytes, 1953525168 sectors Units: sectors of 1 * 512 = 512 bytes Sector size (logical/physical): 512 bytes / 512 bytes I/O size (minimum/optimal): 512 bytes / 512 bytes Disklabel type: dos Disk identifier: 0x000a8556 Device Boot Start End Sectors Size Id Type /dev/sdb1 * 2048 1953791 1951744 953M fd Linux raid autodetect /dev/sdb2 1953792 17577983 15624192 7,5G 82 Linux swap / Solaris /dev/sdb3 17577984 154296319 136718336 65,2G fd Linux raid autodetect Command (m for help): w The partition table has been altered. Calling ioctl() to re-read partition table. Re-reading the partition table failed.: Das Gerät oder die Ressource ist belegt The kernel still uses the old table. The new table will be used at the next reboot or after you run partprobe(8) or kpartx(8). /root/cleanlogs.sh; reboot ### ### ### ### ### ### ### ### ### [root@flatbert:~]# fdisk /dev/sdb Welcome to fdisk (util-linux 2.25.2). Changes will remain in memory only, until you decide to write them. Be careful before using the write command. Command (m for help): m Help: DOS (MBR) a toggle a bootable flag b edit nested BSD disklabel c toggle the dos compatibility flag Generic d delete a partition l list known partition types n add a new partition p print the partition table t change a partition type v verify the partition table Misc m print this menu u change display/entry units x extra functionality (experts only) Save & Exit w write table to disk and exit q quit without saving changes Create a new label g create a new empty GPT partition table G create a new empty SGI (IRIX) partition table o create a new empty DOS partition table s create a new empty Sun partition table Command (m for help): n Partition type p primary (3 primary, 0 extended, 1 free) e extended (container for logical partitions) Select (default e): p Selected partition 4 First sector (154296320-1953525167, default 154296320): Last sector, +sectors or +size{K,M,G,T,P} (154296320-1953525167, default 1953525167): Created a new partition 4 of type 'Linux' and of size 858 GiB. Command (m for help): p Disk /dev/sdb: 931,5 GiB, 1000204886016 bytes, 1953525168 sectors Units: sectors of 1 * 512 = 512 bytes Sector size (logical/physical): 512 bytes / 512 bytes I/O size (minimum/optimal): 512 bytes / 512 bytes Disklabel type: dos Disk identifier: 0x000a8556 Device Boot Start End Sectors Size Id Type /dev/sdb1 * 2048 1953791 1951744 953M fd Linux raid autodetect /dev/sdb2 1953792 17577983 15624192 7,5G 82 Linux swap / Solaris /dev/sdb3 17577984 154296319 136718336 65,2G fd Linux raid autodetect /dev/sdb4 154296320 1953525167 1799228848 858G 83 Linux Command (m for help): t Partition number (1-4, default 4): 4 Hex code (type L to list all codes): L Changed type of partition 'Linux' to 'Solaris'. Command (m for help): p Disk /dev/sdb: 931,5 GiB, 1000204886016 bytes, 1953525168 sectors Units: sectors of 1 * 512 = 512 bytes Sector size (logical/physical): 512 bytes / 512 bytes I/O size (minimum/optimal): 512 bytes / 512 bytes Disklabel type: dos Disk identifier: 0x000a8556 Device Boot Start End Sectors Size Id Type /dev/sdb1 * 2048 1953791 1951744 953M fd Linux raid autodetect /dev/sdb2 1953792 17577983 15624192 7,5G 82 Linux swap / Solaris /dev/sdb3 17577984 154296319 136718336 65,2G fd Linux raid autodetect /dev/sdb4 154296320 1953525167 1799228848 858G bf Solaris Command (m for help): w The partition table has been altered. Calling ioctl() to re-read partition table. Re-reading the partition table failed.: Das Gerät oder die Ressource ist belegt The kernel still uses the old table. The new table will be used at the next reboot or after you run partprobe(8) or kpartx(8). [root@flatbert:~]# ### ### ### ### ### ### ### ### ### /dev/sdb4 154296320 1953525167 1799228848 858G bf Solaris /dev/sdc4 154296320 1953525167 1799228848 858G bf Solaris ### ### ### ### ### ### ### ### ### rm -rfv /rpool zpool create -o ashift=12 rpool mirror /dev/sdb4 /dev/sdc4 zfs set checksum=sha256 rpool zfs set compression=lz4 rpool zfs set reservation=1G rpool zfs create -o checksum=sha256 -o compression=lz4 -o reservation=1G -o mountpoint=/rpool/disk1 rpool/disk1 zfs create -o checksum=sha256 -o compression=lz4 -o reservation=1G -o mountpoint=/rpool/disk2 rpool/disk2 zfs create -o checksum=sha256 -o compression=lz4 -o reservation=1G -o mountpoint=/rpool/disk3 rpool/disk3 zfs create -o checksum=sha256 -o compression=lz4 -o reservation=1G -o mountpoint=/rpool/disk4 rpool/disk4 zfs create -o checksum=sha256 -o compression=lz4 -o reservation=1G -o mountpoint=/rpool/disk5 rpool/disk5 zfs create -o checksum=sha256 -o compression=lz4 -o reservation=1G -o mountpoint=/rpool/disk6 rpool/disk6 zfs create -o checksum=sha256 -o compression=lz4 -o reservation=1G -o mountpoint=/rpool/disk7 rpool/disk7 zfs create -o checksum=sha256 -o compression=lz4 -o reservation=1G -o mountpoint=/rpool/disk8 rpool/disk8 zfs create -o checksum=sha256 -o compression=lz4 -o reservation=1G -o mountpoint=/rpool/disk9 rpool/disk9 zfs create -o checksum=sha256 -o compression=lz4 -o reservation=1G -o mountpoint=/rpool/disk10 rpool/disk10 zfs create -o checksum=sha256 -o compression=lz4 -o reservation=1G -o mountpoint=/rpool/disk11 rpool/disk11 zfs create -o checksum=sha256 -o compression=lz4 -o reservation=1G -o mountpoint=/rpool/disk12 rpool/disk12 zfs create -o checksum=sha256 -o compression=lz4 -o reservation=1G -o mountpoint=/rpool/disk13 rpool/disk13 zfs create -o checksum=sha256 -o compression=lz4 -o reservation=1G -o mountpoint=/rpool/disk14 rpool/disk14 zfs create -o checksum=sha256 -o compression=lz4 -o reservation=1G -o mountpoint=/rpool/disk15 rpool/disk15 zfs create -o checksum=sha256 -o compression=lz4 -o reservation=1G -o mountpoint=/rpool/disk16 rpool/disk16 zfs create -o checksum=sha256 -o compression=lz4 -o reservation=1G -o mountpoint=/rpool/disk17 rpool/disk17 zfs create -o checksum=sha256 -o compression=lz4 -o reservation=1G -o mountpoint=/rpool/disk18 rpool/disk18 zfs create -o checksum=sha256 -o compression=lz4 -o reservation=1G -o mountpoint=/rpool/disk19 rpool/disk19 zfs create -o checksum=sha256 -o compression=lz4 -o reservation=1G -o mountpoint=/rpool/disk20 rpool/disk20 zfs create -o checksum=sha256 -o compression=lz4 -o reservation=1G -o mountpoint=/rpool/disk21 rpool/disk21 zfs create -o checksum=sha256 -o compression=lz4 -o reservation=1G -o mountpoint=/rpool/disk22 rpool/disk22 zfs create -o checksum=sha256 -o compression=lz4 -o reservation=1G -o mountpoint=/rpool/disk23 rpool/disk23 zfs create -o checksum=sha256 -o compression=lz4 -o reservation=1G -o mountpoint=/rpool/disk24 rpool/disk24 zfs create -o checksum=sha256 -o compression=lz4 -o reservation=1G -o mountpoint=/rpool/disk25 rpool/disk25 zfs create -o checksum=sha256 -o compression=lz4 -o reservation=1G -o mountpoint=/rpool/disk26 rpool/disk26 zfs create -o checksum=sha256 -o compression=lz4 -o reservation=1G -o mountpoint=/rpool/disk27 rpool/disk27 zfs create -o checksum=sha256 -o compression=lz4 -o reservation=1G -o mountpoint=/rpool/disk28 rpool/disk28 zfs create -o checksum=sha256 -o compression=lz4 -o reservation=1G -o mountpoint=/rpool/disk29 rpool/disk29 zfs create -o checksum=sha256 -o compression=lz4 -o reservation=1G -o mountpoint=/rpool/disk30 rpool/disk30 ### ### ### ### ### ### ### ### ### zpool import backup-pool ### ### ### ### ### ### ### ### ### rsync -aAX --progress /backup-pool/broken/disk1/ /rpool/disk1/ rsync -aAX --progress /backup-pool/broken/disk2/ /rpool/disk2/ rsync -aAX --progress /backup-pool/broken/disk3/ /rpool/disk3/ rsync -aAX --progress /backup-pool/broken/disk4/ /rpool/disk4/ rsync -aAX --progress /backup-pool/broken/disk5/ /rpool/disk5/ rsync -aAX --progress /backup-pool/broken/disk6_last/ /rpool/disk6/ rsync -aAX --progress /backup-pool/broken/disk7/ /rpool/disk7/ rsync -aAX --progress /backup-pool/broken/disk8/ /rpool/disk8/ rsync -aAX --progress /backup-pool/broken/disk9/ /rpool/disk9/ rsync -aAX --progress /backup-pool/broken/disk10/ /rpool/disk10/ rsync -aAX --progress /backup-pool/broken/disk11/ /rpool/disk11/ rsync -aAX --progress /backup-pool/broken/disk12/ /rpool/disk12/ rsync -aAX --progress /backup-pool/broken/disk13/ /rpool/disk13/ rsync -aAX --progress /backup-pool/broken/disk14/ /rpool/disk14/ rsync -aAX --progress /backup-pool/broken/disk15/ /rpool/disk15/ rsync -aAX --progress /backup-pool/broken/disk16/ /rpool/disk16/ rsync -aAX --progress /backup-pool/broken/disk17/ /rpool/disk17/ rsync -aAX --progress /backup-pool/broken/disk18/ /rpool/disk18/ rsync -aAX --progress /backup-pool/broken/disk19_last/ /rpool/disk19/ rsync -aAX --progress /backup-pool/broken/disk20/ /rpool/disk20/ rsync -aAX --progress /backup-pool/broken/disk21/ /rpool/disk21/ rsync -aAX --progress /backup-pool/broken/disk22/ /rpool/disk22/ rsync -aAX --progress /backup-pool/broken/disk23/ /rpool/disk23/ rsync -aAX --progress /backup-pool/broken/disk25/ /rpool/disk25/ rsync -aAX --progress /backup-pool/broken/disk26/ /rpool/disk26/ rsync -aAX --progress /backup-pool/broken/disk27/ /rpool/disk27/ rsync -aAX --progress /backup-pool/broken/disk28/ /rpool/disk28/ rsync -aAX --progress /backup-pool/broken/disk29/ /rpool/disk29/ rsync -aAX --progress /backup-pool/broken/disk30/ /rpool/disk30/ ### ### ### ### ### ### ### ### ### zpool import backup-pool ### ### ### ### ### ### ### ### ###