Hello, guys!
I'm looking for solution.
We have a storage with 60x4TB zfs pool wich split on three raidz2 arrays + 256 Gb RAM. We have to connect around 30 Mac OS clients for online video streaming (~7-8Gbits/sec) via NFSv3
Some times we have a users complain about freezing.
Our configuration.
loader.conf:
zpool get all trunk:
zfs get all trunk:
sysctl vfs.zfs:
Who have an idea, where we mistaken.
I'm looking for solution.
We have a storage with 60x4TB zfs pool wich split on three raidz2 arrays + 256 Gb RAM. We have to connect around 30 Mac OS clients for online video streaming (~7-8Gbits/sec) via NFSv3
Some times we have a users complain about freezing.
Our configuration.
loader.conf:
Code:
zfs_load="YES"
ipmi_load="YES"
if_lagg_load="YES"
aio_load="YES"
ahci_load="YES"
autoboot_delay="3"
cc_htcp_load="YES"
net.inet.tcp.hostcache.cachelimit="0"
net.link.ifqmaxlen="6144"
net.isr.maxthreads="8"
net.isr.defaultqlimit="1024"
kern.hz=2000
kern.ipc.nmbclusters="32675708"
kern.ipc.nmbjumbop="16337852"
hw.intr_storm_threshold="9000"
net.inet.tcp.syncache.hashsize="1024"
net.inet.tcp.syncache.bucketlimit="100"
net.inet.tcp.tcbhashsize="65536"
loader_logo="orb"
vfs.zfs.trim.txg_delay="2000"
vfs.zfs.zio.use_uma="0"zpool get all trunk:
Code:
NAME PROPERTY VALUE
data size 218T
data capacity 58%
data altroot -
data health ONLINE
data guid 44535528225
data version -
data bootfs -
data delegation on
data autoreplace off
data cachefile -
data failmode wait
data listsnapshots off
data autoexpand off
data dedupditto 0
data dedupratio 1.00x
data free 90.4T
data allocated 127T
data readonly off
data comment -
data expandsize -
data freeing 0
data fragmentation 19%
data leaked 0
data feature@async_destroy enabled
data feature@empty_bpobj active
data feature@lz4_compress active
data feature@multi_vdev_crash_dump enabled
data feature@spacemap_histogram active
data feature@enabled_txg active
data feature@hole_birth active
data feature@extensible_dataset enabled
data feature@embedded_data active
data feature@bookmarks enabled
data feature@filesystem_limits enabled
data feature@large_blocks enabled
Code:
NAME PROPERTY VALUE
data type filesystem
data used 115T
data available 71.8T
data referenced 109T
data compressratio 1.00x
data mounted yes
data quota none
data reservation none
data recordsize 128K
data mountpoint /trunk
data sharenfs off
data checksum on
data compression off
data atime off
data devices on
data exec on
data setuid on
data readonly off
data jailed off
data snapdir hidden
data aclmode passthrough
data aclinherit passthrough
data canmount on
data xattr off
data copies 1
data version 5
data utf8only off
data normalization none
data casesensitivity sensitive
data vscan off
data nbmand off
data sharesmb off
data refquota none
data refreservation none
data primarycache all
data secondarycache all
data usedbysnapshots 0
data usedbydataset 109T
data usedbychildren 6.00T
data usedbyrefreservation 0
data logbias latency
data dedup off
data mlslabel
data sync standard
data refcompressratio 1.00x
data written 109T
data logicalused 113T
data logicalreferenced 109T
data volmode default
data filesystem_limit none
data snapshot_limit none
data filesystem_count none
data snapshot_count none
data redundant_metadata all
Code:
vfs.zfs.trim.max_interval: 1
vfs.zfs.trim.timeout: 30
vfs.zfs.trim.txg_delay: 2000
vfs.zfs.trim.enabled: 1
vfs.zfs.vol.unmap_enabled: 1
vfs.zfs.vol.mode: 1
vfs.zfs.version.zpl: 5
vfs.zfs.version.spa: 5000
vfs.zfs.version.acl: 1
vfs.zfs.version.ioctl: 5
vfs.zfs.debug: 1
vfs.zfs.super_owner: 0
vfs.zfs.sync_pass_rewrite: 2
vfs.zfs.sync_pass_dont_compress: 5
vfs.zfs.sync_pass_deferred_free: 2
vfs.zfs.zio.exclude_metadata: 0
vfs.zfs.zio.use_uma: 0
vfs.zfs.cache_flush_disable: 0
vfs.zfs.zil_replay_disable: 0
vfs.zfs.min_auto_ashift: 9
vfs.zfs.max_auto_ashift: 13
vfs.zfs.vdev.trim_max_pending: 10000
vfs.zfs.vdev.bio_delete_disable: 0
vfs.zfs.vdev.bio_flush_disable: 0
vfs.zfs.vdev.write_gap_limit: 4096
vfs.zfs.vdev.read_gap_limit: 131072
vfs.zfs.vdev.aggregation_limit: 131072
vfs.zfs.vdev.trim_max_active: 32
vfs.zfs.vdev.trim_min_active: 1
vfs.zfs.vdev.scrub_max_active: 2
vfs.zfs.vdev.scrub_min_active: 1
vfs.zfs.vdev.async_write_max_active: 10
vfs.zfs.vdev.async_write_min_active: 1
vfs.zfs.vdev.async_read_max_active: 10
vfs.zfs.vdev.async_read_min_active: 1
vfs.zfs.vdev.sync_write_max_active: 10
vfs.zfs.vdev.sync_write_min_active: 1
vfs.zfs.vdev.sync_read_max_active: 10
vfs.zfs.vdev.sync_read_min_active: 1
vfs.zfs.vdev.max_active: 32
vfs.zfs.vdev.async_write_active_max_dirty_percent: 60
vfs.zfs.vdev.async_write_active_min_dirty_percent: 30
vfs.zfs.vdev.mirror.non_rotating_seek_inc: 1
vfs.zfs.vdev.mirror.non_rotating_inc: 0
vfs.zfs.vdev.mirror.rotating_seek_offset: 2097152
vfs.zfs.vdev.mirror.rotating_seek_inc: 2
vfs.zfs.vdev.mirror.rotating_inc: 0
vfs.zfs.vdev.trim_on_init: 1
vfs.zfs.vdev.cache.bshift: 16
vfs.zfs.vdev.cache.size: 0
vfs.zfs.vdev.cache.max: 16384
vfs.zfs.vdev.metaslabs_per_vdev: 200
vfs.zfs.txg.timeout: 5
vfs.zfs.spa_asize_inflation: 24
vfs.zfs.deadman_enabled: 1
vfs.zfs.deadman_checktime_ms: 5000
vfs.zfs.deadman_synctime_ms: 1000000
vfs.zfs.recover: 0
vfs.zfs.spa_load_verify_data: 1
vfs.zfs.spa_load_verify_metadata: 1
vfs.zfs.spa_load_verify_maxinflight: 10000
vfs.zfs.check_hostid: 1
vfs.zfs.mg_fragmentation_threshold: 85
vfs.zfs.mg_noalloc_threshold: 0
vfs.zfs.condense_pct: 200
vfs.zfs.metaslab.bias_enabled: 1
vfs.zfs.metaslab.lba_weighting_enabled: 1
vfs.zfs.metaslab.fragmentation_factor_enabled: 1
vfs.zfs.metaslab.preload_enabled: 1
vfs.zfs.metaslab.preload_limit: 3
vfs.zfs.metaslab.unload_delay: 8
vfs.zfs.metaslab.load_pct: 50
vfs.zfs.metaslab.min_alloc_size: 33554432
vfs.zfs.metaslab.df_free_pct: 4
vfs.zfs.metaslab.df_alloc_threshold: 131072
vfs.zfs.metaslab.debug_unload: 0
vfs.zfs.metaslab.debug_load: 0
vfs.zfs.metaslab.fragmentation_threshold: 70
vfs.zfs.metaslab.gang_bang: 16777217
vfs.zfs.free_bpobj_enabled: 1
vfs.zfs.free_max_blocks: 18446744073709551615
vfs.zfs.no_scrub_prefetch: 0
vfs.zfs.no_scrub_io: 0
vfs.zfs.resilver_min_time_ms: 3000
vfs.zfs.free_min_time_ms: 50
vfs.zfs.scan_min_time_ms: 1000
vfs.zfs.scan_idle: 50
vfs.zfs.scrub_delay: 4
vfs.zfs.resilver_delay: 2
vfs.zfs.top_maxinflight: 60
vfs.zfs.zfetch.array_rd_sz: 4194304
vfs.zfs.zfetch.max_distance: 33554432
vfs.zfs.zfetch.min_sec_reap: 1
vfs.zfs.zfetch.max_streams: 32
vfs.zfs.prefetch_disable: 0
vfs.zfs.delay_scale: 10000
vfs.zfs.delay_min_dirty_percent: 60
vfs.zfs.dirty_data_sync: 67108864
vfs.zfs.dirty_data_max_percent: 10
vfs.zfs.dirty_data_max_max: 4294967296
vfs.zfs.dirty_data_max: 4294967296
vfs.zfs.max_recordsize: 1048576
vfs.zfs.mdcomp_disable: 0
vfs.zfs.nopwrite_enabled: 1
vfs.zfs.dedup.prefetch: 0
vfs.zfs.l2c_only_size: 0
vfs.zfs.mfu_ghost_data_lsize: 193108771328
vfs.zfs.mfu_ghost_metadata_lsize: 0
vfs.zfs.mfu_ghost_size: 193108771328
vfs.zfs.mfu_data_lsize: 50815461888
vfs.zfs.mfu_metadata_lsize: 3890364416
vfs.zfs.mfu_size: 54778102272
vfs.zfs.mru_ghost_data_lsize: 46535283712
vfs.zfs.mru_ghost_metadata_lsize: 0
vfs.zfs.mru_ghost_size: 46535283712
vfs.zfs.mru_data_lsize: 183730082816
vfs.zfs.mru_metadata_lsize: 7763743232
vfs.zfs.mru_size: 192730621952
vfs.zfs.anon_data_lsize: 0
vfs.zfs.anon_metadata_lsize: 0
vfs.zfs.anon_size: 172032
vfs.zfs.l2arc_norw: 0
vfs.zfs.l2arc_feed_again: 1
vfs.zfs.l2arc_noprefetch: 0
vfs.zfs.l2arc_feed_min_ms: 20
vfs.zfs.l2arc_feed_secs: 1
vfs.zfs.l2arc_headroom: 4
vfs.zfs.l2arc_write_boost: 8388608
vfs.zfs.l2arc_write_max: 8388608
vfs.zfs.arc_meta_limit: 66651366400
vfs.zfs.arc_free_target: 453079
vfs.zfs.arc_shrink_shift: 7
vfs.zfs.arc_average_blocksize: 8192
vfs.zfs.arc_min: 33325683200
vfs.zfs.arc_max: 266605465600Who have an idea, where we mistaken.
Last edited:
