diff --git a/README.md b/README.md index 77d97f8..7d267e1 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Then, add the flake to your `configuration.nix`: # Use accessKeyFile and secretKeyFile instead: accessKeyFile = "/run/secrets/rustfs-access-key"; # or use sops-nix, agenix, etc. secretKeyFile = "/run/secrets/rustfs-secret-key"; - volumes = "/var/lib/rustfs"; # Use a persistent location + pools = [ { volumes = [ "/var/lib/rustfs" ]; } ]; # Use a persistent location address = ":9000"; consoleEnable = true; consoleAddress = ":9001"; @@ -94,7 +94,7 @@ Then, add the flake to your `configuration.nix`: package = inputs.rustfs.packages.${pkgs.stdenv.hostPlatform.system}.default; accessKeyFile = config.sops.secrets.rustfs-access-key.path; secretKeyFile = config.sops.secrets.rustfs-secret-key.path; - volumes = "/var/lib/rustfs"; + pools = [ { volumes = [ "/var/lib/rustfs" ]; } ]; address = ":9000"; consoleEnable = true; }; @@ -168,68 +168,81 @@ User account under which RustFS runs. The service runs as a dedicated non-root u Group under which RustFS runs. -### services.rustfs.volumes +### services.rustfs.pools -**Type:** `string` or `list of strings` +**Type:** `list of submodules ({ nodes, volumes })` -**Default:** `["/var/lib/rustfs"]` +**Default:** `[ { volumes = [ "/var/lib/rustfs" ]; } ]` -List of paths or comma-separated string where RustFS stores data. Use persistent locations, not /tmp. Each entry must be -its own filesystem; several entries on one disk give no redundancy. Erasure coding needs at least 4 drives. Ignored when -`distributed.enable` is set — use `distributed.volumes` instead. +**Example:** -### services.rustfs.distributed.enable +```nix +services.rustfs.pools = [ + { nodes = [ "node1" "node2" "node3" "node4" ]; volumes = [ "/mnt/disk0" "/mnt/disk1" "/mnt/disk2" "/mnt/disk3" ]; } + { nodes = [ "node5" "node6" "node7" "node8" ]; volumes = [ "/mnt/disk0" "/mnt/disk1" "/mnt/disk2" "/mnt/disk3" ]; } +]; +``` -**Type:** `bool` +Server pools, in order. Every RustFS deployment is a pool no matter if it is a single drive, one node of four, four nodes of four, +so this is the only place drives are declared. A pool with an empty `nodes` uses local paths; give it hostnames and its +drives become URL endpoints, which is what makes a deployment distributed. + +Each entry takes `nodes` (hostnames making up the pool, resolvable from every node of it) and `volumes` (drive paths, +each on its own filesystem, with every node of the pool using the same layout). Use persistent locations, not /tmp: +several entries on one disk give no redundancy. -**Default:** `false` +Appending a pool is how a cluster grows without rebalancing what it already stores, and the only route from a +single-node deployment to a distributed one. Draining the old pool afterwards is `rc admin decommission`. -Whether to run as part of a distributed RustFS cluster spanning several nodes. When enabled, the module renders the -shared endpoint list (`http://:` for every node × volume pair) that all nodes must agree on, ordered -drive-major so an erasure set spans nodes instead of sitting on one. +A lone pool is listed drive by drive, so its names take any shape. Several cannot be: RustFS reads plain arguments as +one pool and rejects mixing the two forms, so each pool has to collapse into a single ellipsis expression such as +`node{2...5}`. That needs a common prefix and a contiguous numeric range, and padding that is all or nothing `disk01` +alongside `disk2` would expand to a drive you never declared, so the module refuses it. + +Keep the list identical and in the same order on every node: RustFS derives pool identity from it, so a divergent list +is a different cluster. See [examples/distributed-cluster.nix](./examples/distributed-cluster.nix) for a complete four-node configuration. -### services.rustfs.distributed.nodes +### services.rustfs.port + +**Type:** `port` -**Type:** `list of strings` +**Default:** `9000` -**Default:** `[]` +Port peers reach each other on. Must match the port in `address`, and be open between nodes in the firewall. -**Example:** `["node1" "node2" "node3" "node4"]` +### services.rustfs.erasureSetDriveCount -Hostnames of every node in the cluster, resolvable from each of them. Set identically on all nodes — the endpoint list is -rendered from this and must come out byte-identical cluster-wide. At least 4 nodes are required. +**Type:** `null or positive integer` -### services.rustfs.distributed.volumes +**Default:** `null` -**Type:** `list of strings` +**Example:** `4` -**Default:** `[]` +Drives per erasure set. Left null RustFS picks a divisor of the pool's drive count itself; set it when the split matters, +such as one set spanning all four nodes of a pool rather than sitting inside one. A pool's drive count must divide by it. -**Example:** `["/mnt/disk0" "/mnt/disk1" "/mnt/disk2" "/mnt/disk3"]` +### services.rustfs.storageClassStandardParity -Drive paths present on each node, each on its own filesystem. Every node uses the same layout, so this replaces -`volumes` in distributed mode and is what gets created and made writable locally. At least 4 drives per node are -required. +**Type:** `null or positive integer` -### services.rustfs.distributed.port +**Default:** `null` -**Type:** `port` +**Example:** `2` -**Default:** `9000` +Parity drives per erasure set for the STANDARD storage class. Two of four tolerates one node of a four-node set going +away while writes continue. Must be below `erasureSetDriveCount`. -Port peers reach each other on. Must match the port in `address`, and be open between nodes in the firewall. +### services.rustfs.storageClassRrsParity -### services.rustfs.distributed.localEndpointHost +**Type:** `null or positive integer` -**Type:** `string` +**Default:** `null` -**Default:** `config.networking.hostName` +**Example:** `1` -Which entry of `nodes` identifies this machine, so it claims its own drives instead of reaching them over RPC. Required -whenever `address` binds a wildcard such as `0.0.0.0`, since RustFS cannot infer its identity from that and would -otherwise treat every drive as remote. +Parity drives per erasure set for the REDUCED_REDUNDANCY class. Must be below `erasureSetDriveCount`. > **Note**: All nodes must share the *same* access/secret key pair, and it must not be the default > `rustfsadmin`/`rustfsadmin` — RustFS derives the inter-node RPC secret from the credentials and refuses to derive one diff --git a/docs/IMPROVEMENTS.md b/docs/IMPROVEMENTS.md index bade162..018479c 100644 --- a/docs/IMPROVEMENTS.md +++ b/docs/IMPROVEMENTS.md @@ -344,11 +344,11 @@ tlsDirectory = "/etc/rustfs/tls"; ```nix services.rustfs = { enable = true; - accessKey = "rustfsadmin"; # ❌ In Nix store (world-readable) - secretKey = "rustfsadmin"; # ❌ In Nix store (world-readable) - volumes = "/tmp/rustfs"; # ❌ Temporary storage - # Running as root ❌ Excessive privileges - # No systemd hardening ❌ No sandboxing + accessKey = "rustfsadmin"; # ❌ In Nix store (world-readable) + secretKey = "rustfsadmin"; # ❌ In Nix store (world-readable) + pools = [ { volumes = [ "/tmp/rustfs" ]; } ]; # ❌ Temporary storage + # Running as root ❌ Excessive privileges + # No systemd hardening ❌ No sandboxing }; ``` @@ -367,9 +367,9 @@ services.rustfs = { enable = true; accessKeyFile = config.sops.secrets.rustfs-access-key.path; # ✅ Encrypted secretKeyFile = config.sops.secrets.rustfs-secret-key.path; # ✅ Encrypted - volumes = "/var/lib/rustfs"; # ✅ Persistent - # Runs as unprivileged user ✅ Least privilege - # Comprehensive systemd hardening ✅ Defense in depth + pools = [ { volumes = [ "/var/lib/rustfs" ]; } ]; # ✅ Persistent + # Runs as unprivileged user ✅ Least privilege + # Comprehensive systemd hardening ✅ Defense in depth consoleAddress = "127.0.0.1:9001"; # ✅ Localhost only }; ``` diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index faee271..d8069ce 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -123,7 +123,7 @@ services.rustfs = { enable = true; accessKey = "rustfsadmin"; # ❌ INSECURE! secretKey = "rustfsadmin"; # ❌ INSECURE! - volumes = "/tmp/rustfs"; + pools = [ { volumes = [ "/tmp/rustfs" ]; } ]; address = ":9000"; }; ``` @@ -159,7 +159,7 @@ services.rustfs = { enable = true; accessKeyFile = config.sops.secrets.rustfs-access-key.path; # ✅ SECURE secretKeyFile = config.sops.secrets.rustfs-secret-key.path; # ✅ SECURE - volumes = "/var/lib/rustfs"; # Use persistent storage + pools = [ { volumes = [ "/var/lib/rustfs" ]; } ]; # Use persistent storage address = ":9000"; consoleAddress = "127.0.0.1:9001"; # Localhost only }; @@ -194,7 +194,7 @@ services.rustfs = { enable = true; accessKeyFile = config.age.secrets.rustfs-access-key.path; secretKeyFile = config.age.secrets.rustfs-secret-key.path; - volumes = "/var/lib/rustfs"; + pools = [ { volumes = [ "/var/lib/rustfs" ]; } ]; address = ":9000"; }; } @@ -207,7 +207,7 @@ services.rustfs = { enable = true; accessKeyFile = "/run/secrets/rustfs-access-key"; secretKeyFile = "/run/secrets/rustfs-secret-key"; - volumes = "/var/lib/rustfs"; + pools = [ { volumes = [ "/var/lib/rustfs" ]; } ]; address = ":9000"; }; ``` diff --git a/docs/SECURITY.md b/docs/SECURITY.md index aafb1b4..09111c5 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -186,10 +186,10 @@ Use appropriate volume locations: ```nix services.rustfs = { # ❌ Bad - temporary storage - volumes = "/tmp/rustfs"; + pools = [ { volumes = [ "/tmp/rustfs" ]; } ]; # ✅ Good - persistent storage with proper permissions - volumes = "/var/lib/rustfs"; + pools = [ { volumes = [ "/var/lib/rustfs" ]; } ]; # ✅ Also good - multiple volumes volumes = [ "/mnt/storage1" "/mnt/storage2" ]; diff --git a/examples/distributed-cluster.nix b/examples/distributed-cluster.nix index 85792e3..c195a3d 100644 --- a/examples/distributed-cluster.nix +++ b/examples/distributed-cluster.nix @@ -15,17 +15,15 @@ # RustFS distributed cluster NixOS Configuration Example # # Four nodes with four drives each. This exact file is imported by every node — -# `services.rustfs.distributed` is deliberately identical everywhere, and the +# `services.rustfs.pools` is deliberately identical everywhere, and the # module derives each node's identity from `networking.hostName`. # # Requirements the module asserts at evaluation time: -# - at least 4 nodes in `distributed.nodes` -# - at least 4 drives in `distributed.volumes` -# - `distributed.localEndpointHost` is one of `distributed.nodes` +# - at least 4 nodes and 4 drives in the pool # # Additional requirements the module cannot check for you: # - every hostname in `nodes` resolves from every other node (DNS or -# `networking.hosts`), and `distributed.port` is reachable between them +# `networking.hosts`), and `port` is reachable between them # - the access/secret key pair is identical on all nodes, and is *not* the # default rustfsadmin/rustfsadmin — RustFS derives the inter-node RPC secret # from the credentials and refuses to derive one from the defaults @@ -68,25 +66,22 @@ in services.rustfs = { enable = true; - distributed = { - enable = true; + # One pool spanning the fleet. The module expands it into the shared + # endpoint list (http://:) every node must agree on. + # Appending a second pool here is how the cluster grows later; RustFS works + # out which drives are this machine's by resolving the endpoint hosts. + pools = [{ inherit nodes volumes; }]; - # Replaces `volumes` when distributed mode is on: these are the local - # drives, and the module expands them into the shared endpoint list - # (http://:) that every node must agree on. - inherit nodes volumes; + port = 9000; - port = 9000; + # Sets of four across sixteen drives, two of them parity. Because the module + # lays the endpoints out drive-major, a set spans the four nodes rather than + # sitting on one, so a whole node can go away and writes still have quorum. + erasureSetDriveCount = 4; + storageClassStandardParity = 2; + storageClassRrsParity = 1; - # Which entry of `nodes` is this machine. The default is - # config.networking.hostName, which is correct as long as the hostname - # matches the name used in `nodes`. Set it explicitly when it does not — - # required whenever `address` binds a wildcard, since RustFS cannot infer - # its own identity from 0.0.0.0 and would treat every drive as remote. - localEndpointHost = config.networking.hostName; - }; - - # Must bind an address peers can reach, on `distributed.port`. + # Must bind an address peers can reach, on `port`. address = "0.0.0.0:9000"; # SECURITY: Bind console to localhost only, access via SSH tunnel diff --git a/examples/flake.nix b/examples/flake.nix index 5ef3046..d0a8f47 100644 --- a/examples/flake.nix +++ b/examples/flake.nix @@ -52,7 +52,7 @@ enable = true; package = rustfs-flake.packages.${pkgs.stdenv.hostPlatform.system}.default; - volumes = "/var/lib/rustfs/data"; + pools = [{ volumes = [ "/var/lib/rustfs/data" ]; }]; address = "0.0.0.0:9000"; consoleEnable = true; consoleAddress = "0.0.0.0:9001"; diff --git a/examples/nixos-configuration.nix b/examples/nixos-configuration.nix index ef5b1ed..3e16484 100644 --- a/examples/nixos-configuration.nix +++ b/examples/nixos-configuration.nix @@ -24,7 +24,7 @@ enable = true; # Storage path - use persistent storage, not /tmp - volumes = "/var/lib/rustfs/data"; + pools = [{ volumes = [ "/var/lib/rustfs/data" ]; }]; # API server address (Port 9000) # Use "0.0.0.0:9000" or ":9000" to listen on all interfaces diff --git a/examples/single-node-multi-disk.nix b/examples/single-node-multi-disk.nix index a635e97..bac4567 100644 --- a/examples/single-node-multi-disk.nix +++ b/examples/single-node-multi-disk.nix @@ -51,15 +51,26 @@ services.rustfs = { enable = true; - # One entry per drive. A comma-separated string works too, but the list form - # is easier to read and to generate. - volumes = [ - "/mnt/rustfs0" - "/mnt/rustfs1" - "/mnt/rustfs2" - "/mnt/rustfs3" + # One pool, one entry per drive. Giving the pool `nodes` as well is what + # turns this into a distributed deployment. + pools = [ + { + volumes = [ + "/mnt/rustfs0" + "/mnt/rustfs1" + "/mnt/rustfs2" + "/mnt/rustfs3" + ]; + } ]; + # One erasure set across all four drives, two of them parity: any one drive + # may fail while reads and writes continue. Left unset RustFS picks a split + # itself, which works but leaves the redundancy implicit. + erasureSetDriveCount = 4; + storageClassStandardParity = 2; + storageClassRrsParity = 1; + address = ":9000"; # SECURITY: Bind console to localhost only, access via SSH tunnel diff --git a/flake.nix b/flake.nix index 8ad5e9c..6ce0d05 100644 --- a/flake.nix +++ b/flake.nix @@ -21,9 +21,12 @@ nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; }; - outputs = { self, nixpkgs }: + outputs = + { self, nixpkgs }: let sources = builtins.fromJSON (builtins.readFile ./sources.json); + # Separate file: the CLI releases on its own cadence, in its own repository. + cliSources = builtins.fromJSON (builtins.readFile ./sources-cli.json); supportedSystems = [ "x86_64-linux" "aarch64-linux" @@ -44,10 +47,12 @@ rustfs = self.packages.${prev.stdenv.hostPlatform.system}.default; }; - packages = forAllSystems (system: + packages = forAllSystems ( + system: let pkgs = import nixpkgs { inherit system; }; srcInfo = sources.files.${system} or (throw "Unsupported system: ${system}"); + cliSrcInfo = cliSources.files.${system} or (throw "Unsupported system: ${system}"); isDarwin = pkgs.stdenvNoCC.hostPlatform.isDarwin; in { @@ -60,11 +65,12 @@ sha256 = srcInfo.sha256; }; - nativeBuildInputs = - [ pkgs.unzip ] - ++ pkgs.lib.optionals isDarwin [ - pkgs.darwin.cctools - ]; + nativeBuildInputs = [ + pkgs.unzip + ] + ++ pkgs.lib.optionals isDarwin [ + pkgs.darwin.cctools + ]; # The archive contains the binary at the root, so we set sourceRoot to current dir sourceRoot = "."; @@ -98,22 +104,62 @@ sourceProvenance = [ sourceTypes.binaryNativeCode ]; }; }; + + cli = pkgs.stdenvNoCC.mkDerivation { + pname = "rustfs-cli"; + version = cliSources.version; + + src = pkgs.fetchurl { + url = "${cliSources.downloadBase}/v${cliSources.version}/${cliSrcInfo.name}"; + sha256 = cliSrcInfo.sha256; + }; + + sourceRoot = "."; + + installPhase = '' + runHook preInstall + + mkdir -p $out/bin + install -m755 rc $out/bin/rc + + runHook postInstall + ''; + + meta = with pkgs.lib; { + description = "S3-compatible command-line client and admin tool for RustFS"; + homepage = "https://github.com/rustfs/cli"; + license = with licenses; [ + mit + asl20 + ]; + platforms = supportedSystems; + mainProgram = "rc"; + sourceProvenance = [ sourceTypes.binaryNativeCode ]; + }; + }; } ); - checks = forLinuxSystems (system: + checks = forLinuxSystems ( + system: import ./tests { inherit self; pkgs = import nixpkgs { inherit system; }; } ); - devShells = forAllSystems (system: - let pkgs = import nixpkgs { inherit system; }; - in { + devShells = forAllSystems ( + system: + let + pkgs = import nixpkgs { inherit system; }; + in + { default = pkgs.mkShell { # Security: Include shellcheck to assist in writing secure shell scripts during development. - buildInputs = with pkgs; [ nixpkgs-fmt shellcheck ]; + buildInputs = with pkgs; [ + nixpkgs-fmt + shellcheck + ]; }; } ); diff --git a/nixos/pool-naming.nix b/nixos/pool-naming.nix new file mode 100644 index 0000000..517d834 --- /dev/null +++ b/nixos/pool-naming.nix @@ -0,0 +1,59 @@ +# Collapsing a pool's names into the one ellipsis expression RustFS accepts. +# Separate from the module so it can be tested directly. +{ lib }: + +lib.fix (finalAttrs: { + # Longest leading run of characters every string shares. + commonPrefix = + strs: + let + first = builtins.head strs; + shortest = lib.foldl' + ( + n: s: lib.min n (builtins.stringLength s) + ) + (builtins.stringLength first) + strs; + matchesAt = n: lib.all (s: builtins.substring n 1 s == builtins.substring n 1 first) strs; + go = n: if n >= shortest || !(matchesAt n) then n else go (n + 1); + in + if strs == [ ] then "" else builtins.substring 0 (go 0) first; + + # Names collapse only if they share a prefix and count up without gaps. + rangeable = + items: + let + suffixes = map (lib.removePrefix (finalAttrs.commonPrefix items)) items; + numbers = map lib.toIntBase10 suffixes; + in + items != [ ] + && ( + builtins.length items == 1 + || ( + lib.all (s: builtins.match "[0-9]+" s != null) suffixes + && lib.all (x: x) (lib.imap0 (i: n: n == builtins.head numbers + i) numbers) + ) + ); + + # An ellipsis carries the low bound's width across the range, so "01" … "2" would expand to "01" "02" + consistentlyPadded = + items: + let + suffixes = map (lib.removePrefix (finalAttrs.commonPrefix items)) items; + widths = map builtins.stringLength suffixes; + padded = s: builtins.stringLength s > 1 && lib.hasPrefix "0" s; + in + !(lib.any padded suffixes) || lib.all (n: n == builtins.head widths) widths; + + # "/mnt/rustfs0" … "/mnt/rustfs3" -> "/mnt/rustfs{0…3}" + ellipsisOf = + items: + let + prefix = finalAttrs.commonPrefix items; + suffixes = map (lib.removePrefix prefix) items; + in + if builtins.length items == 1 then + builtins.head items + else + "${prefix}{${builtins.head suffixes}...${lib.last suffixes}}"; +}) diff --git a/nixos/rustfs.nix b/nixos/rustfs.nix index 45fbc06..d553b03 100644 --- a/nixos/rustfs.nix +++ b/nixos/rustfs.nix @@ -21,25 +21,61 @@ let cfg = config.services.rustfs; - dist = cfg.distributed; + # A literal, so it carries 'nodes' itself: the submodule's defaults never reach it. + pools = + if cfg.pools != [ ] then + cfg.pools + else + [ + { + nodes = [ ]; + volumes = [ "/var/lib/rustfs" ]; + } + ]; + + inherit (import ./pool-naming.nix { inherit lib; }) + rangeable + consistentlyPadded + ellipsisOf + ; + + # Both only matter past a single pool, where each has to be one expression. + nameLists = lib.concatMap + ( + pool: [ pool.volumes ] ++ lib.optional (pool.nodes != [ ]) pool.nodes + ) + pools; + + unrangeable = builtins.filter (items: !rangeable items) nameLists; + mixedPadding = builtins.filter (items: rangeable items && !consistentlyPadded items) nameLists; + + # An IPv6 literal needs brackets or its colons run into the port separator. + bracketIfIpv6 = host: if lib.hasInfix ":" host then "[${host}]" else host; + urlFor = host: volume: "http://${bracketIfIpv6 host}:${toString cfg.port}${volume}"; - configuredVolumes = - if builtins.isList cfg.volumes then - cfg.volumes + # Drive-major, so an erasure set spans nodes instead of sitting on one. + endpointsOf = + pool: + if pool.nodes == [ ] then + pool.volumes else - lib.filter (v: v != "") (lib.splitString "," cfg.volumes); + lib.concatMap (volume: map (node: urlFor node volume) pool.nodes) pool.volumes; - localVolumes = if dist.enable then dist.volumes else configuredVolumes; + ellipsisPool = + pool: + if pool.nodes == [ ] then + ellipsisOf pool.volumes + else + urlFor (ellipsisOf pool.nodes) (ellipsisOf pool.volumes); - # Every node must be given the identical endpoint list, ordered drive-major so an - # erasure set spans nodes instead of sitting on one. - endpoints = lib.concatMap - ( - volume: map (node: "http://${node}:${toString dist.port}${volume}") dist.nodes - ) - dist.volumes; + localVolumes = lib.unique (lib.concatMap (pool: pool.volumes) pools); + driveCount = pool: builtins.length pool.volumes * (lib.max 1 (builtins.length pool.nodes)); - volumesStr = lib.concatStringsSep " " (if dist.enable then endpoints else configuredVolumes); + # One pool can be listed drive by drive, which puts no shape on the names. Several + # cannot: rustfs reads plain arguments as a single pool and refuses the mixture. + volumesStr = lib.concatStringsSep " " ( + if builtins.length pools <= 1 then lib.concatMap endpointsOf pools else map ellipsisPool pools + ); in { imports = [ @@ -51,6 +87,11 @@ in [ "services" "rustfs" "secretKey" ] [ "services" "rustfs" "secretKeyFile" ] ) + + # volumes predates the pool model + (lib.mkChangedOptionModule [ "services" "rustfs" "volumes" ] [ "services" "rustfs" "pools" ] + (config: [{ volumes = config.services.rustfs.volumes; }]) + ) ]; options.services.rustfs = { @@ -104,63 +145,118 @@ in ''; }; - volumes = lib.mkOption { - type = lib.types.either lib.types.str (lib.types.listOf lib.types.str); - default = [ "/var/lib/rustfs" ]; - description = "List of paths or comma-separated string where RustFS stores data."; + pools = lib.mkOption { + type = lib.types.listOf ( + lib.types.submodule { + options = { + nodes = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + example = [ + "node1" + "node2" + "node3" + "node4" + ]; + description = '' + Hostnames making up this pool, resolvable from every node of it. + Left empty the drives are local paths, which is the single-node case. + ''; + }; + volumes = lib.mkOption { + type = lib.types.listOf lib.types.str; + example = [ + "/mnt/disk0" + "/mnt/disk1" + "/mnt/disk2" + "/mnt/disk3" + ]; + description = '' + Drive paths, each on its own filesystem. Every node of the pool uses + the same layout. + ''; + }; + }; + } + ); + default = [ ]; + defaultText = lib.literalExpression ''[ { volumes = [ "/var/lib/rustfs" ]; } ]''; + example = [ + { + volumes = [ + "/mnt/disk0" + "/mnt/disk1" + "/mnt/disk2" + "/mnt/disk3" + ]; + } + { + nodes = [ + "node1" + "node2" + "node3" + "node4" + ]; + volumes = [ + "/mnt/disk0" + "/mnt/disk1" + "/mnt/disk2" + "/mnt/disk3" + ]; + } + ]; + description = '' + Server pools, in order. Every RustFS deployment is one of these -- a single + drive, one node of four, four nodes of four -- so this is the only place + drives are declared. + + Appending a pool is how a cluster grows without rebalancing what it already + stores, and the only route from a single-node deployment to a distributed + one; draining the old pool afterwards is `rc admin decommission`. + + A lone pool is listed drive by drive, so its names take any shape. Several + cannot be: RustFS reads plain arguments as one pool and rejects mixing the + two forms, so each pool has to collapse into a single ellipsis expression + such as `node{2...5}`. That needs a common prefix and a contiguous numeric + range, which a single-node pool does not need for its hostname. + + Keep the list identical and in the same order on every node: RustFS derives + pool identity from it, so a divergent list is a different cluster. + ''; }; - distributed = { - enable = lib.mkEnableOption "a distributed RustFS cluster spanning several nodes"; - - nodes = lib.mkOption { - type = lib.types.listOf lib.types.str; - default = [ ]; - example = [ - "node1" - "node2" - "node3" - "node4" - ]; - description = '' - Hostnames of every node in the cluster, resolvable from each of them. - Used to render the shared endpoint list; set identically on all nodes. - ''; - }; + port = lib.mkOption { + type = lib.types.port; + default = 9000; + description = "Port peers reach each other on, matching `address`."; + }; - volumes = lib.mkOption { - type = lib.types.listOf lib.types.str; - default = [ ]; - example = [ - "/mnt/disk0" - "/mnt/disk1" - "/mnt/disk2" - "/mnt/disk3" - ]; - description = '' - Drive paths present on each node, each on its own filesystem. Every node - uses the same layout, so this replaces `volumes` when distributed mode is - enabled and is what gets created and made writable locally. - ''; - }; + erasureSetDriveCount = lib.mkOption { + type = lib.types.nullOr lib.types.ints.positive; + default = null; + example = 4; + description = '' + Drives per erasure set. Left null RustFS picks a divisor of the pool's drive + count itself; set it when the split matters, such as one set spanning all + four nodes of a pool rather than sitting inside one. + ''; + }; - port = lib.mkOption { - type = lib.types.port; - default = 9000; - description = "Port peers reach each other on, matching `address`."; - }; + storageClassStandardParity = lib.mkOption { + type = lib.types.nullOr lib.types.ints.positive; + default = null; + example = 2; + description = '' + Parity drives per erasure set for the STANDARD storage class. Two of four + tolerates one node of a four-node set going away while writes continue. + ''; + }; - localEndpointHost = lib.mkOption { - type = lib.types.str; - default = config.networking.hostName; - defaultText = lib.literalExpression "config.networking.hostName"; - description = '' - Which entry of `nodes` identifies this machine, so it claims its own - drives instead of reaching them over RPC. Required whenever `address` - binds a wildcard such as `0.0.0.0`, since RustFS cannot infer its - identity from that and would otherwise treat every drive as remote. - ''; - }; + storageClassRrsParity = lib.mkOption { + type = lib.types.nullOr lib.types.ints.positive; + default = null; + example = 1; + description = "Parity drives per erasure set for the REDUCED_REDUNDANCY class."; }; address = lib.mkOption { @@ -205,19 +301,42 @@ in }; config = lib.mkIf cfg.enable { - # Erasure coding needs 4 drives; a distributed cluster needs 4 nodes of 4. assertions = [ { - assertion = dist.enable -> builtins.length dist.nodes >= 4; - message = "services.rustfs.distributed.nodes needs at least 4 nodes, got ${toString (builtins.length dist.nodes)}."; + assertion = pools != [ ]; + message = "services.rustfs.pools cannot be empty -- RustFS needs at least one drive."; + } + { + assertion = builtins.length pools <= 1 || unrangeable == [ ]; + message = "services.rustfs.pools: ${builtins.toJSON unrangeable} cannot each be named as one rustfs pool. Past a single pool every name list has to collapse to an ellipsis expression, so it needs a common prefix and a contiguous numeric range such as node{2...5}."; + } + { + assertion = builtins.length pools <= 1 || mixedPadding == [ ]; + message = "services.rustfs.pools: ${builtins.toJSON mixedPadding} mixes zero-padded and bare numbers. An ellipsis carries the low bound's width across the range, so these would expand to drive names you never declared -- pad all of them or none."; } { - assertion = dist.enable -> builtins.length dist.volumes >= 4; - message = "services.rustfs.distributed.volumes needs at least 4 drives per node, got ${toString (builtins.length dist.volumes)}."; + assertion = lib.all (pool: pool.volumes != [ ]) pools; + message = "every services.rustfs.pools entry needs at least one drive."; } + # A pool's drives are dealt into erasure sets, so the count has to divide. { - assertion = dist.enable -> builtins.elem dist.localEndpointHost dist.nodes; - message = "services.rustfs.distributed.localEndpointHost is ${dist.localEndpointHost}, which is not one of nodes (${lib.concatStringsSep ", " dist.nodes})."; + assertion = + cfg.erasureSetDriveCount != null + -> lib.all (pool: lib.mod (driveCount pool) cfg.erasureSetDriveCount == 0) pools; + message = "every services.rustfs.pools entry needs a drive count divisible by erasureSetDriveCount (${toString cfg.erasureSetDriveCount}); got ${ + lib.concatMapStringsSep ", " (pool: toString (driveCount pool)) pools + }."; + } + # Parity is taken out of the set, so it cannot claim the whole of it. + { + assertion = + lib.all + (parity: parity != null && cfg.erasureSetDriveCount != null -> parity < cfg.erasureSetDriveCount) + [ + cfg.storageClassStandardParity + cfg.storageClassRrsParity + ]; + message = "services.rustfs storage class parity must be below erasureSetDriveCount (${toString cfg.erasureSetDriveCount})."; } ]; @@ -263,8 +382,14 @@ in // lib.optionalAttrs (cfg.logDirectory != null) { RUSTFS_OBS_LOG_DIRECTORY = cfg.logDirectory; } - // lib.optionalAttrs dist.enable { - RUSTFS_LOCAL_ENDPOINT_HOST = dist.localEndpointHost; + // lib.optionalAttrs (cfg.erasureSetDriveCount != null) { + RUSTFS_ERASURE_SET_DRIVE_COUNT = toString cfg.erasureSetDriveCount; + } + // lib.optionalAttrs (cfg.storageClassStandardParity != null) { + RUSTFS_STORAGE_CLASS_STANDARD = "EC:${toString cfg.storageClassStandardParity}"; + } + // lib.optionalAttrs (cfg.storageClassRrsParity != null) { + RUSTFS_STORAGE_CLASS_RRS = "EC:${toString cfg.storageClassRrsParity}"; } // cfg.extraEnvironmentVariables; diff --git a/sources-cli.json b/sources-cli.json new file mode 100644 index 0000000..9f490a7 --- /dev/null +++ b/sources-cli.json @@ -0,0 +1,18 @@ +{ + "version": "0.1.31", + "downloadBase": "https://github.com/rustfs/cli/releases/download", + "files": { + "x86_64-linux": { + "name": "rustfs-cli-linux-amd64-v0.1.31.tar.gz", + "sha256": "129ac825f495fe44583505233d849045f7fc1801edb483a042cee7a563e922d7" + }, + "aarch64-linux": { + "name": "rustfs-cli-linux-arm64-v0.1.31.tar.gz", + "sha256": "04803754ffd8b3699392bacec04276ab75f200b71177e03141dcdf9718f3e8b9" + }, + "aarch64-darwin": { + "name": "rustfs-cli-macos-arm64-v0.1.31.tar.gz", + "sha256": "d8068d9396f14bdf67c3fe2e26c8f3f306b6d22442cd8387e2c12498ab3dad09" + } + } +} diff --git a/tests/default.nix b/tests/default.nix index c013e4a..329b67e 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -3,4 +3,6 @@ { singleNodeMultiDisk = import ./single-node-multi-disk.nix { inherit pkgs self; }; multiNodeMultiDisk = import ./multi-node-multi-disk.nix { inherit pkgs self; }; + multiPool = import ./multi-pool.nix { inherit pkgs self; }; + poolNaming = import ./pool-naming.nix { inherit pkgs self; }; } diff --git a/tests/multi-node-multi-disk.nix b/tests/multi-node-multi-disk.nix index 141e109..f82bb35 100644 --- a/tests/multi-node-multi-disk.nix +++ b/tests/multi-node-multi-disk.nix @@ -37,12 +37,13 @@ let services.rustfs = { enable = true; package = self.packages.${pkgs.stdenv.hostPlatform.system}.default; - distributed = { - enable = true; - nodes = hosts; - volumes = localVolumes; - port = 9000; - }; + pools = [ + { + nodes = hosts; + volumes = localVolumes; + } + ]; + port = 9000; address = "0.0.0.0:9000"; consoleEnable = false; accessKeyFile = "/etc/rustfs-access-key"; diff --git a/tests/multi-pool.nix b/tests/multi-pool.nix new file mode 100644 index 0000000..f7af3ec --- /dev/null +++ b/tests/multi-pool.nix @@ -0,0 +1,101 @@ +{ pkgs, self }: + +let + disks = [ + "vdb" + "vdc" + "vdd" + "vde" + "vdf" + "vdg" + "vdh" + "vdi" + ]; + volumes = pkgs.lib.imap0 (i: _: "/mnt/rustfs${toString i}") disks; + firstPool = pkgs.lib.take 4 volumes; + secondPool = pkgs.lib.drop 4 volumes; +in +pkgs.testers.runNixOSTest { + name = "rustfs-multi-pool"; + + nodes.machine = + { lib, ... }: + { + imports = [ self.nixosModules.rustfs ]; + + virtualisation.emptyDiskImages = map (_: 1024) disks; + virtualisation.memorySize = 2048; + boot.supportedFilesystems = [ "xfs" ]; + + virtualisation.fileSystems = lib.listToAttrs ( + lib.imap0 + ( + i: disk: + lib.nameValuePair (builtins.elemAt volumes i) { + device = "/dev/${disk}"; + fsType = "xfs"; + autoFormat = true; + } + ) + disks + ); + + services.rustfs = { + enable = true; + package = self.packages.${pkgs.stdenv.hostPlatform.system}.default; + # A second pool is how a cluster grows: the first keeps what it stores and + # the new one takes the writes, rather than everything rebalancing at once. + pools = [ + { volumes = firstPool; } + { volumes = secondPool; } + ]; + address = "127.0.0.1:9000"; + consoleEnable = false; + accessKeyFile = "/etc/rustfs-access-key"; + secretKeyFile = "/etc/rustfs-secret-key"; + }; + + environment.etc."rustfs-access-key".text = "rustfsadmin"; + environment.etc."rustfs-secret-key".text = "rustfsadmin"; + environment.systemPackages = [ + pkgs.awscli2 + self.packages.${pkgs.stdenv.hostPlatform.system}.cli + ]; + }; + + testScript = '' + machine.wait_for_unit("rustfs.service") + machine.wait_for_open_port(9000) + + # Each pool collapses to its own ellipsis expression. Listing the drives plainly + # would make rustfs read all eight as a single pool. + machine.succeed( + "systemctl show -p Environment rustfs.service | " + "grep -q 'RUSTFS_VOLUMES=/mnt/rustfs{0...3} /mnt/rustfs{4...7}'" + ) + + aws = ( + "AWS_ACCESS_KEY_ID=rustfsadmin AWS_SECRET_ACCESS_KEY=rustfsadmin " + "AWS_DEFAULT_REGION=us-east-1 aws --endpoint-url http://127.0.0.1:9000" + ) + machine.wait_until_succeeds(aws + " s3 ls", timeout=120) + + # What rustfs itself believes, rather than what we asked for. + machine.succeed( + "rc alias set t http://127.0.0.1:9000 rustfsadmin rustfsadmin" + ) + pools = machine.succeed("rc admin pool list t --json") + import json + listed = json.loads(pools)["pools"] + assert len(listed) == 2, f"expected two pools, got {listed}" + assert all(p["status"] == "active" for p in listed), f"pool not active: {listed}" + + machine.succeed(aws + " s3 mb s3://testbucket") + machine.succeed("echo hello > /tmp/obj.txt") + machine.succeed(aws + " s3 cp /tmp/obj.txt s3://testbucket/obj.txt") + machine.succeed(aws + " s3 cp s3://testbucket/obj.txt /tmp/roundtrip.txt") + machine.succeed("grep -q hello /tmp/roundtrip.txt") + + machine.succeed("systemctl is-active rustfs.service") + ''; +} diff --git a/tests/pool-naming.nix b/tests/pool-naming.nix new file mode 100644 index 0000000..655bb26 --- /dev/null +++ b/tests/pool-naming.nix @@ -0,0 +1,155 @@ +{ pkgs, self }: + +let + lib = pkgs.lib; + naming = import ../nixos/pool-naming.nix { inherit lib; }; + + # Pads to width, but lets a number that outgrew it through unchanged, the way + # 98…105 at width 2 has to behave. + pad = + width: n: + let + s = toString n; + in + if width == 0 || builtins.stringLength s >= width then s else lib.fixedWidthNumber width n; + + expand = + rendered: + let + m = builtins.match "(.*)[{]([0-9]+)[.][.][.]([0-9]+)[}]" rendered; + prefix = builtins.elemAt m 0; + lo = builtins.elemAt m 1; + hi = builtins.elemAt m 2; + width = if lib.hasPrefix "0" lo then builtins.stringLength lo else 0; + show = pad width; + from = lib.toIntBase10 lo; + in + if m == null then + [ rendered ] + else + map (i: "${prefix}${show (from + i)}") (lib.range 0 (lib.toIntBase10 hi - from)); + + # Every shape a real pool takes, swept rather than sampled. + bases = [ + "/mnt/rustfs" + "node" + "recorder-a-" + "192.168.1." + "2001:db8::" + ]; + starts = [ + 0 + 1 + 8 + 9 + 10 + 98 + ]; + counts = [ + 1 + 2 + 4 + 8 + ]; + widths = [ + 0 + 2 + 3 + ]; + + nameList = + base: start: count: width: + map (i: "${base}${pad width (start + i)}") (lib.range 0 (count - 1)); + + swept = lib.concatMap + ( + base: + lib.concatMap + ( + start: lib.concatMap (count: map (width: nameList base start count width) widths) counts + ) + starts + ) + bases; + + # Both predicates together are what the module asserts before rendering. + nameable = items: naming.rangeable items && naming.consistentlyPadded items; + + # Shapes that must be refused, including both bugs the fuzzing turned up. + refused = [ + [ ] + [ + "d1" + "d3" + ] # gap + [ + "d3" + "d2" + "d1" + ] # descending + [ + "d1" + "d1" + ] # duplicate + [ + "d1" + "d2" + "x9" + ] # prefix breaks + [ + "a1b" + "a2b" + ] # digits not at the end + [ + "d" + "d1" + ] # empty suffix + [ + "d01" + "d2" + ] # mixed padding -- rendered d{01…2} => d01,d02 + [ + "d8" + "d09" + ] # mixed padding, other way round + [ + "" + "" + ] + [ + "d1.5" + "d2.5" + ] + ]; + + roundTripFailures = lib.concatMap + ( + items: + let + got = expand (naming.ellipsisOf items); + in + lib.optional (!(nameable items) || got != items) { + inherit items got; + rangeable = naming.rangeable items; + consistentlyPadded = naming.consistentlyPadded items; + } + ) + swept; + + wronglyAccepted = builtins.filter nameable refused; + + failures = { + roundTrip = roundTripFailures; + accepted = wronglyAccepted; + }; + ok = roundTripFailures == [ ] && wronglyAccepted == [ ]; +in +pkgs.runCommand "rustfs-pool-naming-test" { } '' + ${lib.optionalString (!ok) '' + echo 'pool naming failures:' >&2 + echo ${lib.escapeShellArg (builtins.toJSON failures)} >&2 + exit 1 + ''} + echo "checked ${toString (builtins.length swept)} rangeable shapes and ${toString (builtins.length refused)} refusals" + touch $out +'' diff --git a/tests/single-node-multi-disk.nix b/tests/single-node-multi-disk.nix index e6d4700..38fd78c 100644 --- a/tests/single-node-multi-disk.nix +++ b/tests/single-node-multi-disk.nix @@ -27,7 +27,7 @@ pkgs.testers.runNixOSTest { services.rustfs = { enable = true; package = self.packages.${pkgs.stdenv.hostPlatform.system}.default; - inherit volumes; + pools = [{ inherit volumes; }]; address = "127.0.0.1:9000"; consoleEnable = false; accessKeyFile = "/etc/rustfs-access-key";