From d9d32e019469e96a0e326ffc42ad306cfd86c04e Mon Sep 17 00:00:00 2001 From: Raito Bezarius Date: Thu, 3 Aug 2023 22:56:37 +0200 Subject: [PATCH] epyc: init hydra settings properly --- hosts/epyc.nix | 16 ++++++++ modules/hydra/coordinator.nix | 76 ++++++++++++++++++++++++++++++++++- modules/nix-daemon.nix | 6 ++- 3 files changed, 95 insertions(+), 3 deletions(-) diff --git a/hosts/epyc.nix b/hosts/epyc.nix index fda7667..ac0864b 100644 --- a/hosts/epyc.nix +++ b/hosts/epyc.nix @@ -1,3 +1,8 @@ +{ lib, ... }: +let + gcc-system-features = arch: lib.optionals (arch != null) ([ "gccarch-${arch}" ] + ++ map (x: "gccarch-${x}") lib.systems.architectures.inferiors.${arch}); +in { imports = [ ../modules/ipmi-supermicro.nix @@ -19,6 +24,17 @@ }; }; + nix.buildMachines = [ + { hostName = "localhost"; + systems = [ + "x86_64-linux" + "riscv64-linux" + ]; + supportedFeatures = [ "kvm" "nixos-test" "big-parallel" "benchmark" ] ++ gcc-system-features "znver3"; + maxJobs = 1; + } + ]; + boot.binfmt.emulatedSystems = [ "riscv64-linux" "aarch64-linux" "riscv64-linux" ]; simd.arch = "znver3"; diff --git a/modules/hydra/coordinator.nix b/modules/hydra/coordinator.nix index 77c1ceb..55dda02 100644 --- a/modules/hydra/coordinator.nix +++ b/modules/hydra/coordinator.nix @@ -1,9 +1,81 @@ -{ ... }: { +{ pkgs, ... }: { services.hydra = { enable = true; hydraURL = "https://hydra.newtype.fr"; notificationSender = "hydra@localhost"; - buildMachinesFiles = [ ]; + buildMachinesFiles = [ "/etc/nix/machines" ]; useSubstitutes = true; }; + + environment.systemPackages = [ pkgs.nix-prefetch-git ]; + nix.trustedUsers = [ "hydra" "hydra-www" ]; + + services.postgresql = { + enableJIT = true; + settings = { + checkpoint_completion_target = "0.9"; + default_statistics_target = 100; + + max_connections = 500; + work_mem = "20MB"; + maintenance_work_mem = "2GB"; + + shared_buffers = "8GB"; + + min_wal_size = "1GB"; + max_wal_size = "2GB"; + wal_buffers = "16MB"; + + max_worker_processes = 16; + max_parallel_workers_per_gather = 8; + max_parallel_workers = 16; + + # NVMe related performance tuning + effective_io_concurrency = 200; + random_page_cost = "1.1"; + + # We can risk losing some transactions. + synchronous_commit = "off"; + + effective_cache_size = "16GB"; + + # autovacuum and autoanalyze much more frequently: + # at these values vacuum should run approximately + # every 2 mass rebuilds, or a couple times a day + # on the builds table. Some of those queries really + # benefit from frequent vacuums, so this should + # help. In particular, I'm thinking the jobsets + # pages. + autovacuum_vacuum_scale_factor = 0.002; + autovacuum_analyze_scale_factor = 0.001; + + shared_preload_libraries = "pg_stat_statements"; + compute_query_id = "on"; + }; + }; + + security.acme = { + acceptTerms = true; + defaults.email = "ryan@lahfa.xyz"; + }; + + services.nginx = { + enable = true; + + recommendedZstdSettings = true; + recommendedBrotliSettings = true; + recommendedGzipSettings = true; + recommendedOptimisation =true; + recommendedTlsSettings = true; + recommendedProxySettings = true; + }; + + services.nginx.virtualHosts."hydra.newtype.fr" = { + forceSSL = true; + enableACME = true; + # TODO: remove compression for some locations + locations."/".proxyPass = "http://localhost:3000"; + }; + + networking.firewall.allowedTCPPorts = [ 80 443 ]; } diff --git a/modules/nix-daemon.nix b/modules/nix-daemon.nix index b45d3a8..3120c3d 100644 --- a/modules/nix-daemon.nix +++ b/modules/nix-daemon.nix @@ -42,13 +42,17 @@ in package = pkgs.nixVersions.nix_2_13; # should be enough? - nrBuildUsers = lib.mkDefault 32; + nrBuildUsers = 128; # https://github.com/NixOS/nix/issues/719 + daemonCPUSchedPolicy = "batch"; + daemonIOSchedClass = "best-effort"; + daemonIOSchedPriority = 5; settings = { keep-outputs = true; keep-derivations = true; + max-jobs = 64; # in zfs we trust fsync-metadata = lib.boolToString (!config.boot.isContainer or config.fileSystems."/".fsType != "zfs"); substituters = [