From d9d32e019469e96a0e326ffc42ad306cfd86c04e Mon Sep 17 00:00:00 2001
From: Raito Bezarius <masterancpp@gmail.com>
Date: Thu, 3 Aug 2023 22:56:37 +0200
Subject: [PATCH] epyc: init hydra settings properly

---
 hosts/epyc.nix                | 16 ++++++++
 modules/hydra/coordinator.nix | 76 ++++++++++++++++++++++++++++++++++-
 modules/nix-daemon.nix        |  6 ++-
 3 files changed, 95 insertions(+), 3 deletions(-)

diff --git a/hosts/epyc.nix b/hosts/epyc.nix
index fda7667..ac0864b 100644
--- a/hosts/epyc.nix
+++ b/hosts/epyc.nix
@@ -1,3 +1,8 @@
+{ lib, ... }:
+let
+  gcc-system-features = arch: lib.optionals (arch != null) ([ "gccarch-${arch}" ]
+    ++ map (x: "gccarch-${x}") lib.systems.architectures.inferiors.${arch});
+in
 {
   imports = [
     ../modules/ipmi-supermicro.nix
@@ -19,6 +24,17 @@
     };
   };
 
+  nix.buildMachines = [
+    { hostName = "localhost";
+      systems = [
+        "x86_64-linux"
+        "riscv64-linux"
+      ];
+      supportedFeatures = [ "kvm" "nixos-test" "big-parallel" "benchmark" ] ++ gcc-system-features "znver3";
+      maxJobs = 1;
+    }
+  ];
+
   boot.binfmt.emulatedSystems = [ "riscv64-linux" "aarch64-linux" "riscv64-linux" ];
 
   simd.arch = "znver3";
diff --git a/modules/hydra/coordinator.nix b/modules/hydra/coordinator.nix
index 77c1ceb..55dda02 100644
--- a/modules/hydra/coordinator.nix
+++ b/modules/hydra/coordinator.nix
@@ -1,9 +1,81 @@
-{ ... }: {
+{ pkgs, ... }: {
   services.hydra = {
     enable = true;
     hydraURL = "https://hydra.newtype.fr";
     notificationSender = "hydra@localhost";
-    buildMachinesFiles = [ ];
+    buildMachinesFiles = [ "/etc/nix/machines" ];
     useSubstitutes = true;
   };
+
+  environment.systemPackages = [ pkgs.nix-prefetch-git ];
+  nix.trustedUsers = [ "hydra" "hydra-www" ];
+
+  services.postgresql = {
+    enableJIT = true;
+    settings = {
+      checkpoint_completion_target = "0.9";
+      default_statistics_target = 100;
+
+      max_connections = 500;
+      work_mem = "20MB";
+      maintenance_work_mem = "2GB";
+
+      shared_buffers = "8GB";
+
+      min_wal_size = "1GB";
+      max_wal_size = "2GB";
+      wal_buffers = "16MB";
+
+      max_worker_processes = 16;
+      max_parallel_workers_per_gather = 8;
+      max_parallel_workers = 16;
+
+      # NVMe related performance tuning
+      effective_io_concurrency = 200;
+      random_page_cost = "1.1";
+
+      # We can risk losing some transactions.
+      synchronous_commit = "off";
+
+      effective_cache_size = "16GB";
+
+      # autovacuum and autoanalyze much more frequently:
+      # at these values vacuum should run approximately
+      # every 2 mass rebuilds, or a couple times a day
+      # on the builds table. Some of those queries really
+      # benefit from frequent vacuums, so this should
+      # help. In particular, I'm thinking the jobsets
+      # pages.
+      autovacuum_vacuum_scale_factor = 0.002;
+      autovacuum_analyze_scale_factor = 0.001;
+
+      shared_preload_libraries = "pg_stat_statements";
+      compute_query_id = "on";
+    };
+  };
+
+  security.acme = {
+    acceptTerms = true;
+    defaults.email = "ryan@lahfa.xyz";
+  };
+
+  services.nginx = {
+    enable = true;
+
+    recommendedZstdSettings = true;
+    recommendedBrotliSettings = true;
+    recommendedGzipSettings = true;
+    recommendedOptimisation =true;
+    recommendedTlsSettings = true;
+    recommendedProxySettings = true;
+  };
+
+  services.nginx.virtualHosts."hydra.newtype.fr" = {
+    forceSSL = true;
+    enableACME = true;
+    # TODO: remove compression for some locations
+    locations."/".proxyPass = "http://localhost:3000";
+  };
+
+  networking.firewall.allowedTCPPorts = [ 80 443 ];
 }
diff --git a/modules/nix-daemon.nix b/modules/nix-daemon.nix
index b45d3a8..3120c3d 100644
--- a/modules/nix-daemon.nix
+++ b/modules/nix-daemon.nix
@@ -42,13 +42,17 @@ in
       package = pkgs.nixVersions.nix_2_13;
 
       # should be enough?
-      nrBuildUsers = lib.mkDefault 32;
+      nrBuildUsers = 128;
 
       # https://github.com/NixOS/nix/issues/719
+      daemonCPUSchedPolicy = "batch";
+      daemonIOSchedClass = "best-effort";
+      daemonIOSchedPriority = 5;
 
       settings = {
         keep-outputs = true;
         keep-derivations = true;
+        max-jobs = 64;
         # in zfs we trust
         fsync-metadata = lib.boolToString (!config.boot.isContainer or config.fileSystems."/".fsType != "zfs");
         substituters = [