infra: steal from the best

(Mic92's cluster TUM's infra.)

modules/auto-upgrade.nix

@@ -0,0 +1,25 @@
+{ pkgs, ... }: {
+  system.autoUpgrade.enable = true;
+  system.autoUpgrade.flake = "git:git.newtype.fr/newtype/newtype-org-configurations";
+  system.autoUpgrade.flags = [ "--option" "accept-flake-config" "true" ];
+
+  # add a random jitter so not all machines reboot at the same time.
+  systemd.timers.auto-reboot.timerConfig.RandomizedDelaySec = 60 * 20;
+
+  systemd.services.auto-reboot = {
+    path = [ pkgs.systemd pkgs.util-linux ];
+    # The last saturday in a month
+    startAt = "Sat *-*~07/1";
+    script = ''
+      booted="$(readlink /run/booted-system/{initrd,kernel,kernel-modules})"
+      built="$(readlink /nix/var/nix/profiles/system/{initrd,kernel,kernel-modules})"
+      if [ "''${booted}" = "''${built}" ]; then
+        echo "No kernel update... skipping reboot"
+      else
+        # reboot in 24 hours
+        msg=$(shutdown -r +${toString (60 * 24)} 2>&1)
+        echo "$msg" | wall
+      fi
+    '';
+  };
+}

modules/builder.nix

@@ -0,0 +1,11 @@
+{
+  users.extraUsers.nix = {
+    isNormalUser = true;
+    home = "/home/nix";
+    openssh.authorizedKeys.keys = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAZpEtSfB0GDwcELc5/AKNiBZJV9OVfQ0BMFzBlF+8Yd raito@everywhere"
+    ];
+    uid = 5001;
+  };
+  nix.settings.trusted-users = [ "nix" ];
+}

modules/fck-spectr.nix

@@ -0,0 +1,16 @@
+{
+  # It may leak your data, but look how FAST it is!1!!
+  # https://make-linux-fast-again.com/
+  boot.kernelParams = [
+    "noibrs"
+    "noibpb"
+    "nopti"
+    "nospectre_v2"
+    "nospectre_v1"
+    "l1tf=off"
+    "nospec_store_bypass_disable"
+    "no_stf_barrier"
+    "mds=off"
+    "mitigations=off"
+  ];
+}

modules/hardware/supermicro-H12SSL-i.nix

@@ -0,0 +1,41 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.kernelParams = [ "pci=realloc" ];
+  boot.initrd.availableKernelModules = [ "xhci_pci" "nvme" "usbhid" "usb_storage" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  boot.initrd.systemd.enable = lib.mkForce false;
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/3a81ba8f-f5bb-446c-89a3-ad77e354dae0";
+      fsType = "btrfs";
+    };
+
+  boot.initrd.luks.devices."nixroot" = {
+   device = "/dev/disk/by-uuid/c10d2822-cb83-4666-98f8-0aa04be259bc";
+   keyFile = "/dev/zero";
+   keyFileSize = 1;
+  };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/AFF2-3149";
+      fsType = "vfat";
+    };
+
+  swapDevices =
+    [ { device = "/dev/disk/by-uuid/93e251e1-1bfc-4bd4-8585-ea2eae7795bf"; }
+    ];
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

modules/hosts.nix

@@ -0,0 +1,47 @@
+{ lib, config, ... }:
+let
+  hostOptions = with lib; {
+    ipv4 = mkOption {
+      default = null;
+      type = types.nullOr types.str;
+      description = ''
+        own ipv4 address
+      '';
+    };
+
+    ipv6 = mkOption {
+      type = types.nullOr types.str;
+      default = null;
+      description = ''
+        own ipv6 address
+      '';
+    };
+  };
+in
+{
+  options = with lib; {
+    networking.newtype.hosts = mkOption {
+      type = with types; attrsOf (submodule [{ options = hostOptions; }]);
+      description = "A host in our cluster";
+    };
+    networking.newtype.currentHost = mkOption {
+      type = with types; submodule [{ options = hostOptions; }];
+      default = config.networking.newtype.hosts.${config.networking.hostName};
+      description = "The host that is described by this configuration";
+    };
+  };
+  config = {
+    warnings =
+      lib.optional (!(config.networking.newtype.hosts ? ${config.networking.hostName}) && 
+        config.networking.hostName != "nixos" # we dont care about nixos netboot/installer images
+      )
+        "Please add network configuration for ${config.networking.hostName}. None found in ${./hosts.nix}";
+
+    # usually, for each host there is a hostname.dse.in.tum.de and hostname.r domain
+    networking.newtype.hosts = {
+      epyc = {
+        ipv6 = "2001:470:ca5e:dee:587c:7a50:f36c:cae8";
+      };
+    };
+  };
+}

modules/ipmi-supermicro.nix

@@ -0,0 +1,7 @@
+{
+  # supermicro uses ttyS1 for SOL
+  boot.kernelParams = [
+    "console=ttyS1,115200n8"
+    "console=tty1"
+  ];
+}

modules/network.nix

@@ -0,0 +1,49 @@
+{ config
+, lib
+, ...
+}: {
+  # use networkd
+  networking.dhcpcd.enable = false;
+  systemd.network.enable = true;
+
+  # add an entry to /etc/hosts for each host
+  networking.extraHosts = lib.concatStringsSep "\n" (lib.mapAttrsToList
+    (name: host: ''
+      ${lib.optionalString (host.ipv4 != null) "${host.ipv4} ${name}"}
+      ${lib.optionalString (host.ipv6 != null) "${host.ipv6} ${name}"}
+    '')
+    config.networking.newtype.hosts);
+
+  # leave container interfaces alone
+  systemd.network.networks."05-veth".extraConfig = ''
+    [Match]
+    Driver = veth
+
+    [Link]
+    Unmanaged = yes
+    '';
+
+  systemd.network.networks."10-nat-lan" = {
+    matchConfig.Name = "nat-lan";
+    linkConfig.RequiredForOnline = true;
+    DHCP = "yes";
+  };
+
+  systemd.network.links."10-nat-lan" = {
+    matchConfig.MACAddress = "3c:ec:ef:7e:bd:c8";
+    linkConfig.Name = "nat-lan";
+  };
+
+  systemd.network.networks."10-wan" = {
+    matchConfig.Name = "wan";
+    linkConfig.RequiredForOnline = true;
+    networkConfig.Address = [ config.networking.newtype.currentHost.ipv6 ];
+  };
+
+  systemd.network.links."10-wan" = {
+    matchConfig.MACAddress = "3c:ec:ef:7e:bd:c9";
+    linkConfig.Name = "wan";
+  };
+
+  deployment.targetHost = "${config.networking.hostName}.infra.newtype.fr";
+}

modules/nix-daemon.nix

@@ -0,0 +1,60 @@
+{ lib
+, config
+, pkgs
+, ...
+}:
+
+let
+  gcc-system-features = arch: lib.optionals (arch != null) ([ "gccarch-${arch}" ]
+    ++ map (x: "gccarch-${x}") lib.systems.architectures.inferiors.${arch});
+in
+{
+  options = {
+    simd.arch = lib.mkOption {
+      type = with lib.types; nullOr str;
+      default = null;
+      description = ''
+        Microarchitecture string for nixpkgs.hostPlatform.gcc.march and to generate system-features.
+        Can be determined with: gcc -march=native -Q --help=target | grep march
+      '';
+    };
+  };
+
+  imports = [ ./builder.nix ];
+
+  config = {
+    warnings = lib.optionals (config.simd.arch == null) [ "Please set simd.arch for ${config.networking.hostName}" ];
+
+    nix = {
+      gc.automatic = true;
+      gc.dates = "03:15";
+      gc.options = "--delete-older-than 30d";
+
+      # 2.11, 2.12 suffers from a bug with remote builders…
+      package = pkgs.nixVersions.nix_2_13;
+
+      # should be enough?
+      nrBuildUsers = lib.mkDefault 32;
+
+      # https://github.com/NixOS/nix/issues/719
+
+      settings = {
+        keep-outputs = true;
+        keep-derivations = true;
+        # in zfs we trust
+        fsync-metadata = lib.boolToString (!config.boot.isContainer or config.fileSystems."/".fsType != "zfs");
+        substituters = [
+          "https://nix-community.cachix.org"
+          "https://tum-dse.cachix.org"
+        ];
+        system-features = [ "benchmark" "big-parallel" "kvm" "nixos-test" ] ++ gcc-system-features config.simd.arch;
+        trusted-public-keys = [
+          "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
+          "tum-dse.cachix.org-1:v67rK18oLwgO0Z4b69l30SrV1yRtqxKpiHodG4YxhNM="
+        ];
+      };
+    };
+
+    nixpkgs.config.allowUnfree = true;
+  };
+}

modules/packages.nix

@@ -0,0 +1,39 @@
+{ pkgs, ... }: {
+  # this extends the list from:
+  # https://github.com/numtide/srvos/blob/master/server.nix#L10
+  environment.systemPackages = with pkgs; [
+    socat
+    whois
+
+    jq
+    psmisc
+    libarchive
+    sipcalc
+    iperf
+    openssl
+    binutils
+    file
+    wget
+    htop
+    ripgrep
+    lsof
+    tcpdump
+    rsync
+    git
+    tig
+    lazygit
+    python3
+    iotop
+    man-pages
+    netcat
+    mtr
+    (neovim.override { vimAlias = true; })
+
+    pciutils
+    ethtool
+    usbutils
+
+    ipmitool
+    # tries to default to soft-float due to out-dated cc-rs
+  ] ++ lib.optional (!stdenv.hostPlatform.isRiscV) bandwhich;
+}

modules/tor-ssh.nix

@@ -0,0 +1,18 @@
+{
+  # imports = [ ./sshd ];
+  services.tor = {
+    enable = true;
+    relay.onionServices."ssh".map = [{ port = 22; }];
+
+    settings = {
+      DnsPort = 9053;
+      AutomapHostsOnResolve = true;
+      AutomapHostsSuffixes = [ ".exit" ".onion" ];
+      EnforceDistinctSubnets = true;
+      ExitNodes = "{fr}";
+      EntryNodes = "{fr}";
+      NewCircuitPeriod = 120;
+      DNSPort = 9053;
+    };
+  };
+}

modules/users/admins.nix

@@ -0,0 +1,70 @@
+{ config
+, lib
+, ...
+}:
+let
+  extraGroups = [ "wheel" "docker" "plugdev" "vboxusers" "adbusers" "input" ];
+in
+{
+  config = {
+    users.users = {
+      # Ryan Lahfa
+      raito = {
+        isNormalUser = true;
+        home = "/home/raito";
+        inherit extraGroups;
+        shell = "/run/current-system/sw/bin/zsh";
+        uid = 1000;
+        openssh.authorizedKeys.keyFiles = [ ./keys/raito.keys ];
+      };
+
+      # Julien Malka
+      luj = {
+        isNormalUser = true;
+        home = "/home/luj";
+        inherit (config.users.users.raito) extraGroups;
+        shell = "/run/current-system/sw/bin/zsh";
+        uid = 1001;
+        openssh.authorizedKeys.keyFiles = [ ./keys/luj.keys ];
+      };
+
+      # Gabriel Doriath Döhler
+      gdd = {
+        isNormalUser = true;
+        home = "/home/gdd";
+        inherit (config.users.users.raito) extraGroups;
+        shell = "/run/current-system/sw/bin/zsh";
+        uid = 1002;
+        openssh.authorizedKeys.keyFiles = [ ./keys/gdd.keys ];
+      };
+
+      # Samy Lahfa
+      akechi = {
+        isNormalUser = true;
+        home = "/home/akechi";
+        inherit (config.users.users.raito) extraGroups;
+        shell = "/run/current-system/sw/bin/zsh";
+        uid = 1003;
+        openssh.authorizedKeys.keyFiles = [ ./keys/akechi.keys ];
+      };
+
+      # Tom Hubrecht
+      tomate = {
+        isNormalUser = true;
+        home = "/home/tomate";
+        inherit (config.users.users.raito) extraGroups;
+        shell = "/run/current-system/sw/bin/zsh";
+        uid = 1004;
+        openssh.authorizedKeys.keyFiles = [ ./keys/tomate.keys ];
+      };
+
+      root = {
+        hashedPassword = "$y$j9T$LiCWsEVrg9FlcEwuDGsol.$ghfkPkQGoAt23hI6.vWNLrSdHDnVwxg8EE/2w2pRbT6";
+        # passwordFile = lib.mkIf config.users.withSops config.sops.secrets.root-password-hash.path;
+        openssh.authorizedKeys.keyFiles = lib.concatMap (user: config.users.users.${user}.openssh.authorizedKeys.keyFiles) [ "raito" "luj" "gdd" "akechi" "tomate" ];
+      };
+    };
+
+    nix.settings.trusted-users = [ "raito" "luj" "gdd" "akechi" "tomate" ];
+  };
+}

modules/users/keys/akechi.keys

@@ -0,0 +1,2 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDmC/lBooMyJZnyuU0/f6Qufi48DCEQ1RNT1l25cbvZWCXRrAJuQUTgb7u2nl8SEiQTRbZy/FTesaoPNBRWtWXK/8M0K58DeTMdNeTV8joW+vwEzRF3LPb9gU3FqCuuFFRqhJNC16cFBxXMCyjmktl8EG9o5yVcT/6HdwvXRKlas+EqXHlZblXxYQfCjWkpd3vzIoEfPwkUpQFlvLltTBT+EUWTiysvF8cCCuBx7EXHX++fNNySDoB9bQJ+sYoAvuhtfZ5eYNePpYQCV71KMEPYBqJ7AQpvdbMa34Rx7mghGCLk580qWTuXf/vqKRD4EAsvPYtK/xJNyjrCTX99TL60NIsBgNzQvvFFIaAsD91nD13gaEvybOjIo1lbzSZhbreX1qdWeUCR32HDUH4fEVv81VSaPbH2zxWqIf8bLDMJ7pCLKncyCZD1yVFCXsDV/J8BQq0c3GtFh3eEt59bkk6iU+LXv9dvwJtCvsp087Yh4qhj0L9z2zgczeljKJtFFTZhWbY3WEKiureoI7iPEJvLB8B6pGF0qQmIjbIIx6gtBoJc891w+XUmaXfHhWsVV7eq2y/AU7BvXAZp3LigWvosmPpoYsgjS0RNp2RJo2ufN6wKBtq+qv0xM+S6JvXo5pf2Oe11qp8hxF32MNv+djFK3Qf/JMkZAQd1Y/vW/Gg8Xw==
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK5vbxUd8I+uF/OY/PpPhSzrLN14Waq82uyQXNPYpHjA

modules/users/keys/gdd.keys

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICE7TN5NQKGojNGIeTFiHjLHTDQGT8i05JFqX/zLW2zc

modules/users/keys/luj.keys

@@ -0,0 +1,11 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM9Uzb7szWlux7HuxLZej9cBR5MhLz/vaAPPfSoozt2k
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDCKfPoMNrnyNWH6J1OvQ+n1rvSS9Sc2iZf6E1JQC+L4
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIESMWr29i3rhj32oLV3DKe57YI+jvNaKjZhhpq6dEjsn
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJOCKgHRHAJDSgKqYNfWboL04mnEOM0m0K3TGxBhBNDR
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOpGHx430EpJmbtJc8+lF1CpQ1gXeHT9OeZ08O8yzohF
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEaCGndojnmS5IoqHVMEPRfKuBZotMyqo7wNkAZJWigp
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILxfFq8wx5Bet5Q0gI28/lc9ryYYFQelpZdPPdzxGBbA
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGa+7n7kNzb86pTqaMn554KiPrkHRGeTJ0asY1NjSbpr
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILKIDLmQQ+P+jE4zVRpdVp8fmYEe4nzPDqYZt6A4eyIi
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAkj2xsN7Qt/Ew2QO+HiF2yOjXPRucZ3SbIdPDLJoh22
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCUt5I3IgONzYsMOFnRXtvR/uLXlIs6oWsCmh6YGgnpGD4M9lFdoYAOeC1faQUnP66sNs6AoacrGlPZ1UkVUqYEoIr2hiNCDRzzLCQ2J/sSaw7Hv0PKT7MWMo8R076M3TrdunCchBJI1noez3waM9aL4b/iYVhxym28ET55QrWjyMQfZL9PXzOKZatNVcK8AmdtSbI+pFrm/tTZPa321drm9PHOo9CL+lG4YmVZcXa0bVfVtk1GXlWwNpCj2ExLmbF1rRpAa05khfnbg3sBSklwf5NRXj11KneodKRF81ji7MtBhIIfoEXSYht7yspdkkS9e9mv16VGV+2ziM8zG3MK/iUq7fg5ksN54D3DNrd9iI5WjQZsLUrK0ypxO2NtvupWGYt3rCyKA/QvynbxOWFp6cy3Evej142hsfbiOcPIgCtGdHIBevp+KmPxkHBqsJPBqb3Y7nOMT1/ggDMtvHZEZJjEI2D2RjZNEXGbq63OPAqEkgmecW0cXlrjLEGhF2E=

modules/users/keys/raito.keys

@@ -0,0 +1,3 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0xMwWedkKosax9+7D2OlnMxFL/eV4CvFZLsbLptpXr
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU

modules/users/keys/tomate.keys

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL+EZXYziiaynJX99EW8KesnmRTZMof3BoIs3mdEl8L3

modules/zsh.nix

@@ -0,0 +1,8 @@
+{ pkgs, ... }: {
+  programs.zsh.enable = true;
+  programs.zsh.enableCompletion = true;
+  programs.zsh.enableGlobalCompInit = false;
+  programs.zsh.interactiveShellInit = ''
+    source ${pkgs.zsh-nix-shell}/share/zsh-nix-shell/nix-shell.plugin.zsh
+  '';
+}