diff --git a/.gitignore b/.gitignore index 7a709d6..4aed93c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ +.DS_Store +.direnv .hypothesis __pycache__ diff --git a/README.md b/README.md index 958571a..c29cb1b 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,135 @@ # esgi-devops -Contenus & ressources pour le cours de 4A de DevOps \ No newline at end of file +Contenus & ressources pour le cours de 4A de DevOps + +# Plan + +- 1 Introduction (c'est quoi le DevOps ? TP -> installation d'une VM NixOS) + + rappels de base sur Linux / BONUS: présentation de nix-shell, nixos-rebuild + +- 2 Tests (unitaires & integrations, TP -> utilisation de pytest, hypothesis) + + utilisation de nix-env et direnv (change vos variables d'env de votre shell en fonction du dossier dans lequel vous vous trouvez) + => utilise Nix pour installer l'ensemble des dépendances de dev requise !! (pratique pour les illustrations / TPs) + +- 3 Deploiements (TP -> déployer un site static dans une machine NixOS / Debian 9 avec Fabric) + + BONUS: Terraform sur t2.macro AWS (gratuit), NixOps + +... en cas de pépin ~> ngrock !! + + +- 4 Containers (Docker & autres) +- 5 Haute Disponibilité (autoscaling) + +--- + +# 🕸️ DevOps with NixOS + +Contenus & ressources pour le cours de 4A de DevOps + +## ProTip Nix ™ + +[Search options](https://nixos.org/nixos/options.html#) (chercher des options de module Nix à utiliser prêt à l'emploi, le système expert ™) + +[Search code](https://search.nix.gsc.io/) (lire du code qui fait un truc, genre "matomo" et voir comment c'est impléménté; lire la doc de la lib standard) + +[Package search](https://nixos.org/nixos/packages.html) (chercher des pkg à installer) + +[NixOS Infra Status](https://status.nixos.org/) (pour voir s'il y a des trucs kassé) + +["How up to date are NixOS channels?"](https://howoldis.herokuapp.com/) (pour voir à quel point master de nixpkgs lag derrière) + +[Useful Nix Hacks](http://chriswarbo.net/projects/nixos/useful_hacks.html) (super useful pour le coup) + +[Nix in 1 page](https://github.com/tazjin/nix-1p) (cheatsheet du langage) + +[A tour of Nix](https://nixcloud.io/tour/) (jeu emscripten du langage où tu as des exercices de trucs à faire, très rapide à faire) + +[Pull Requests · NixOS/rfcs](https://github.com/NixOS/rfcs/pulls) (parce que c'est important de savoir comment ça évolue) + +## Vidéo + +[cirocosta/asciinema-edit](https://github.com/cirocosta/asciinema-edit) + +[untitled](https://asciinema.org/a/jikcJpkl09MoakqkYFTBGfakT) + +[](https://clbin.com/uFXdw) + +[https://www.youtube.com/watch?v=Q4WUCRkM_I8](https://www.youtube.com/watch?v=Q4WUCRkM_I8) + +[https://www.youtube.com/watch?v=ZxD3LMPl5_4](https://www.youtube.com/watch?v=ZxD3LMPl5_4) + +[https://www.youtube.com/watch?v=95Zs95HXFS4](https://www.youtube.com/watch?v=95Zs95HXFS4) + +## Cool env + +[Install Nix](https://nixos.org/guides/install-nix.html) + +[Zsh](https://grml.org/zsh/) + +[Starship: Cross-Shell Prompt](https://starship.rs/guide/#🚀-installation) + +## Network + +[ngrok - download](https://ngrok.com/download) + +## TDD + +[pytest-cov](https://pypi.org/project/pytest-cov/) + +## Deployment + +[](https://clbin.com/wwgTu) + +[Fabric](https://docs.fabfile.org/en/2.5/api/transfer.html?highlight=put#fabric.transfer.Transfer.put) + +[Fabric](http://www.fabfile.org) + +## Reproducible + +[Is NixOS Reproducible?](https://r13y.com) + +[Reproducible Builds](https://wiki.archlinux.org/index.php/Reproducible_Builds) + +[Projects · Reproducible Builds / disorderfs](https://salsa.debian.org/reproducible-builds/disorderfs) + +## Docker + +[Docker](https://nixos.wiki/wiki/Docker) + +### Issues + +[Computational overhead due to Docker under macOS](https://lemire.me/blog/2020/06/19/computational-overhead-due-to-docker-under-macos/) + +[Docker daemon is leaking file descriptors (with reproduction) · Issue #37182 · moby/moby](https://github.com/moby/moby/issues/37182) + +[the container's netns fds leak, causing the container netns to not clean up successfully after the container stops · Issue #41136 · moby/moby](https://github.com/moby/moby/issues/41136) + +[[Windows] mounted volumes behave differently · Issue #41121 · moby/moby](https://github.com/moby/moby/issues/41121) + +## ☹️ + +[CoreOS](https://coreos.com/rkt/docs/latest/) + +[README and Docs need to state that RancherOS is dead · Issue #3000 · rancher/os](https://github.com/rancher/os/issues/3000) + +### Better than Docker + +[Firejail](https://wiki.archlinux.org/index.php/firejail) + +[Lightweight Kubernetes](https://k3s.io) + +[Dokku](http://dokku.viewdocs.io/dokku/) + +[Docker 19.03: Rootless Mode (Experimental) | Hacker News](https://news.ycombinator.com/item?id=20542915) + +[google/nixery](https://github.com/google/nixery) + +## IaC + +[Install Terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli) + +[Chef: Enabling the Coded Enterprise through Infrastructure, Security and Application Automation](https://www.chef.io) + +[Powerful infrastructure automation and delivery | Puppet](https://puppet.com) + +[Ansible is Simple IT Automation](https://www.ansible.com) \ No newline at end of file diff --git a/illustrations/3_terraform-docker-demo/.terraform/plugins/linux_amd64/lock.json b/illustrations/3_terraform-docker-demo/.terraform/plugins/linux_amd64/lock.json new file mode 100755 index 0000000..6f18c56 --- /dev/null +++ b/illustrations/3_terraform-docker-demo/.terraform/plugins/linux_amd64/lock.json @@ -0,0 +1,3 @@ +{ + "docker": "4813d00312683bbce8b765ad0b876c14f43efa46b6eb0e9a2370e906470b0cc6" +} \ No newline at end of file diff --git a/illustrations/3_terraform-docker-demo/.terraform/plugins/linux_amd64/terraform-provider-docker_v2.7.1_x4 b/illustrations/3_terraform-docker-demo/.terraform/plugins/linux_amd64/terraform-provider-docker_v2.7.1_x4 new file mode 100755 index 0000000..6539bc2 Binary files /dev/null and b/illustrations/3_terraform-docker-demo/.terraform/plugins/linux_amd64/terraform-provider-docker_v2.7.1_x4 differ diff --git a/illustrations/3_terraform-docker-demo/main.tf b/illustrations/3_terraform-docker-demo/main.tf new file mode 100644 index 0000000..7dbd2fc --- /dev/null +++ b/illustrations/3_terraform-docker-demo/main.tf @@ -0,0 +1,13 @@ +resource "docker_image" "nginx" { + name = "nginx:latest" + keep_locally = false +} + +resource "docker_container" "nginx" { + image = docker_image.nginx.latest + name = "tutorial" + ports { + internal = 80 + external = 8000 + } +} diff --git a/illustrations/3_terraform-docker-demo/terraform.tfstate b/illustrations/3_terraform-docker-demo/terraform.tfstate new file mode 100644 index 0000000..eeb83c2 --- /dev/null +++ b/illustrations/3_terraform-docker-demo/terraform.tfstate @@ -0,0 +1,8 @@ +{ + "version": 4, + "terraform_version": "0.12.26", + "serial": 6, + "lineage": "e51c7988-e2ed-5428-5618-66f774fa7d46", + "outputs": {}, + "resources": [] +} diff --git a/illustrations/3_terraform-docker-demo/terraform.tfstate.backup b/illustrations/3_terraform-docker-demo/terraform.tfstate.backup new file mode 100644 index 0000000..90bf427 --- /dev/null +++ b/illustrations/3_terraform-docker-demo/terraform.tfstate.backup @@ -0,0 +1,125 @@ +{ + "version": 4, + "terraform_version": "0.12.26", + "serial": 3, + "lineage": "e51c7988-e2ed-5428-5618-66f774fa7d46", + "outputs": {}, + "resources": [ + { + "mode": "managed", + "type": "docker_container", + "name": "nginx", + "provider": "provider.docker", + "instances": [ + { + "schema_version": 2, + "attributes": { + "attach": false, + "bridge": "", + "capabilities": [], + "command": [ + "nginx", + "-g", + "daemon off;" + ], + "container_logs": null, + "cpu_set": "", + "cpu_shares": 0, + "destroy_grace_seconds": null, + "devices": [], + "dns": null, + "dns_opts": null, + "dns_search": null, + "domainname": "", + "entrypoint": [ + "/docker-entrypoint.sh" + ], + "env": [], + "exit_code": null, + "gateway": "172.17.0.1", + "group_add": null, + "healthcheck": [], + "host": [], + "hostname": "5f48f0bc50f0", + "id": "5f48f0bc50f0133b10831f7bb1334ce7e54b4e2ecb37b9ce0be8e866bc60c9cf", + "image": "sha256:2622e6cca7ebbb6e310743abce3fc47335393e79171b9d76ba9d4f446ce7b163", + "ip_address": "172.17.0.2", + "ip_prefix_length": 16, + "ipc_mode": "private", + "labels": [], + "links": null, + "log_driver": "journald", + "log_opts": {}, + "logs": false, + "max_retry_count": 0, + "memory": 0, + "memory_swap": 0, + "mounts": [], + "must_run": true, + "name": "tutorial", + "network_alias": null, + "network_data": [ + { + "gateway": "172.17.0.1", + "ip_address": "172.17.0.2", + "ip_prefix_length": 16, + "network_name": "bridge" + } + ], + "network_mode": "default", + "networks": null, + "networks_advanced": [], + "pid_mode": "", + "ports": [ + { + "external": 8000, + "internal": 80, + "ip": "0.0.0.0", + "protocol": "tcp" + } + ], + "privileged": false, + "publish_all_ports": false, + "read_only": false, + "restart": "no", + "rm": false, + "shm_size": 64, + "start": true, + "sysctls": null, + "tmpfs": null, + "ulimit": [], + "upload": [], + "user": "", + "userns_mode": "", + "volumes": [], + "working_dir": "" + }, + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjIifQ==", + "dependencies": [ + "docker_image.nginx" + ] + } + ] + }, + { + "mode": "managed", + "type": "docker_image", + "name": "nginx", + "provider": "provider.docker", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "sha256:2622e6cca7ebbb6e310743abce3fc47335393e79171b9d76ba9d4f446ce7b163nginx:latest", + "keep_locally": false, + "latest": "sha256:2622e6cca7ebbb6e310743abce3fc47335393e79171b9d76ba9d4f446ce7b163", + "name": "nginx:latest", + "pull_trigger": null, + "pull_triggers": null + }, + "private": "bnVsbA==" + } + ] + } + ] +} diff --git a/illustrations/old_project/.envrc b/illustrations/old_project/.envrc new file mode 100644 index 0000000..2d28b7e --- /dev/null +++ b/illustrations/old_project/.envrc @@ -0,0 +1 @@ +export CI=true diff --git a/slides/.envrc b/slides/.envrc new file mode 100644 index 0000000..65326bb --- /dev/null +++ b/slides/.envrc @@ -0,0 +1 @@ +use nix \ No newline at end of file diff --git a/slides/0_nixcon.md b/slides/0_nixcon.md new file mode 100644 index 0000000..3feb4e5 --- /dev/null +++ b/slides/0_nixcon.md @@ -0,0 +1,110 @@ +--- +title: Host a DevOps exam using NixOS +subtitle: \footnotesize Deploy a Bastion host and a KVM virtual machine per student in order for them to deploy a website which will be auto-validated. +author: Yvan SRAKA, Ryan LAHFA +theme: default +--- + +## Host a DevOps exam using NixOS + +- DevOps students have to test their skills in a real-world exam, what’s better than scripting it using NixOps in order to reuse it? + +- They will be provided with their own user account on the Bastion and access to a jump user account, WireGuard will be auto-configured on the host to enable access to their KVM host directly. + +- The main point is controlling the KVM guests “declaratively” and giving choice to the students to choose their guest OS: Debian or NixOS. + +- We will see how we can easily generate Nix expression from a scripting language, e.g. Python, and feed it to a NixOps deployment. + +## Disclaimer: _all this is incomplete and experimental!_ + +This was tried during a real exam this summer, but it's very alpha-quality and had to get a lot of last-minute fixes due to external issues (e.g. disks dying…) + +## Git repository + +The full code can be found here\footnote{https://git.newtype.fr/yvan/devops-exam-model}, it's still a bit messy... + +`\nix` folder contains files on which will focus this talk: + +- \color{red}`student-setup.nix` \color{black} is generated by \color{red} `generate-setup.py` \color{black} (that rely on \color{red} `nix-expr.py`\color{black}) from students data. +- \color{red}`deploy.sh` \color{black} push on our setup \color{red} `exam.nix`\color{black}, \color{red} `student.nix`\color{black}, \color{red} `kvm-guests.nix` \color{black} and \color{red} `student-setup.nix` \color{black} + +## `deploy.sh` + +```bash +##!/usr/bin/env bash + +echo "[+] Regenerating of the setup" +python3 generate_setup.py + +echo "[+] Sending Nix files" +rsync --inplace --temp-dir=/tmp -avPz *.nix \ + yvan@bastion:/etc/nixos/ + +echo "[+] Rebuilding of the exam machine" +ssh -t yvan@bastion "sudo nixos-rebuild switch" +``` + +## `generate_setup.py` _(subset of)_ + +```python +def students(csv_filename): + with open(csv_filename, 'r', newline='') as csvfile: + ereader = csv.reader(csvfile, delimiter=';') + next(ereader) # exhaust header. + for index, student in enumerate(ereader): + name, surname, email, username = student + wg_pubkey, wg_privkey = wireguard_parameters() + keys = read_keys(username) + yield { + "surname": surname, + "name": name, + "email": email, + "username": username, + "keys": keys, + "wireguardPublicKey": wg_pubkey, + "wireguardPrivateKey": wg_privkey, + } +``` + +## `student-setup.nix` _(generated)_ + +```nix +ltorvalds = { + email = "torvalds@linux-foundation.org"; + guestOperatingSystem = "debian"; + index = 42; + keys = [ + "ssh-ed25519 ... torvalds@linux-foundation.org" + ]; + name = "Linus"; + surname = "Torvalds"; + username = "ltorvalds"; + wireguardPrivateKey = "0ECE2Js+RkxQVTyJ9BvZB0DjpEGnWMy1X5cI8R2RdHA="; + wireguardPublicKey = "3o9Dhmrrql/5PZEhi5kS+Fob1m8rN70SXzDGy48bMR0="; + }; +rstallman = { ... } +``` + +## `student.nix` _(subset of)_ + +```nix +mkGuest = name: student: { + memory = "1G"; + netDevice = "tap${toString student.index}"; + vncDisplay = "localhost:${toString student.index}"; + operatingSystem = student.guestOperatingSystem; +}; + +services.kvmGuests = { + enable = true; + guests = mapAttrs mkGuest cfg.students; +}; + +# Create users for each student + management/jump accounts. +users.users = (mapAttrs mkBastionUser cfg.students) // ({ + jump = mkJumpUser cfg.students; + admin = adminUser; +}); +``` + +# \Huge Q/A \ No newline at end of file diff --git a/slides/0_nixcon.md.handout.pdf b/slides/0_nixcon.md.handout.pdf new file mode 100644 index 0000000..8842fe5 Binary files /dev/null and b/slides/0_nixcon.md.handout.pdf differ diff --git a/slides/0_nixcon.md.slides.pdf b/slides/0_nixcon.md.slides.pdf new file mode 100644 index 0000000..19d1659 Binary files /dev/null and b/slides/0_nixcon.md.slides.pdf differ diff --git a/slides/Makefile b/slides/Makefile index d607e92..c3bbe7e 100644 --- a/slides/Makefile +++ b/slides/Makefile @@ -4,15 +4,14 @@ HANDOUTS := $(patsubst %.md,%.md.handout.pdf,$(wildcard *.md)) all : $(SLIDES) $(HANDOUTS) %.md.slides.pdf : %.md - pandoc $^ -t beamer --slide-level 2 -o $@ + pandoc $^ --pdf-engine=xelatex -t beamer --slide-level 2 -o $@ %.md.handout.pdf : %.md - pandoc $^ -t beamer --slide-level 2 -V handout -o $@ + pandoc $^ --pdf-engine=xelatex -t beamer --slide-level 2 -V handout -o $@ pdfjam $@ --nup 1x2 --no-landscape --keepinfo \ --paper letterpaper --frame true --scale 0.9 \ --suffix "nup" mv $*.md.handout-nup.pdf $@ - clobber : rm -f $(SLIDES) diff --git a/slides/shell.nix b/slides/shell.nix new file mode 100644 index 0000000..4f297ba --- /dev/null +++ b/slides/shell.nix @@ -0,0 +1,7 @@ +with (import {}); +mkShell { + buildInputs = [ + pandoc + texlive.combined.scheme-full + ]; +} \ No newline at end of file