We put Ceph on Nitro. It’s faster, but how much faster?

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

import * as fs from "fs";
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";

///////////////////////////
// Setup & Configuration //
///////////////////////////

const config = new pulumi.Config();

const environment = process.env.ENVIRONMENT;
if (!environment) { throw new Error("ENV variable [ENVIRONMENT] missing"); }

const sshKeyAbsolutePath = process.env.SSH_PUB_KEY_ABS_PATH;
if (!sshKeyAbsolutePath) { throw new Error("ENV variable [SSH_PUB_KEY_ABS_PATH] missing"); }

const ec2InstanceType = config.require("ec2-instance-type");
const ec2Node0AZ = config.require("ec2-node0-az");
const ec2SSHKeyName = config.require("ec2-ssh-key-name");

// ... snip ...

const adminSSHKey = new aws.ec2.KeyPair(
  "admin-ssh-key",
  {
    keyName: ec2SSHKeyName,
    publicKey: fs.readFileSync(sshKeyAbsolutePath).toString() },
);

export const adminSSHKeyKeyName = adminSSHKey.keyName;

const node0 = new aws.ec2.Instance(
  "node-0",
  {
    ami: ami.then(ami => ami.id),
    instanceType: ec2InstanceType,
    availabilityZone: ec2Node0AZ,
    keyName: adminSSHKeyKeyName,
    tags: {
      NodeId: "0",
      Environment: environment,
    },
  },
);

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88

k0sctl.yaml
---
apiVersion: k0sctl.k0sproject.io/v1beta1
kind: Cluster
metadata:
  name: k0s-cluster
spec:
  k0s:
    version: 1.23.5+k0s.0
    config:
      spec:
        api:
          externalAddress: ${CTRL_0_PUBLIC_IP}
          address: ${CTRL_0_PRIVATE_IP}
          sans:
            - ${CTRL_0_PUBLIC_IP}
          port: 6443
          k0sApiPort: 9443
          extraArgs:
            # see: https://github.com/kubernetes/kubernetes/issues/74302
            http2-max-streams-per-connection: "1000"

        storage:
          type: etcd

        network:
          podCIDR: 10.244.0.0/16
          serviceCIDR: 10.96.0.0/12
          provider: calico
          calico:
            mode: vxlan
            vxlanPort: 4789
            vxlanVNI: 4096
            mtu: 1450
            wireguard: true
            flexVolumeDriverPath: /usr/libexec/k0s/kubelet-plugins/volume/exec/nodeagent~uds
            withWindowsNodes: false
            overlay: Always

        podSecurityPolicy:
          defaultPolicy: 00-k0s-privileged

        installConfig:
          users:
            etcdUser: etcd
            kineUser: kube-apiserver
            konnectivityUser: konnectivity-server
            kubeAPIserverUser: kube-apiserver
            kubeSchedulerUser: kube-scheduler

        konnectivity:
          agentPort: 8132
          adminPort: 8133

        images:
          default_pull_policy: IfNotPresent

  hosts:
    ##############
    # Controller #
    ##############
    - role: controller
      ssh:
        address: ${CTRL_0_PUBLIC_IP} # envsubst
        user: ubuntu
        port: 22
        keyPath: ~/.ssh/id_rsa

    ################
    # Worker Nodes #
    ################
    - role: worker
      ssh:
        address: ${WORKER_0_PUBLIC_IP} # envsubst
        user: ubuntu
        keyPath: ~/.ssh/id_rsa

    - role: worker
      ssh:
        address: ${WORKER_1_PUBLIC_IP} # envsubst
        user: ubuntu
        keyPath: ~/.ssh/id_rsa

    - role: worker
      ssh:
        address: ${WORKER_2_PUBLIC_IP} # envsubst
        user: ubuntu
        keyPath: ~/.ssh/id_rsa

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

K0SCTL ?= k0sctl
ENVSUBST ?= envsubst

K0SCTL_YAML_PATH ?= path/to/your/k0sctl.yaml

## Generate the k0sctl YAML file
generate: generated-folder
    @echo -e "=> Generating k0sctl.yaml based on template @ [$(K0SCTL_YAML_TEMPLATE_PATH)]"
    @export CTRL_0_PUBLIC_IP=$(CTRL_0_PUBLIC_IP) \
        && export CTRL_0_PRIVATE_IP=$(CTRL_0_PRIVATE_IP) \
        && export WORKER_0_PUBLIC_IP=$(WORKER_0_PUBLIC_IP) \
        && export WORKER_1_PUBLIC_IP=$(WORKER_1_PUBLIC_IP) \
        && export WORKER_2_PUBLIC_IP=$(WORKER_2_PUBLIC_IP) \
        && cat $(K0SCTL_YAML_TEMPLATE_PATH) | $(ENVSUBST) > $(K0SCTL_YAML_PATH)

## Install k0s
deploy-k8s: generate
  @echo -e "=> Running k0sctl..."
  $(K0SCTL) -c $(K0SCTL_YAML_PATH)

1
2
3
4
5
6
7

rook:
    @echo "=> Installing CRDs for rook..."
    $(KUBECTL) apply -f crds.yaml
    @echo "=> Installing common resources for rook..."
    $(KUBECTL) apply -f common.yaml
    @echo "=> Installing rook operator..."
    $(KUBECTL) apply -f operator.yaml