1  resource "kubernetes_namespace" "longhorn" {
  2    metadata {
  3      name = "longhorn-system"
  4      labels = {
  5        "pod-security.kubernetes.io/enforce"         = "privileged"
  6        "pod-security.kubernetes.io/enforce-version" = "latest"
  7        "pod-security.kubernetes.io/audit"           = "privileged"
  8        "pod-security.kubernetes.io/audit-version"   = "latest"
  9        "pod-security.kubernetes.io/warn"            = "privileged"
 10        "pod-security.kubernetes.io/warn-version"    = "latest"
 11      }
 12    }
 13  }
 14  
 15  resource "helm_release" "longhorn" {
 16    name       = "longhorn"
 17    namespace  = kubernetes_namespace.longhorn.metadata[0].name
 18    repository = "https://charts.longhorn.io"
 19    chart      = "longhorn"
 20    version    = "1.6.0"
 21  
 22    values = [<<YAML
 23      defaultSettings:
 24        defaultDataLocality: best-effort
 25        replicaAutoBalance: best-effort
 26        createDefaultDiskLabeledNodes: false
 27  
 28        backupTarget: s3://${b2_bucket.backup_target.bucket_name}@us-east-1/
 29        backupTargetCredentialSecret: ${kubernetes_secret.longhorn_backup_auth.metadata[0].name}
 30    
 31      networkPolicies:
 32        enabled: true
 33      persistence:
 34        defaultClass: false
 35      metrics:
 36        serviceMonitor:
 37          enabled: true
 38    YAML
 39    ]
 40  
 41    depends_on = [helm_release.cilium, kubectl_manifest.coreos_crds]
 42  }
 43  
 44  # Nodes
 45  resource "kubectl_manifest" "longhorn_node" {
 46    count      = var.node_count
 47    yaml_body  = <<YAML
 48      apiVersion: longhorn.io/v1beta2
 49      kind: Node
 50      metadata:
 51        name: nereid-${count.index + 1}
 52        namespace: longhorn-system
 53      spec:
 54        name: nereid-${count.index + 1}
 55        allowScheduling: true
 56        evictionRequested: false
 57        instanceManagerCPURequest: 0
 58        disks:
 59          ssd-1:
 60            allowScheduling: true
 61            diskType: filesystem
 62            evictionRequested: false
 63            path: /var/lib/longhorn
 64            storageReserved: 21474836480
 65            tags:
 66            - ssd
 67          ssd-2:
 68            allowScheduling: true
 69            diskType: filesystem
 70            evictionRequested: false
 71            path: /var/mnt/ssd-2
 72            storageReserved: 21474836480
 73            tags:
 74            - ssd
 75          hdd-1:
 76            allowScheduling: true
 77            diskType: filesystem
 78            evictionRequested: false
 79            path: /var/mnt/hdd
 80            storageReserved: 21474836480
 81            tags:
 82            - hdd
 83    YAML
 84    depends_on = [helm_release.longhorn]
 85  }
 86  
 87  
 88  # Backup target
 89  resource "b2_bucket" "backup_target" {
 90    bucket_name = "super-cluster-backup-longhorn"
 91    bucket_type = "allPrivate"
 92  }
 93  
 94  resource "b2_application_key" "backup_target_key" {
 95    key_name     = "super-cluster-backup-longhorn"
 96    bucket_id    = b2_bucket.backup_target.id
 97    capabilities = ["readBuckets", "listBuckets", "listFiles", "readFiles", "writeFiles", "deleteFiles"]
 98  }
 99  
100  # Storage classes
101  resource "kubernetes_secret" "longhorn_backup_auth" {
102    metadata {
103      name      = "longhorn-backup-auth"
104      namespace = kubernetes_namespace.longhorn.metadata[0].name
105    }
106  
107    data = {
108      AWS_ACCESS_KEY_ID     = b2_application_key.backup_target_key.application_key_id
109      AWS_SECRET_ACCESS_KEY = b2_application_key.backup_target_key.application_key
110      AWS_ENDPOINTS         = "https://s3.us-west-004.backblazeb2.com"
111    }
112  }
113  
114  resource "kubernetes_storage_class" "longhorn_ssd" {
115    metadata {
116      name = "ssd"
117      annotations = {
118        "storageclass.kubernetes.io/is-default-class" = "true"
119      }
120    }
121    storage_provisioner = "driver.longhorn.io"
122    parameters = {
123      numberOfReplicas     = "2"
124      staleReplicaTimeout  = "2880"
125      diskSelector         = "ssd"
126      recurringJobSelector = "[{ \"name\": \"ssd-backup\", \"isGroup\": true }]"
127    }
128  }
129  
130  resource "kubernetes_storage_class" "longhorn_hdd" {
131    metadata {
132      name = "hdd"
133    }
134    storage_provisioner = "driver.longhorn.io"
135    parameters = {
136      numberOfReplicas     = "2"
137      staleReplicaTimeout  = "2880"
138      diskSelector         = "hdd"
139      recurringJobSelector = "[{ \"name\": \"hdd-backup\", \"isGroup\": true }]"
140    }
141  }
142  
143  resource "kubernetes_storage_class" "longhorn_hdd_unsafe" {
144    metadata {
145      name = "hdd-unsafe"
146    }
147    storage_provisioner = "driver.longhorn.io"
148    parameters = {
149      numberOfReplicas    = "1"
150      staleReplicaTimeout = "2880"
151      diskSelector        = "hdd"
152    }
153  }
154  
155  # Backups
156  resource "kubectl_manifest" "longhorn_backup_ssd" {
157    yaml_body = <<YAML
158      apiVersion: longhorn.io/v1beta1
159      kind: RecurringJob
160      metadata:
161        name: ssd-backup
162        namespace: ${kubernetes_namespace.longhorn.metadata[0].name}
163      spec:
164        cron: "0 0 */2 * *"
165        task: "backup"
166        groups:
167          - ssd-backup
168        retain: 3
169        concurrency: 5
170    YAML
171  
172    depends_on = [helm_release.longhorn]
173  }
174  
175  resource "kubectl_manifest" "longhorn_backup_hdd" {
176    yaml_body  = <<YAML
177      apiVersion: longhorn.io/v1beta1
178      kind: RecurringJob
179      metadata:
180        name: hdd-backup
181        namespace: ${kubernetes_namespace.longhorn.metadata[0].name}
182      spec:
183        cron: "0 0 */2 * *"
184        task: "backup"
185        groups:
186          - hdd-backup
187        retain: 3
188        concurrency: 5
189    YAML
190    depends_on = [helm_release.longhorn]
191  }
192  
193  # Periodic trim
194  resource "kubectl_manifest" "longhorn_periodic_trim" {
195    yaml_body  = <<YAML
196      apiVersion: longhorn.io/v1beta1
197      kind: RecurringJob
198      metadata:
199        name: trim
200        namespace: ${kubernetes_namespace.longhorn.metadata[0].name}
201      spec:
202        cron: "0 0 * * *"
203        task: trim-filesystem
204        groups:
205          - default
206    YAML
207    depends_on = [helm_release.longhorn]
208  }