From BOSH to Kubernetes

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340

---
# The name of the deployment. Replace the name with the name of the BOSHDeployment resource
# It's used to namespace resources created for this deployment.
# Based on docs [1], names should be less than 253 characters. We should limit this to
# characters in the operator, to make sure that with any suffix, we won't go beyond the limit.
name: "foo"
# Not used by the quarks-operator.
# A warning is printed in the logs if this is present.
director_uuid: "bar"
# A hash of director features. We could use this to control operator features as well.
features:
  # Enable variables to be regenerated by the config server (e.g. CredHub) when the variable options change. Default false.
  # In the quarks-operator, if an QuarksSecret is changed, (e.g. a new domain is added to a cert),
  # the value will be automatically updated.
  # The operator won't be able to control this behavior.
  # A warning is printed in the logs if this is present.
  converge_variables: true
  # Randomizes AZs for left over instances that cannot be distributed equally between AZs.
  # Not currently used. It's likely that we'll be able to support this.
  randomize_az_placement: false
  # Enables or disables returning of DNS addresses in links.
  # In Kubernetes we always use DNS addresses.
  # An error should be returned if this value is set to false.
  use_dns_addresses: true
# A list of all releases used in this deployment.
# Required.
# Each release's image reference is constructed from this information like this:
# <url>/<name>:<stemcell.os>-<stemcell.version>-<version>
releases:
  # Name of a release used in the deployment.
  name: "capi-release"
  # The version of the release to be used.
  # "latest" is not supported by the quarks-operator. An error is thrown if "latest" is used.
  version: "1.0"
  # Required for the operator. Link to the registry and organization containing the image.
  url: "docker.io/cloudfoundry"
  # Not used by the quarks-operator.
  # Integrity of the image itself is handled by whatever
  # container runtime and the image registry.
  sha1: "332ac15609b220a3fdf5efad0e0aa069d8235788"
  # Required by the operator
  stemcell:
    # OS of the stemcell used by the release. Used to construct the image name.
    os: "opensuse"
    # Version of the OS of the stemcell used by the release.
    version: "42.3"
  # Only used by the quarks-operator.
  # A secret is created with the credentials [2], used by the pods
  # that reference this release.
  credentials:
    username: "foo"
    password: "secret"
# Not used by the quarks-operator.
# A warning is logged if this is set
stemcells: []
# Specifies how updates are handled
# The quarks-operator uses some of these settings.
update:
  # The number of pods to deploy in the new version of an QuarksStatefulSet
  # Once canaries are running, deployment can continue.
  canaries: 2
  # Time to wait for canary pods to be ready in a new version of an QuarksStatefulSet
  canary_watch_time: 100
  # The maximum number of non-canary instances to update in parallel for an QuarksStatefulSet.
  # TODO: Support for this needs to be implemented in the controller.
  max_in_flight: 2
  # This key must be set in the manifest for the canary deployment to work.
  update_watch_time: 0
  # Not used in quarks-operator.
  # All instance groups are deployed at the same time.
  # If set to true, a warning is logged.
  serial: false
  # Not used in quarks-operator.
  # If set, a warning is logged.
  vm_strategy: ""
# Each instance group is converted into an QuarksStatefulSet
instance_groups:
  # Used to name the QuarksStatefulSet or QuarksJob
- name: "api-az1"
  # Support for AZs is implemented in the QuarksStatefulSet
  azs: ["az1"]
  # Number of replicas for the StatefulSets in an QuarksStatefulSet
  # If this instance group defines an QuarksJob, this value must be 1. An error is thrown otherwise
  instances: 3
  # Each job results in a rendered bpm.yml file.
  # BPM information is required - the deployment fails if it's missing.
  # Each job has one or more processes (defined in bpm.yml), and each   corresponds to a container of a pod in a StatefulSet or Job
  jobs:
    # It's used to name the container
  - name: "cloud_controller_ng"
    # The name of a release that must exist in the releases block.
    # If it doesn't exist in the releases block, an error is thrown.
    # The docker image used for the container is resolved using this release name.
    release: "capi-release"
    # Used by the quarks-operator to calculate links before rendering templates.
    # All resources in the quarks-operator are deterministic (IP addresses are not used),
    # So they can be calculated before template rendering occurs.
    consumes: {}
    # Same as the consumes block above.
    provides: {}
    # Defines all properties, used to render job templates.
    # Job templates are rendered as Secrets, and then mounted into pod containers.
    # If a property is changed, the operator runs rendering in an QuarksJob, and the
    # template's secret is (re)generated.
    # All properties are input to this QuarksJob that does rendering.
    # Some properties can reference variables, which can be generated. The quarks-operator
    # collects values for all properties before starting the rendering process.
    properties:
      domain: "mycf.com"
      admin_password: "((adminpass))"
      # Extra information specific to the quarks-operator
      quarks:
        run:
          # Hints for pod replica count
          scaling:
            min: 1
            max: 3
            ha: 2
          # Extra capabilities required by the containers of this job
          capabilities: []
          # Memory used by each container. Overrides info from vm_resources.
          memory: 128
          # Number of vCPUs used by each container. Overrides info from vm_resources.
          virtual-cpus: 2
          # Healthcheck information for the containers in this job.
          healthcheck:
            some_process_name:
              readiness:
                exec:
                  command:
                  - "curl --silent --fail --head http://${HOSTNAME}:8080/health"
        # List of ports to be opened up for this job.
        ports:
        - name: "health-port"
          protocol: "TCP"
          internal: 8080
  # Not used by the quarks-operator.
  # A warning is logged if this is set.
  vm_type: ""
  # Not used by the quarks-operator.
  # A warning is logged if this is set.
  vm_extensions: []
  # Used by the quarks-operator to limit the resources used by a container in a pod
  vm_resources:
    # Number of vCPUs used by a container
    cpu: 4
    # Memory used by a container
    ram: 1024
    # Used for PVC sizes if `ephemeralAsPVC` is set to true
    ephemeral_disk_size: 4096
  # Not used by the quarks-operator.
  # A warning is logged if this is set.
  stemcell: ""
  # Size of the volume attached to a pod container.
  persistent_disk: 4096
  # This must be the name of a StorageClass used by the quarks-operator to create volumes.
  persistent_disk_type: "default"
  # Not used by the quarks-operator.
  # A warning is logged if this key is set.
  networks:
    # Not used by the quarks-operator
    - name: "foo"
      # Not used by the quarks-operator
      static_ips: []
      # Not used by the quarks-operator
      default: []
  # Specific update settings for this instance group. Use this to override global job update settings on a per-instance-group basis.
  update: {}
  # TODO: understand how instance group renames can occur in an QuarksStatefulSet or QuarksJob
  migrated_from:
  - cloud_controller
  # This is the key that controls how an instance group is treated by the quarks-operator.
  # If lifecycle is "service", an QuarksStatefulSet is created for the instance group.
  # Otherwise, if it's "errand", an QuarksJob is created. As with normal BOSH, errands have a
  # manual trigger, so QuarksJobs have to support this (manual triggers).
  # In Kubernetes we also need errands that can run on a trigger. These are not supported by BOSH.
  # The lifecycle for such an QuarksJob is "auto-errand".
  # Manual triggers are supported by QuarksJobs
  lifecycle: "service"
  # Deprecated - the quarks-operator does not support this key.
  # An error is thrown if this is set.
  properties: {}
  # Usually used for BOSH Agent configuration.
  # We can use this hash to control how the operator generates resources, however
  # none of the settings used by the Agent are supported by the operator.
  env:
    # Not used by the quarks-operator.
    # A warning is logged if this is set.
    persistent_disk_fs: "ext4"
    # Not used by the quarks-operator.
    # A warning is logged if this is set.
    persistent_disk_mount_options: []
    # Not used by the quarks-operator.
    # A warning is logged if this is set.
    bosh [Hash, optional]:
      # Not used by the quarks-operator.
      # A warning is logged if this is set.
      password: "foo"
      # Not used by the quarks-operator.
      # A warning is logged if this is set.
      keep_root_password: vcap
      # Not used by the quarks-operator.
      # A warning is logged if this is set.
      remove_dev_tools: false
      # Not used by the quarks-operator.
      # A warning is logged if this is set.
      remove_static_libraries: false.
      # Not used by the quarks-operator.
      # A warning is logged if this is set.
      swap_size: 100
      # Not used by the quarks-operator.
      # A warning is logged if this is set.
      ipv6:
        # Not used by the quarks-operator.
        # A warning is logged if this is set.
        enable: false
      # Not used by the quarks-operator.
      # A warning is logged if this is set.
      job_dir:
        # Not used by the quarks-operator.
        # A warning is logged if this is set.
        tmpfs: false
        # Not used by the quarks-operator.
        # A warning is logged if this is set.
        tmpfs_size: "0m"
      agent:
        # Not used by the quarks-operator.
        # A warning is logged if this is set.
        tmpfs: false
        # Used by the quarks-operator to set kubernetes-specific information
        # for the resources representing this instance group.
        settings:
          # Affinity information for this instance group's pod.
          # These definitions are merged directly into the pod's definition.
          # The structure is the same as the one used by Kube [3].
          affinity: {}
          # Labels to add to the resources representing the instance group
          labels: {}
          # Annotations to add to the resources representing the instance group
          annotations: {}
          # Ops files that are applied on top of instance group properties yaml or BPM data yaml
          preRenderOps:
            bpm:
            - type: replace
              path: /foo
              value: bar
            instanceGroup:
            - type: replace
              path: /foo
              value: bar
          # disable_log_sidecar is an option to disable log sidecar
          disable_log_sidecar: false
          # serviceAccountName is the name of the ServiceAccount to use to run this pod.
          serviceAccountName: kubecf
          # automountServiceAccountToken indicates whether a service account token should be automatically mounted
          automountServiceAccountToken: false
          # ImagePullSecrets is an optional list of references to secrets to use for pulling any of the images.
          # This field in PodSpec can be automated by setting the imagePullSecrets in a serviceAccount.
          imagePullSecrets: {}
          # Tolerations and taints are a concept defined in kubernetes to repel pods from nodes. [4]
          tolerations: []
          # If this is set to true, the operator will define a PersistentVolumeClaim template
          # on the QuarksStatefulSet of the instance group, and it will use that PVC for all volume
          # mounts for ephemeral disks
          ephemeralAsPVC: false
          # This sets the backoffLimit for the jobs running errands. If not set, it will use the Kube default which is 6.
          # https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/#handling-pod-and-container-failures
          jobBackoffLimit: 6
          # An array of disks to be mounted on the containers
          disks:
            # A PersistentVolumeClaim to be used as a template in the StatefulSet of the instance group.
          - pvc:
              name: foo
              storageClassName: persistent
            # Volume definition to be included in the pod.
            volume:
              name: extravolume
              emptyDir: {}
            # Volume mounts to be set on the containers that match the job and process set in "filters".
            volumeMount:
              name: extravolume
              mountPath: /var/vcap/data/rep
            # Filters to identify on which containers to apply the volume mounts.
            filters:
              job_name: "cflinuxfs3-rootfs-setup"
              process_name: "test-server"
# Each addon job is added to the desired manifest before it's persisted
# Not all placement rules are supported, see below for more details.
addons:
  # The name of the addon is not used by the operator.
  # TODO: investigate whether it's useful to  set this in an annotation of the instance group sts/pod
- name: foo
  # All jobs are added to instance groups based on placement rules before the desired manifest is persisted
  jobs:
  - name: metron
    release: loggregator-release
    properties:
      loggregator:
        metron:
          log_level: debug
  include:
    # Supported
    stemcell:
    - os: opensuse
    # Not supported, addons are used per-deployment
    deployments: []
    # Supported
    jobs:
      name: cloud_controller_ng
      release: capi-release
    # Supported
    instance_groups:
    - api
    - diego-cell
    # Not supported
    networks: []
    # Not supported
    teams: []
  # The same matchers are supported as the "include" key
  exclude: {}
# Deprecated - the quarks-operator does not support this key.
# An error is thrown if this is set.
properties: {}
# For each variable, the quarks-operator creates QuarksSecrets
# As with normal BOSH, variables are referenced by job properties.
# Each variable's generated secret is mounted in the container that renders each job's
# templates. They are then used by the rendering process.
# This means that the operator needs to look at the job's properties, and parse any references
# to variables, so it knows what it needs to mount.
variables:
    # Unique name used to identify a variable. Used to name the QuarksSecret
  - name: "adminPass"
    # As with normal BOSH, supported types are certificate, password, rsa, and ssh.
    type: "password"
    # Specifies generation options
    options: {is_ca: true, common_name: "some-ca"}
# Tags are transformed into annotations for the resources created
# by this deployment.
tags:
  maintainer: "Philip J. Fry"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

stemcells:
- alias: default
  os: opensuse-42.3
  version: 28.g837c5b3-30.263-7.0.0_234.gcd7d1132
instance_groups:
- name: diego-cell
  stemcell: default
  jobs:
  - name: cflinuxfs3-rootfs-setup
    release: cflinuxfs3
releases:
- name: cflinuxfs3
  version: 0.62.0
  url: hub.docker.com/cfcontainerization
  sha1: 6466c44827c3493645ca34b084e7c21de23272b4
  stemcell:
    os: opensuse-15.0
    version: 28.g837c5b3-30.263-7.0.0_233.gde0accd0

1

var-<VARIABLE_NAME>

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101

---
apiVersion: v1
kind: Secret
metadata:
  name: var-system-domain
type: Opaque
stringData:
  value: foo.com
---
apiVersion: v1
kind: Secret
metadata:
  name: my-nats-password
type: Opaque
stringData:
  password: deadbeef
---
apiVersion: v1
kind: Secret
metadata:
  name: my-ca
type: Opaque
stringData:
  certificate: "my-ca-cert-data"
  private_key: "my-ca-private-key"
  is_ca: "true"
---
apiVersion: v1
kind: Secret
metadata:
  name: my-cert
type: Opaque
stringData:
  certificate: "my-cert-data"
  private_key: "my-private-key-data"
  is_ca: "false"
  ca: "my-ca-cert-data"
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: nats-manifest
data:
  manifest: |
    ---
    name: nats-deployment
    releases:
    - name: nats
      version: "33"
      url: ghcr.io/cloudfoundry-incubator
      stemcell:
        os: SLE_15_SP1
        version: 27.8-7.0.0_374.gb8e8e6af
    instance_groups:
    - name: nats
      instances: 1
      jobs:
      - name: nats
        release: nats
        properties:
          nats:
            user: admin
            password: ((nats_password))
            domain: ((system_domain))
            ca_cert: ((nats_ca.certificate))
            ca_key: ((nats_ca.private_key))
            cert: ((nats_cert.certificate))
            key: ((nats_cert.private_key))
            cert_ca: ((nats_cert.ca))
    variables:
    - name: nats_password
      type: password
    - name: nats_ca
      type: certificate
      options:
        is_ca: true
        common_name: routerCA
    - name: nats_cert
      type: certificate
      options:
        ca: nats_ca
        common_name: routerSSL
        alternative_names:
        - "foo.((system_domain))"
        - "*.foo.bar"
---
apiVersion: quarks.cloudfoundry.org/v1alpha1
kind: BOSHDeployment
metadata:
  name: nats-deployment
spec:
  manifest:
    name: nats-manifest
    type: configmap
  vars:
  - name: nats_password
    secret: my-nats-password
  - name: nats_ca
    secret: my-ca
  - name: nats_cert
    secret: my-cert

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

instance_groups:
- name: "api-az1"
  process.
    properties:
      quarks:
        run:
          healthcheck:
            bpm-process-name:
              readiness:
              liveness:

1

var-<variable-name>

1
2
3
4
5
6
7
8

---
apiVersion: v1
kind: Secret
metadata:
  name: var-system-domain
type: Opaque
stringData:
  value: example.com

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

---
apiVersion: v1
kind: Secret
metadata:
  name: var-ssl
type: Opaque
stringData:
  ca: ...
  cert: ...
  key: ...

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

instance_groups:
- name: redis-slave
  instances: 2
  lifecycle: errand
  azs: [z1, z2]
  jobs:
  - name: redis-server
    release: redis
    properties:
      quarks:
        pre_render_scripts:
          bpm:
          - |
            touch /tmp

1
2
3
4
5
6
7

  - domain: blobstore.service.cf.internal
    targets:
    - deployment: cf
      domain: bosh
      instance_group: singleton-blobstore
      network: default
      query: '*'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

addons:
- name: bosh-dns
  jobs:
  - name: bosh-dns
    release: bosh-dns
    properties:
      handlers:
      - domain: corp.intranet.local.
        source:
          recursors:
          - 10.0.0.2
          type: dns