Example: Configuring Log Rotation and Syslog Running in a Sidecar Container

The following procedure shows how to configure log rotation and syslog running in a sidecar container.

Creating the Log Directory and Configuring Log Rotation

• Create the log directory on all the nodes, including the master, and change its owner to whatever user has ID 1000.

  mkdir /var/log/nsx-ujo
  chown localadmin:localadmin /var/log/nsx-ujo

• Configure log rotation on all the nodes for the /var/log/nsx-ujo directory.

  cat <<EOF >  /etc/logrotate.d/nsx-ujo
  /var/log/nsx-ujo/*.log {
         copytruncate
         daily
         size 100M
         rotate 4
         delaycompress
         compress
         notifempty
         missingok
  }
  EOF

Creating the NCP Replication Controller

• Create the ncp.ini file for NCP.

  cat <<EOF > /tmp/ncp.ini
  [DEFAULT]
  log_dir = /var/log/nsx-ujo
  [coe]
  cluster = k8s-cl1
  [k8s]
  apiserver_host_ip = 10.114.209.77
  apiserver_host_port = 6443
  ca_file = /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
  client_token_file = /var/run/secrets/kubernetes.io/serviceaccount/token
  insecure = True
  ingress_mode = nat
  [nsx_v3]
  nsx_api_user = admin
  nsx_api_password = Password1!
  nsx_api_managers = 10.114.209.68
  insecure = True
  subnet_prefix = 29
  [nsx_node_agent]
  [nsx_kube_proxy]
  ovs_uplink_port = ens192
  EOF

• Create the config map from the ini file.

  kubectl create configmap nsx-ncp-config-with-logging --from-file=/tmp/ncp.ini

• Create the NCP rsyslog config.

  cat <<EOF > /tmp/nsx-ncp-rsyslog.conf
  # yaml template for NCP ReplicationController
  # Correct kubernetes API and NSX API parameters, and NCP Docker image
  # must be specified.
  apiVersion: v1
  kind: ConfigMap
  metadata:
      name: rsyslog-config
      labels:
          version: v1
  data:
      ncp.conf: |
          module(load="imfile")
  
          ruleset(name="remote") {
              action(type="omfwd"
                     Protocol="tcp"
                     Target="nsx.licf.vmware.com"
                     Port="514")
  
              stop
          }
  
          input(type="imfile"
                File="/var/log/nsx-ujo/ncp.log"
                Tag="ncp"
                Ruleset="remote")
  EOF

• Create the config map from the above.

  kubectl create -f /tmp/nsx-ncp-rsyslog.conf

• Create the NCP replication controller with the rsyslog sidecar.

  cat <<EOF > /tmp/ncp-rc-with-logging.yml
  # Replication Controller yaml for NCP
  apiVersion: v1
  kind: ReplicationController
  metadata:
    # VMware NSX Container Plugin
    name: nsx-ncp
    labels:
      tier: nsx-networking
      component: nsx-ncp
      version: v1
  spec:
    # Active-Active/Active-Standby is not supported in current release.
    # so replica *must be* 1.
    replicas: 1
    template:
      metadata:
        labels:
          tier: nsx-networking
          component: nsx-ncp
          version: v1
      spec:
        # NCP shares the host management network.
        hostNetwork: true
        nodeSelector:
          kubernetes.io/hostname: k8s-master
        tolerations:
        - key: "node-role.kubernetes.io/master"
          operator: "Exists"
          effect: "NoSchedule"
        containers:
          - name: nsx-ncp
            # Docker image for NCP
            image: nsx-ujo-docker-local.artifactory.eng.vmware.com/nsx-ncp:ob-6236425
            imagePullPolicy: IfNotPresent
            readinessProbe:
              exec:
                command:
                - cat
                - /tmp/ncp_ready
              initialDelaySeconds: 5
              periodSeconds: 5
              failureThreshold: 5
            securityContext:
              capabilities:
                add:
                  - NET_ADMIN
                  - SYS_ADMIN
                  - SYS_PTRACE
                  - DAC_READ_SEARCH
            volumeMounts:
            - name: config-volume
              # NCP expects ncp.ini is present in /etc/nsx-ujo
              mountPath: /etc/nsx-ujo
            - name: log-volume
              mountPath: /var/log/nsx-ujo
          - name: rsyslog
            image: jumanjiman/rsyslog
            imagePullPolicy: IfNotPresent
            volumeMounts:
            - name: rsyslog-config-volume
              mountPath: /etc/rsyslog.d
              readOnly: true
            - name: log-volume
              mountPath: /var/log/nsx-ujo
        volumes:
          - name: config-volume
            # ConfigMap nsx-ncp-config is expected to supply ncp.ini
            configMap:
              name: nsx-ncp-config-with-logging
          - name: rsyslog-config-volume
            configMap:
              name: rsyslog-config
          - name: log-volume
            hostPath:
              path: /var/log/nsx-ujo/
  EOF

• Create NCP with the above specification.

  kubectl apply -f /tmp/ncp-rc-with-logging.yml

Creating the NSX Node Agent Daemon Set

• Create the rsyslog configuration for the node agents.

  cat <<EOF > /tmp/nsx-node-agent-rsyslog.conf
  # yaml template for NCP ReplicationController
  # Correct kubernetes API and NSX API parameters, and NCP Docker image
  # must be specified.
  apiVersion: v1
  kind: ConfigMap
  metadata:
      name: rsyslog-config-node-agent
      labels:
          version: v1
  data:
      ncp.conf: |
          module(load="imfile")
  
          ruleset(name="remote") {
              action(type="omfwd"
                     Protocol="tcp"
                     Target="nsx.licf.vmware.com"
                     Port="514")
  
              stop
          }
  
          input(type="imfile"
                File="/var/log/nsx-ujo/nsx_kube_proxy.log"
                Tag="nsx_kube_proxy"
                Ruleset="remote")
  
          input(type="imfile"
                File="/var/log/nsx-ujo/nsx_node_agent.log"
                Tag="nsx_node_agent"
                Ruleset="remote")
  EOF

• Create the configmap from the above.

  kubectl create -f /tmp/nsx-node-agent-rsyslog.conf

• Create the DaemonSet with the configmap sidecar.

  cat <<EOF > /tmp/nsx-node-agent-rsyslog.yml
  # nsx-node-agent DaemonSet
  apiVersion: extensions/v1beta1
  kind: DaemonSet
  metadata:
    name: nsx-node-agent
    labels:
      tier: nsx-networking
      component: nsx-node-agent
      version: v1
  spec:
    template:
      metadata:
        annotations:
          container.apparmor.security.beta.kubernetes.io/nsx-node-agent: localhost/node-agent-apparmor
        labels:
          tier: nsx-networking
          component: nsx-node-agent
          version: v1
      spec:
        hostNetwork: true
        tolerations:
        - key: "node-role.kubernetes.io/master"
          operator: "Exists"
          effect: "NoSchedule"
        containers:
          - name: nsx-node-agent
            # Docker image for NCP
            image: nsx-ujo-docker-local.artifactory.eng.vmware.com/nsx-ncp:ob-6236425
            imagePullPolicy: IfNotPresent
            # override NCP image entrypoint
            command: ["nsx_node_agent"]
            livenessProbe:
              exec:
                command:
                  - /bin/sh
                  - -c
                  - ps aux | grep [n]sx_node_agent
              initialDelaySeconds: 5
              periodSeconds: 5
            securityContext:
              capabilities:
                add:
                  - NET_ADMIN
                  - SYS_ADMIN
                  - SYS_PTRACE
                  - DAC_READ_SEARCH
            volumeMounts:
            # ncp.ini
            - name: config-volume
              mountPath: /etc/nsx-ujo
            # mount openvswitch dir
            - name: openvswitch
              mountPath: /var/run/openvswitch
            # mount CNI socket path
            - name: cni-sock
              mountPath: /var/run/nsx-ujo
            # mount container namespace
            - name: netns
              mountPath: /var/run/netns
            # mount host proc
            - name: proc
              mountPath: /host/proc
              readOnly: true
            - name: log-volume
              mountPath: /var/log/nsx-ujo
          - name: nsx-kube-proxy
            # Docker image for NCP
            image: nsx-ujo-docker-local.artifactory.eng.vmware.com/nsx-ncp:ob-6236425
            imagePullPolicy: IfNotPresent
            # override NCP image entrypoint
            command: ["nsx_kube_proxy"]
            livenessProbe:
              exec:
                command:
                  - /bin/sh
                  - -c
                  - ps aux | grep [n]sx_kube_proxy
              initialDelaySeconds: 5
              periodSeconds: 5
            securityContext:
              capabilities:
                add:
                  - NET_ADMIN
                  - SYS_ADMIN
                  - SYS_PTRACE
                  - DAC_READ_SEARCH
            volumeMounts:
            # ncp.ini
            - name: config-volume
              mountPath: /etc/nsx-ujo
            # mount openvswitch dir
            - name: openvswitch
              mountPath: /var/run/openvswitch
            - name: log-volume
              mountPath: /var/log/nsx-ujo
          - name: rsyslog
            image: jumanjiman/rsyslog
            imagePullPolicy: IfNotPresent
            volumeMounts:
            - name: rsyslog-config-volume
              mountPath: /etc/rsyslog.d
              readOnly: true
            - name: log-volume
              mountPath: /var/log/nsx-ujo
        volumes:
          - name: config-volume
            configMap:
              name: nsx-ncp-config-with-logging
          - name: cni-sock
            hostPath:
              path: /var/run/nsx-ujo
          - name: netns
            hostPath:
              path: /var/run/netns
          - name: proc
            hostPath:
              path: /proc
          - name: openvswitch
            hostPath:
              path: /var/run/openvswitch
          - name: rsyslog-config-volume
            configMap:
              name: rsyslog-config-node-agent
          - name: log-volume
            hostPath:
              path: /var/log/nsx-ujo/
  EOF

• Create the DaemonSet.

  kubectl apply -f /tmp/nsx-node-agent-rsyslog.yml