以下过程说明如何配置在 sidecar 容器中运行的日志轮换和 syslog。

创建日志目录和配置日志轮换

  • 在所有节点(包括主节点)上创建日志目录,并将其所有者更改为 ID 为 1000 的任何用户。
    mkdir /var/log/nsx-ujo
    chown localadmin:localadmin /var/log/nsx-ujo
  • /var/log/nsx-ujo 目录的所有节点上配置日志轮换。
    cat <<EOF >  /etc/logrotate.d/nsx-ujo
    /var/log/nsx-ujo/*.log {
           copytruncate
           daily
           size 100M
           rotate 4
           delaycompress
           compress
           notifempty
           missingok
    }
    EOF

创建 NCP 复制控制器

  • 为 NCP 创建 ncp.ini 文件。
    cat <<EOF > /tmp/ncp.ini
    [DEFAULT]
    log_dir = /var/log/nsx-ujo
    [coe]
    cluster = k8s-cl1
    [k8s]
    apiserver_host_ip = 10.114.209.77
    apiserver_host_port = 6443
    ca_file = /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
    client_token_file = /var/run/secrets/kubernetes.io/serviceaccount/token
    insecure = True
    ingress_mode = nat
    [nsx_v3]
    nsx_api_user = admin
    nsx_api_password = Password1!
    nsx_api_managers = 10.114.209.68
    insecure = True
    subnet_prefix = 29
    [nsx_node_agent]
    [nsx_kube_proxy]
    ovs_uplink_port = ens192
    EOF
  • 基于 ini 文件创建配置映射。
    kubectl create configmap nsx-ncp-config-with-logging --from-file=/tmp/ncp.ini
  • 创建 NCP rsyslog 配置。
    cat <<EOF > /tmp/nsx-ncp-rsyslog.conf
    # yaml template for NCP ReplicationController
    # Correct kubernetes API and NSX API parameters, and NCP Docker image
    # must be specified.
    apiVersion: v1
    kind: ConfigMap
    metadata:
        name: rsyslog-config
        labels:
            version: v1
    data:
        ncp.conf: |
            module(load="imfile")
    
            ruleset(name="remote") {
                action(type="omfwd"
                       Protocol="tcp"
                       Target="nsx.licf.vmware.com"
                       Port="514")
    
                stop
            }
    
            input(type="imfile"
                  File="/var/log/nsx-ujo/ncp.log"
                  Tag="ncp"
                  Ruleset="remote")
    EOF
  • 基于上述情况创建配置映射。
    kubectl create -f /tmp/nsx-ncp-rsyslog.conf
  • 使用 rsyslog sidecar 创建 NCP 复制控制器。
    cat <<EOF > /tmp/ncp-rc-with-logging.yml
    # Replication Controller yaml for NCP
    apiVersion: v1
    kind: ReplicationController
    metadata:
      # VMware NSX Container Plugin
      name: nsx-ncp
      labels:
        tier: nsx-networking
        component: nsx-ncp
        version: v1
    spec:
      # Active-Active/Active-Standby is not supported in current release.
      # so replica *must be* 1.
      replicas: 1
      template:
        metadata:
          labels:
            tier: nsx-networking
            component: nsx-ncp
            version: v1
        spec:
          # NCP shares the host management network.
          hostNetwork: true
          nodeSelector:
            kubernetes.io/hostname: k8s-master
          tolerations:
          - key: "node-role.kubernetes.io/master"
            operator: "Exists"
            effect: "NoSchedule"
          containers:
            - name: nsx-ncp
              # Docker image for NCP
              image: nsx-ujo-docker-local.artifactory.eng.vmware.com/nsx-ncp:ob-6236425
              imagePullPolicy: IfNotPresent
              readinessProbe:
                exec:
                  command:
                  - cat
                  - /tmp/ncp_ready
                initialDelaySeconds: 5
                periodSeconds: 5
                failureThreshold: 5
              securityContext:
                capabilities:
                  add:
                    - NET_ADMIN
                    - SYS_ADMIN
                    - SYS_PTRACE
                    - DAC_READ_SEARCH
              volumeMounts:
              - name: config-volume
                # NCP expects ncp.ini is present in /etc/nsx-ujo
                mountPath: /etc/nsx-ujo
              - name: log-volume
                mountPath: /var/log/nsx-ujo
            - name: rsyslog
              image: jumanjiman/rsyslog
              imagePullPolicy: IfNotPresent
              volumeMounts:
              - name: rsyslog-config-volume
                mountPath: /etc/rsyslog.d
                readOnly: true
              - name: log-volume
                mountPath: /var/log/nsx-ujo
          volumes:
            - name: config-volume
              # ConfigMap nsx-ncp-config is expected to supply ncp.ini
              configMap:
                name: nsx-ncp-config-with-logging
            - name: rsyslog-config-volume
              configMap:
                name: rsyslog-config
            - name: log-volume
              hostPath:
                path: /var/log/nsx-ujo/
    EOF
  • 使用上面的规范创建 NCP。
    kubectl apply -f /tmp/ncp-rc-with-logging.yml

创建 NSX 节点代理 DaemonSet

  • 为节点代理创建 rsyslog 配置。
    cat <<EOF > /tmp/nsx-node-agent-rsyslog.conf
    # yaml template for NCP ReplicationController
    # Correct kubernetes API and NSX API parameters, and NCP Docker image
    # must be specified.
    apiVersion: v1
    kind: ConfigMap
    metadata:
        name: rsyslog-config-node-agent
        labels:
            version: v1
    data:
        ncp.conf: |
            module(load="imfile")
    
            ruleset(name="remote") {
                action(type="omfwd"
                       Protocol="tcp"
                       Target="nsx.licf.vmware.com"
                       Port="514")
    
                stop
            }
    
            input(type="imfile"
                  File="/var/log/nsx-ujo/nsx_kube_proxy.log"
                  Tag="nsx_kube_proxy"
                  Ruleset="remote")
    
            input(type="imfile"
                  File="/var/log/nsx-ujo/nsx_node_agent.log"
                  Tag="nsx_node_agent"
                  Ruleset="remote")
    EOF
  • 基于上述情况创建 configmap
    kubectl create -f /tmp/nsx-node-agent-rsyslog.conf
  • 使用 configmap sidecar 创建 DaemonSet。
    cat <<EOF > /tmp/nsx-node-agent-rsyslog.yml
    # nsx-node-agent DaemonSet
    apiVersion: extensions/v1beta1
    kind: DaemonSet
    metadata:
      name: nsx-node-agent
      labels:
        tier: nsx-networking
        component: nsx-node-agent
        version: v1
    spec:
      template:
        metadata:
          annotations:
            container.apparmor.security.beta.kubernetes.io/nsx-node-agent: localhost/node-agent-apparmor
          labels:
            tier: nsx-networking
            component: nsx-node-agent
            version: v1
        spec:
          hostNetwork: true
          tolerations:
          - key: "node-role.kubernetes.io/master"
            operator: "Exists"
            effect: "NoSchedule"
          containers:
            - name: nsx-node-agent
              # Docker image for NCP
              image: nsx-ujo-docker-local.artifactory.eng.vmware.com/nsx-ncp:ob-6236425
              imagePullPolicy: IfNotPresent
              # override NCP image entrypoint
              command: ["nsx_node_agent"]
              livenessProbe:
                exec:
                  command:
                    - /bin/sh
                    - -c
                    - ps aux | grep [n]sx_node_agent
                initialDelaySeconds: 5
                periodSeconds: 5
              securityContext:
                capabilities:
                  add:
                    - NET_ADMIN
                    - SYS_ADMIN
                    - SYS_PTRACE
                    - DAC_READ_SEARCH
              volumeMounts:
              # ncp.ini
              - name: config-volume
                mountPath: /etc/nsx-ujo
              # mount openvswitch dir
              - name: openvswitch
                mountPath: /var/run/openvswitch
              # mount CNI socket path
              - name: cni-sock
                mountPath: /var/run/nsx-ujo
              # mount container namespace
              - name: netns
                mountPath: /var/run/netns
              # mount host proc
              - name: proc
                mountPath: /host/proc
                readOnly: true
              - name: log-volume
                mountPath: /var/log/nsx-ujo
            - name: nsx-kube-proxy
              # Docker image for NCP
              image: nsx-ujo-docker-local.artifactory.eng.vmware.com/nsx-ncp:ob-6236425
              imagePullPolicy: IfNotPresent
              # override NCP image entrypoint
              command: ["nsx_kube_proxy"]
              livenessProbe:
                exec:
                  command:
                    - /bin/sh
                    - -c
                    - ps aux | grep [n]sx_kube_proxy
                initialDelaySeconds: 5
                periodSeconds: 5
              securityContext:
                capabilities:
                  add:
                    - NET_ADMIN
                    - SYS_ADMIN
                    - SYS_PTRACE
                    - DAC_READ_SEARCH
              volumeMounts:
              # ncp.ini
              - name: config-volume
                mountPath: /etc/nsx-ujo
              # mount openvswitch dir
              - name: openvswitch
                mountPath: /var/run/openvswitch
              - name: log-volume
                mountPath: /var/log/nsx-ujo
            - name: rsyslog
              image: jumanjiman/rsyslog
              imagePullPolicy: IfNotPresent
              volumeMounts:
              - name: rsyslog-config-volume
                mountPath: /etc/rsyslog.d
                readOnly: true
              - name: log-volume
                mountPath: /var/log/nsx-ujo
          volumes:
            - name: config-volume
              configMap:
                name: nsx-ncp-config-with-logging
            - name: cni-sock
              hostPath:
                path: /var/run/nsx-ujo
            - name: netns
              hostPath:
                path: /var/run/netns
            - name: proc
              hostPath:
                path: /proc
            - name: openvswitch
              hostPath:
                path: /var/run/openvswitch
            - name: rsyslog-config-volume
              configMap:
                name: rsyslog-config-node-agent
            - name: log-volume
              hostPath:
                path: /var/log/nsx-ujo/
    EOF
  • 创建 DaemonSet。
    kubectl apply -f /tmp/nsx-node-agent-rsyslog.yml