controller-0:/var/log/containers$ kubectl -n openstack get po horizon-7f6499b6df-b2468 -o yaml apiVersion: v1 kind: Pod metadata: annotations: cni.projectcalico.org/podIP: 172.16.192.109/32 configmap-bin-hash: ce6561f9e653a214ddd8f4a449c143c47dc17ea2180f38a2be6cf76441c7e742 configmap-etc-hash: 0c1c8d13646f2f63e06ca6cfd35c690268b26143161283aa2c8dca2be8fbf2b1 k8s.v1.cni.cncf.io/networks-status: |- [{ "name": "chain", "ips": [ "172.16.192.109" ], "default": true, "dns": {} }] openstackhelm.openstack.org/release_uuid: "" creationTimestamp: "2019-09-29T12:04:29Z" generateName: horizon-7f6499b6df- labels: application: horizon component: server pod-template-hash: 7f6499b6df release_group: osh-openstack-horizon name: horizon-7f6499b6df-b2468 namespace: openstack ownerReferences: - apiVersion: apps/v1 blockOwnerDeletion: true controller: true kind: ReplicaSet name: horizon-7f6499b6df uid: 1021f90d-0177-46fc-aedf-f1dc2e97fcff resourceVersion: "198076" selfLink: /api/v1/namespaces/openstack/pods/horizon-7f6499b6df-b2468 uid: 33e2ce54-2e94-4bb3-bcbc-1d3b2a8f1c8b spec: affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - podAffinityTerm: labelSelector: matchExpressions: - key: release_group operator: In values: - osh-openstack-horizon - key: application operator: In values: - horizon - key: component operator: In values: - server topologyKey: kubernetes.io/hostname weight: 10 containers: - command: - /tmp/horizon.sh - start env: - name: MY_POD_IP valueFrom: fieldRef: apiVersion: v1 fieldPath: status.podIP image: registry.local:9001/docker.io/starlingx/stx-horizon:master-centos-stable-latest imagePullPolicy: IfNotPresent lifecycle: preStop: exec: command: - /tmp/horizon.sh - stop livenessProbe: failureThreshold: 3 httpGet: path: / port: 80 scheme: HTTP initialDelaySeconds: 180 periodSeconds: 60 successThreshold: 1 timeoutSeconds: 5 name: horizon ports: - containerPort: 80 name: web protocol: TCP readinessProbe: failureThreshold: 3 httpGet: path: / port: 80 scheme: HTTP initialDelaySeconds: 15 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 1 resources: {} securityContext: allowPrivilegeEscalation: false runAsUser: 0 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - mountPath: /tmp name: pod-tmp - mountPath: /var/www/html/ name: static-horizon - mountPath: /tmp/horizon.sh name: horizon-bin readOnly: true subPath: horizon.sh - mountPath: /tmp/manage.py name: horizon-bin readOnly: true subPath: manage.py - mountPath: /etc/apache2/sites-available/000-default.conf name: horizon-etc readOnly: true subPath: horizon.conf - mountPath: /etc/apache2/conf-available/security.conf name: horizon-etc readOnly: true subPath: security.conf - mountPath: /var/www/cgi-bin/horizon/django.wsgi name: horizon-bin readOnly: true subPath: django.wsgi - mountPath: /etc/openstack-dashboard/local_settings name: horizon-etc readOnly: true subPath: local_settings - mountPath: /etc/openstack-dashboard/ceilometer_policy.json name: horizon-etc readOnly: true subPath: ceilometer_policy.json - mountPath: /etc/openstack-dashboard/cinder_policy.json name: horizon-etc readOnly: true subPath: cinder_policy.json - mountPath: /etc/openstack-dashboard/glance_policy.json name: horizon-etc readOnly: true subPath: glance_policy.json - mountPath: /etc/openstack-dashboard/heat_policy.json name: horizon-etc readOnly: true subPath: heat_policy.json - mountPath: /etc/openstack-dashboard/keystone_policy.json name: horizon-etc readOnly: true subPath: keystone_policy.json - mountPath: /etc/openstack-dashboard/neutron_policy.json name: horizon-etc readOnly: true subPath: neutron_policy.json - mountPath: /etc/openstack-dashboard/nova_policy.json name: horizon-etc readOnly: true subPath: nova_policy.json - mountPath: /opt/branding name: horizon-branding - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: horizon-token-7vkwp readOnly: true dnsPolicy: ClusterFirst enableServiceLinks: true imagePullSecrets: - name: default-registry-key initContainers: - command: - kubernetes-entrypoint env: - name: POD_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace - name: INTERFACE_NAME value: eth0 - name: PATH value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/ - name: DEPENDENCY_SERVICE value: openstack:memcached,openstack:mariadb,openstack:keystone-api - name: DEPENDENCY_JOBS value: horizon-db-sync - name: DEPENDENCY_DAEMONSET - name: DEPENDENCY_CONTAINER - name: DEPENDENCY_POD_JSON - name: DEPENDENCY_CUSTOM_RESOURCE - name: COMMAND value: echo done image: registry.local:9001/quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 imagePullPolicy: IfNotPresent name: init resources: {} securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true runAsUser: 65534 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: horizon-token-7vkwp readOnly: true nodeName: controller-0 nodeSelector: openstack-control-plane: enabled priority: 0 restartPolicy: Always schedulerName: default-scheduler securityContext: runAsUser: 42424 serviceAccount: horizon serviceAccountName: horizon terminationGracePeriodSeconds: 30 tolerations: - effect: NoExecute key: node.kubernetes.io/not-ready operator: Exists tolerationSeconds: 30 - effect: NoExecute key: node.kubernetes.io/unreachable operator: Exists tolerationSeconds: 30 volumes: - emptyDir: {} name: pod-tmp - emptyDir: {} name: wsgi-horizon - emptyDir: {} name: static-horizon - configMap: defaultMode: 365 name: horizon-bin name: horizon-bin - name: horizon-etc secret: defaultMode: 292 secretName: horizon-etc - hostPath: path: /opt/branding type: Directory name: horizon-branding - name: horizon-token-7vkwp secret: defaultMode: 420 secretName: horizon-token-7vkwp status: conditions: - lastProbeTime: null lastTransitionTime: "2019-09-29T12:04:57Z" status: "True" type: Initialized - lastProbeTime: null lastTransitionTime: "2019-09-29T12:04:29Z" message: 'containers with unready status: [horizon]' reason: ContainersNotReady status: "False" type: Ready - lastProbeTime: null lastTransitionTime: "2019-09-29T12:04:29Z" message: 'containers with unready status: [horizon]' reason: ContainersNotReady status: "False" type: ContainersReady - lastProbeTime: null lastTransitionTime: "2019-09-29T12:04:29Z" status: "True" type: PodScheduled containerStatuses: - containerID: docker://805a8504b91821d116e29e3c9d3b35836a49e606ffb2788a285576744c4a0e60 image: registry.dcp-dev.intel.com/starlingx/stx-horizon:master-centos-stable-latest imageID: docker-pullable://registry.dcp-dev.intel.com/starlingx/stx-horizon@sha256:6d54a00b9bafb34f84fec72fdfa00405514a6be8d602053e0f4752b3caa6baf7 lastState: terminated: containerID: docker://8fe3be2a9ff96f7bb3e6601b13d68c02213dba1f7df643fdcf3a388b5357a9b4 exitCode: 0 finishedAt: "2019-09-30T02:34:30Z" reason: Completed startedAt: "2019-09-30T02:28:32Z" name: horizon ready: false restartCount: 145 state: running: startedAt: "2019-09-30T02:34:31Z" hostIP: 192.168.204.3 initContainerStatuses: - containerID: docker://6a10738cee864205ab29f03ea860ef5e09d8a26697c7111834097cf8d2ba0536 image: registry.dcp-dev.intel.com/stackanetes/kubernetes-entrypoint:v0.3.1 imageID: docker-pullable://registry.dcp-dev.intel.com/stackanetes/kubernetes-entrypoint@sha256:2cad635d9079ee130f7f288b2ffb9c65c90fc7711364dcf438973ddf579b7fed lastState: {} name: init ready: true restartCount: 0 state: terminated: containerID: docker://6a10738cee864205ab29f03ea860ef5e09d8a26697c7111834097cf8d2ba0536 exitCode: 0 finishedAt: "2019-09-29T12:04:56Z" reason: Completed startedAt: "2019-09-29T12:04:38Z" phase: Running podIP: 172.16.192.109 qosClass: BestEffort startTime: "2019-09-29T12:04:29Z" controller-0:/var/log/containers$ kubectl -n openstack describe po horizon-7f6499b6df-b2468 Name: horizon-7f6499b6df-b2468 Namespace: openstack Priority: 0 Node: controller-0/192.168.204.3 Start Time: Sun, 29 Sep 2019 12:04:29 +0000 Labels: application=horizon component=server pod-template-hash=7f6499b6df release_group=osh-openstack-horizon Annotations: cni.projectcalico.org/podIP: 172.16.192.109/32 configmap-bin-hash: ce6561f9e653a214ddd8f4a449c143c47dc17ea2180f38a2be6cf76441c7e742 configmap-etc-hash: 0c1c8d13646f2f63e06ca6cfd35c690268b26143161283aa2c8dca2be8fbf2b1 k8s.v1.cni.cncf.io/networks-status: [{ "name": "chain", "ips": [ "172.16.192.109" ], "default": true, "dns": {} }] openstackhelm.openstack.org/release_uuid: Status: Running IP: 172.16.192.109 Controlled By: ReplicaSet/horizon-7f6499b6df Init Containers: init: Container ID: docker://6a10738cee864205ab29f03ea860ef5e09d8a26697c7111834097cf8d2ba0536 Image: registry.local:9001/quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 Image ID: docker-pullable://registry.dcp-dev.intel.com/stackanetes/kubernetes-entrypoint@sha256:2cad635d9079ee130f7f288b2ffb9c65c90fc7711364dcf438973ddf579b7fed Port: Host Port: Command: kubernetes-entrypoint State: Terminated Reason: Completed Exit Code: 0 Started: Sun, 29 Sep 2019 12:04:38 +0000 Finished: Sun, 29 Sep 2019 12:04:56 +0000 Ready: True Restart Count: 0 Environment: POD_NAME: horizon-7f6499b6df-b2468 (v1:metadata.name) NAMESPACE: openstack (v1:metadata.namespace) INTERFACE_NAME: eth0 PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/ DEPENDENCY_SERVICE: openstack:memcached,openstack:mariadb,openstack:keystone-api DEPENDENCY_JOBS: horizon-db-sync DEPENDENCY_DAEMONSET: DEPENDENCY_CONTAINER: DEPENDENCY_POD_JSON: DEPENDENCY_CUSTOM_RESOURCE: COMMAND: echo done Mounts: /var/run/secrets/kubernetes.io/serviceaccount from horizon-token-7vkwp (ro) Containers: horizon: Container ID: docker://805a8504b91821d116e29e3c9d3b35836a49e606ffb2788a285576744c4a0e60 Image: registry.local:9001/docker.io/starlingx/stx-horizon:master-centos-stable-latest Image ID: docker-pullable://registry.dcp-dev.intel.com/starlingx/stx-horizon@sha256:6d54a00b9bafb34f84fec72fdfa00405514a6be8d602053e0f4752b3caa6baf7 Port: 80/TCP Host Port: 0/TCP Command: /tmp/horizon.sh start State: Running Started: Mon, 30 Sep 2019 02:34:31 +0000 Last State: Terminated Reason: Completed Exit Code: 0 Started: Mon, 30 Sep 2019 02:28:32 +0000 Finished: Mon, 30 Sep 2019 02:34:30 +0000 Ready: False Restart Count: 145 Liveness: http-get http://:80/ delay=180s timeout=5s period=60s #success=1 #failure=3 Readiness: http-get http://:80/ delay=15s timeout=1s period=10s #success=1 #failure=3 Environment: MY_POD_IP: (v1:status.podIP) Mounts: /etc/apache2/conf-available/security.conf from horizon-etc (ro,path="security.conf") /etc/apache2/sites-available/000-default.conf from horizon-etc (ro,path="horizon.conf") /etc/openstack-dashboard/ceilometer_policy.json from horizon-etc (ro,path="ceilometer_policy.json") /etc/openstack-dashboard/cinder_policy.json from horizon-etc (ro,path="cinder_policy.json") /etc/openstack-dashboard/glance_policy.json from horizon-etc (ro,path="glance_policy.json") /etc/openstack-dashboard/heat_policy.json from horizon-etc (ro,path="heat_policy.json") /etc/openstack-dashboard/keystone_policy.json from horizon-etc (ro,path="keystone_policy.json") /etc/openstack-dashboard/local_settings from horizon-etc (ro,path="local_settings") /etc/openstack-dashboard/neutron_policy.json from horizon-etc (ro,path="neutron_policy.json") /etc/openstack-dashboard/nova_policy.json from horizon-etc (ro,path="nova_policy.json") /opt/branding from horizon-branding (rw) /tmp from pod-tmp (rw) /tmp/horizon.sh from horizon-bin (ro,path="horizon.sh") /tmp/manage.py from horizon-bin (ro,path="manage.py") /var/run/secrets/kubernetes.io/serviceaccount from horizon-token-7vkwp (ro) /var/www/cgi-bin/horizon/django.wsgi from horizon-bin (ro,path="django.wsgi") /var/www/html/ from static-horizon (rw) Conditions: Type Status Initialized True Ready False ContainersReady False PodScheduled True Volumes: pod-tmp: Type: EmptyDir (a temporary directory that shares a pod's lifetime) Medium: SizeLimit: wsgi-horizon: Type: EmptyDir (a temporary directory that shares a pod's lifetime) Medium: SizeLimit: static-horizon: Type: EmptyDir (a temporary directory that shares a pod's lifetime) Medium: SizeLimit: horizon-bin: Type: ConfigMap (a volume populated by a ConfigMap) Name: horizon-bin Optional: false horizon-etc: Type: Secret (a volume populated by a Secret) SecretName: horizon-etc Optional: false horizon-branding: Type: HostPath (bare host directory volume) Path: /opt/branding HostPathType: Directory horizon-token-7vkwp: Type: Secret (a volume populated by a Secret) SecretName: horizon-token-7vkwp Optional: false QoS Class: BestEffort Node-Selectors: openstack-control-plane=enabled Tolerations: node.kubernetes.io/not-ready:NoExecute for 30s node.kubernetes.io/unreachable:NoExecute for 30s Events: Type Reason Age From Message ---- ------ ---- ---- ------- Warning Unhealthy 2m27s (x4966 over 14h) kubelet, controller-0 Readiness probe failed: Get http://172.16.192.109:80/: dial tcp 172.16.192.109:80: connect: connection refused controller-0:/var/log/containers$ curl 172.16.192.109:80 curl: (7) Failed connect to 172.16.192.109:80; Connection refused