Merge pull request #87 from hjacobs/json-delta

JSON delta

app.py

@@ -9,6 +9,7 @@ import functools
 import gevent
 import gevent.wsgi
 import json
+import json_delta
 import logging
 import os
 import re
@@ -60,9 +61,16 @@ class MemoryStore:
     '''Memory-only backend, mostly useful for local debugging'''
 
     def __init__(self):
+        self._data = {}
         self._queues = []
         self._screen_tokens = {}
 
+    def set(self, key, value):
+        self._data[key] = value
+
+    def get(self, key):
+        return self._data.get(key)
+
     def acquire_lock(self):
         # no-op for memory store
         return 'fake-lock'
@@ -105,6 +113,14 @@ class RedisStore:
         self._redis = redis.StrictRedis.from_url(url)
         self._redlock = Redlock([url])
 
+    def set(self, key, value):
+        self._redis.set(key, json.dumps(value, separators=(',', ':')))
+
+    def get(self, key):
+        value = self._redis.get(key)
+        if value:
+            return json.loads(value.decode('utf-8'))
+
     def acquire_lock(self):
         return self._redlock.lock('update', 10000)
 
@@ -147,6 +163,7 @@ def get_bool(name: str):
     return os.getenv(name, '').lower() in ('1', 'true')
 
 
+DEBUG = get_bool('DEBUG')
 SERVER_PORT = int(os.getenv('SERVER_PORT', 8080))
 SERVER_STATUS = {'shutdown': False}
 DEFAULT_CLUSTERS = 'http://localhost:8001/'
@@ -158,7 +175,7 @@ REDIS_URL = os.getenv('REDIS_URL')
 STORE = RedisStore(REDIS_URL) if REDIS_URL else MemoryStore()
 
 app = Flask(__name__)
-app.debug = get_bool('DEBUG')
+app.debug = DEBUG
 app.secret_key = os.getenv('SECRET_KEY', 'development')
 
 oauth = OAuth(app)
@@ -250,7 +267,7 @@ def generate_mock_pod(index: int, i: int, j: int):
         'agent-cooper',
         'black-lodge',
         'bob',
-        'bobby-briggs'
+        'bobby-briggs',
         'laura-palmer',
         'leland-palmer',
         'log-lady',
@@ -261,14 +278,18 @@ def generate_mock_pod(index: int, i: int, j: int):
     phase = pod_phases[hash_int((index + 1) * (i + 1) * (j + 1)) % len(pod_phases)]
     containers = []
     for k in range(1 + j % 2):
-        containers.append({'name': 'myapp', 'image': 'foo/bar/{}'.format(j), 'resources': {'requests': {'cpu': '100m', 'memory': '100Mi'}, 'limits': {}}})
+        container = {
-    status = {'phase': phase}
+            'name': 'myapp', 'image': 'foo/bar/{}'.format(j), 'resources': {'requests': {'cpu': '100m', 'memory': '100Mi'}, 'limits': {}},
-    if phase == 'Running':
+            'ready': True,
-        if j % 13 == 0:
+            'state': {'running': {}}
-            status['containerStatuses'] = [{'ready': False, 'state': {'waiting': {'reason': 'CrashLoopBackOff'}}}]
+        }
-        elif j % 7 == 0:
+        if phase == 'Running':
-            status['containerStatuses'] = [{'ready': True, 'state': {'running': {}}, 'restartCount': 3}]
+            if j % 13 == 0:
-    pod = {'name': '{}-{}-{}'.format(names[hash_int((i + 1) * (j + 1)) % len(names)], i, j), 'namespace': 'kube-system' if j < 3 else 'default', 'labels': labels, 'status': status, 'containers': containers}
+                container.update(**{'ready': False, 'state': {'waiting': {'reason': 'CrashLoopBackOff'}}})
+            elif j % 7 == 0:
+                container.update(**{'ready': True, 'state': {'running': {}}, 'restartCount': 3})
+        containers.append(container)
+    pod = {'name': '{}-{}-{}'.format(names[hash_int((i + 1) * (j + 1)) % len(names)], i, j), 'namespace': 'kube-system' if j < 3 else 'default', 'labels': labels, 'phase': phase, 'containers': containers}
     if phase == 'Running' and j % 17 == 0:
         pod['deleted'] = 123
 
@@ -285,7 +306,7 @@ def generate_cluster_id(url: str):
 
 def generate_mock_cluster_data(index: int):
     '''Generate deterministic (no randomness!) mock data'''
-    nodes = []
+    nodes = {}
     for i in range(10):
         # add/remove the second to last node every 13 seconds
         if i == 8 and int(time.time() / 13) % 2 == 0:
@@ -293,15 +314,18 @@ def generate_mock_cluster_data(index: int):
         labels = {}
         if i < 2:
             labels['master'] = 'true'
-        pods = []
+        pods = {}
         for j in range(hash_int((index + 1) * (i + 1)) % 32):
-            # add/remove some pods every 6 seconds
+            # add/remove some pods every 7 seconds
-            if j % 17 == 0 and int(time.time() / 6) % 2 == 0:
+            if j % 17 == 0 and int(time.time() / 7) % 2 == 0:
                 pass
             else:
-                pods.append(generate_mock_pod(index, i, j))
+                pod = generate_mock_pod(index, i, j)
-        nodes.append({'name': 'node-{}'.format(i), 'labels': labels, 'status': {'capacity': {'cpu': '4', 'memory': '32Gi', 'pods': '110'}}, 'pods': pods})
+                pods['{}/{}'.format(pod['namespace'], pod['name'])] = pod
-    unassigned_pods = [generate_mock_pod(index, 11, index)]
+        node = {'name': 'node-{}'.format(i), 'labels': labels, 'status': {'capacity': {'cpu': '4', 'memory': '32Gi', 'pods': '110'}}, 'pods': pods}
+        nodes[node['name']] = node
+    pod = generate_mock_pod(index, 11, index)
+    unassigned_pods = {'{}/{}'.format(pod['namespace'], pod['name']): pod}
     return {
         'id': 'mock-cluster-{}'.format(index),
         'api_server_url': 'https://kube-{}.example.org'.format(index),
@@ -316,6 +340,41 @@ def get_mock_clusters():
         yield data
 
 
+def map_node_status(status: dict):
+    return {
+        'addresses': status.get('addresses'),
+        'capacity': status.get('capacity'),
+    }
+
+
+def map_node(node: dict):
+    return {
+        'name': node['metadata']['name'],
+        'labels': node['metadata']['labels'],
+        'status': map_node_status(node['status']),
+        'pods': {}
+    }
+
+
+def map_pod(pod: dict):
+    return {
+        'name': pod['metadata']['name'],
+        'namespace': pod['metadata']['namespace'],
+        'labels': pod['metadata'].get('labels', {}),
+        'phase': pod['status'].get('phase'),
+        'startTime': pod['status']['startTime'] if 'startTime' in pod['status'] else '',
+        'containers': []
+    }
+
+
+def map_container(cont: dict, pod: dict):
+    obj = {'name': cont['name'], 'image': cont['image'], 'resources': cont['resources']}
+    status = list([s for s in pod.get('status', {}).get('containerStatuses', []) if s['name'] == cont['name']])
+    if status:
+        obj.update(**status[0])
+    return obj
+
+
 def get_kubernetes_clusters():
     for api_server_url in (os.getenv('CLUSTERS') or DEFAULT_CLUSTERS).split(','):
         cluster_id = generate_cluster_id(api_server_url)
@@ -324,36 +383,28 @@ def get_kubernetes_clusters():
             session.headers['Authorization'] = 'Bearer {}'.format(tokens.get('read-only'))
         response = session.get(urljoin(api_server_url, '/api/v1/nodes'), timeout=5)
         response.raise_for_status()
-        nodes = []
+        nodes = {}
-        nodes_by_name = {}
         pods_by_namespace_name = {}
-        unassigned_pods = []
+        unassigned_pods = {}
         for node in response.json()['items']:
-            obj = {'name': node['metadata']['name'], 'labels': node['metadata']['labels'], 'status': node['status'],
+            obj = map_node(node)
-                   'pods': []}
+            nodes[obj['name']] = obj
-            nodes.append(obj)
-            nodes_by_name[obj['name']] = obj
         response = session.get(urljoin(api_server_url, '/api/v1/pods'), timeout=5)
         response.raise_for_status()
         for pod in response.json()['items']:
-            obj = {'name': pod['metadata']['name'],
+            obj = map_pod(pod)
-                   'namespace': pod['metadata']['namespace'],
-                   'labels': pod['metadata'].get('labels', {}),
-                   'status': pod['status'],
-                   'startTime': pod['status']['startTime'] if 'startTime' in pod['status'] else '',
-                   'containers': []
-                   }
             if 'deletionTimestamp' in pod['metadata']:
                 obj['deleted'] = datetime.datetime.strptime(pod['metadata']['deletionTimestamp'],
                                                             '%Y-%m-%dT%H:%M:%SZ').replace(
                     tzinfo=datetime.timezone.utc).timestamp()
             for cont in pod['spec']['containers']:
-                obj['containers'].append({'name': cont['name'], 'image': cont['image'], 'resources': cont['resources']})
+                obj['containers'].append(map_container(cont, pod))
             pods_by_namespace_name[(obj['namespace'], obj['name'])] = obj
-            if 'nodeName' in pod['spec'] and pod['spec']['nodeName'] in nodes_by_name:
+            pod_key = '{}/{}'.format(obj['namespace'], obj['name'])
-                nodes_by_name[pod['spec']['nodeName']]['pods'].append(obj)
+            if 'nodeName' in pod['spec'] and pod['spec']['nodeName'] in nodes:
+                nodes[pod['spec']['nodeName']]['pods'][pod_key] = obj
             else:
-                unassigned_pods.append(obj)
+                unassigned_pods[pod_key] = obj
 
         try:
             response = session.get(urljoin(api_server_url, '/api/v1/namespaces/kube-system/services/heapster/proxy/apis/metrics/v1alpha1/nodes'), timeout=5)
@@ -363,7 +414,7 @@ def get_kubernetes_clusters():
                 logging.info('Heapster node metrics not available (yet)')
             else:
                 for metrics in data['items']:
-                    nodes_by_name[metrics['metadata']['name']]['usage'] = metrics['usage']
+                    nodes[metrics['metadata']['name']]['usage'] = metrics['usage']
         except:
             logging.exception('Failed to get node metrics')
         try:
@@ -388,10 +439,16 @@ def get_kubernetes_clusters():
 
 
 def event(cluster_ids: set):
+    # first sent full data once
+    for cluster_id in (STORE.get('cluster-ids') or []):
+        if not cluster_ids or cluster_id in cluster_ids:
+            cluster = STORE.get(cluster_id)
+            yield 'event: clusterupdate\ndata: ' + json.dumps(cluster, separators=(',', ':')) + '\n\n'
     while True:
-        for event_type, cluster in STORE.listen():
+        for event_type, event_data in STORE.listen():
-            if not cluster_ids or cluster['id'] in cluster_ids:
+            # hacky, event_data can be delta or full cluster object
-                yield 'event: ' + event_type + '\ndata: ' + json.dumps(cluster, separators=(',', ':')) + '\n\n'
+            if not cluster_ids or event_data.get('cluster_id', event_data.get('id')) in cluster_ids:
+                yield 'event: ' + event_type + '\ndata: ' + json.dumps(event_data, separators=(',', ':')) + '\n\n'
 
 
 @app.route('/events')
@@ -461,8 +518,19 @@ def update():
                     clusters = get_mock_clusters()
                 else:
                     clusters = get_kubernetes_clusters()
+                cluster_ids = []
                 for cluster in clusters:
-                    STORE.publish('clusterupdate', cluster)
+                    old_data = STORE.get(cluster['id'])
+                    if old_data:
+                        # https://pikacode.com/phijaro/json_delta/ticket/11/
+                        # diff is extremely slow without array_align=False
+                        delta = json_delta.diff(old_data, cluster, verbose=DEBUG, array_align=False)
+                        STORE.publish('clusterdelta', {'cluster_id': cluster['id'], 'delta': delta})
+                    else:
+                        STORE.publish('clusterupdate', cluster)
+                    STORE.set(cluster['id'], cluster)
+                    cluster_ids.append(cluster['id'])
+                STORE.set('cluster-ids', cluster_ids)
             except:
                 logging.exception('Failed to update')
             finally:

app/.eslintignore

@@ -0,0 +1 @@
+src/vendor/*.js

app/.eslintrc.yml

@@ -21,6 +21,7 @@ rules:
     - error
   no-unused-vars:
     - warn
+    - argsIgnorePattern: "^_"
   semi:
     - error
     - never

app/package.json

@@ -27,7 +27,7 @@
   "homepage": "https://github.com/hjacobs/kube-ops-view#readme",
   "dependencies": {
     "pixi-display": "^1.0.1",
-    "pixi.js": "^4.3.0",
+    "pixi.js": "^4.3.2",
     "webpack-dev-server": "^1.16.2"
   },
   "devDependencies": {
@@ -36,7 +36,7 @@
     "babel-preset-es2015": "^6.18.0",
     "babel-runtime": "^6.20.0",
     "brfs": "^1.4.3",
-    "eslint": "^3.12.2",
+    "eslint": "^3.13.1",
     "eslint-loader": "^1.6.1",
     "rimraf": "^2.5.4",
     "transform-loader": "^0.2.3",

app/src/app.js

@@ -4,6 +4,7 @@ import {Pod, ALL_PODS, sortByName, sortByMemory, sortByCPU, sortByAge} from './p
 import SelectBox from './selectbox'
 import { Theme, ALL_THEMES} from './themes.js'
 import { DESATURATION_FILTER } from './filters.js'
+import { JSON_delta } from './vendor/json_delta.js'
 
 const PIXI = require('pixi.js')
 
@@ -18,6 +19,12 @@ export default class App {
         this.sorterFn = ''
         this.theme = Theme.get(localStorage.getItem('theme'))
         this.eventSource = null
+        this.connectTime = null
+        this.keepAliveTimer = null
+        // make sure we got activity at least every 20 seconds
+        this.keepAliveSeconds = 20
+        // always reconnect after 5 minutes
+        this.maxConnectionLifetimeSeconds = 300
         this.clusters = new Map()
     }
 
@@ -262,17 +269,19 @@ export default class App {
         this.stage.addChild(pod)
     }
     update() {
+        // make sure we create a copy (this.clusters might get modified)
+        const clusters = Array.from(this.clusters.entries()).sort().map(idCluster => idCluster[1])
         const that = this
         let changes = 0
         const firstTime = this.seenPods.size == 0
         const podKeys = new Set()
-        for (const cluster of this.clusters.values()) {
+        for (const cluster of clusters) {
-            for (const node of cluster.nodes) {
+            for (const node of Object.values(cluster.nodes)) {
-                for (const pod of node.pods) {
+                for (const pod of Object.values(node.pods)) {
                     podKeys.add(cluster.id + '/' + pod.namespace + '/' + pod.name)
                 }
             }
-            for (const pod of cluster.unassigned_pods) {
+            for (const pod of Object.values(cluster.unassigned_pods)) {
                 podKeys.add(cluster.id + '/' + pod.namespace + '/' + pod.name)
             }
         }
@@ -301,10 +310,10 @@ export default class App {
         }
         let y = 0
         const clusterIds = new Set()
-        for (const [clusterId, cluster] of Array.from(this.clusters.entries()).sort()) {
+        for (const cluster of clusters) {
-            if (!this.selectedClusters.size || this.selectedClusters.has(clusterId)) {
+            if (!this.selectedClusters.size || this.selectedClusters.has(cluster.id)) {
-                clusterIds.add(clusterId)
+                clusterIds.add(cluster.id)
-                let clusterBox = clusterComponentById[clusterId]
+                let clusterBox = clusterComponentById[cluster.id]
                 if (!clusterBox) {
                     clusterBox = new Cluster(cluster, this.tooltip)
                     this.viewContainer.addChild(clusterBox)
@@ -340,7 +349,7 @@ export default class App {
         }
     }
 
-    tick(time) {
+    tick(_time) {
         this.renderer.render(this.stage)
     }
 
@@ -364,15 +373,35 @@ export default class App {
         }
         this.changeLocationHash('clusters', Array.from(this.selectedClusters).join(','))
         // make sure we are updating our EventSource filter
-        this.listen()
+        this.connect()
         this.update()
     }
 
-    listen() {
+    keepAlive() {
+        if (this.keepAliveTimer != null) {
+            clearTimeout(this.keepAliveTimer)
+        }
+        this.keepAliveTimer = setTimeout(this.connect.bind(this), this.keepAliveSeconds * 1000)
+        if (this.connectTime != null) {
+            const now = Date.now()
+            if (now - this.connectTime > this.maxConnectionLifetimeSeconds * 1000) {
+                // maximum connection lifetime exceeded => reconnect
+                this.connect()
+            }
+        }
+    }
+
+    disconnect() {
         if (this.eventSource != null) {
             this.eventSource.close()
             this.eventSource = null
+            this.connectTime = null
         }
+    }
+
+    connect() {
+        // first close the old connection
+        this.disconnect()
         const that = this
         // NOTE: path must be relative to work with kubectl proxy out of the box
         let url = 'events'
@@ -381,20 +410,44 @@ export default class App {
             url += '?cluster_ids=' + clusterIds
         }
         const eventSource = this.eventSource = new EventSource(url, {credentials: 'include'})
-        eventSource.onerror = function(event) {
+        this.keepAlive()
-            that.listen()
+        eventSource.onerror = function(_event) {
+            that._errors++
+            if (that._errors <= 1) {
+                // immediately reconnect on first error
+                that.connect()
+            } else {
+                // rely on keep-alive timer to reconnect
+                that.disconnect()
+            }
         }
         eventSource.addEventListener('clusterupdate', function(event) {
+            that._errors = 0
+            that.keepAlive()
             const cluster = JSON.parse(event.data)
             that.clusters.set(cluster.id, cluster)
             that.update()
         })
+        eventSource.addEventListener('clusterdelta', function(event) {
+            that._errors = 0
+            that.keepAlive()
+            const data = JSON.parse(event.data)
+            let cluster = that.clusters.get(data.cluster_id)
+            if (cluster && data.delta) {
+                // deep copy cluster object (patch function mutates inplace!)
+                cluster = JSON.parse(JSON.stringify(cluster))
+                cluster = JSON_delta.patch(cluster, data.delta)
+                that.clusters.set(cluster.id, cluster)
+                that.update()
+            }
+        })
+        this.connectTime = Date.now()
     }
 
     run() {
         this.initialize()
         this.draw()
-        this.listen()
+        this.connect()
 
         PIXI.ticker.shared.add(this.tick, this)
     }

app/src/cluster.js

@@ -26,7 +26,8 @@ export default class Cluster extends PIXI.Graphics {
         let workerHeight = 0
         const workerNodes = []
         const maxWidth = window.innerWidth - 130
-        for (const node of this.cluster.nodes) {
+        for (const nodeName of Object.keys(this.cluster.nodes).sort()) {
+            const node = this.cluster.nodes[nodeName]
             var nodeBox = new Node(node, this, this.tooltip)
             nodeBox.draw()
             if (nodeBox.isMaster()) {
@@ -64,7 +65,7 @@ export default class Cluster extends PIXI.Graphics {
         }
 
 
-        for (const pod of this.cluster.unassigned_pods) {
+        for (const pod of Object.values(this.cluster.unassigned_pods)) {
             var podBox = Pod.getOrCreate(pod, this, this.tooltip)
             podBox.x = masterX
             podBox.y = masterY
@@ -86,7 +87,7 @@ export default class Cluster extends PIXI.Graphics {
         topHandle.interactive = true
         topHandle.buttonMode = true
         const that = this
-        topHandle.on('click', function(event) {
+        topHandle.on('click', function(_event) {
             App.current.toggleCluster(that.cluster.id)
         })
         var text = new PIXI.Text(this.cluster.api_server_url, {fontFamily: 'ShareTechMono', fontSize: 10, fill: 0x000000})

app/src/node.js

@@ -30,7 +30,9 @@ export default class Node extends PIXI.Graphics {
                 resources[key]['used'] = parseResource(this.node.usage[key])
             }
         }
-        for (const pod of this.node.pods) {
+        let numberOfPods = 0
+        for (const pod of Object.values(this.node.pods)) {
+            numberOfPods++
             for (const container of pod.containers) {
                 if (container.resources && container.resources.requests) {
                     for (const key of Object.keys(container.resources.requests)) {
@@ -39,8 +41,8 @@ export default class Node extends PIXI.Graphics {
                 }
             }
         }
-        resources['pods'].requested = this.node.pods.length
+        resources['pods'].requested = numberOfPods
-        resources['pods'].used = this.node.pods.length
+        resources['pods'].used = numberOfPods
         return resources
     }
 
@@ -89,7 +91,7 @@ export default class Node extends PIXI.Graphics {
         const nodeBox = this
         let px = 24
         let py = 20
-        const pods = this.node.pods.sort(sorterFn)
+        const pods = Object.values(this.node.pods).sort(sorterFn)
         for (const pod of pods) {
             if (pod.namespace != 'kube-system') {
                 const podBox = Pod.getOrCreate(pod, this.cluster, this.tooltip)

app/src/pod.js

@@ -10,8 +10,8 @@ const sortByName = (a, b) => {
 }
 
 const sortByAge = (a, b) => {
-    const dateA = new Date(a.status.startTime)
+    const dateA = new Date(a.startTime)
-    const dateB = new Date(b.status.startTime)
+    const dateB = new Date(b.startTime)
     if (dateA.getTime() < dateB.getTime()) {
         return -1
     } else if (dateA.getTime() === dateB.getTime())
@@ -54,6 +54,7 @@ export class Pod extends PIXI.Graphics {
         if (this.tick) {
             PIXI.ticker.shared.remove(this.tick, this)
         }
+        PIXI.ticker.shared.remove(this.animateMove, this)
         super.destroy()
     }
 
@@ -122,39 +123,37 @@ export class Pod extends PIXI.Graphics {
         }
     }
 
-    pulsate(time) {
+    pulsate(_time) {
         const v = Math.sin((PIXI.ticker.shared.lastTime % 1000) / 1000. * Math.PI)
         this.alpha = v * this._progress
     }
 
-    crashing(time) {
+    crashing(_time) {
         const v = Math.sin((PIXI.ticker.shared.lastTime % 1000) / 1000. * Math.PI)
         this.tint = PIXI.utils.rgb2hex([1, v, v])
     }
 
-    terminating(time) {
+    terminating(_time) {
         const v = Math.sin(((1000 + PIXI.ticker.shared.lastTime) % 1000) / 1000. * Math.PI)
         this.cross.alpha = v
     }
 
     draw() {
 
-        // pod.status.containerStatuses might be undefined!
-        const containerStatuses = this.pod.status.containerStatuses || []
         let ready = 0
         let running = 0
         let restarts = 0
-        for (const containerStatus of containerStatuses) {
+        for (const container of this.pod.containers) {
-            if (containerStatus.ready) {
+            if (container.ready) {
                 ready++
             }
-            if (containerStatus.state.running) {
+            if (container.state && container.state.running) {
                 running++
             }
-            restarts += containerStatus.restartCount || 0
+            restarts += container.restartCount || 0
         }
-        const allReady = ready >= containerStatuses.length
+        const allReady = ready >= this.pod.containers.length
-        const allRunning = running >= containerStatuses.length
+        const allRunning = running >= this.pod.containers.length
         const resources = this.getResourceUsage()
 
         let newTick = null
@@ -164,8 +163,8 @@ export class Pod extends PIXI.Graphics {
         podBox.on('mouseover', function () {
             podBox.filters = podBox.filters.filter(x => x != BRIGHTNESS_FILTER).concat([BRIGHTNESS_FILTER])
             let s = this.pod.name
-            s += '\nStatus    : ' + this.pod.status.phase + ' (' + ready + '/' + containerStatuses.length + ' ready)'
+            s += '\nStatus    : ' + this.pod.phase + ' (' + ready + '/' + this.pod.containers.length + ' ready)'
-            s += '\nStart Time: ' + this.pod.status.startTime
+            s += '\nStart Time: ' + this.pod.startTime
             s += '\nLabels    :'
             for (var key of Object.keys(this.pod.labels).sort()) {
                 if (key !== 'pod-template-hash') {
@@ -173,15 +172,18 @@ export class Pod extends PIXI.Graphics {
                 }
             }
             s += '\nContainers:'
-            for (const containerStatus of containerStatuses) {
+            for (const container of this.pod.containers) {
-                const key = Object.keys(containerStatus.state)[0]
+                s += '\n  ' + container.name + ': '
-                s += '\n  ' + containerStatus.name + ': ' + key
+                if (container.state) {
-                if (containerStatus.state[key].reason) {
+                    const key = Object.keys(container.state)[0]
-                    // "CrashLoopBackOff"
+                    s += key
-                    s += ': ' + containerStatus.state[key].reason
+                    if (container.state[key].reason) {
+                        // "CrashLoopBackOff"
+                        s += ': ' + container.state[key].reason
+                    }
                 }
-                if (containerStatus.restartCount) {
+                if (container.restartCount) {
-                    s += ' (' + containerStatus.restartCount + ' restarts)'
+                    s += ' (' + container.restartCount + ' restarts)'
                 }
             }
             s += '\nCPU:'
@@ -202,23 +204,21 @@ export class Pod extends PIXI.Graphics {
             this.tooltip.visible = false
         })
         podBox.lineStyle(1, App.current.theme.primaryColor, 1)
-        let i = 0
         const w = 10 / this.pod.containers.length
-        for (const container of this.pod.containers) {
+        for (let i = 0; i < this.pod.containers.length; i++) {
             podBox.drawRect(i * w, 0, w, 10)
-            i++
         }
         let color
-        if (this.pod.status.phase == 'Succeeded') {
+        if (this.pod.phase == 'Succeeded') {
             // completed Job
             color = 0xaaaaff
-        } else if (this.pod.status.phase == 'Running' && allReady) {
+        } else if (this.pod.phase == 'Running' && allReady) {
             color = 0xaaffaa
-        } else if (this.pod.status.phase == 'Running' && allRunning && !allReady) {
+        } else if (this.pod.phase == 'Running' && allRunning && !allReady) {
             // all containers running, but some not ready (readinessProbe)
             newTick = this.pulsate
             color = 0xaaffaa
-        } else if (this.pod.status.phase == 'Pending') {
+        } else if (this.pod.phase == 'Pending') {
             newTick = this.pulsate
             color = 0xffffaa
         } else {

app/src/utils.js

@@ -73,7 +73,7 @@ const metric = (metric, type) =>
 
 const podResource = type => (containers, resource) =>
     containers
-        .map(({resources}) => metric(resources[resource], type))
+        .map(({resources}) => resources ? metric(resources[resource], type) : 0)
         .reduce((a, b) => a + b, 0)
 
 export {FACTORS, hsvToRgb, getBarColor, parseResource, metric, podResource}

app/src/vendor/json_delta.js

@@ -0,0 +1,533 @@
+/* JSON-delta v2.0 - A diff/patch pair for JSON-serialized data
+structures.
+
+Copyright 2013-2015 Philip J. Roberts <himself@phil-roberts.name>.
+All rights reserved
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+This implementation is based heavily on the original python2 version:
+see http://www.phil-roberts.name/json-delta/ for further
+documentation.  */
+
+export const JSON_delta = {
+    // Main entry points: ======================================================
+    patch: function(struc, diff) {
+        /* Apply the sequence of diff stanzas diff to the structure
+         struc, and returns the patched structure. */
+	var stan_key;
+        for (stan_key = 0; stan_key < diff.length; stan_key++) {
+            struc = this.patchStanza(struc, diff[stan_key]);
+        }
+        return struc;
+    },
+
+    diff: function(left, right, minimal, key) {
+	/* Build a diff between the structures left and right.
+
+	 Parameters:
+	     key: this is used for mutual recursion between this
+	         function and those it calls.  Normally it should be
+	         left unset or set as its default [].
+
+	     minimal: if this flag is set true, the function will try
+                 harder to find the diff that encodes as the shortest
+                 possible JSON string, at the expense of using more of
+                 both memory and processor time (as alternatives are
+                 computed and compared).
+	*/
+        key = key !== undefined ? key : [];
+        minimal = minimal !== undefined ? minimal : true;
+        var dumbdiff = [[key, right]], my_diff = [], common;
+
+	if (this.structureWorthInvestigating(left, right)) {
+	    common = this.commonality(left, right);
+            if (minimal) {
+		my_diff = this.needleDiff(left, right, minimal, key);
+            } else if (common < 0.5) {
+		my_diff = this.thisLevelDiff(left, right, key, common);
+            } else {
+		my_diff = this.keysetDiff(left, right, minimal, key);
+            }
+	} else {
+	    my_diff = this.thisLevelDiff(left, right, key, 0.0);
+	}
+
+        if (minimal) {
+            if (JSON.stringify(dumbdiff).length <
+                JSON.stringify(my_diff).length) {
+                my_diff = dumbdiff;
+            }
+        }
+
+        if (key.length === 0) {
+            if (my_diff.length > 1) {
+                my_diff = this.sortStanzas(my_diff);
+            }
+        }
+        return my_diff;
+    },
+
+    // =========================================================================
+
+    isStrictlyEqual: function(left, right) {
+	/* Recursively compare the (potentially nested) objects left
+	 * and right */
+	var idx, ks, key;
+        if (this.isTerminal(left) && this.isTerminal(right)) {
+            return (left === right);
+        }
+        if (this.isTerminal(left) || this.isTerminal(right)) {
+            return false;
+        }
+        if (left instanceof Array && right instanceof Array) {
+            if (left.length !== right.length) {
+                return false;
+            }
+            for (idx = 0; idx < left.length; idx++) {
+                if (! this.isStrictlyEqual(left[idx], right[idx])) {
+                    return false;
+                }
+            }
+            return true;
+        }
+        if (left instanceof Array || right instanceof Array) {
+            return false;
+        }
+        ks = this.computeKeysets(left, right);
+        if (ks[1].length !== 0 || ks[2].length !== 0) {
+            return false;
+        }
+        for (idx = 0; idx < ks[0].length; idx++) {
+            key = ks[0][idx];
+            if (! this.isStrictlyEqual(left[key], right[key])) {
+                return false;
+            }
+        }
+        return true;
+    },
+
+    isTerminal: function(obj) {
+	/* Test whether obj will be a terminal node in the tree when
+	 * serialized as JSON. */
+        if (typeof obj === 'string' || typeof obj === 'number' ||
+	    typeof obj === 'boolean' || obj === null) {
+            return true;
+        }
+        return false;
+    },
+
+    appendKey: function(stanzas, arr, key) {
+	/* Get the appropriate key for appending to the array arr,
+	 * assuming that stanzas will also be applied, and arr appears
+	 * at key within the overall structure. */
+	key = key !== undefined ? key : [];
+	var addition_key = arr.length, prior_key, i;
+	for (i = 0; i < stanzas.length; i++) {
+	    prior_key = stanzas[i][0];
+	    if (stanzas[i].length > 1 &&
+		prior_key.length === key.length + 1 &&
+		prior_key[prior_key.length-1] >= addition_key)
+	    { addition_key = prior_key[prior_key.length-1] + 1; }
+	}
+	return addition_key;
+    },
+
+    loopOver: function(obj, callback) {
+	/* Helper function for looping over obj.  Does the Right Thing
+	 * whether obj is an array or not. */
+	var i, key;
+	if (obj instanceof Array) {
+	    for (i = 0; i < obj.length; i++) {
+		callback(obj, i);
+	    }
+	} else {
+	    for (key in obj) {
+		if (obj.hasOwnProperty(key)) {
+		    callback(obj, key);
+		}
+	    }
+	}
+    },
+
+    inArray: function(keypath) {
+	var terminal = keypath[keypath.length - 1];
+	return (typeof terminal === 'number')
+    },
+
+    inObject: function(keypath) {
+	var terminal = keypath[keypath.length - 1];
+	return (typeof terminal === 'string')
+    },
+
+    splitDiff: function(diff) {
+	/* Split the stanzas in diff into an array of three arrays:
+	 * [modifications, deletions, insertions]. */
+	var idx, objs = [], mods = [], dels = [], inss = [];
+	var dests = {3: inss, 1: dels}, stanza, keypath;
+        if (diff.length === 0) {return [[], diff];}
+        for (idx = 0; idx < diff.length; idx++) {
+	    stanza = diff[idx]
+	    if (stanza.length === 2) {
+		if (this.inObject(stanza[0])) {
+		    objs.push(stanza);
+		} else {
+		    mods.push(stanza);
+		}
+	    } else {
+		dests[stanza.length].push(stanza)
+	    }
+        }
+        return [objs, mods, dels, inss];
+    },
+
+    stableKeypathLengthSort: function(stanzas) {
+	var comparator = function (a, b) {
+	    var swap;
+	    if (a[0].length === b[0].length) {
+		return a[0][0] - b[0][0];
+	    }
+	    return b[0].length - a[0].length;
+	}
+	for (var i = 0; i < stanzas.length; i++) {
+	    stanzas[i][0].unshift(i)
+	}
+	stanzas.sort(comparator)
+	for (i = 0; i < stanzas.length; i++) {
+	    stanzas[i][0].shift()
+	}
+	return stanzas
+    },
+
+    keypathCompare: function(a, b) {
+	a = a[0]; b = b[0];
+	if (a.length !== b.length) {
+	    return a.length - b.length;
+	}
+	for (var i = 0; i < a.length; i++) {
+	    if (typeof a[i] === 'number' && a[i] !== b[i]) {
+		return a[i] - b[i];
+	    }
+	}
+	return 0;
+    },
+
+    keypathCompareReverse: function(a, b) {
+	a = a[0]; b = b[0];
+	if (a.length !== b.length) {
+	    return b.length - a.length;
+	}
+	for (var i = 0; i < a.length; i++) {
+	    if (typeof a[i] === 'number' && a[i] !== b[i]) {
+		return b[i] - a[i];
+	    }
+	}
+	return 0;
+    },
+
+    sortStanzas: function(diff) {
+        /* Sorts the stanzas in a diff: object changes can occur in
+	 * any order, but deletions from arrays have to happen last
+	 * node first: ['foo', 'bar', 'baz'] -> ['foo', 'bar'] ->
+	 * ['foo'] -> []; additions to sequences have to happen
+	 * leftmost-node-first: [] -> ['foo'] -> ['foo', 'bar'] ->
+	 * ['foo', 'bar', 'baz'], and insert-and-shift alterations to
+	 * arrays must happen last. */
+
+        // First we divide the stanzas using splitDiff():
+        var split_thing = this.splitDiff(diff);
+	// Then we sort modifications of arrays in ascending order of keypath
+	// (note that we can?t tell appends from mods on the info available):
+        split_thing[1].sort(this.keypathCompare);
+        // Deletions from arrays in descending order of keypath:
+        split_thing[2].sort(this.keypathCompareReverse);
+	// And insert-and-shifts in ascending order of keypath:
+        split_thing[3].sort(this.keypathCompare)
+        diff = split_thing[0].concat(
+	    split_thing[1], split_thing[2], split_thing[3]
+	);
+	// Finally, we sort by length of keypath:
+	diff = this.stableKeypathLengthSort(diff, true)
+	return diff
+    },
+
+    computeKeysets: function(left, right) {
+        /* Returns an array of three arrays (overlap, left_only,
+         * right_only), representing the properties common to left and
+         * right, only defined for left, and only defined for right,
+         * respectively. */
+        var overlap = [], left_only = [], right_only = [];
+        var target = overlap;
+
+        this.loopOver(left, function(obj, key) {
+            if (right[key] !== undefined) {
+                target = overlap;
+            }
+            else {
+                target = left_only;
+            }
+            target.push(key);
+        });
+        this.loopOver(right, function(obj, key) {
+            if (left[key] === undefined) {
+                right_only.push(key);
+            }
+        });
+        return [overlap, left_only, right_only];
+    },
+
+    structureWorthInvestigating: function(left, right) {
+	/* Test whether it is worth looking at the internal structure
+	 * of `left` and `right` to see if they can be efficiently
+	 * diffed. */
+        if (this.isTerminal(left) || this.isTerminal(right)) {
+            return false;
+        }
+	if ((left.length === 0) || (right.length === 0)) {
+	    return false;
+	}
+        if ((left instanceof Array) && (right instanceof Array)) {
+	    return true;
+	}
+	if ((left instanceof Array) || (right instanceof Array)) {
+	    return false;
+	}
+	if ((typeof left === 'object') && (typeof right === 'object')) {
+	    return true;
+	}
+	return false;
+    },
+
+    commonality: function(left, right) {
+	/* Calculate the amount that the structures left and right
+	 * have in common */
+        var com = 0, tot = 0;
+        var elem, keysets, o, l, r, idx;
+        if (this.isTerminal(left) || this.isTerminal(right)) {
+            return 0;
+        }
+
+        if ((left instanceof Array) && (right instanceof Array)) {
+            for (idx = 0; idx < left.length; idx++) {
+                elem = left[idx];
+                if (right.indexOf(elem) !== -1) {
+                    com++;
+                }
+            }
+            tot = Math.max(left.length, right.length);
+        }
+        else {
+	    if ((left instanceof Array) || (right instanceof Array)) {
+		return 0;
+            }
+            keysets = this.computeKeysets(left, right);
+            o = keysets[0]; l = keysets[1]; r = keysets[2];
+            com = o.length;
+            tot = o.length + l.length + r.length;
+            for (idx = 0; idx < r.length; idx++) {
+                elem = r[idx];
+                if (l.indexOf(elem) === -1) {
+                    tot++;
+                }
+            }
+        }
+        if (tot === 0) {return 0;}
+        return com / tot;
+    },
+
+    thisLevelDiff: function(left, right, key, common) {
+        /* Returns a sequence of diff stanzas between the objects left
+	 * and right, assuming that they are each at the position key
+	 * within the overall structure. */
+        var out = [], idx, okey;
+        key = key !== undefined ? key : [];
+
+        if (common === undefined) {
+            common = this.commonality(left, right);
+        }
+
+        if (common) {
+            var ks = this.computeKeysets(left, right);
+            for (idx = 0; idx < ks[0].length; idx++) {
+                okey = ks[0][idx];
+                if (left[okey] !== right[okey]) {
+                    out.push([key.concat([okey]), right[okey]]);
+                }
+            }
+            for (idx = 0; idx < ks[1].length; idx++) {
+                okey = ks[1][idx];
+                out.push([key.concat([okey])]);
+            }
+            for (idx = 0; idx < ks[2].length; idx++) {
+                okey = ks[2][idx];
+                out.push([key.concat([okey]), right[okey]]);
+            }
+            return out;
+        }
+        if (! this.isStrictlyEqual(left, right)) {
+            return [[key, right]];
+        }
+        return [];
+    },
+
+    keysetDiff: function(left, right, minimal, key) {
+	/* Compute a diff between left and right, without treating
+	 * arrays differently from objects. */
+        minimal = minimal !== undefined ? minimal : true;
+        var out = [], k;
+        var ks = this.computeKeysets(left, right);
+        for (k = 0; k < ks[1].length; k++) {
+            out.push([key.concat(ks[1][k])]);
+        }
+        for (k = 0; k < ks[2].length; k++) {
+            out.push([key.concat(ks[2][k]), right[ks[2][k]]]);
+        }
+        for (k = 0; k < ks[0].length; k++) {
+            out = out.concat(this.diff(left[ks[0][k]], right[ks[0][k]],
+                                       minimal, key.concat([ks[0][k]])));
+        }
+        return out;
+    },
+
+    needleDiff: function(left, right, minimal, key) {
+	/* Compute a diff between left and right.  If both are arrays,
+	 * a variant of Needleman-Wunsch sequence alignment is used to
+	 * make the diff minimal (at a significant cost in both
+	 * storage and processing).  Otherwise, the parms are passed on
+	 * to keysetDiff.*/
+        if (! (left instanceof Array && right instanceof Array)) {
+	    return this.keysetDiff(left, right, minimal, key);
+	}
+        minimal = minimal !== undefined ? minimal : true;
+	var down_col = 0, lastrow = [], i, sub_i, left_i, right_i, col_i;
+	var row, first_left_i, left_elem, right_elem;
+	var cand_length, win_length, cand, winner;
+
+	var modify_cand = function () {
+	    if (col_i + 1 < lastrow.length) {
+		return lastrow[col_i+1].concat(
+		    JSON_delta.diff(left_elem, right_elem,
+				    minimal, key.concat([left_i]))
+		);
+	    }
+	};
+
+	var delete_cand = function () {
+	    if (row.length > 0) {
+		return row[0].concat([[key.concat([left_i])]]);
+	    }
+	};
+
+	var append_cand = function () {
+	    if (col_i === down_col) {
+		return lastrow[col_i].concat(
+		    [[key.concat([JSON_delta.appendKey(lastrow[col_i], left, key)]),
+		      right_elem]]
+		);
+	    }
+	};
+
+	var insert_cand = function () {
+	    if (col_i !== down_col) {
+		return lastrow[col_i].concat(
+		    [[key.concat([right_i]), right_elem, "i"]]
+		);
+	    }
+	};
+
+	var cand_funcs = [modify_cand, delete_cand, append_cand, insert_cand];
+
+	for (i = 0; i <= left.length; i++) {
+	    lastrow.unshift([]);
+	    for (sub_i = 0; sub_i < i; sub_i++) {
+		lastrow[0].push([key.concat([sub_i])]);
+	    }
+	}
+
+	for (right_i = 0; right_i < right.length; right_i++) {
+	    right_elem = right[right_i];
+	    row = []
+	    for (left_i = 0; left_i < left.length; left_i++) {
+		left_elem = left[left_i];
+		col_i = left.length - left_i - 1;
+		win_length = Infinity;
+		for (i = 0; i < cand_funcs.length; i++) {
+		    cand = cand_funcs[i]();
+		    if (cand !== undefined) {
+			cand_length = JSON.stringify(cand).length;
+			if (cand_length < win_length) {
+			    winner = cand;
+			    win_length = cand_length;
+			}
+		    }
+		}
+		row.unshift(winner);
+	    }
+	    lastrow = row;
+	}
+	return winner;
+    },
+
+    patchStanza: function(struc, diff) {
+        /* Applies the diff stanza diff to the structure struc.
+         Returns the modified structure. */
+        var key = diff[0];
+        switch (key.length) {
+        case 0:
+            struc = diff[1];
+            break;
+        case 1:
+            if (diff.length === 1) {
+                if (struc.splice === undefined) {
+                    delete struc[key[0]];
+                }
+                else {
+                    struc.splice(key[0], 1);
+                }
+            } else if (diff.length === 3) {
+		if (struc.splice === undefined) {
+                    struc[key[0]] = diff[1];
+		} else {
+		    struc.splice(key[0], 0, diff[1]);
+		}
+	    }
+            else {
+                struc[key[0]] = diff[1];
+            }
+            break;
+        default:
+            var pass_key = key.slice(1), pass_struc = struc[key[0]];
+            var pass_diff = [pass_key].concat(diff.slice(1));
+	    if (pass_struc === undefined) {
+		if (typeof pass_key[0] === 'string') {
+		    pass_struc = {};
+		} else {
+		    pass_struc = [];
+		}
+	    }
+            struc[key[0]] = this.patchStanza(pass_struc, pass_diff);
+        }
+        return struc;
+    }
+};

requirements.txt

@@ -4,3 +4,4 @@ gevent
 requests
 stups-tokens>=1.1.19
 redlock-py
+json_delta>=2.0