lnmch/kops-arm64
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #87 from hjacobs/json-delta

Browse Source 
JSON delta
This commit is contained in:
Henning Jacobs
2017-01-14 02:16:12 +01:00
committed by GitHub
parent c7ca6d3492 5eddd970d2
commit b6d75bf240
11 changed files with 754 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

140
app.py
View File

@@ -9,6 +9,7 @@ import functools
import gevent
import gevent.wsgi
import json
import json_delta
import logging
import os
import re
@@ -60,9 +61,16 @@ class MemoryStore:
'''Memory-only backend, mostly useful for local debugging'''
def __init__(self):
self._data = {}
self._queues = []
self._screen_tokens = {}
def set(self, key, value):
self._data[key] = value
def get(self, key):
return self._data.get(key)
def acquire_lock(self):
# no-op for memory store
return 'fake-lock'
@@ -105,6 +113,14 @@ class RedisStore:
self._redis = redis.StrictRedis.from_url(url)
self._redlock = Redlock([url])
def set(self, key, value):
self._redis.set(key, json.dumps(value, separators=(',', ':')))
def get(self, key):
value = self._redis.get(key)
if value:
return json.loads(value.decode('utf-8'))
def acquire_lock(self):
return self._redlock.lock('update', 10000)
@@ -147,6 +163,7 @@ def get_bool(name: str):
return os.getenv(name, '').lower() in ('1', 'true')
DEBUG = get_bool('DEBUG')
SERVER_PORT = int(os.getenv('SERVER_PORT', 8080))
SERVER_STATUS = {'shutdown': False}
DEFAULT_CLUSTERS = 'http://localhost:8001/'
@@ -158,7 +175,7 @@ REDIS_URL = os.getenv('REDIS_URL')
STORE = RedisStore(REDIS_URL) if REDIS_URL else MemoryStore()
app = Flask(__name__)
app.debug = get_bool('DEBUG')
app.debug = DEBUG
app.secret_key = os.getenv('SECRET_KEY', 'development')
oauth = OAuth(app)
@@ -250,7 +267,7 @@ def generate_mock_pod(index: int, i: int, j: int):
'agent-cooper',
'black-lodge',
'bob',
'bobby-briggs'
'bobby-briggs',
'laura-palmer',
'leland-palmer',
'log-lady',
@@ -261,14 +278,18 @@ def generate_mock_pod(index: int, i: int, j: int):
phase = pod_phases[hash_int((index + 1) * (i + 1) * (j + 1)) % len(pod_phases)]
containers = []
for k in range(1 + j % 2):
containers.append({'name': 'myapp', 'image': 'foo/bar/{}'.format(j), 'resources': {'requests': {'cpu': '100m', 'memory': '100Mi'}, 'limits': {}}})
status = {'phase': phase}
container = {
'name': 'myapp', 'image': 'foo/bar/{}'.format(j), 'resources': {'requests': {'cpu': '100m', 'memory': '100Mi'}, 'limits': {}},
'ready': True,
'state': {'running': {}}
}
if phase == 'Running':
if j % 13 == 0:
status['containerStatuses'] = [{'ready': False, 'state': {'waiting': {'reason': 'CrashLoopBackOff'}}}]
container.update(**{'ready': False, 'state': {'waiting': {'reason': 'CrashLoopBackOff'}}})
elif j % 7 == 0:
status['containerStatuses'] = [{'ready': True, 'state': {'running': {}}, 'restartCount': 3}]
pod = {'name': '{}-{}-{}'.format(names[hash_int((i + 1) * (j + 1)) % len(names)], i, j), 'namespace': 'kube-system' if j < 3 else 'default', 'labels': labels, 'status': status, 'containers': containers}
container.update(**{'ready': True, 'state': {'running': {}}, 'restartCount': 3})
containers.append(container)
pod = {'name': '{}-{}-{}'.format(names[hash_int((i + 1) * (j + 1)) % len(names)], i, j), 'namespace': 'kube-system' if j < 3 else 'default', 'labels': labels, 'phase': phase, 'containers': containers}
if phase == 'Running' and j % 17 == 0:
pod['deleted'] = 123
@@ -285,7 +306,7 @@ def generate_cluster_id(url: str):
def generate_mock_cluster_data(index: int):
'''Generate deterministic (no randomness!) mock data'''
nodes = []
nodes = {}
for i in range(10):
# add/remove the second to last node every 13 seconds
if i == 8 and int(time.time() / 13) % 2 == 0:
@@ -293,15 +314,18 @@ def generate_mock_cluster_data(index: int):
labels = {}
if i < 2:
labels['master'] = 'true'
pods = []
pods = {}
for j in range(hash_int((index + 1) * (i + 1)) % 32):
# add/remove some pods every 6 seconds
if j % 17 == 0 and int(time.time() / 6) % 2 == 0:
# add/remove some pods every 7 seconds
if j % 17 == 0 and int(time.time() / 7) % 2 == 0:
pass
else:
pods.append(generate_mock_pod(index, i, j))
nodes.append({'name': 'node-{}'.format(i), 'labels': labels, 'status': {'capacity': {'cpu': '4', 'memory': '32Gi', 'pods': '110'}}, 'pods': pods})
unassigned_pods = [generate_mock_pod(index, 11, index)]
pod = generate_mock_pod(index, i, j)
pods['{}/{}'.format(pod['namespace'], pod['name'])] = pod
node = {'name': 'node-{}'.format(i), 'labels': labels, 'status': {'capacity': {'cpu': '4', 'memory': '32Gi', 'pods': '110'}}, 'pods': pods}
nodes[node['name']] = node
pod = generate_mock_pod(index, 11, index)
unassigned_pods = {'{}/{}'.format(pod['namespace'], pod['name']): pod}
return {
'id': 'mock-cluster-{}'.format(index),
'api_server_url': 'https://kube-{}.example.org'.format(index),
@@ -316,6 +340,41 @@ def get_mock_clusters():
yield data
def map_node_status(status: dict):
return {
'addresses': status.get('addresses'),
'capacity': status.get('capacity'),
}
def map_node(node: dict):
return {
'name': node['metadata']['name'],
'labels': node['metadata']['labels'],
'status': map_node_status(node['status']),
'pods': {}
}
def map_pod(pod: dict):
return {
'name': pod['metadata']['name'],
'namespace': pod['metadata']['namespace'],
'labels': pod['metadata'].get('labels', {}),
'phase': pod['status'].get('phase'),
'startTime': pod['status']['startTime'] if 'startTime' in pod['status'] else '',
'containers': []
}
def map_container(cont: dict, pod: dict):
obj = {'name': cont['name'], 'image': cont['image'], 'resources': cont['resources']}
status = list([s for s in pod.get('status', {}).get('containerStatuses', []) if s['name'] == cont['name']])
if status:
obj.update(**status[0])
return obj
def get_kubernetes_clusters():
for api_server_url in (os.getenv('CLUSTERS') or DEFAULT_CLUSTERS).split(','):
cluster_id = generate_cluster_id(api_server_url)
@@ -324,36 +383,28 @@ def get_kubernetes_clusters():
session.headers['Authorization'] = 'Bearer {}'.format(tokens.get('read-only'))
response = session.get(urljoin(api_server_url, '/api/v1/nodes'), timeout=5)
response.raise_for_status()
nodes = []
nodes_by_name = {}
nodes = {}
pods_by_namespace_name = {}
unassigned_pods = []
unassigned_pods = {}
for node in response.json()['items']:
obj = {'name': node['metadata']['name'], 'labels': node['metadata']['labels'], 'status': node['status'],
'pods': []}
nodes.append(obj)
nodes_by_name[obj['name']] = obj
obj = map_node(node)
nodes[obj['name']] = obj
response = session.get(urljoin(api_server_url, '/api/v1/pods'), timeout=5)
response.raise_for_status()
for pod in response.json()['items']:
obj = {'name': pod['metadata']['name'],
'namespace': pod['metadata']['namespace'],
'labels': pod['metadata'].get('labels', {}),
'status': pod['status'],
'startTime': pod['status']['startTime'] if 'startTime' in pod['status'] else '',
'containers': []
}
obj = map_pod(pod)
if 'deletionTimestamp' in pod['metadata']:
obj['deleted'] = datetime.datetime.strptime(pod['metadata']['deletionTimestamp'],
'%Y-%m-%dT%H:%M:%SZ').replace(
tzinfo=datetime.timezone.utc).timestamp()
for cont in pod['spec']['containers']:
obj['containers'].append({'name': cont['name'], 'image': cont['image'], 'resources': cont['resources']})
obj['containers'].append(map_container(cont, pod))
pods_by_namespace_name[(obj['namespace'], obj['name'])] = obj
if 'nodeName' in pod['spec'] and pod['spec']['nodeName'] in nodes_by_name:
nodes_by_name[pod['spec']['nodeName']]['pods'].append(obj)
pod_key = '{}/{}'.format(obj['namespace'], obj['name'])
if 'nodeName' in pod['spec'] and pod['spec']['nodeName'] in nodes:
nodes[pod['spec']['nodeName']]['pods'][pod_key] = obj
else:
unassigned_pods.append(obj)
unassigned_pods[pod_key] = obj
try:
response = session.get(urljoin(api_server_url, '/api/v1/namespaces/kube-system/services/heapster/proxy/apis/metrics/v1alpha1/nodes'), timeout=5)
@@ -363,7 +414,7 @@ def get_kubernetes_clusters():
logging.info('Heapster node metrics not available (yet)')
else:
for metrics in data['items']:
nodes_by_name[metrics['metadata']['name']]['usage'] = metrics['usage']
nodes[metrics['metadata']['name']]['usage'] = metrics['usage']
except:
logging.exception('Failed to get node metrics')
try:
@@ -388,10 +439,16 @@ def get_kubernetes_clusters():
def event(cluster_ids: set):
# first sent full data once
for cluster_id in (STORE.get('cluster-ids') or []):
if not cluster_ids or cluster_id in cluster_ids:
cluster = STORE.get(cluster_id)
yield 'event: clusterupdate\ndata: ' + json.dumps(cluster, separators=(',', ':')) + '\n\n'
while True:
for event_type, cluster in STORE.listen():
if not cluster_ids or cluster['id'] in cluster_ids:
yield 'event: ' + event_type + '\ndata: ' + json.dumps(cluster, separators=(',', ':')) + '\n\n'
for event_type, event_data in STORE.listen():
# hacky, event_data can be delta or full cluster object
if not cluster_ids or event_data.get('cluster_id', event_data.get('id')) in cluster_ids:
yield 'event: ' + event_type + '\ndata: ' + json.dumps(event_data, separators=(',', ':')) + '\n\n'
@app.route('/events')
@@ -461,8 +518,19 @@ def update():
clusters = get_mock_clusters()
else:
clusters = get_kubernetes_clusters()
cluster_ids = []
for cluster in clusters:
old_data = STORE.get(cluster['id'])
if old_data:
# https://pikacode.com/phijaro/json_delta/ticket/11/
# diff is extremely slow without array_align=False
delta = json_delta.diff(old_data, cluster, verbose=DEBUG, array_align=False)
STORE.publish('clusterdelta', {'cluster_id': cluster['id'], 'delta': delta})
else:
STORE.publish('clusterupdate', cluster)
STORE.set(cluster['id'], cluster)
cluster_ids.append(cluster['id'])
STORE.set('cluster-ids', cluster_ids)
except:
logging.exception('Failed to update')
finally:

1
app/.eslintignore Normal file
View File

@@ -0,0 +1 @@
src/vendor/*.js

1
app/.eslintrc.yml
View File

@@ -21,6 +21,7 @@ rules:
- error
no-unused-vars:
- warn
- argsIgnorePattern: "^_"
semi:
- error
- never

4
app/package.json
View File

@@ -27,7 +27,7 @@
"homepage": "https://github.com/hjacobs/kube-ops-view#readme",
"dependencies": {
"pixi-display": "^1.0.1",
"pixi.js": "^4.3.0",
"pixi.js": "^4.3.2",
"webpack-dev-server": "^1.16.2"
},
"devDependencies": {
@@ -36,7 +36,7 @@
"babel-preset-es2015": "^6.18.0",
"babel-runtime": "^6.20.0",
"brfs": "^1.4.3",
"eslint": "^3.12.2",
"eslint": "^3.13.1",
"eslint-loader": "^1.6.1",
"rimraf": "^2.5.4",
"transform-loader": "^0.2.3",

81
app/src/app.js
View File

@@ -4,6 +4,7 @@ import {Pod, ALL_PODS, sortByName, sortByMemory, sortByCPU, sortByAge} from './p
import SelectBox from './selectbox'
import { Theme, ALL_THEMES} from './themes.js'
import { DESATURATION_FILTER } from './filters.js'
import { JSON_delta } from './vendor/json_delta.js'
const PIXI = require('pixi.js')
@@ -18,6 +19,12 @@ export default class App {
this.sorterFn = ''
this.theme = Theme.get(localStorage.getItem('theme'))
this.eventSource = null
this.connectTime = null
this.keepAliveTimer = null
// make sure we got activity at least every 20 seconds
this.keepAliveSeconds = 20
// always reconnect after 5 minutes
this.maxConnectionLifetimeSeconds = 300
this.clusters = new Map()
}
@@ -262,17 +269,19 @@ export default class App {
this.stage.addChild(pod)
}
update() {
// make sure we create a copy (this.clusters might get modified)
const clusters = Array.from(this.clusters.entries()).sort().map(idCluster => idCluster[1])
const that = this
let changes = 0
const firstTime = this.seenPods.size == 0
const podKeys = new Set()
for (const cluster of this.clusters.values()) {
for (const node of cluster.nodes) {
for (const pod of node.pods) {
for (const cluster of clusters) {
for (const node of Object.values(cluster.nodes)) {
for (const pod of Object.values(node.pods)) {
podKeys.add(cluster.id + '/' + pod.namespace + '/' + pod.name)
}
}
for (const pod of cluster.unassigned_pods) {
for (const pod of Object.values(cluster.unassigned_pods)) {
podKeys.add(cluster.id + '/' + pod.namespace + '/' + pod.name)
}
}
@@ -301,10 +310,10 @@ export default class App {
}
let y = 0
const clusterIds = new Set()
for (const [clusterId, cluster] of Array.from(this.clusters.entries()).sort()) {
if (!this.selectedClusters.size || this.selectedClusters.has(clusterId)) {
clusterIds.add(clusterId)
let clusterBox = clusterComponentById[clusterId]
for (const cluster of clusters) {
if (!this.selectedClusters.size || this.selectedClusters.has(cluster.id)) {
clusterIds.add(cluster.id)
let clusterBox = clusterComponentById[cluster.id]
if (!clusterBox) {
clusterBox = new Cluster(cluster, this.tooltip)
this.viewContainer.addChild(clusterBox)
@@ -340,7 +349,7 @@ export default class App {
}
}
tick(time) {
tick(_time) {
this.renderer.render(this.stage)
}
@@ -364,15 +373,35 @@ export default class App {
}
this.changeLocationHash('clusters', Array.from(this.selectedClusters).join(','))
// make sure we are updating our EventSource filter
this.listen()
this.connect()
this.update()
}
listen() {
keepAlive() {
if (this.keepAliveTimer != null) {
clearTimeout(this.keepAliveTimer)
}
this.keepAliveTimer = setTimeout(this.connect.bind(this), this.keepAliveSeconds * 1000)
if (this.connectTime != null) {
const now = Date.now()
if (now - this.connectTime > this.maxConnectionLifetimeSeconds * 1000) {
// maximum connection lifetime exceeded => reconnect
this.connect()
}
}
}
disconnect() {
if (this.eventSource != null) {
this.eventSource.close()
this.eventSource = null
this.connectTime = null
}
}
connect() {
// first close the old connection
this.disconnect()
const that = this
// NOTE: path must be relative to work with kubectl proxy out of the box
let url = 'events'
@@ -381,20 +410,44 @@ export default class App {
url += '?cluster_ids=' + clusterIds
}
const eventSource = this.eventSource = new EventSource(url, {credentials: 'include'})
eventSource.onerror = function(event) {
that.listen()
this.keepAlive()
eventSource.onerror = function(_event) {
that._errors++
if (that._errors <= 1) {
// immediately reconnect on first error
that.connect()
} else {
// rely on keep-alive timer to reconnect
that.disconnect()
}
}
eventSource.addEventListener('clusterupdate', function(event) {
that._errors = 0
that.keepAlive()
const cluster = JSON.parse(event.data)
that.clusters.set(cluster.id, cluster)
that.update()
})
eventSource.addEventListener('clusterdelta', function(event) {
that._errors = 0
that.keepAlive()
const data = JSON.parse(event.data)
let cluster = that.clusters.get(data.cluster_id)
if (cluster && data.delta) {
// deep copy cluster object (patch function mutates inplace!)
cluster = JSON.parse(JSON.stringify(cluster))
cluster = JSON_delta.patch(cluster, data.delta)
that.clusters.set(cluster.id, cluster)
that.update()
}
})
this.connectTime = Date.now()
}
run() {
this.initialize()
this.draw()
this.listen()
this.connect()
PIXI.ticker.shared.add(this.tick, this)
}

7
app/src/cluster.js
View File

@@ -26,7 +26,8 @@ export default class Cluster extends PIXI.Graphics {
let workerHeight = 0
const workerNodes = []
const maxWidth = window.innerWidth - 130
for (const node of this.cluster.nodes) {
for (const nodeName of Object.keys(this.cluster.nodes).sort()) {
const node = this.cluster.nodes[nodeName]
var nodeBox = new Node(node, this, this.tooltip)
nodeBox.draw()
if (nodeBox.isMaster()) {
@@ -64,7 +65,7 @@ export default class Cluster extends PIXI.Graphics {
}
for (const pod of this.cluster.unassigned_pods) {
for (const pod of Object.values(this.cluster.unassigned_pods)) {
var podBox = Pod.getOrCreate(pod, this, this.tooltip)
podBox.x = masterX
podBox.y = masterY
@@ -86,7 +87,7 @@ export default class Cluster extends PIXI.Graphics {
topHandle.interactive = true
topHandle.buttonMode = true
const that = this
topHandle.on('click', function(event) {
topHandle.on('click', function(_event) {
App.current.toggleCluster(that.cluster.id)
})
var text = new PIXI.Text(this.cluster.api_server_url, {fontFamily: 'ShareTechMono', fontSize: 10, fill: 0x000000})

10
app/src/node.js
View File

@@ -30,7 +30,9 @@ export default class Node extends PIXI.Graphics {
resources[key]['used'] = parseResource(this.node.usage[key])
}
}
for (const pod of this.node.pods) {
let numberOfPods = 0
for (const pod of Object.values(this.node.pods)) {
numberOfPods++
for (const container of pod.containers) {
if (container.resources && container.resources.requests) {
for (const key of Object.keys(container.resources.requests)) {
@@ -39,8 +41,8 @@ export default class Node extends PIXI.Graphics {
}
}
}
resources['pods'].requested = this.node.pods.length
resources['pods'].used = this.node.pods.length
resources['pods'].requested = numberOfPods
resources['pods'].used = numberOfPods
return resources
}
@@ -89,7 +91,7 @@ export default class Node extends PIXI.Graphics {
const nodeBox = this
let px = 24
let py = 20
const pods = this.node.pods.sort(sorterFn)
const pods = Object.values(this.node.pods).sort(sorterFn)
for (const pod of pods) {
if (pod.namespace != 'kube-system') {
const podBox = Pod.getOrCreate(pod, this.cluster, this.tooltip)

58
app/src/pod.js
View File

@@ -10,8 +10,8 @@ const sortByName = (a, b) => {
}
const sortByAge = (a, b) => {
const dateA = new Date(a.status.startTime)
const dateB = new Date(b.status.startTime)
const dateA = new Date(a.startTime)
const dateB = new Date(b.startTime)
if (dateA.getTime() < dateB.getTime()) {
return -1
} else if (dateA.getTime() === dateB.getTime())
@@ -54,6 +54,7 @@ export class Pod extends PIXI.Graphics {
if (this.tick) {
PIXI.ticker.shared.remove(this.tick, this)
}
PIXI.ticker.shared.remove(this.animateMove, this)
super.destroy()
}
@@ -122,39 +123,37 @@ export class Pod extends PIXI.Graphics {
}
}
pulsate(time) {
pulsate(_time) {
const v = Math.sin((PIXI.ticker.shared.lastTime % 1000) / 1000. * Math.PI)
this.alpha = v * this._progress
}
crashing(time) {
crashing(_time) {
const v = Math.sin((PIXI.ticker.shared.lastTime % 1000) / 1000. * Math.PI)
this.tint = PIXI.utils.rgb2hex([1, v, v])
}
terminating(time) {
terminating(_time) {
const v = Math.sin(((1000 + PIXI.ticker.shared.lastTime) % 1000) / 1000. * Math.PI)
this.cross.alpha = v
}
draw() {
// pod.status.containerStatuses might be undefined!
const containerStatuses = this.pod.status.containerStatuses || []
let ready = 0
let running = 0
let restarts = 0
for (const containerStatus of containerStatuses) {
if (containerStatus.ready) {
for (const container of this.pod.containers) {
if (container.ready) {
ready++
}
if (containerStatus.state.running) {
if (container.state && container.state.running) {
running++
}
restarts += containerStatus.restartCount || 0
restarts += container.restartCount || 0
}
const allReady = ready >= containerStatuses.length
const allRunning = running >= containerStatuses.length
const allReady = ready >= this.pod.containers.length
const allRunning = running >= this.pod.containers.length
const resources = this.getResourceUsage()
let newTick = null
@@ -164,8 +163,8 @@ export class Pod extends PIXI.Graphics {
podBox.on('mouseover', function () {
podBox.filters = podBox.filters.filter(x => x != BRIGHTNESS_FILTER).concat([BRIGHTNESS_FILTER])
let s = this.pod.name
s += '\nStatus : ' + this.pod.status.phase + ' (' + ready + '/' + containerStatuses.length + ' ready)'
s += '\nStart Time: ' + this.pod.status.startTime
s += '\nStatus : ' + this.pod.phase + ' (' + ready + '/' + this.pod.containers.length + ' ready)'
s += '\nStart Time: ' + this.pod.startTime
s += '\nLabels :'
for (var key of Object.keys(this.pod.labels).sort()) {
if (key !== 'pod-template-hash') {
@@ -173,15 +172,18 @@ export class Pod extends PIXI.Graphics {
}
}
s += '\nContainers:'
for (const containerStatus of containerStatuses) {
const key = Object.keys(containerStatus.state)[0]
s += '\n ' + containerStatus.name + ': ' + key
if (containerStatus.state[key].reason) {
for (const container of this.pod.containers) {
s += '\n ' + container.name + ': '
if (container.state) {
const key = Object.keys(container.state)[0]
s += key
if (container.state[key].reason) {
// "CrashLoopBackOff"
s += ': ' + containerStatus.state[key].reason
s += ': ' + container.state[key].reason
}
if (containerStatus.restartCount) {
s += ' (' + containerStatus.restartCount + ' restarts)'
}
if (container.restartCount) {
s += ' (' + container.restartCount + ' restarts)'
}
}
s += '\nCPU:'
@@ -202,23 +204,21 @@ export class Pod extends PIXI.Graphics {
this.tooltip.visible = false
})
podBox.lineStyle(1, App.current.theme.primaryColor, 1)
let i = 0
const w = 10 / this.pod.containers.length
for (const container of this.pod.containers) {
for (let i = 0; i < this.pod.containers.length; i++) {
podBox.drawRect(i * w, 0, w, 10)
i++
}
let color
if (this.pod.status.phase == 'Succeeded') {
if (this.pod.phase == 'Succeeded') {
// completed Job
color = 0xaaaaff
} else if (this.pod.status.phase == 'Running' && allReady) {
} else if (this.pod.phase == 'Running' && allReady) {
color = 0xaaffaa
} else if (this.pod.status.phase == 'Running' && allRunning && !allReady) {
} else if (this.pod.phase == 'Running' && allRunning && !allReady) {
// all containers running, but some not ready (readinessProbe)
newTick = this.pulsate
color = 0xaaffaa
} else if (this.pod.status.phase == 'Pending') {
} else if (this.pod.phase == 'Pending') {
newTick = this.pulsate
color = 0xffffaa
} else {

2
app/src/utils.js
View File

@@ -73,7 +73,7 @@ const metric = (metric, type) =>
const podResource = type => (containers, resource) =>
containers
.map(({resources}) => metric(resources[resource], type))
.map(({resources}) => resources ? metric(resources[resource], type) : 0)
.reduce((a, b) => a + b, 0)
export {FACTORS, hsvToRgb, getBarColor, parseResource, metric, podResource}

533
app/src/vendor/json_delta.js vendored Normal file
View File

@@ -0,0 +1,533 @@
/* JSON-delta v2.0 - A diff/patch pair for JSON-serialized data
structures.
Copyright 2013-2015 Philip J. Roberts <himself@phil-roberts.name>.
All rights reserved
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
This implementation is based heavily on the original python2 version:
see http://www.phil-roberts.name/json-delta/ for further
documentation. */
export const JSON_delta = {
// Main entry points: ======================================================
patch: function(struc, diff) {
/* Apply the sequence of diff stanzas diff to the structure
struc, and returns the patched structure. */
var stan_key;
for (stan_key = 0; stan_key < diff.length; stan_key++) {
struc = this.patchStanza(struc, diff[stan_key]);
}
return struc;
},
diff: function(left, right, minimal, key) {
/* Build a diff between the structures left and right.
Parameters:
key: this is used for mutual recursion between this
function and those it calls. Normally it should be
left unset or set as its default [].
minimal: if this flag is set true, the function will try
harder to find the diff that encodes as the shortest
possible JSON string, at the expense of using more of
both memory and processor time (as alternatives are
computed and compared).
*/
key = key !== undefined ? key : [];
minimal = minimal !== undefined ? minimal : true;
var dumbdiff = [[key, right]], my_diff = [], common;
if (this.structureWorthInvestigating(left, right)) {
common = this.commonality(left, right);
if (minimal) {
my_diff = this.needleDiff(left, right, minimal, key);
} else if (common < 0.5) {
my_diff = this.thisLevelDiff(left, right, key, common);
} else {
my_diff = this.keysetDiff(left, right, minimal, key);
}
} else {
my_diff = this.thisLevelDiff(left, right, key, 0.0);
}
if (minimal) {
if (JSON.stringify(dumbdiff).length <
JSON.stringify(my_diff).length) {
my_diff = dumbdiff;
}
}
if (key.length === 0) {
if (my_diff.length > 1) {
my_diff = this.sortStanzas(my_diff);
}
}
return my_diff;
},
// =========================================================================
isStrictlyEqual: function(left, right) {
/* Recursively compare the (potentially nested) objects left
* and right */
var idx, ks, key;
if (this.isTerminal(left) && this.isTerminal(right)) {
return (left === right);
}
if (this.isTerminal(left) || this.isTerminal(right)) {
return false;
}
if (left instanceof Array && right instanceof Array) {
if (left.length !== right.length) {
return false;
}
for (idx = 0; idx < left.length; idx++) {
if (! this.isStrictlyEqual(left[idx], right[idx])) {
return false;
}
}
return true;
}
if (left instanceof Array || right instanceof Array) {
return false;
}
ks = this.computeKeysets(left, right);
if (ks[1].length !== 0 || ks[2].length !== 0) {
return false;
}
for (idx = 0; idx < ks[0].length; idx++) {
key = ks[0][idx];
if (! this.isStrictlyEqual(left[key], right[key])) {
return false;
}
}
return true;
},
isTerminal: function(obj) {
/* Test whether obj will be a terminal node in the tree when
* serialized as JSON. */
if (typeof obj === 'string' || typeof obj === 'number' ||
typeof obj === 'boolean' || obj === null) {
return true;
}
return false;
},
appendKey: function(stanzas, arr, key) {
/* Get the appropriate key for appending to the array arr,
* assuming that stanzas will also be applied, and arr appears
* at key within the overall structure. */
key = key !== undefined ? key : [];
var addition_key = arr.length, prior_key, i;
for (i = 0; i < stanzas.length; i++) {
prior_key = stanzas[i][0];
if (stanzas[i].length > 1 &&
prior_key.length === key.length + 1 &&
prior_key[prior_key.length-1] >= addition_key)
{ addition_key = prior_key[prior_key.length-1] + 1; }
}
return addition_key;
},
loopOver: function(obj, callback) {
/* Helper function for looping over obj. Does the Right Thing
* whether obj is an array or not. */
var i, key;
if (obj instanceof Array) {
for (i = 0; i < obj.length; i++) {
callback(obj, i);
}
} else {
for (key in obj) {
if (obj.hasOwnProperty(key)) {
callback(obj, key);
}
}
}
},
inArray: function(keypath) {
var terminal = keypath[keypath.length - 1];
return (typeof terminal === 'number')
},
inObject: function(keypath) {
var terminal = keypath[keypath.length - 1];
return (typeof terminal === 'string')
},
splitDiff: function(diff) {
/* Split the stanzas in diff into an array of three arrays:
* [modifications, deletions, insertions]. */
var idx, objs = [], mods = [], dels = [], inss = [];
var dests = {3: inss, 1: dels}, stanza, keypath;
if (diff.length === 0) {return [[], diff];}
for (idx = 0; idx < diff.length; idx++) {
stanza = diff[idx]
if (stanza.length === 2) {
if (this.inObject(stanza[0])) {
objs.push(stanza);
} else {
mods.push(stanza);
}
} else {
dests[stanza.length].push(stanza)
}
}
return [objs, mods, dels, inss];
},
stableKeypathLengthSort: function(stanzas) {
var comparator = function (a, b) {
var swap;
if (a[0].length === b[0].length) {
return a[0][0] - b[0][0];
}
return b[0].length - a[0].length;
}
for (var i = 0; i < stanzas.length; i++) {
stanzas[i][0].unshift(i)
}
stanzas.sort(comparator)
for (i = 0; i < stanzas.length; i++) {
stanzas[i][0].shift()
}
return stanzas
},
keypathCompare: function(a, b) {
a = a[0]; b = b[0];
if (a.length !== b.length) {
return a.length - b.length;
}
for (var i = 0; i < a.length; i++) {
if (typeof a[i] === 'number' && a[i] !== b[i]) {
return a[i] - b[i];
}
}
return 0;
},
keypathCompareReverse: function(a, b) {
a = a[0]; b = b[0];
if (a.length !== b.length) {
return b.length - a.length;
}
for (var i = 0; i < a.length; i++) {
if (typeof a[i] === 'number' && a[i] !== b[i]) {
return b[i] - a[i];
}
}
return 0;
},
sortStanzas: function(diff) {
/* Sorts the stanzas in a diff: object changes can occur in
* any order, but deletions from arrays have to happen last
* node first: ['foo', 'bar', 'baz'] -> ['foo', 'bar'] ->
* ['foo'] -> []; additions to sequences have to happen
* leftmost-node-first: [] -> ['foo'] -> ['foo', 'bar'] ->
* ['foo', 'bar', 'baz'], and insert-and-shift alterations to
* arrays must happen last. */
// First we divide the stanzas using splitDiff():
var split_thing = this.splitDiff(diff);
// Then we sort modifications of arrays in ascending order of keypath
// (note that we can?t tell appends from mods on the info available):
split_thing[1].sort(this.keypathCompare);
// Deletions from arrays in descending order of keypath:
split_thing[2].sort(this.keypathCompareReverse);
// And insert-and-shifts in ascending order of keypath:
split_thing[3].sort(this.keypathCompare)
diff = split_thing[0].concat(
split_thing[1], split_thing[2], split_thing[3]
);
// Finally, we sort by length of keypath:
diff = this.stableKeypathLengthSort(diff, true)
return diff
},
computeKeysets: function(left, right) {
/* Returns an array of three arrays (overlap, left_only,
* right_only), representing the properties common to left and
* right, only defined for left, and only defined for right,
* respectively. */
var overlap = [], left_only = [], right_only = [];
var target = overlap;
this.loopOver(left, function(obj, key) {
if (right[key] !== undefined) {
target = overlap;
}
else {
target = left_only;
}
target.push(key);
});
this.loopOver(right, function(obj, key) {
if (left[key] === undefined) {
right_only.push(key);
}
});
return [overlap, left_only, right_only];
},
structureWorthInvestigating: function(left, right) {
/* Test whether it is worth looking at the internal structure
* of `left` and `right` to see if they can be efficiently
* diffed. */
if (this.isTerminal(left) || this.isTerminal(right)) {
return false;
}
if ((left.length === 0) || (right.length === 0)) {
return false;
}
if ((left instanceof Array) && (right instanceof Array)) {
return true;
}
if ((left instanceof Array) || (right instanceof Array)) {
return false;
}
if ((typeof left === 'object') && (typeof right === 'object')) {
return true;
}
return false;
},
commonality: function(left, right) {
/* Calculate the amount that the structures left and right
* have in common */
var com = 0, tot = 0;
var elem, keysets, o, l, r, idx;
if (this.isTerminal(left) || this.isTerminal(right)) {
return 0;
}
if ((left instanceof Array) && (right instanceof Array)) {
for (idx = 0; idx < left.length; idx++) {
elem = left[idx];
if (right.indexOf(elem) !== -1) {
com++;
}
}
tot = Math.max(left.length, right.length);
}
else {
if ((left instanceof Array) || (right instanceof Array)) {
return 0;
}
keysets = this.computeKeysets(left, right);
o = keysets[0]; l = keysets[1]; r = keysets[2];
com = o.length;
tot = o.length + l.length + r.length;
for (idx = 0; idx < r.length; idx++) {
elem = r[idx];
if (l.indexOf(elem) === -1) {
tot++;
}
}
}
if (tot === 0) {return 0;}
return com / tot;
},
thisLevelDiff: function(left, right, key, common) {
/* Returns a sequence of diff stanzas between the objects left
* and right, assuming that they are each at the position key
* within the overall structure. */
var out = [], idx, okey;
key = key !== undefined ? key : [];
if (common === undefined) {
common = this.commonality(left, right);
}
if (common) {
var ks = this.computeKeysets(left, right);
for (idx = 0; idx < ks[0].length; idx++) {
okey = ks[0][idx];
if (left[okey] !== right[okey]) {
out.push([key.concat([okey]), right[okey]]);
}
}
for (idx = 0; idx < ks[1].length; idx++) {
okey = ks[1][idx];
out.push([key.concat([okey])]);
}
for (idx = 0; idx < ks[2].length; idx++) {
okey = ks[2][idx];
out.push([key.concat([okey]), right[okey]]);
}
return out;
}
if (! this.isStrictlyEqual(left, right)) {
return [[key, right]];
}
return [];
},
keysetDiff: function(left, right, minimal, key) {
/* Compute a diff between left and right, without treating
* arrays differently from objects. */
minimal = minimal !== undefined ? minimal : true;
var out = [], k;
var ks = this.computeKeysets(left, right);
for (k = 0; k < ks[1].length; k++) {
out.push([key.concat(ks[1][k])]);
}
for (k = 0; k < ks[2].length; k++) {
out.push([key.concat(ks[2][k]), right[ks[2][k]]]);
}
for (k = 0; k < ks[0].length; k++) {
out = out.concat(this.diff(left[ks[0][k]], right[ks[0][k]],
minimal, key.concat([ks[0][k]])));
}
return out;
},
needleDiff: function(left, right, minimal, key) {
/* Compute a diff between left and right. If both are arrays,
* a variant of Needleman-Wunsch sequence alignment is used to
* make the diff minimal (at a significant cost in both
* storage and processing). Otherwise, the parms are passed on
* to keysetDiff.*/
if (! (left instanceof Array && right instanceof Array)) {
return this.keysetDiff(left, right, minimal, key);
}
minimal = minimal !== undefined ? minimal : true;
var down_col = 0, lastrow = [], i, sub_i, left_i, right_i, col_i;
var row, first_left_i, left_elem, right_elem;
var cand_length, win_length, cand, winner;
var modify_cand = function () {
if (col_i + 1 < lastrow.length) {
return lastrow[col_i+1].concat(
JSON_delta.diff(left_elem, right_elem,
minimal, key.concat([left_i]))
);
}
};
var delete_cand = function () {
if (row.length > 0) {
return row[0].concat([[key.concat([left_i])]]);
}
};
var append_cand = function () {
if (col_i === down_col) {
return lastrow[col_i].concat(
[[key.concat([JSON_delta.appendKey(lastrow[col_i], left, key)]),
right_elem]]
);
}
};
var insert_cand = function () {
if (col_i !== down_col) {
return lastrow[col_i].concat(
[[key.concat([right_i]), right_elem, "i"]]
);
}
};
var cand_funcs = [modify_cand, delete_cand, append_cand, insert_cand];
for (i = 0; i <= left.length; i++) {
lastrow.unshift([]);
for (sub_i = 0; sub_i < i; sub_i++) {
lastrow[0].push([key.concat([sub_i])]);
}
}
for (right_i = 0; right_i < right.length; right_i++) {
right_elem = right[right_i];
row = []
for (left_i = 0; left_i < left.length; left_i++) {
left_elem = left[left_i];
col_i = left.length - left_i - 1;
win_length = Infinity;
for (i = 0; i < cand_funcs.length; i++) {
cand = cand_funcs[i]();
if (cand !== undefined) {
cand_length = JSON.stringify(cand).length;
if (cand_length < win_length) {
winner = cand;
win_length = cand_length;
}
}
}
row.unshift(winner);
}
lastrow = row;
}
return winner;
},
patchStanza: function(struc, diff) {
/* Applies the diff stanza diff to the structure struc.
Returns the modified structure. */
var key = diff[0];
switch (key.length) {
case 0:
struc = diff[1];
break;
case 1:
if (diff.length === 1) {
if (struc.splice === undefined) {
delete struc[key[0]];
}
else {
struc.splice(key[0], 1);
}
} else if (diff.length === 3) {
if (struc.splice === undefined) {
struc[key[0]] = diff[1];
} else {
struc.splice(key[0], 0, diff[1]);
}
}
else {
struc[key[0]] = diff[1];
}
break;
default:
var pass_key = key.slice(1), pass_struc = struc[key[0]];
var pass_diff = [pass_key].concat(diff.slice(1));
if (pass_struc === undefined) {
if (typeof pass_key[0] === 'string') {
pass_struc = {};
} else {
pass_struc = [];
}
}
struc[key[0]] = this.patchStanza(pass_struc, pass_diff);
}
return struc;
}
};

1
requirements.txt
View File

@@ -4,3 +4,4 @@ gevent
requests
stups-tokens>=1.1.19
redlock-py
json_delta>=2.0