SAN-4866 Cleanup Old Instances

index.js

@@ -31,6 +31,7 @@ const tasks = {
  'khronos:images:prune': require('tasks/images/prune'),
  'khronos:images:prune-dock': require('tasks/images/prune-dock'),
  'khronos:images:remove': require('tasks/images/remove'),
+ 'khronos:instances:cleanup': require('tasks/instances/cleanup'),
  'khronos:metrics:container-status': require('tasks/metrics/container-status'),
  'khronos:metrics:report-org-container-status': require('tasks/metrics/report-org-container-status'),
  'khronos:weave:prune': require('tasks/weave/prune'),

lib/models/rabbitmq.js

@@ -16,7 +16,8 @@ const publisher = new RabbitMQ({
  username: process.env.RABBITMQ_USERNAME,
  password: process.env.RABBITMQ_PASSWORD,
  events: [
- 'instance.container.health-check.failed'
+ 'instance.container.health-check.failed',
+ 'instance.expired'
  ],
  tasks: [
  'khronos:canary:build',
@@ -39,6 +40,7 @@ const publisher = new RabbitMQ({
  'khronos:images:prune',
  'khronos:images:prune-dock',
  'khronos:images:remove',
+ 'khronos:instances:cleanup',
  'khronos:metrics:container-status',
  'khronos:metrics:report-org-container-status',
  'khronos:weave:prune',

lib/tasks/instances/cleanup.js

@@ -0,0 +1,44 @@
+'use strict'
+
+// external
+const Promise = require('bluebird')
+const rabbitmq = require('models/rabbitmq')
+const mongodbHelper = require('tasks/utils/mongodb')
+const moment = require('moment')
+
+// internal
+const logger = require('logger').getChild(__filename)
+
+module.exports = CleanupInstances
+
+function CleanupInstances () {
+ const cleanupDate = moment().subtract(7, 'days').toDate()
+ var log = logger.child({
+ cleanupCutoff: cleanupDate
+ })
+ log.info('CleanupInstances')
+
+ return Promise.using(mongodbHelper(), function (mongoClient) {
+ return mongoClient.fetchInstancesAsync({
+ masterPod: false,
+ 'contextVersion.created': { $lt: cleanupDate },
+ $or: [
+ { isolated: { $exists: false } },
+ { isIsolationGroupMaster: true }
+ ]
+ })
+ })
+ .then(function (instances) {
+ log.info({
+ instanceCount: instances.length,
+ instanceIds: instances.map(function (instance) {
+ return instance._id
+ })
+ }, 'Found instances to cleanup')
+ instances.forEach(function (instance) {
+ rabbitmq.publishEvent('instance.expired', {
+ instanceId: instance._id
+ })
+ })
+ })
+}

package.json

@@ -64,6 +64,7 @@
  "keypather": "^1.10.1",
  "loadenv": "^2.2.0",
  "lodash": "^4.14.0",
+ "moment": "^2.14.1",
  "mongodb": "2.1.7",
  "monitor-dog": "^1.5.0",
  "ponos": "^4.6.0",

test/unit/tasks/instances/cleanup.js

@@ -0,0 +1,87 @@
+'use strict'
+
+require('loadenv')({ debugName: 'khronos:test' })
+
+var chai = require('chai')
+var assert = chai.assert
+chai.use(require('chai-as-promised'))
+
+// external
+var rabbitmq = require('models/rabbitmq')
+var sinon = require('sinon')
+require('sinon-as-promised')(require('bluebird'))
+
+// Internal
+const MongoDB = require('models/mongodb')
+
+// internal (being tested)
+var CleanupInstances = require('tasks/instances/cleanup')
+
+describe('khronos:instances:cleanup', function () {
+ var mockInstances
+
+ beforeEach(function () {
+ mockInstances = [
+ {
+ _id: '1234'
+ },
+ {
+ _id: '5678'
+ }
+ ]
+ sinon.stub(MongoDB.prototype, 'close').yieldsAsync()
+ sinon.stub(MongoDB.prototype, 'connect').yieldsAsync()
+ sinon.stub(MongoDB.prototype, 'fetchInstances').yieldsAsync(null, mockInstances)
+ sinon.stub(rabbitmq, 'publishEvent').resolves()
+ })
+
+ afterEach(function () {
+ MongoDB.prototype.close.restore()
+ MongoDB.prototype.connect.restore()
+ MongoDB.prototype.fetchInstances.restore()
+ rabbitmq.publishEvent.restore()
+ })
+
+ describe('when there are instances to cleanup', function () {
+ it('should fetch instances with the propery query parameters', function (done) {
+ return assert.isFulfilled(CleanupInstances({}))
+ .then(function () {
+ sinon.assert.calledOnce(MongoDB.prototype.fetchInstances)
+ sinon.assert.calledWith(
+ MongoDB.prototype.fetchInstances,
+ {
+ masterPod: false,
+ 'contextVersion.created': { $lt: sinon.match.date },
+ $or: [
+ { isolated: { $exists: false } },
+ { isIsolationGroupMaster: true }
+ ]
+ }
+ )
+ })
+ .asCallback(done)
+ })
+
+ it('should cleanup the old instances', function (done) {
+ return assert.isFulfilled(CleanupInstances({}))
+ .then(function () {
+ sinon.assert.calledTwice(rabbitmq.publishEvent)
+ sinon.assert.calledWith(
+ rabbitmq.publishEvent,
+ 'instance.expired',
+ {
+ instanceId: '1234'
+ }
+ )
+ sinon.assert.calledWith(
+ rabbitmq.publishEvent,
+ 'instance.expired',
+ {
+ instanceId: '5678'
+ }
+ )
+ })
+ .asCallback(done)
+ })
+ })
+})