Skip to content

SAN-5817 - Store container logs in s3#1903

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Myztiq merged 10 commits into from
Mar 8, 2017
Merged

SAN-5817 - Store container logs in s3 #1903

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
3a73e54
Added logic for container logs!
Feb 28, 2017
07d2fa9
Updated tests
Feb 28, 2017
1f8cc69
Open buff2String to everyone
Nathan219 Mar 1, 2017
03e373d
Merge remote-tracking branch 'origin/master' into SAN-5817-s3-logs
Mar 1, 2017
b5360a0
Stubbed out new methods in integraiton tests
Mar 1, 2017
6247b0e
Removed error handling
Mar 2, 2017
4176711
Merge remote-tracking branch 'origin/master' into SAN-5817-s3-logs
Mar 2, 2017
995fb96
Merge branch 'convert_names_to_use_underscore' into SAN-5817-s3-logs
Mar 3, 2017
8b19c3f
Merge remote-tracking branch 'origin/master' into SAN-5817-s3-logs
Mar 3, 2017
1dfa084
Merge branch 'master' into SAN-5817-s3-logs
Mar 7, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions lib/models/rabbitmq/index.js
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -53,6 +53,7 @@ RabbitMQ.prototype.connect = function () {
'cluster.delete',
'cluster.update',
'container.delete',
'container.logs.store',
'container.resource.clear',
'context-version.delete',
'image.push',
Expand DownExpand Up@@ -367,6 +368,10 @@ RabbitMQ.prototype.autoDeployInstance = function (data) {
return this._publisher.publishTask('instance.auto-deploy', data)
}

RabbitMQ.prototype.publishContainerLogsStore = function (data) {
return this._publisher.publishTask('container.logs.store', data)
}

RabbitMQ.prototype.publishTerminalConnected = function (data) {
return this._publisher.publishEvent('terminal.connected', data)
}
Expand Down
1 change: 1 addition & 0 deletions lib/socket/common-stream.js
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -152,6 +152,7 @@ function pipeLogsToClient (clientStream, baseDataName, tags, containerId, opts)
}

module.exports = {
buff2StringTransform: buff2StringTransform,
onValidateFailure: onValidateFailure,
pipeLogsToClient: pipeLogsToClient,
pipeAndEndOnError: pipeAndEndOnError,
Expand Down
1 change: 1 addition & 0 deletions lib/worker-server.js
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -33,6 +33,7 @@ module.exports = new ponos.Server({
'cluster.delete': require('workers/cluster.delete'),
'cluster.update': require('workers/cluster.update'),
'container.delete': require('workers/container.delete'),
'container.logs.store': require('workers/container.logs.store'),
'container.resource.clear': require('workers/container.resource.clear'),
'context-version.delete': require('workers/context-version.delete'),
'image.push': require('workers/image.push'),
Expand Down
1 change: 1 addition & 0 deletions lib/workers/application.container.died.js
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -56,6 +56,7 @@ module.exports.task = (job) => {
const sessionUserGithubId = job.inspectData.Config.Labels.sessionUserGithubId
return InstanceService.emitInstanceUpdate(instance, sessionUserGithubId, 'update')
})
.tap(rabbitMQ.publishContainerLogsStore.bind(rabbitMQ, {containerId: job.id}))
.tap(function (instance) {
// should only run once test is complete.
// will clean all instances part of this test
Expand Down
4 changes: 3 additions & 1 deletion lib/workers/build.container.died.js
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -64,8 +64,10 @@ class Worker {
* @returns {Promise}
*/
task () {
return Promise.try(this._clearBuildResources.bind(this)).bind(this)
return Promise.try(this._clearBuildResources.bind(this))
.bind(this)
.then(this._updateModelsAndGetUpdatedContextVersions)
.tap(rabbitMQ.publishContainerLogsStore.bind(rabbitMQ, {containerId: this.id}))

@Nathan219Nathan219Mar 2, 2017

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

() => rabbitMQ.publishContainerLogsStore({containerId: this.id})

@MyztiqMyztiqMar 2, 2017

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why?

@Nathan219Nathan219Mar 2, 2017

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cuz it looks better

@MyztiqMyztiqMar 2, 2017

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I disagree. Not going to change it unless we've a style guide thing that prevents it.

.then(this._emitUpdateAndGetInstances)
.spread(this._updateAndGetAutoDeployedInstances)
.then(this._filterOutAndKillIsolatedInstances)
Expand Down
52 changes: 52 additions & 0 deletions lib/workers/container.logs.store.js
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
/**
* Handle instance container log store events
* @module lib/workers/container.log.store
*/
'use strict'

require('loadenv')()

const joi = require('utils/joi')
const logger = require('logger')
const commonStream = require('socket/common-stream')
const Through2 = require('through2')
const aws = require('aws-sdk')
const Docker = require('models/apis/docker')

const s3 = new aws.S3()

module.exports.maxNumRetries = 5

module.exports.jobSchema = joi.object({
containerId: joi.string().required()
}).unknown().required()

module.exports.task = (job) => {
const log = logger.child({
container: job.containerId,
method: 'ContainerLogsStore'
})
log.trace('called')
const docker = new Docker({ timeout: 0 })
return docker.getLogsAsync(job.containerId)
.then(function (dockerLogStream) {
log.trace({
bucket: `${process.env.NODE_ENV}.container-logs`,
key: job.containerId
}, 'Began piping logs')
const cleanedStream = new Through2({}, commonStream.buff2StringTransform)
commonStream.connectStream(dockerLogStream, cleanedStream, log)

dockerLogStream.on('end', () => {
log.trace('Dockerlog stream ended')
cleanedStream.end()
})
// Send stream to s3
return s3.upload({
Bucket: `${process.env.NODE_ENV}.container-logs`,
Key: job.containerId,
Body: cleanedStream
})
.promise()
})
}
2 changes: 2 additions & 0 deletions test/integration/workers/build.container.died.js
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -50,12 +50,14 @@ describe('ImageBuilderContainerDied Integration Tests', function () {

before(function (done) {
sinon.stub(rabbitMQ, 'clearContainerMemory')
sinon.stub(rabbitMQ, 'publishContainerLogsStore')
sinon.stub(rabbitMQ, 'pushImage')
done()
})

after(function (done) {
rabbitMQ.pushImage.restore()
rabbitMQ.publishContainerLogsStore.restore()
rabbitMQ.clearContainerMemory.restore()
done()
})
Expand Down
16 changes: 16 additions & 0 deletions unit/workers/application.container.died.js
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -92,6 +92,7 @@ describe('ApplicationContainerDiedWorker', function () {
sinon.stub(IsolationService, 'isTestingIsolation').resolves(false)
sinon.stub(rabbitMQ, 'killIsolation')
sinon.stub(rabbitMQ, 'clearContainerMemory')
sinon.stub(rabbitMQ, 'publishContainerLogsStore')
done()
})
afterEach(function (done) {
Expand All@@ -101,6 +102,7 @@ describe('ApplicationContainerDiedWorker', function () {
IsolationService.isTestingIsolation.restore()
rabbitMQ.killIsolation.restore()
rabbitMQ.clearContainerMemory.restore()
rabbitMQ.publishContainerLogsStore.restore()
done()
})
describe('success', function () {
Expand All@@ -110,12 +112,14 @@ describe('ApplicationContainerDiedWorker', function () {
sinon.assert.calledOnce(InstanceService.modifyExistingContainerInspect)
sinon.assert.notCalled(rabbitMQ.clearContainerMemory)
sinon.assert.calledOnce(rabbitMQ.killIsolation)
sinon.assert.calledOnce(rabbitMQ.publishContainerLogsStore)
sinon.assert.calledOnce(InstanceService.emitInstanceUpdate)
sinon.assert.calledOnce(IsolationService.isTestingIsolation)
sinon.assert.calledOnce(IsolationService.redeployIfAllKilled)
sinon.assert.callOrder(
InstanceService.modifyExistingContainerInspect,
InstanceService.emitInstanceUpdate,
rabbitMQ.publishContainerLogsStore,
rabbitMQ.killIsolation,
IsolationService.isTestingIsolation,
IsolationService.redeployIfAllKilled)
Expand All@@ -132,6 +136,7 @@ describe('ApplicationContainerDiedWorker', function () {
sinon.assert.calledOnce(InstanceService.modifyExistingContainerInspect)
sinon.assert.notCalled(rabbitMQ.clearContainerMemory)
sinon.assert.notCalled(rabbitMQ.killIsolation)
sinon.assert.calledOnce(rabbitMQ.publishContainerLogsStore)
sinon.assert.calledOnce(InstanceService.emitInstanceUpdate)
sinon.assert.calledOnce(IsolationService.isTestingIsolation)
sinon.assert.calledOnce(IsolationService.redeployIfAllKilled)
Expand All@@ -153,6 +158,7 @@ describe('ApplicationContainerDiedWorker', function () {
sinon.assert.calledOnce(InstanceService.modifyExistingContainerInspect)
sinon.assert.notCalled(rabbitMQ.clearContainerMemory)
sinon.assert.notCalled(rabbitMQ.killIsolation)
sinon.assert.calledOnce(rabbitMQ.publishContainerLogsStore)
sinon.assert.calledOnce(InstanceService.emitInstanceUpdate)
sinon.assert.notCalled(IsolationService.isTestingIsolation)
sinon.assert.notCalled(IsolationService.redeployIfAllKilled)
Expand All@@ -170,6 +176,7 @@ describe('ApplicationContainerDiedWorker', function () {
sinon.assert.calledOnce(InstanceService.modifyExistingContainerInspect)
sinon.assert.calledOnce(rabbitMQ.clearContainerMemory)
sinon.assert.calledOnce(rabbitMQ.killIsolation)
sinon.assert.calledOnce(rabbitMQ.publishContainerLogsStore)
sinon.assert.calledOnce(InstanceService.emitInstanceUpdate)
sinon.assert.calledOnce(IsolationService.isTestingIsolation)
sinon.assert.calledOnce(IsolationService.redeployIfAllKilled)
Expand DownExpand Up@@ -197,6 +204,10 @@ describe('ApplicationContainerDiedWorker', function () {
isolationId: ctx.mockInstance.isolated,
triggerRedeploy: false
})
sinon.assert.calledOnce(rabbitMQ.publishContainerLogsStore)
sinon.assert.calledWith(rabbitMQ.publishContainerLogsStore, {
containerId: ctx.data.id
})
done()
})
})
Expand All@@ -222,6 +233,9 @@ describe('ApplicationContainerDiedWorker', function () {
expect(err).to.exist()
expect(err).to.be.instanceOf(WorkerStopError)
expect(err.message).to.equal('Instance not found')
sinon.assert.notCalled(InstanceService.emitInstanceUpdate)
sinon.assert.notCalled(rabbitMQ.killIsolation)
sinon.assert.notCalled(rabbitMQ.publishContainerLogsStore)
sinon.assert.calledOnce(InstanceService.modifyExistingContainerInspect)
sinon.assert.calledWith(InstanceService.modifyExistingContainerInspect,
ctx.mockInstance._id, ctx.data.id, ctx.data.inspectData)
Expand All@@ -237,6 +251,8 @@ describe('ApplicationContainerDiedWorker', function () {
ApplicationContainerDied(ctx.data).asCallback(function (err) {
expect(err).to.exist()
expect(err.message).to.equal(mongoError.message)
sinon.assert.notCalled(rabbitMQ.killIsolation)
sinon.assert.notCalled(rabbitMQ.publishContainerLogsStore)
sinon.assert.calledOnce(InstanceService.emitInstanceUpdate)
sinon.assert.calledWith(InstanceService.emitInstanceUpdate,
ctx.mockInstance, ctx.sessionUserGithubId, 'update')
Expand Down