[Vector clock] Clock work (2/3)

creator-node/package.json

@@ -61,7 +61,8 @@
  "sinon": "^7.0.0",
  "standard": "^12.0.1",
  "fs-extra": "^9.0.1",
- "supertest": "^3.3.0"
+ "supertest": "^3.3.0",
+ "proxyquire": "^2.1.3"
  },
  "//": {
  "dependenciesComments": {
@@ -73,13 +74,8 @@
  }
  },
  "standard": {
- "globals": [
- "assert",
- "beforeEach",
- "it",
- "before",
- "describe",
- "afterEach"
+ "env": [
+ "mocha"
  ]
  }
 }

creator-node/scripts/run-tests.sh

@@ -28,12 +28,12 @@ tear_down () {
 
 run_unit_tests () {
  echo Running unit tests...
- ./node_modules/mocha/bin/mocha --recursive 'src/**/*.test.js'
+ ./node_modules/mocha/bin/mocha --recursive 'src/**/*.test.js' --exit
 }
 
 run_integration_tests () {
  echo Running integration tests...
- ./node_modules/mocha/bin/mocha --timeout 30000 --exit
+ ./node_modules/mocha/bin/mocha test/*.test.js --timeout 30000 --exit
 }
 
 if [ "$1" == "standalone_creator" ]; then

creator-node/sequelize/migrations/20200911004845-allow-track-and-audiusUsers-appends.js

@@ -0,0 +1,74 @@
+'use strict'
+
+module.exports = {
+ up: async (queryInterface, Sequelize) => {
+ // Scope of the migration is:
+ // Remove trackUUID from Files and replace it with trackBlockchainId and migrate all existing values over
+ // Remove trackUUID field from Tracks table
+ // Remove UNIQUE constraint for blockchainId, trackUUID in Tracks table
+ // Add NOT NULL constraint for blockchainId in Tracks table
+ // Remove audiusUserUUID field from AudiusUsers table
+ // Remove UNIQUE constraint for audiusUserUUID in AudiusUsers table (no unique constraint on blockchainId)
+ // Add NOT NULL constraint for blockchainId in AudiusUsers table
+
+ console.log('STARTING MIGRATION 20200911004845-allow-track-and-audiusUsers-appends')
+ await queryInterface.sequelize.query(`
+ BEGIN;
+ -- replace Files table in place with extra trackBlockchainId column and drops the trackUUID column
+ CREATE TABLE "Files_new" ( like "Files" );
+ ALTER TABLE "Files_new" ADD COLUMN "trackBlockchainId" INTEGER;
+ INSERT INTO "Files_new" ("multihash", "sourceFile", "storagePath", "createdAt", "updatedAt", "fileUUID", "cnodeUserUUID", "trackUUID", "type", "fileName", "dirMultihash", "trackBlockchainId") SELECT f."multihash", f."sourceFile", f."storagePath", f."createdAt", f."updatedAt", f."fileUUID", f."cnodeUserUUID", f."trackUUID", f."type", f."fileName", f."dirMultihash", t."blockchainId" FROM "Files" f LEFT OUTER JOIN "Tracks" t ON f."trackUUID" = t."trackUUID";
+ ALTER TABLE "Files_new" DROP COLUMN "trackUUID";
+ ALTER TABLE "Files" RENAME TO "Files_old";
+ ALTER TABLE "Files_new" RENAME TO "Files";
+ DROP TABLE "Files_old" CASCADE;
+
+ -- add pkey
+ -- ALTER TABLE "Files" ADD CONSTRAINT "Files_fileUUID_key" UNIQUE ("fileUUID");
+ ALTER TABLE "Files" ADD PRIMARY KEY ("fileUUID");
+
+ -- add back indexes
+ CREATE INDEX "Files_multihash_idx" ON public."Files" USING btree (multihash);
+ CREATE INDEX "Files_cnodeUserUUID_idx" ON public."Files" USING btree ("cnodeUserUUID");
+ CREATE INDEX "Files_dir_multihash_idx" ON public."Files" USING btree ("dirMultihash");
+ CREATE INDEX "Files_trackBlockchainId_idx" ON public."Files" USING btree ("trackBlockchainId");
+
+ -- add in the foreign key constraint from Files to other tables
+ -- No fkey from Files to Tracks because we don't have a unique constraint on trackUUID or blockchainId on Tracks so postgres would reject the fkey
+ ALTER TABLE "Files" ADD CONSTRAINT "Files_cnodeUserUUID_fkey" FOREIGN KEY ("cnodeUserUUID") REFERENCES "CNodeUsers" ("cnodeUserUUID") ON DELETE RESTRICT;
+
+ -- remove the unique constraints from Tracks
+ ALTER TABLE "Tracks" DROP CONSTRAINT "Tracks_trackUUID_key";
+ ALTER TABLE "Tracks" DROP CONSTRAINT "blockchainId_unique_idx";
+
+ -- add a not null constraint to Tracks blockchainId, just run a delete query to remove any outstanding Tracks without a blockchainId in case there are any
+ DELETE FROM "Tracks" WHERE "blockchainId" IS NULL;
+ ALTER TABLE "Tracks" ALTER COLUMN "blockchainId" SET NOT NULL;
+
+ -- remove the trackUUID column from Tracks
+ ALTER TABLE "Tracks" DROP COLUMN "trackUUID";
+
+ -- remove the unique constraint from AudiusUsers
+ ALTER TABLE "AudiusUsers" DROP CONSTRAINT "AudiusUsers_audiusUserUUID_key";
+
+ -- add a not null constraint to AudiusUsers blockchainId, just run a delete query to remove any outstanding AudiusUsers without a blockchainId in case there are any
+ DELETE FROM "AudiusUsers" WHERE "blockchainId" IS NULL;
+ ALTER TABLE "AudiusUsers" ALTER COLUMN "blockchainId" SET NOT NULL;
+
+ -- remove the audiusUserUUID field as the AudiusUsers pkey
+ ALTER TABLE "AudiusUsers" DROP CONSTRAINT "AudiusUsers_pkey";
+
+ -- remove the audiusUserUUID column from AudiusUsers
+ ALTER TABLE "AudiusUsers" DROP COLUMN "audiusUserUUID";
+
+ COMMIT;
+ `)
+ console.log('FINISHED MIGRATION 20200911004845-allow-track-and-audiusUsers-appends')
+ },
+
+ down: (queryInterface, Sequelize) => {
+ /*
+ The up migration destroys tons of information, if we need to revert the best option is to restore from a snapshot
+ */
+ }
+}

creator-node/sequelize/migrations/20200918150546-add-clock.js

@@ -0,0 +1,116 @@
+'use strict'
+
+/**
+ * Content Tables = AudiusUsers, Tracks, Files
+ * CNodeUsers Table considered a Reference Table only
+ */
+
+module.exports = {
+ up: async (queryInterface, Sequelize) => {
+ console.log('STARTING MIGRATION 20200918150546-add-clock')
+ const transaction = await queryInterface.sequelize.transaction()
+
+ // Add 'clock' column to all 4 tables
+ await addClockColumn(queryInterface, Sequelize, transaction)
+
+ // Create Clock table
+ await createClockRecordsTable(queryInterface, Sequelize, transaction)
+
+ // Add composite unique constraint on (blockchainId, clock) to AudiusUsers and Tracks
+ // Add composite unique constraint on (cnodeUserUUID, clock) to Files
+ await addCompositeUniqueConstraints(queryInterface, Sequelize, transaction)
+
+ await transaction.commit()
+ console.log('FINISHED MIGRATION 20200918150546-add-clock')
+ },
+
+ down: async (queryInterface, Sequelize) => { }
+}
+
+async function addClockColumn (queryInterface, Sequelize, transaction) {
+ await queryInterface.addColumn('CNodeUsers', 'clock', {
+ type: Sequelize.INTEGER,
+ unique: false,
+ allowNull: true
+ }, { transaction })
+ await queryInterface.addColumn('AudiusUsers', 'clock', {
+ type: Sequelize.INTEGER,
+ unique: false,
+ allowNull: true
+ }, { transaction })
+ await queryInterface.addColumn('Tracks', 'clock', {
+ type: Sequelize.INTEGER,
+ unique: false,
+ allowNull: true
+ }, { transaction })
+ await queryInterface.addColumn('Files', 'clock', {
+ type: Sequelize.INTEGER,
+ unique: false,
+ allowNull: true
+ }, { transaction })
+}
+
+async function addCompositeUniqueConstraints (queryInterface, Sequelize, transaction) {
+ await queryInterface.addConstraint(
+ 'AudiusUsers',
+ {
+ type: 'UNIQUE',
+ fields: ['blockchainId', 'clock'],
+ name: 'AudiusUsers_unique_(blockchainId,clock)',
+ transaction
+ }
+ )
+ await queryInterface.addConstraint(
+ 'Tracks',
+ {
+ type: 'UNIQUE',
+ fields: ['blockchainId', 'clock'],
+ name: 'Tracks_unique_(blockchainId,clock)',
+ transaction
+ }
+ )
+ await queryInterface.addConstraint(
+ 'Files',
+ {
+ type: 'UNIQUE',
+ fields: ['cnodeUserUUID', 'clock'],
+ name: 'Files_unique_(cnodeUserUUID,clock)',
+ transaction
+ }
+ )
+}
+
+async function createClockRecordsTable (queryInterface, Sequelize, transaction) {
+ await queryInterface.createTable('ClockRecords', {
+ cnodeUserUUID: {
+ type: Sequelize.UUID,
+ primaryKey: true, // composite primary key (cnodeUserUUID, clock)
+ unique: false,
+ allowNull: false,
+ references: {
+ model: 'CNodeUsers',
+ key: 'cnodeUserUUID',
+ as: 'cnodeUserUUID'
+ },
+ onDelete: 'RESTRICT'
+ },
+ clock: {
+ type: Sequelize.INTEGER,
+ primaryKey: true, // composite primary key (cnodeUserUUID, clock)
+ unique: false,
+ allowNull: false
+ },
+ sourceTable: {
+ type: Sequelize.ENUM('AudiusUser', 'Track', 'File'),
+ allowNull: false
+ },
+ createdAt: {
+ allowNull: false,
+ type: Sequelize.DATE
+ },
+ updatedAt: {
+ allowNull: false,
+ type: Sequelize.DATE
+ }
+ }, { transaction })
+}

creator-node/src/config.js

@@ -203,6 +203,12 @@ const config = convict({
  env: 'hlsSegmentType',
  default: 'mpegts'
  },
+ printSequelizeLogs: {
+ doc: 'If we should print logs from sequelize',
+ format: Boolean,
+ env: 'printSequelizeLogs',
+ default: true
+ },
 
  // Transcoding settings
  transcodingMaxConcurrency: {

creator-node/src/dbManager.js

@@ -0,0 +1,54 @@
+const models = require('./models')
+const sequelize = models.sequelize
+
+class DBManager {
+ /**
+ * Given file insert query object and cnodeUserUUID, inserts new file record in DB
+ * and handles all required clock management.
+ * Steps:
+ * 1. increments cnodeUser clock value by 1
+ * 2. insert new ClockRecord entry with new clock value
+ * 3. insert new Data Table (File, Track, AudiusUser) entry with queryObj and new clock value
+ * In steps 2 and 3, clock values are read as subquery to guarantee atomicity
+ */
+ static async createNewDataRecord (queryObj, cnodeUserUUID, sequelizeTableInstance, transaction) {
+ // Increment CNodeUser.clock value by 1
+ await models.CNodeUser.increment('clock', {
+ where: { cnodeUserUUID },
+ by: 1,
+ transaction
+ })
+
+ const selectCNodeUserClockSubqueryLiteral = _getSelectCNodeUserClockSubqueryLiteral(cnodeUserUUID)
+
+ // Add row in ClockRecords table using new CNodeUser.clock
+ await models.ClockRecord.create({
+ cnodeUserUUID,
+ clock: selectCNodeUserClockSubqueryLiteral,
+ sourceTable: sequelizeTableInstance.name
+ }, { transaction })
+
+ // Add cnodeUserUUID + clock value to queryObj
+ queryObj.cnodeUserUUID = cnodeUserUUID
+ queryObj.clock = selectCNodeUserClockSubqueryLiteral
+
+ // Create new Data table entry with queryObj using new CNodeUser.clock
+ const file = await sequelizeTableInstance.create(queryObj, { transaction })
+
+ return file.dataValues
+ }
+}
+
+/**
+ * returns string literal `select "clock" from "CNodeUsers" where "cnodeUserUUID" = '${cnodeUserUUID}'`
+ * @dev source: https://stackoverflow.com/questions/36164694/sequelize-subquery-in-where-clause
+ */
+function _getSelectCNodeUserClockSubqueryLiteral (cnodeUserUUID) {
+ const subquery = sequelize.dialect.QueryGenerator.selectQuery('CNodeUsers', {
+ attributes: ['clock'],
+ where: { cnodeUserUUID }
+ }).slice(0, -1) // removes trailing ';'
+ return sequelize.literal(`(${subquery})`)
+}
+
+module.exports = DBManager