CreatorNode Content Blacklist

creator-node/default-config.json

@@ -33,5 +33,7 @@
  "spOwnerWalletIndex": 1,
  "spOwnerWallet": "",
  "debounceTime": 30000,
- "discoveryProviderWhitelist": "http://docker.for.mac.localhost:5000"
+ "discoveryProviderWhitelist": "http://docker.for.mac.localhost:5000",
+ "userBlacklist": "",
+ "trackBlacklist": ""
 }

creator-node/docker-compose/development.env

@@ -40,4 +40,8 @@ isUserMetadataNode=false
 debounceTime=5000
 
 # not part of the config itself, but required for docker to know when ports are available
-WAIT_HOSTS=docker.for.mac.localhost:4379,docker.for.mac.localhost:4432
+WAIT_HOSTS=docker.for.mac.localhost:4379,docker.for.mac.localhost:4432
+
+# content blacklist
+userBlacklist=
+trackBlacklist=

creator-node/src/app.js

@@ -39,11 +39,12 @@ function errorHandler (err, req, res, next) {
 }
 app.use(errorHandler)
 
-const initializeApp = (port, storageDir, ipfsAPI, audiusLibs) => {
+const initializeApp = (port, storageDir, ipfsAPI, audiusLibs, blacklistManager) => {
  app.set('ipfsAPI', ipfsAPI)
  app.set('storagePath', storageDir)
  app.set('redisClient', redisClient)
  app.set('audiusLibs', audiusLibs)
+ app.set('blacklistManager', blacklistManager)
 
  const server = app.listen(port, () => logger.info(`Listening on port ${port}...`))
 

creator-node/src/blacklistManager.js

@@ -0,0 +1,111 @@
+const { logger } = require('./logging')
+const config = require('./config')
+const models = require('./models')
+const redis = require('./redis')
+
+const REDIS_SET_BLACKLIST_TRACKID_KEY = 'SET.BLACKLIST.TRACKID'
+const REDIS_SET_BLACKLIST_USERID_KEY = 'SET.BLACKLIST.USERID'
+const REDIS_SET_BLACKLIST_SEGMENTCID_KEY = 'SET.BLACKLIST.SEGMENTCID'
+
+class BlacklistManager {
+ static async blacklist (ipfs) {
+ try {
+ const { trackIdsToBlacklist, userIdsToBlacklist } = await _buildBlacklist()
+ await _processBlacklist(ipfs, trackIdsToBlacklist, userIdsToBlacklist)
+ } catch (e) {
+ throw new Error(`BLACKLIST ERROR ${e}`)
+ }
+ }
+
+ static async userIdIsInBlacklist (userId) {
+ return redis.sismember(REDIS_SET_BLACKLIST_USERID_KEY, userId)
+ }
+
+ static async trackIdIsInBlacklist (trackId) {
+ return redis.sismember(REDIS_SET_BLACKLIST_TRACKID_KEY, trackId)
+ }
+
+ static async CIDIsInBlacklist (CID) {
+ return redis.sismember(REDIS_SET_BLACKLIST_SEGMENTCID_KEY, CID)
+ }
+}
+
+/** Return list of trackIds and userIds to be blacklisted. */
+async function _buildBlacklist () {
+ const trackBlacklist = config.get('trackBlacklist') === '' ? [] : config.get('trackBlacklist').split(',')
+ const userBlacklist = config.get('userBlacklist') === '' ? [] : config.get('userBlacklist').split(',')
+
+ const trackIds = new Set(trackBlacklist)
+
+ // Fetch all tracks created by users in userBlacklist
+ let trackBlockchainIds = []
+ if (userBlacklist.length > 0) {
+ trackBlockchainIds = (await models.sequelize.query(
+ 'select "blockchainId" from "Tracks" where "cnodeUserUUID" in (' +
+ 'select "cnodeUserUUID" from "AudiusUsers" where "blockchainId" in (:userBlacklist)' +
+ ');'
+ , { replacements: { userBlacklist } }
+ ))[0]
+ }
+ if (trackBlockchainIds) {
+ for (const trackObj of trackBlockchainIds) {
+ if (trackObj.blockchainId) {
+ trackIds.add(trackObj.blockchainId)
+ }
+ }
+ }
+
+ return { trackIdsToBlacklist: [...trackIds], userIdsToBlacklist: userBlacklist }
+}
+
+/**
+ * Given trackIds and userIds to blacklist, fetch all segmentCIDs and unpin from IPFS.
+ * Also add all trackIds, userIds, and segmentCIDs to redis blacklist sets to prevent future interaction.
+ */
+async function _processBlacklist (ipfs, trackIdsToBlacklist, userIdsToBlacklist) {
+ const tracks = await models.Track.findAll({ where: { blockchainId: trackIdsToBlacklist } })
+
+ let segmentCIDsToBlacklist = new Set()
+
+ for (const track of tracks) {
+ if (!track.metadataJSON || !track.metadataJSON.track_segments) continue
+
+ for (const segment of track.metadataJSON.track_segments) {
+ const CID = segment.multihash
+ if (!CID) continue
+
+ // unpin from IPFS
+ try {
+ await ipfs.pin.rm(CID)
+ } catch (e) {
+ if (e.message.indexOf('not pinned') === -1) {
+ throw new Error(e)
+ }
+ }
+ logger.info(`unpinned ${CID}`)
+ segmentCIDsToBlacklist.add(CID)
+ }
+ }
+ segmentCIDsToBlacklist = [...segmentCIDsToBlacklist]
+
+ // Add all trackIds, userIds, and CIDs to redis blacklist sets.
+ try {
+ if (trackIdsToBlacklist.length > 0) {
+ const resp = await redis.sadd('SET.BLACKLIST.TRACKID', trackIdsToBlacklist)
+ logger.info(`redis set add SET.BLACKLIST.TRACKID response: ${resp}.`)
+ }
+ if (userIdsToBlacklist.length > 0) {
+ const resp = await redis.sadd('SET.BLACKLIST.USERID', userIdsToBlacklist)
+ logger.info(`redis set add SET.BLACKLIST.USERID response: ${resp}.`)
+ }
+ if (segmentCIDsToBlacklist.length > 0) {
+ const resp = await redis.sadd('SET.BLACKLIST.SEGMENTCID', segmentCIDsToBlacklist)
+ logger.info(`redis set add SET.BLACKLIST.SEGMENTCID response: ${resp}.`)
+ }
+ logger.info('Completed Processing trackId, userId, and segmentCid blacklists.')
+ } catch (e) {
+ throw new Error('Failed to process blacklist.', e)
+ }
+}
+
+module.exports = BlacklistManager

creator-node/src/config.js

@@ -249,6 +249,20 @@ const config = convict({
  format: String,
  env: 'discoveryProviderWhitelist',
  default: ''
+ },
+
+ /** Manual content blacklists */
+ userBlacklist: {
+ doc: 'Comma-separated list of user blockchain IDs that creator node should avoid serving / storing',
+ format: String,
+ env: 'userBlacklist',
+ default: ''
+ },
+ trackBlacklist: {
+ doc: 'Comma-separated list of track blockchain IDs that creator node should avoid serving / storing',
+ format: String,
+ env: 'trackBlacklist',
+ default: ''
  }
 
  // unsupported options at the moment
@@ -294,6 +308,7 @@ if (fs.existsSync('eth-contract-config.json')) {
 // Perform validation and error any properties are not present on schema
 config.validate()
 
+// Retrieves and populates IP info configs
 const asyncConfig = async () => {
  const ipinfo = await axios.get('https://ipinfo.io')
  const country = ipinfo.data.country

creator-node/src/index.js

@@ -10,6 +10,7 @@ const config = require('./config')
 const { sequelize } = require('./models')
 const { runMigrations } = require('./migrationManager')
 const { logger } = require('./logging')
+const BlacklistManager = require('./blacklistManager')
 
 const initAudiusLibs = async () => {
  const ethWeb3 = await AudiusLibs.Utils.configureWeb3(
@@ -34,41 +35,50 @@ const initAudiusLibs = async () => {
  return audiusLibs
 }
 
-const startApp = async () => {
- // configure file storage
+const configFileStorage = () => {
  if (!config.get('storagePath')) {
  logger.error('Must set storagePath to use for content repository.')
  process.exit(1)
  }
- const storagePath = path.resolve('./', config.get('storagePath'))
-
- // run config
- logger.info('Configuring service...')
- config.asyncConfig().then(() => {
- logger.info('Service configured')
- })
+ return (path.resolve('./', config.get('storagePath')))
+}
 
- // connect to IPFS
-let ipfsAddr = config.get('ipfsHost')
+const initIPFS = async () => {
+const ipfsAddr = config.get('ipfsHost')
  if (!ipfsAddr) {
  logger.error('Must set ipfsAddr')
  process.exit(1)
  }
- let ipfs = ipfsClient(ipfsAddr, config.get('ipfsPort'))
+ const ipfs = ipfsClient(ipfsAddr, config.get('ipfsPort'))
+ const identity = await ipfs.id()
+ logger.info(`Current IPFS Peer ID: ${JSON.stringify(identity)}`)
+ return ipfs
+}
 
- // run all migrations
- logger.info('Executing database migrations...')
- runMigrations().then(async () => {
+const runDBMigrations = async () => {
+ try {
+ logger.info('Executing database migrations...')
+ await runMigrations()
  logger.info('Migrations completed successfully')
- }).error((err) => {
+ } catch (err) {
  logger.error('Error in migrations: ', err)
  process.exit(1)
- })
+ }
+}
+
+const startApp = async () => {
+ logger.info('Configuring service...')
+ await config.asyncConfig()
+ const storagePath = configFileStorage()
+ const ipfs = await initIPFS()
+ await runDBMigrations()
+
+ await BlacklistManager.blacklist(ipfs)
 
  const audiusLibs = (config.get('isUserMetadataNode')) ? null : await initAudiusLibs()
  logger.info('Initialized audius libs')
 
- const appInfo = initializeApp(config.get('port'), storagePath, ipfs, audiusLibs)
+ const appInfo = initializeApp(config.get('port'), storagePath, ipfs, audiusLibs, BlacklistManager)
 
  // when app terminates, close down any open DB connections gracefully
  ON_DEATH((signal, error) => {

creator-node/src/routes/files.js

@@ -6,7 +6,7 @@ const writeFile = promisify(fs.writeFile)
 const mkdir = promisify(fs.mkdir)
 
 const { upload } = require('../fileManager')
-const { handleResponse, sendResponse, successResponse, errorResponseBadRequest, errorResponseServerError, errorResponseNotFound } = require('../apiHelpers')
+const { handleResponse, sendResponse, successResponse, errorResponseBadRequest, errorResponseServerError, errorResponseNotFound, errorResponseForbidden } = require('../apiHelpers')
 
 const models = require('../models')
 const { logger } = require('../logging')
@@ -145,10 +145,16 @@ module.exports = function (app) {
  if (!(req.params && req.params.CID)) {
  return sendResponse(req, res, errorResponseBadRequest(`Invalid request, no CID provided`))
  }
- req.logger.info(req.params.CID)
 
  // Do not act as a public gateway. Only serve IPFS files that are hosted by this creator node.
  const CID = req.params.CID
+
+ // Don't serve if blacklisted
+ if (await req.app.get('blacklistManager').CIDIsInBlacklist(CID)) {
+ return sendResponse(req, res, errorResponseForbidden(`CID ${CID} has been blacklisted by this node.`))
+ }
+
+ // Don't serve if not found in DB
  const queryResults = await models.File.findOne({ where: { multihash: CID } })
  if (!queryResults) {
  return sendResponse(req, res, errorResponseNotFound(`No file found for provided CID: ${CID}`))

creator-node/src/routes/tracks.js

@@ -6,7 +6,7 @@ const ffmpeg = require('../ffmpeg')
 const { getSegmentsDuration } = require('../segmentDuration')
 const models = require('../models')
 const { saveFileFromBuffer, saveFileToIPFSFromFS, removeTrackFolder, trackFileUpload } = require('../fileManager')
-const { handleResponse, successResponse, errorResponseBadRequest, errorResponseServerError } = require('../apiHelpers')
+const { handleResponse, successResponse, errorResponseBadRequest, errorResponseServerError, errorResponseForbidden } = require('../apiHelpers')
 const { getFileUUIDForImageCID } = require('../utils')
 const { authMiddleware, syncLockMiddleware, ensurePrimaryMiddleware, triggerSecondarySyncs } = require('../middlewares')
 
@@ -79,6 +79,23 @@ module.exports = function (app) {
 
  // exclude 0-length segments that are sometimes outputted by ffmpeg segmentation
  trackSegments = trackSegments.filter(trackSegment => trackSegment.duration)
+
+ // Don't allow if any segment CID is in blacklist.
+ try {
+ await Promise.all(trackSegments.map(async segmentObj => {
+ if (await req.app.get('blacklistManager').CIDIsInBlacklist(segmentObj.multihash)) {
+ throw new Error(`Track upload failed - part or all of this track has been blacklisted by this node.`)
+ }
+ }))
+ } catch (e) {
+ if (e.message.indexOf('blacklisted') >= 0) {
+ // TODO clean up orphaned content
+ return errorResponseForbidden(`Track upload failed - part or all of this track has been blacklisted by this node.`)
+ } else {
+ return errorResponseServerError(e.message)
+ }
+ }
+
  req.logger.info(`Time taken in /track_content for full route: ${Date.now() - routeTimeStart}ms for file ${req.fileName}`)
  return successResponse({ 'track_segments': trackSegments })
  }))
@@ -96,16 +113,30 @@ module.exports = function (app) {
  }
 
  // Ensure each segment multihash in metadata obj has an associated file, else error.
- await Promise.all(metadataJSON.track_segments.map(async segment => {
- const file = await models.File.findOne({ where: {
- multihash: segment.multihash,
- cnodeUserUUID: req.session.cnodeUserUUID,
- trackUUID: null
- } })
- if (!file) {
- return errorResponseBadRequest(`No file found for provided segment multihash: ${segment.multihash}`)
+ try {
+ await Promise.all(metadataJSON.track_segments.map(async segment => {
+ if (await req.app.get('blacklistManager').CIDIsInBlacklist(segment.multihash)) {
+ throw new Error(`Segment CID ${segment.multihash} has been blacklisted by this node.`)
+ }
+
+ const file = await models.File.findOne({ where: {
+ multihash: segment.multihash,
+ cnodeUserUUID: req.session.cnodeUserUUID,
+ trackUUID: null
+ } })
+ if (!file) {
+ throw new Error(`No file found for provided segment CID: ${segment.multihash}.`)
+ }
+ }))
+ } catch (e) {
+ if (e.message.indexOf('blacklisted') >= 0) {
+ return errorResponseForbidden(e.message)
+ } else if (e.message.indexOf('No file found') >= 0) {
+ return errorResponseBadRequest(e.message)
+ } else {
+ return errorResponseServerError(e.message)
  }
- }))
+ }
 
  // Store + pin metadata multihash to disk + IPFS.
  const metadataBuffer = Buffer.from(JSON.stringify(metadataJSON))
@@ -206,7 +237,10 @@ module.exports = function (app) {
  }
  }))
 
- /** Returns all tracks for cnodeUser. */
+ /**
+ * Returns all tracks for cnodeUser.
+ * @notice DEPRECATED.
+ */
  app.get('/tracks', authMiddleware, handleResponse(async (req, res) => {
  const tracks = await models.Track.findAll({
  where: { cnodeUserUUID: req.session.cnodeUserUUID }

creator-node/test/audiusUsers.js

@@ -1,20 +1,27 @@
 const request = require('supertest')
 
+const BlacklistManager = require('../src/blacklistManager')
+
 const { getApp } = require('./lib/app')
 const { createStarterCNodeUser } = require('./lib/dataSeeds')
 const { getIPFSMock } = require('./lib/ipfsMock')
 const { getLibsMock } = require('./lib/libsMock')
 
 describe('test AudiusUsers', function () {
  let app, server, session, ipfsMock, libsMock
+
  beforeEach(async () => {
  ipfsMock = getIPFSMock()
  libsMock = getLibsMock()
- const appInfo = await getApp(ipfsMock, libsMock)
+
+ const appInfo = await getApp(ipfsMock, libsMock, BlacklistManager)
+ await BlacklistManager.blacklist(ipfsMock)
+
  app = appInfo.app
  server = appInfo.server
  session = await createStarterCNodeUser()
  })
+
  afterEach(async () => {
  await server.close()
  })