From a9b14856642373942cb9628bc826d6d5dbbfdb19 Mon Sep 17 00:00:00 2001 From: William Jackson Date: Tue, 10 Nov 2020 09:00:25 -0600 Subject: [PATCH 01/64] started EventsController --- app/controllers/events_controller.rb | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index 3dba2a8..cd8030e 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -1,2 +1,26 @@ class EventsController < ApplicationController -end \ No newline at end of file + def index + + end +end + + + + + + + + +# SCHEMA +# create_table "events", force: :cascade do |t| +# t.boolean "completed", null: false +# t.binary "data", null: false +# t.string "inbound_url", null: false +# t.string "outbound_url", null: false +# t.integer "status_code", null: false +# t.datetime "completed_at" +# t.bigint "bridge_id", null: false +# t.datetime "created_at", precision: 6, null: false +# t.datetime "updated_at", precision: 6, null: false +# t.index ["bridge_id"], name: "index_events_on_bridge_id" +# end \ No newline at end of file From a4da75c39e3991e9a0df3355eb4aa38b4ec50fa9 Mon Sep 17 00:00:00 2001 From: William Jackson Date: Tue, 10 Nov 2020 10:27:00 -0600 Subject: [PATCH 02/64] latest controller change --- .ruby-version | 2 +- Gemfile | 2 +- Gemfile.lock | 11 +---------- app/controllers/events_controller.rb | 12 ++++++++++-- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.ruby-version b/.ruby-version index 860487c..338a5b5 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -2.7.1 +2.6.6 diff --git a/Gemfile b/Gemfile index c93bb1d..7f8bc64 100644 --- a/Gemfile +++ b/Gemfile @@ -1,7 +1,7 @@ source 'https://rubygems.org' git_source(:github) { |repo| "https://github.com/#{repo}.git" } -ruby '2.7.1' +ruby '2.6.6' # Bundle edge Rails instead: gem 'rails', github: 'rails/rails' gem 'rails', '~> 6.0.3', '>= 6.0.3.4' diff --git a/Gemfile.lock b/Gemfile.lock index 8fb7016..c4a820a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -57,12 +57,6 @@ GEM tzinfo (~> 1.1) zeitwerk (~> 2.2, >= 2.2.2) bcrypt (3.1.16) - better_errors (2.7.1) - coderay (>= 1.0.0) - erubi (>= 1.0.0) - rack (>= 0.9.0) - binding_of_caller (0.8.0) - debug_inspector (>= 0.0.1) bootsnap (1.5.0) msgpack (~> 1.0) builder (3.2.4) @@ -70,7 +64,6 @@ GEM coderay (1.1.2) concurrent-ruby (1.1.7) crass (1.0.6) - debug_inspector (0.0.3) diff-lcs (1.4.4) erubi (1.9.0) ffi (1.13.1) @@ -180,8 +173,6 @@ PLATFORMS DEPENDENCIES bcrypt (~> 3.1.7) - better_errors - binding_of_caller bootsnap (>= 1.4.2) byebug jbuilder (~> 2.7) @@ -196,7 +187,7 @@ DEPENDENCIES tzinfo-data RUBY VERSION - ruby 2.7.1p83 + ruby 2.6.6p146 BUNDLED WITH 2.1.4 diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index cd8030e..1c002b1 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -1,7 +1,15 @@ class EventsController < ApplicationController def index - - end + # events = Event.all + restricted_events = Event.all.map do |event| + date = event.updated_at .... + time = event.updated_at .... + + { updated_at: event.updated_at, + status_code: event.status_code + } + end + end From c93f54351c02c1585d05adf853e7e64762b687c9 Mon Sep 17 00:00:00 2001 From: William Jackson Date: Tue, 10 Nov 2020 12:05:06 -0600 Subject: [PATCH 03/64] . --- app/controllers/application_controller.rb | 1 + app/controllers/events_controller.rb | 47 ++++++++++++++--------- app/controllers/users_controller.rb | 23 ++++++----- app/models/user.rb | 10 ++--- config/routes.rb | 5 ++- 5 files changed, 49 insertions(+), 37 deletions(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 8a8853e..9f04df0 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -1,5 +1,6 @@ class ApplicationController < ActionController::API attr_accessor :current_user + TOKEN_HEADER = 'BRIDGE-JWT'.freeze USER_ERROR_MSG = 'ERROR: Could not find user with decoded JWT. This should not happen.'.freeze diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index 1c002b1..8cd5c66 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -1,25 +1,36 @@ -class EventsController < ApplicationController - def index - # events = Event.all - restricted_events = Event.all.map do |event| - date = event.updated_at .... - time = event.updated_at .... - - { updated_at: event.updated_at, - status_code: event.status_code - } +class EventsController < ApplicationController + before_action :authorize_request + # Needs to find all events based on bridge_id or event_id + # Needs to return id for each event as well + def index + binding.pry + events = @current_user.Event.all + binding.pry + events.map do |event| + updated_at = String(event.updated_at) + date = date_format(updated_at.split(' ')[1]) + time = updated_at.split(' ')[0] + + { time: time, + date: date, + status_code: event.status_code } end - -end - - - - + binding.pry + end + def show; end + private + def date_format(_date) + year = time.split('-')[0] + month = time.split('-')[1] + day = time.split('-')[2] + "#{year}-#{month}-#{day}" + end +end -# SCHEMA +# SCHEMA # create_table "events", force: :cascade do |t| # t.boolean "completed", null: false # t.binary "data", null: false @@ -31,4 +42,4 @@ def index # t.datetime "created_at", precision: 6, null: false # t.datetime "updated_at", precision: 6, null: false # t.index ["bridge_id"], name: "index_events_on_bridge_id" -# end \ No newline at end of file +# end diff --git a/app/controllers/users_controller.rb b/app/controllers/users_controller.rb index 8cac383..8f80d0e 100644 --- a/app/controllers/users_controller.rb +++ b/app/controllers/users_controller.rb @@ -1,26 +1,25 @@ class UsersController < ApplicationController - before_action :authorize_request, only: [:show, :destroy, :update] + before_action :authorize_request, only: %i[show destroy update] def show - render json: @current_user.safe_json, status: 200 + render json: @current_user.safe_json, status: 200 end def create user = User.new(user_params) user.save! - token = JsonWebToken.encode( {user_id: user.id} ) - render json: {user: user.safe_json, token: token}, status: 201 # Created - - rescue ActiveRecord::RecordInvalid - render json: {error: 'email or password is invalid'}, status: 422 # Unprocessable Entity - rescue JWT::EncodeError - render json: {}, status: 422 # Unprocessable Entity + token = JsonWebToken.encode({ user_id: user.id }) + render json: { user: user.safe_json, token: token }, status: 201 # Created + rescue ActiveRecord::RecordInvalid + render json: { error: 'email or password is invalid' }, status: 422 # Unprocessable Entity + rescue JWT::EncodeError + render json: {}, status: 422 # Unprocessable Entity end def destroy - @current_user.destroy + @current_user.destroy end - + def update if @current_user.update(user_params) render json: @current_user.safe_json, status: 204 # No Content @@ -34,4 +33,4 @@ def update def user_params params.require(:user).permit(:email, :password, :notifications) end -end +end diff --git a/app/models/user.rb b/app/models/user.rb index 5beaccb..ed4410a 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -1,11 +1,11 @@ class User < ApplicationRecord has_secure_password :password has_secure_password :recovery_password, validations: false - validates :email, presence: true, uniqueness: true, - format: { with: URI::MailTo::EMAIL_REGEXP } - validates :notifications, inclusion: { in: [true, false ]} + validates :email, presence: true, uniqueness: true, + format: { with: URI::MailTo::EMAIL_REGEXP } + validates :notifications, inclusion: { in: [true, false] } - def safe_json - {email: self.email, notifications: self.notifications} + def safe_json + { email: email, notifications: notifications } end end diff --git a/config/routes.rb b/config/routes.rb index 481029e..cab16a7 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,8 +1,9 @@ Rails.application.routes.draw do post 'users', to: 'users#create' - get 'users', to: "users#show" + get 'users', to: 'users#show' patch 'users', to: 'users#update' put 'users', to: 'users#update' delete 'users', to: 'users#destroy' - post "login", to: "sessions#create" + post 'login', to: 'sessions#create' + get 'events', to: 'events#index' end From 2bc02902737d9ff1851594a79ecef54c672d6c5e Mon Sep 17 00:00:00 2001 From: William Jackson Date: Tue, 10 Nov 2020 12:07:37 -0600 Subject: [PATCH 04/64] . --- app/controllers/application_controller.rb | 7 +------ app/controllers/events_controller.rb | 7 ------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 4621ac8..9ac5e93 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -3,13 +3,8 @@ class ApplicationController < ActionController::API attr_accessor :current_user -<<<<<<< HEAD - TOKEN_HEADER = 'BRIDGE-JWT'.freeze + TOKEN_HEADER = 'BRIDGE-JWT'.freeze USER_ERROR_MSG = 'ERROR: Could not find user with decoded JWT. This should not happen.'.freeze -======= - TOKEN_HEADER = 'BRIDGE-JWT' - USER_ERROR_MSG = 'ERROR: Could not find user with decoded JWT. This should not happen.' ->>>>>>> 11d80c27fa5104bb4322e4fb4e3883f0e02bcfa1 # Set this method as a before_action for routes you want # to protect with JWT authentication. Your route action diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index 5ac305e..8cd5c66 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -1,4 +1,3 @@ -<<<<<<< HEAD class EventsController < ApplicationController before_action :authorize_request # Needs to find all events based on bridge_id or event_id @@ -44,9 +43,3 @@ def date_format(_date) # t.datetime "updated_at", precision: 6, null: false # t.index ["bridge_id"], name: "index_events_on_bridge_id" # end -======= -# frozen_string_literal: true - -class EventsController < ApplicationController -end ->>>>>>> 11d80c27fa5104bb4322e4fb4e3883f0e02bcfa1 From 47d290644ec972374c57d77b53c6e5d8e7a5198b Mon Sep 17 00:00:00 2001 From: William Jackson Date: Tue, 10 Nov 2020 15:34:05 -0600 Subject: [PATCH 05/64] updated user model --- app/controllers/events_controller.rb | 8 +++----- app/models/bridge.rb | 1 + app/models/user.rb | 1 + 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index 8cd5c66..a30d746 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -3,9 +3,7 @@ class EventsController < ApplicationController # Needs to find all events based on bridge_id or event_id # Needs to return id for each event as well def index - binding.pry - events = @current_user.Event.all - binding.pry + events = @current_user.bridges events.map do |event| updated_at = String(event.updated_at) date = date_format(updated_at.split(' ')[1]) @@ -15,10 +13,10 @@ def index date: date, status_code: event.status_code } end - binding.pry end - def show; end + def show + end private diff --git a/app/models/bridge.rb b/app/models/bridge.rb index 8fd3460..6d06d4a 100644 --- a/app/models/bridge.rb +++ b/app/models/bridge.rb @@ -36,6 +36,7 @@ class Bridge < ApplicationRecord has_many :environment_variables, dependent: :destroy has_many :headers, dependent: :destroy has_many :events, dependent: :destroy + belongs to :user alias_attribute :env_vars, :environment_variables diff --git a/app/models/user.rb b/app/models/user.rb index bd0d41d..c55ea32 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -6,6 +6,7 @@ class User < ApplicationRecord validates :email, presence: true, uniqueness: true, format: { with: URI::MailTo::EMAIL_REGEXP } validates :notifications, inclusion: { in: [true, false] } + has_many :bridges, dependent: :destroy def safe_json { email: email, notifications: notifications } From 112c6940b63ef8fc87009b740bf7ef3ba5dfed3e Mon Sep 17 00:00:00 2001 From: William Jackson Date: Tue, 10 Nov 2020 15:46:12 -0600 Subject: [PATCH 06/64] . --- app/controllers/application_controller.rb | 4 ++-- app/controllers/events_controller.rb | 21 ++++----------------- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 9ac5e93..2e870c9 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -3,8 +3,8 @@ class ApplicationController < ActionController::API attr_accessor :current_user - TOKEN_HEADER = 'BRIDGE-JWT'.freeze - USER_ERROR_MSG = 'ERROR: Could not find user with decoded JWT. This should not happen.'.freeze + TOKEN_HEADER = 'BRIDGE-JWT' + USER_ERROR_MSG = 'ERROR: Could not find user with decoded JWT. This should not happen.' # Set this method as a before_action for routes you want # to protect with JWT authentication. Your route action diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index a30d746..1ae15a3 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -1,9 +1,11 @@ +# frozen_string_literal: true + class EventsController < ApplicationController before_action :authorize_request # Needs to find all events based on bridge_id or event_id # Needs to return id for each event as well def index - events = @current_user.bridges + events = @current_user.bridges events.map do |event| updated_at = String(event.updated_at) date = date_format(updated_at.split(' ')[1]) @@ -15,8 +17,7 @@ def index end end - def show - end + # def show; end private @@ -27,17 +28,3 @@ def date_format(_date) "#{year}-#{month}-#{day}" end end - -# SCHEMA -# create_table "events", force: :cascade do |t| -# t.boolean "completed", null: false -# t.binary "data", null: false -# t.string "inbound_url", null: false -# t.string "outbound_url", null: false -# t.integer "status_code", null: false -# t.datetime "completed_at" -# t.bigint "bridge_id", null: false -# t.datetime "created_at", precision: 6, null: false -# t.datetime "updated_at", precision: 6, null: false -# t.index ["bridge_id"], name: "index_events_on_bridge_id" -# end From 3b35da629e4c18fa5a4508c78402fdcff40dd898 Mon Sep 17 00:00:00 2001 From: William Jackson Date: Tue, 10 Nov 2020 15:54:34 -0600 Subject: [PATCH 07/64] . --- config/routes.rb | 3 --- 1 file changed, 3 deletions(-) diff --git a/config/routes.rb b/config/routes.rb index 0a934a4..33156ba 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -7,8 +7,5 @@ put 'users', to: 'users#update' delete 'users', to: 'users#destroy' post 'login', to: 'sessions#create' -<<<<<<< HEAD get 'events', to: 'events#index' -======= ->>>>>>> 11d80c27fa5104bb4322e4fb4e3883f0e02bcfa1 end From 14dbaa03df7773de8942e03ee6381186aced2f78 Mon Sep 17 00:00:00 2001 From: William Jackson Date: Tue, 10 Nov 2020 16:07:29 -0600 Subject: [PATCH 08/64] updated db schema --- ...0220329_add_user_foreign_key_to_bridges.rb | 5 + db/schema.rb | 99 ++++++++++--------- 2 files changed, 56 insertions(+), 48 deletions(-) create mode 100644 db/migrate/20201110220329_add_user_foreign_key_to_bridges.rb diff --git a/db/migrate/20201110220329_add_user_foreign_key_to_bridges.rb b/db/migrate/20201110220329_add_user_foreign_key_to_bridges.rb new file mode 100644 index 0000000..fbc4bf2 --- /dev/null +++ b/db/migrate/20201110220329_add_user_foreign_key_to_bridges.rb @@ -0,0 +1,5 @@ +class AddUserForeignKeyToBridges < ActiveRecord::Migration[6.0] + def change + add_reference :bridges, :user, foreign_key: true, null: false + end +end diff --git a/db/schema.rb b/db/schema.rb index f8a4a65..ea8bba3 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1,5 +1,3 @@ -# frozen_string_literal: true - # This file is auto-generated from the current state of the database. Instead # of editing this file, please use the migrations feature of Active Record to # incrementally modify your database, and then regenerate this schema definition. @@ -12,61 +10,66 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20_201_109_184_334) do +ActiveRecord::Schema.define(version: 2020_11_10_220329) do + # These are extensions that must be enabled in order to support this database - enable_extension 'plpgsql' + enable_extension "plpgsql" - create_table 'bridges', force: :cascade do |t| - t.string 'name', null: false - t.string 'inbound_url', null: false - t.string 'outbound_url', null: false - t.string 'method', null: false - t.integer 'retries', null: false - t.integer 'delay', null: false - t.binary 'payload', null: false - t.datetime 'created_at', precision: 6, null: false - t.datetime 'updated_at', precision: 6, null: false - t.index ['inbound_url'], name: 'index_bridges_on_inbound_url', unique: true + create_table "bridges", force: :cascade do |t| + t.string "name", null: false + t.string "inbound_url", null: false + t.string "outbound_url", null: false + t.string "method", null: false + t.integer "retries", null: false + t.integer "delay", null: false + t.binary "payload", null: false + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.bigint "user_id", null: false + t.index ["inbound_url"], name: "index_bridges_on_inbound_url", unique: true + t.index ["user_id"], name: "index_bridges_on_user_id" end - create_table 'environment_variables', force: :cascade do |t| - t.string 'key', null: false - t.string 'value', null: false - t.bigint 'bridge_id', null: false - t.datetime 'created_at', precision: 6, null: false - t.datetime 'updated_at', precision: 6, null: false - t.index ['bridge_id'], name: 'index_environment_variables_on_bridge_id' + create_table "environment_variables", force: :cascade do |t| + t.string "key", null: false + t.string "value", null: false + t.bigint "bridge_id", null: false + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["bridge_id"], name: "index_environment_variables_on_bridge_id" end - create_table 'events', force: :cascade do |t| - t.boolean 'completed', null: false - t.binary 'data', null: false - t.string 'inbound_url', null: false - t.string 'outbound_url', null: false - t.integer 'status_code', null: false - t.datetime 'completed_at' - t.bigint 'bridge_id', null: false - t.datetime 'created_at', precision: 6, null: false - t.datetime 'updated_at', precision: 6, null: false - t.index ['bridge_id'], name: 'index_events_on_bridge_id' + create_table "events", force: :cascade do |t| + t.boolean "completed", null: false + t.binary "data", null: false + t.string "inbound_url", null: false + t.string "outbound_url", null: false + t.integer "status_code", null: false + t.datetime "completed_at" + t.bigint "bridge_id", null: false + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["bridge_id"], name: "index_events_on_bridge_id" end - create_table 'headers', force: :cascade do |t| - t.string 'value', null: false - t.string 'key', null: false - t.bigint 'bridge_id', null: false - t.datetime 'created_at', precision: 6, null: false - t.datetime 'updated_at', precision: 6, null: false - t.index ['bridge_id'], name: 'index_headers_on_bridge_id' + create_table "headers", force: :cascade do |t| + t.string "value", null: false + t.string "key", null: false + t.bigint "bridge_id", null: false + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["bridge_id"], name: "index_headers_on_bridge_id" end - create_table 'users', force: :cascade do |t| - t.string 'email', null: false - t.string 'password_digest', null: false - t.string 'recovery_password_digest' - t.boolean 'notifications', default: false, null: false - t.datetime 'created_at', precision: 6, null: false - t.datetime 'updated_at', precision: 6, null: false - t.index ['email'], name: 'index_users_on_email', unique: true + create_table "users", force: :cascade do |t| + t.string "email", null: false + t.string "password_digest", null: false + t.string "recovery_password_digest" + t.boolean "notifications", default: false, null: false + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["email"], name: "index_users_on_email", unique: true end + + add_foreign_key "bridges", "users" end From 88463d195e29b7852d1d0821f872d1d6f4724dab Mon Sep 17 00:00:00 2001 From: William Jackson Date: Tue, 10 Nov 2020 16:13:06 -0600 Subject: [PATCH 09/64] fixed type --- app/models/bridge.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/bridge.rb b/app/models/bridge.rb index 6d06d4a..8518a45 100644 --- a/app/models/bridge.rb +++ b/app/models/bridge.rb @@ -36,7 +36,7 @@ class Bridge < ApplicationRecord has_many :environment_variables, dependent: :destroy has_many :headers, dependent: :destroy has_many :events, dependent: :destroy - belongs to :user + belongs_to :user alias_attribute :env_vars, :environment_variables From 5bf7e3bc8003d3c1f18500ac181cc7e76fafb91f Mon Sep 17 00:00:00 2001 From: William Jackson Date: Wed, 11 Nov 2020 11:50:33 -0600 Subject: [PATCH 10/64] binary to jsonb --- .../20201111170705_change_binary_columns_to_json_b.rb | 11 +++++++++++ db/migrate/20201111174614_remove_default_attribute.rb | 5 +++++ db/schema.rb | 6 +++--- 3 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 db/migrate/20201111170705_change_binary_columns_to_json_b.rb create mode 100644 db/migrate/20201111174614_remove_default_attribute.rb diff --git a/db/migrate/20201111170705_change_binary_columns_to_json_b.rb b/db/migrate/20201111170705_change_binary_columns_to_json_b.rb new file mode 100644 index 0000000..0e9cb74 --- /dev/null +++ b/db/migrate/20201111170705_change_binary_columns_to_json_b.rb @@ -0,0 +1,11 @@ +class ChangeBinaryColumnsToJsonB < ActiveRecord::Migration[6.0] + def up + change_column :bridges, :payload, :jsonb, using: 'payload::text::jsonb', null: false + change_column :events, :data, :jsonb, using: 'data::text::jsonb', null: false + end + + def down + change_column :bridges, :payload, :text + change_column :events, :data, :text + end +end diff --git a/db/migrate/20201111174614_remove_default_attribute.rb b/db/migrate/20201111174614_remove_default_attribute.rb new file mode 100644 index 0000000..5975a97 --- /dev/null +++ b/db/migrate/20201111174614_remove_default_attribute.rb @@ -0,0 +1,5 @@ +class RemoveDefaultAttribute < ActiveRecord::Migration[6.0] + def change + change_column_default :bridges, :payload, nil + end +end diff --git a/db/schema.rb b/db/schema.rb index ea8bba3..e7c1078 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2020_11_10_220329) do +ActiveRecord::Schema.define(version: 2020_11_11_174614) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -22,7 +22,7 @@ t.string "method", null: false t.integer "retries", null: false t.integer "delay", null: false - t.binary "payload", null: false + t.jsonb "payload", null: false t.datetime "created_at", precision: 6, null: false t.datetime "updated_at", precision: 6, null: false t.bigint "user_id", null: false @@ -41,7 +41,7 @@ create_table "events", force: :cascade do |t| t.boolean "completed", null: false - t.binary "data", null: false + t.jsonb "data", null: false t.string "inbound_url", null: false t.string "outbound_url", null: false t.integer "status_code", null: false From 4f8920ea9cc6e335fb21025826fce700f5c6693a Mon Sep 17 00:00:00 2001 From: William Jackson Date: Wed, 11 Nov 2020 13:28:16 -0600 Subject: [PATCH 11/64] progressing --- app/controllers/events_controller.rb | 63 ++++++++++++++++------- config/routes.rb | 1 + db/schema.rb | 75 ++++++++++++++++++++++++++++ 3 files changed, 120 insertions(+), 19 deletions(-) create mode 100644 db/schema.rb diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index e1ba1d2..da33e36 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true class EventsController < ApplicationController - before_action :authorize_request - before_action :set_user + # before_action :authorize_request + # Needs to find all events based on bridge_id or event_id # Needs to return id for each event as well # def index @@ -19,19 +19,21 @@ class EventsController < ApplicationController # render json: { events: events }, status: 200 # OK # end - def show; end + # def show; end # receive bridge id + data def create - bridge = Bridge.find(params[:id]) - data = { inbound: request.body, outbounds: [] } - event = Event.new(data: data, bridge_id: bridge.id) - if event.save - # EventWorker.perform(id of event just created) - status 201 # Created - else - status 400 # Bad Request - end + # bridge = Bridge.find(event_params[:id]) + payload = JSON.parse(request.body.read) + data = { inbound: payload, outbounds: [] } + binding.pry + # event = Event.new(data: data, bridge_id: bridge.id) + # if event.save + # # EventWorker.perform(id of event just created) + # status 201 # Created + # else + # status 400 # Bad Request + # end end private @@ -47,17 +49,18 @@ def set_user # bridge_id or #event_id # => User end -end -# Step 1 -# Convert binary to jsonb + def event_params + params.require(:bridge).permit(:id) + end +end # Step 2 # payload: -{ - test: 'user entered string from editor', - production: 'user entered string from editor' -} +# { +# test: 'user entered string from editor', +# production: 'user entered string from editor' +# } # Step 3 # @@ -66,3 +69,25 @@ def set_user # localhost:3000/sidekiq to monitor sidekiq while running # after turning it on # => mount Sidekiq::Web => '/sidekiq + +# events_data: { +# inbound: {payload}, +# outbound: [ # 0 - 5 +# { request: {payload}, +# response: {payload} +# }, +# { request: {payload}, +# response: {payload} +# }, +# { request: {payload}, +# response: {payload} +# }, +# { request: {payload}, +# response: {payload} +# }, +# { request: {payload}, +# response: {payload} +# } +# ] + +# } diff --git a/config/routes.rb b/config/routes.rb index 33156ba..eb1aa5d 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -7,5 +7,6 @@ put 'users', to: 'users#update' delete 'users', to: 'users#destroy' post 'login', to: 'sessions#create' + post 'events', to: 'events#create' get 'events', to: 'events#index' end diff --git a/db/schema.rb b/db/schema.rb new file mode 100644 index 0000000..b26c6ef --- /dev/null +++ b/db/schema.rb @@ -0,0 +1,75 @@ +# This file is auto-generated from the current state of the database. Instead +# of editing this file, please use the migrations feature of Active Record to +# incrementally modify your database, and then regenerate this schema definition. +# +# This file is the source Rails uses to define your schema when running `rails +# db:schema:load`. When creating a new database, `rails db:schema:load` tends to +# be faster and is potentially less error prone than running all of your +# migrations from scratch. Old migrations may fail to apply correctly if those +# migrations use external dependencies or application code. +# +# It's strongly recommended that you check this file into your version control system. + +ActiveRecord::Schema.define(version: 2020_11_11_175121) do + + # These are extensions that must be enabled in order to support this database + enable_extension "plpgsql" + + create_table "bridges", force: :cascade do |t| + t.string "name", null: false + t.string "inbound_url", null: false + t.string "outbound_url", null: false + t.string "method", null: false + t.integer "retries", null: false + t.integer "delay", null: false + t.jsonb "payload", null: false + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.bigint "user_id", null: false + t.index ["inbound_url"], name: "index_bridges_on_inbound_url", unique: true + t.index ["user_id"], name: "index_bridges_on_user_id" + end + + create_table "environment_variables", force: :cascade do |t| + t.string "key", null: false + t.string "value", null: false + t.bigint "bridge_id", null: false + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["bridge_id"], name: "index_environment_variables_on_bridge_id" + end + + create_table "events", force: :cascade do |t| + t.boolean "completed", null: false + t.jsonb "data", null: false + t.string "inbound_url", null: false + t.string "outbound_url", null: false + t.integer "status_code", null: false + t.datetime "completed_at" + t.bigint "bridge_id", null: false + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["bridge_id"], name: "index_events_on_bridge_id" + end + + create_table "headers", force: :cascade do |t| + t.string "value", null: false + t.string "key", null: false + t.bigint "bridge_id", null: false + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["bridge_id"], name: "index_headers_on_bridge_id" + end + + create_table "users", force: :cascade do |t| + t.string "email", null: false + t.string "password_digest", null: false + t.string "recovery_password_digest" + t.boolean "notifications", default: false, null: false + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["email"], name: "index_users_on_email", unique: true + end + + add_foreign_key "bridges", "users" +end From 45e38b80ff689202b7b7b4a8f1cb1f79902f460d Mon Sep 17 00:00:00 2001 From: William Jackson Date: Wed, 11 Nov 2020 13:56:01 -0600 Subject: [PATCH 12/64] removed null constraint from status_code attribute --- .../20201111195349_remove_not_null_status_code_in_events.rb | 5 +++++ db/schema.rb | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 db/migrate/20201111195349_remove_not_null_status_code_in_events.rb diff --git a/db/migrate/20201111195349_remove_not_null_status_code_in_events.rb b/db/migrate/20201111195349_remove_not_null_status_code_in_events.rb new file mode 100644 index 0000000..790e3db --- /dev/null +++ b/db/migrate/20201111195349_remove_not_null_status_code_in_events.rb @@ -0,0 +1,5 @@ +class RemoveNotNullStatusCodeInEvents < ActiveRecord::Migration[6.0] + def change + change_column_null :events, :status_code, true + end +end diff --git a/db/schema.rb b/db/schema.rb index 39dcf84..e363f26 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2020_11_11_192930) do +ActiveRecord::Schema.define(version: 2020_11_11_195349) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -44,7 +44,7 @@ t.jsonb "data", null: false t.string "inbound_url", null: false t.string "outbound_url", null: false - t.integer "status_code", null: false + t.integer "status_code" t.datetime "completed_at" t.bigint "bridge_id", null: false t.datetime "created_at", precision: 6, null: false From efc0d249f2310517bac1081e921c482992dba52d Mon Sep 17 00:00:00 2001 From: William Jackson Date: Wed, 11 Nov 2020 18:30:38 -0600 Subject: [PATCH 13/64] http request functionality in sidekiq --- app/controllers/events_controller.rb | 62 ++++++++++++++-------------- app/workers/event_worker.rb | 30 +++++++++++--- 2 files changed, 56 insertions(+), 36 deletions(-) diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index da33e36..21175b5 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -5,35 +5,42 @@ class EventsController < ApplicationController # Needs to find all events based on bridge_id or event_id # Needs to return id for each event as well - # def index - # events = @current_user.bridges - # events.map! do |event| - # updated_at = String(event.updated_at) - # date = date_format(updated_at.split(' ')[1]) - # time = updated_at.split(' ')[0] - # { id: 1, - # time: time, - # date: date, - # status_code: event.status_code } - # end - # render json: { events: events }, status: 200 # OK - # end + def index + # events = @current_user.bridges + # events.map! do |event| + # updated_at = String(event.updated_at) + # date = date_format(updated_at.split(' ')[1]) + # time = updated_at.split(' ')[0] + # { id: 1, + # time: time, + # date: date, + # status_code: event.status_code } + # end + # render json: { events: events }, status: 200 # OK + end - # def show; end + def show; end # receive bridge id + data def create - # bridge = Bridge.find(event_params[:id]) + bridge = Bridge.find(event_params[:id]) payload = JSON.parse(request.body.read) - data = { inbound: payload, outbounds: [] } - binding.pry - # event = Event.new(data: data, bridge_id: bridge.id) - # if event.save - # # EventWorker.perform(id of event just created) - # status 201 # Created - # else - # status 400 # Bad Request - # end + data = { inbound: payload, outbound: [] } + event = Event.new( + data: data, + bridge_id: bridge.id, + inbound_url: bridge.inbound_url, + outbound_url: bridge.outbound_url + ) + event.save! + EventWorker.new.perform(event.id) + # render json: {}, status: 201 # Created + rescue ActiveRecord::RecordNotFound + render json: { error: 'a bridge by that id was not found' }, status: 400 + rescue ActiveRecord::RecordInvalid + render json: { error: 'payload, bridge_id, or urls were invalid' }, status: 400 # Bad Request + rescue ActiveRecord::NotNullViolation + render json: { error: 'payload, bridge_id, or urls fields were not submitted' }, status: 400 # Bad Request end private @@ -45,13 +52,8 @@ def date_format(_date) "#{year}-#{month}-#{day}" end - def set_user - # bridge_id or #event_id - # => User - end - def event_params - params.require(:bridge).permit(:id) + params.require(:bridge).permit(:id, :inbound_url, :outbound_url) end end diff --git a/app/workers/event_worker.rb b/app/workers/event_worker.rb index 76f0ad6..f3abfc3 100644 --- a/app/workers/event_worker.rb +++ b/app/workers/event_worker.rb @@ -1,15 +1,33 @@ # frozen_string_literal: true +require 'net/http' + class EventWorker include Sidekiq::Worker - # takes event id argument - # find event => - # find bridge => - # git payload & outbound url - # => send it + def make_http_uri(uri) + uri =~ %r{\Ahttp://} ? uri : "http://#{uri}" + end def perform(event_id) - # do something + event = Event.find(event_id) + bridge = Bridge.find(event.bridge_id) + method = bridge.method.capitalize + + uri = URI(make_http_uri(bridge.outbound_url)) + http = Net::HTTP.new(uri.host, uri.port) + + request = "Net::HTTP::#{method}".constantize.new(uri, 'Content-Type' => 'application/json') + request.body = bridge.payload.to_json + + response = http.request(req) + response_body = JSON.parse(response.body) + + # payload = { + # test: {}, + # production: {} + # } + + render json: { payload: response_body }, status: 201 # Created end end From e2de02f390b46356df96ccd4f4fe29103e76b45d Mon Sep 17 00:00:00 2001 From: William Jackson Date: Wed, 11 Nov 2020 23:33:38 -0600 Subject: [PATCH 14/64] latest --- Gemfile | 2 +- Gemfile.lock | 1 + app/controllers/events_controller.rb | 11 ++++-- app/workers/event_worker.rb | 52 +++++++++++++++++++++++----- 4 files changed, 54 insertions(+), 12 deletions(-) diff --git a/Gemfile b/Gemfile index 8e06226..61b458f 100644 --- a/Gemfile +++ b/Gemfile @@ -14,7 +14,7 @@ gem 'puma', '~> 4.1' # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder gem 'jbuilder', '~> 2.7' # Use Redis adapter to run Action Cable in production -# gem 'redis', '~> 4.0' +gem 'redis', '~> 4.0' # Use Active Model has_secure_password gem 'bcrypt', '~> 3.1.7' gem 'jwt' diff --git a/Gemfile.lock b/Gemfile.lock index 9461f08..6d6dce7 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -217,6 +217,7 @@ DEPENDENCIES puma (~> 4.1) rack-cors rails (~> 6.0.3, >= 6.0.3.4) + redis (~> 4.0) rspec-rails (~> 4.0.1) rubocop sidekiq diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index 21175b5..e61aa2f 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -25,7 +25,14 @@ def show; end def create bridge = Bridge.find(event_params[:id]) payload = JSON.parse(request.body.read) - data = { inbound: payload, outbound: [] } + data = { 'inbound' => { + 'payload' => payload, + 'date' => DateTime.now.utc.to_s.split(' ').first, + 'time' => DateTime.now.utc.to_s.split(' ')[1], + 'ip' => request.ip, + 'content_length' => request.content_length + }, + outbound: [] } event = Event.new( data: data, bridge_id: bridge.id, @@ -34,7 +41,7 @@ def create ) event.save! EventWorker.new.perform(event.id) - # render json: {}, status: 201 # Created + render json: event, status: 201 # Created rescue ActiveRecord::RecordNotFound render json: { error: 'a bridge by that id was not found' }, status: 400 rescue ActiveRecord::RecordInvalid diff --git a/app/workers/event_worker.rb b/app/workers/event_worker.rb index f3abfc3..aad6aee 100644 --- a/app/workers/event_worker.rb +++ b/app/workers/event_worker.rb @@ -1,33 +1,67 @@ # frozen_string_literal: true require 'net/http' +SCHEME = 'http://' class EventWorker include Sidekiq::Worker + def save_request(request_body, content_length) + request = { + 'payload' => request_body, + 'date' => DateTime.now.utc.to_s.split(' ').first, + 'time' => DateTime.now.utc.to_s.split(' ')[1], + 'content_length' => content_length + } + + @event.data['outbound'].push({ 'request' => request, 'response' => {} }) + end + + def save_response(response) + event_response = @event.data['outbound'].last['response'] + event_response['payload'] = JSON.parse(response.body) + # #=> {"ip" => "153.33.111.24"} + event_response['message'] = response.message + # #=> "OK" + event_response['status_code'] = @event.status_code = response.code + # #=> "200" + + if @event.status_code < 300 + @event.completed = true + @event.completed_at = Time.now.utc + end + @event.save + end + def make_http_uri(uri) - uri =~ %r{\Ahttp://} ? uri : "http://#{uri}" + return uri if uri.starts_with?('https') + + minus_scheme = uri.split('//').last + "#{SCHEME}#{minus_scheme}" end def perform(event_id) - event = Event.find(event_id) - bridge = Bridge.find(event.bridge_id) + @event = Event.find(event_id) + bridge = Bridge.find(@event.bridge_id) method = bridge.method.capitalize uri = URI(make_http_uri(bridge.outbound_url)) http = Net::HTTP.new(uri.host, uri.port) - - request = "Net::HTTP::#{method}".constantize.new(uri, 'Content-Type' => 'application/json') - request.body = bridge.payload.to_json + http.use_ssl = (uri.scheme == 'https') + + req = "Net::HTTP::#{method}".constantize.new(uri, 'Content-Type' => 'application/json') + request_body = bridge.payload # HOW TO DYNAMICALLY GENERATE THIS? + req.body = request_body.to_json + save_request(request_body, req.body.length) response = http.request(req) - response_body = JSON.parse(response.body) + save_response(response) + + # binding.pry # payload = { # test: {}, # production: {} # } - - render json: { payload: response_body }, status: 201 # Created end end From f83a65ff8ddb76344e7396f56135f05d7aa6ecb4 Mon Sep 17 00:00:00 2001 From: William Jackson Date: Thu, 12 Nov 2020 01:23:06 -0600 Subject: [PATCH 15/64] finished draft for index, create and show eventscontroller actions --- app/controllers/events_controller.rb | 60 ++++++++++++++++----------- app/workers/event_worker.rb | 33 ++++++++------- config/routes.rb | 1 + db/schema.rb | 3 ++ dump.rdb | Bin 0 -> 203 bytes 5 files changed, 58 insertions(+), 39 deletions(-) create mode 100644 dump.rdb diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index e61aa2f..9f4f0e0 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -3,25 +3,43 @@ class EventsController < ApplicationController # before_action :authorize_request + def date_format(time) + year = time.split('-')[0] + month = time.split('-')[1] + day = time.split('-')[2] + "#{year}-#{month}-#{day}" + end + # Needs to find all events based on bridge_id or event_id # Needs to return id for each event as well def index - # events = @current_user.bridges - # events.map! do |event| - # updated_at = String(event.updated_at) - # date = date_format(updated_at.split(' ')[1]) - # time = updated_at.split(' ')[0] - # { id: 1, - # time: time, - # date: date, - # status_code: event.status_code } - # end - # render json: { events: events }, status: 200 # OK + # binding.pry + if params[:event_id] + event = Event.find(params[:event_id]) + events = Bridge.find(event.bridge_id).events + elsif params[:bridge_id] + events = Bridge.find(params[:bridge_id]).events + else raise ActiveRecord::RecordNotFound + end + + safe_events = events.map do |event| + updated_at = String(event.updated_at) + time = date_format(updated_at.split(' ')[1]) + date = updated_at.split(' ')[0] + { id: event.id, + time: time.slice(0..-3), + date: date, + status_code: event.status_code } + end + render json: safe_events, status: 200 # OK end - def show; end + def show + event = Event.find(params[:id]) + render json: event, status: 200 # OK + end - # receive bridge id + data + # receives bridge id + data def create bridge = Bridge.find(event_params[:id]) payload = JSON.parse(request.body.read) @@ -32,15 +50,16 @@ def create 'ip' => request.ip, 'content_length' => request.content_length }, - outbound: [] } + 'outbound' => [] } event = Event.new( data: data, bridge_id: bridge.id, inbound_url: bridge.inbound_url, outbound_url: bridge.outbound_url ) - event.save! - EventWorker.new.perform(event.id) + event.save! unless event_params[:test] + EventWorker.new.perform(event, bridge, event_params[:test]) + render json: event, status: 201 # Created rescue ActiveRecord::RecordNotFound render json: { error: 'a bridge by that id was not found' }, status: 400 @@ -52,15 +71,8 @@ def create private - def date_format(_date) - year = time.split('-')[0] - month = time.split('-')[1] - day = time.split('-')[2] - "#{year}-#{month}-#{day}" - end - def event_params - params.require(:bridge).permit(:id, :inbound_url, :outbound_url) + params.permit(:id, :bridge_id, :event_id, :test, :inbound_url, :outbound_url) end end diff --git a/app/workers/event_worker.rb b/app/workers/event_worker.rb index aad6aee..634b4ce 100644 --- a/app/workers/event_worker.rb +++ b/app/workers/event_worker.rb @@ -13,24 +13,26 @@ def save_request(request_body, content_length) 'time' => DateTime.now.utc.to_s.split(' ')[1], 'content_length' => content_length } - @event.data['outbound'].push({ 'request' => request, 'response' => {} }) end def save_response(response) - event_response = @event.data['outbound'].last['response'] - event_response['payload'] = JSON.parse(response.body) - # #=> {"ip" => "153.33.111.24"} - event_response['message'] = response.message - # #=> "OK" - event_response['status_code'] = @event.status_code = response.code - # #=> "200" + response_object = { + 'date' => DateTime.now.utc.to_s.split(' ').first, + 'time' => DateTime.now.utc.to_s.split(' ')[1], + 'status_code' => response.code, + 'message' => response.message, + 'size' => response.size, + 'payload' => JSON.parse(response.body) + } + @event.data['outbound'].last['response'] = response_object + @event.status_code = response.code if @event.status_code < 300 @event.completed = true @event.completed_at = Time.now.utc end - @event.save + @event.save unless @test end def make_http_uri(uri) @@ -40,9 +42,10 @@ def make_http_uri(uri) "#{SCHEME}#{minus_scheme}" end - def perform(event_id) - @event = Event.find(event_id) - bridge = Bridge.find(@event.bridge_id) + def perform(event, bridge, test = false) + @test = test + @event = event + # bridge = Bridge.find(@event.bridge_id) method = bridge.method.capitalize uri = URI(make_http_uri(bridge.outbound_url)) @@ -50,15 +53,15 @@ def perform(event_id) http.use_ssl = (uri.scheme == 'https') req = "Net::HTTP::#{method}".constantize.new(uri, 'Content-Type' => 'application/json') - request_body = bridge.payload # HOW TO DYNAMICALLY GENERATE THIS? + request_body = bridge.payload # Done on the frontend / bridge controller? req.body = request_body.to_json save_request(request_body, req.body.length) response = http.request(req) save_response(response) - # binding.pry - + # Send back test event from here? + # payload = { # test: {}, # production: {} diff --git a/config/routes.rb b/config/routes.rb index eb1aa5d..96000d3 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -9,4 +9,5 @@ post 'login', to: 'sessions#create' post 'events', to: 'events#create' get 'events', to: 'events#index' + get 'events/:id', to: 'events#show' end diff --git a/db/schema.rb b/db/schema.rb index e363f26..9a2d83f 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -72,4 +72,7 @@ end add_foreign_key "bridges", "users" + add_foreign_key "environment_variables", "bridges" + add_foreign_key "events", "bridges" + add_foreign_key "headers", "bridges" end diff --git a/dump.rdb b/dump.rdb new file mode 100644 index 0000000000000000000000000000000000000000..0617018086f58b4e53891b6cbbd8f8ad250e7939 GIT binary patch literal 203 zcmWG?b@2=~Ffg$E#aWb^l3A=^hTkkq41C2Ui6vGAMfu68#X!{u z7#O(0B58@4IUw#I<+UdYXE8D`Fo+?ETNxP`8R!}s>KYm$E0IJs3`N!d5Pp%xwYQ!( F0s!uNNzniR literal 0 HcmV?d00001 From cc00730a623ddbd913bc7d897151d9827df15f08 Mon Sep 17 00:00:00 2001 From: William Jackson Date: Thu, 12 Nov 2020 01:27:59 -0600 Subject: [PATCH 16/64] . --- app/workers/event_worker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/event_worker.rb b/app/workers/event_worker.rb index 634b4ce..a92ad6a 100644 --- a/app/workers/event_worker.rb +++ b/app/workers/event_worker.rb @@ -61,7 +61,7 @@ def perform(event, bridge, test = false) save_response(response) # Send back test event from here? - + # payload = { # test: {}, # production: {} From 216b7e0abec96499ffb14e1e4f095e4aa25ad26b Mon Sep 17 00:00:00 2001 From: William Jackson Date: Thu, 12 Nov 2020 01:59:27 -0600 Subject: [PATCH 17/64] added destroy action --- app/controllers/events_controller.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index 9f4f0e0..67fc62e 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -39,6 +39,11 @@ def show render json: event, status: 200 # OK end + def destroy + event = Event.find(event_params[:id]) + event.destroy + end + # receives bridge id + data def create bridge = Bridge.find(event_params[:id]) From 0649385859437cc2d4756dd498ee758aadd656fe Mon Sep 17 00:00:00 2001 From: William Jackson Date: Fri, 13 Nov 2020 00:24:11 -0600 Subject: [PATCH 18/64] updating retry and delay logic --- app/controllers/events_controller.rb | 56 ++++--- app/lib/exceptions/large_status_code.rb | 6 + app/workers/event_worker.rb | 146 ++++++++++++------ config/routes.rb | 3 + .../20201113001336_add_test_to_events.rb | 5 + db/schema.rb | 3 +- db/seeds.rb | 12 +- dump.rdb | Bin 203 -> 4829 bytes 8 files changed, 155 insertions(+), 76 deletions(-) create mode 100644 app/lib/exceptions/large_status_code.rb create mode 100644 db/migrate/20201113001336_add_test_to_events.rb diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index 67fc62e..773473f 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -1,24 +1,22 @@ # frozen_string_literal: true +def date_format(time) + year = time.split('-')[0] + month = time.split('-')[1] + day = time.split('-')[2] + "#{year}-#{month}-#{day}" +end + class EventsController < ApplicationController # before_action :authorize_request - def date_format(time) - year = time.split('-')[0] - month = time.split('-')[1] - day = time.split('-')[2] - "#{year}-#{month}-#{day}" - end - - # Needs to find all events based on bridge_id or event_id - # Needs to return id for each event as well def index # binding.pry - if params[:event_id] - event = Event.find(params[:event_id]) + if event_params[:event_id] + event = Event.find(event_params[:event_id]) events = Bridge.find(event.bridge_id).events - elsif params[:bridge_id] - events = Bridge.find(params[:bridge_id]).events + elsif event_params[:bridge_id] + events = Bridge.find(event_params[:bridge_id]).events else raise ActiveRecord::RecordNotFound end @@ -35,7 +33,7 @@ def index end def show - event = Event.find(params[:id]) + event = Event.find(event_params[:id]) render json: event, status: 200 # OK end @@ -44,9 +42,9 @@ def destroy event.destroy end - # receives bridge id + data def create - bridge = Bridge.find(event_params[:id]) + bridge = Bridge.find(create_event_params[:id]) + test_mode = create_event_params[:test] payload = JSON.parse(request.body.read) data = { 'inbound' => { 'payload' => payload, @@ -57,15 +55,27 @@ def create }, 'outbound' => [] } event = Event.new( - data: data, + data: data.to_json, bridge_id: bridge.id, + test: test_mode, inbound_url: bridge.inbound_url, outbound_url: bridge.outbound_url ) - event.save! unless event_params[:test] - EventWorker.new.perform(event, bridge, event_params[:test]) + event.save! + # EventWorker.new.perform(event.id) + EventWorker.perform_async(event.id) + # (bridge.retries + 1).times do + # if bridge.delay + # seconds = bridge.delay * 60 + # EventWorker.perform_in(seconds, event.id, event_params[:test_mode]) + # sleep seconds + 5 + # break if event.completed + # else + # EventWorker.perform_async(event.id, event_params[:test_mode]) + # end + # end - render json: event, status: 201 # Created + render json: {}, status: 201 # Created rescue ActiveRecord::RecordNotFound render json: { error: 'a bridge by that id was not found' }, status: 400 rescue ActiveRecord::RecordInvalid @@ -76,8 +86,12 @@ def create private + def create_event_params + params.require(:bridge).permit(:id, :test) + end + def event_params - params.permit(:id, :bridge_id, :event_id, :test, :inbound_url, :outbound_url) + params.permit(:id, :bridge_id, :event_id, :test) end end diff --git a/app/lib/exceptions/large_status_code.rb b/app/lib/exceptions/large_status_code.rb new file mode 100644 index 0000000..9f552dd --- /dev/null +++ b/app/lib/exceptions/large_status_code.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +module Sidekiq + class LargeStatusCode < StandardError + end +end diff --git a/app/workers/event_worker.rb b/app/workers/event_worker.rb index a92ad6a..ac3a7ba 100644 --- a/app/workers/event_worker.rb +++ b/app/workers/event_worker.rb @@ -1,70 +1,120 @@ # frozen_string_literal: true require 'net/http' +require_relative '../lib/exceptions/large_status_code' SCHEME = 'http://' -class EventWorker - include Sidekiq::Worker +HTTP_ERRORS = [ + EOFError, + Errno::ECONNRESET, + Errno::EINVAL, + Net::HTTPBadResponse, + Net::HTTPHeaderSyntaxError, + Net::ProtocolError, + SocketError, + Timeout::Error +].freeze - def save_request(request_body, content_length) - request = { - 'payload' => request_body, - 'date' => DateTime.now.utc.to_s.split(' ').first, - 'time' => DateTime.now.utc.to_s.split(' ')[1], - 'content_length' => content_length - } - @event.data['outbound'].push({ 'request' => request, 'response' => {} }) - end +def make_http_uri(uri) + return uri if uri.starts_with?('https') - def save_response(response) - response_object = { - 'date' => DateTime.now.utc.to_s.split(' ').first, - 'time' => DateTime.now.utc.to_s.split(' ')[1], - 'status_code' => response.code, - 'message' => response.message, - 'size' => response.size, - 'payload' => JSON.parse(response.body) - } - @event.data['outbound'].last['response'] = response_object - @event.status_code = response.code + minus_scheme = uri.split('//').last + "#{SCHEME}#{minus_scheme}" +end - if @event.status_code < 300 - @event.completed = true - @event.completed_at = Time.now.utc - end - @event.save unless @test - end +def save_request(event, req) + request = { + payload: JSON.parse(req.body), + date: DateTime.now.utc.to_s.split(' ').first, + time: DateTime.now.utc.to_s.split(' ')[1], + content_length: req.length + } + event_data = JSON.parse(event.data) + event_data['outbound'].push({ 'request' => request, 'response' => {} }) + event.data = event_data.to_json + event.save +end - def make_http_uri(uri) - return uri if uri.starts_with?('https') +def save_response(event, resp) + response = { + date: DateTime.now.utc.to_s.split(' ').first, + time: DateTime.now.utc.to_s.split(' ')[1], + status_code: resp.code, + message: resp.message, + size: resp.size, + payload: resp.body + } + event_data = JSON.parse(event.data) + event_data['outbound'].last['response'] = response + event.data = event_data.to_json + event.status_code = resp.code - minus_scheme = uri.split('//').last - "#{SCHEME}#{minus_scheme}" - end + # if event.status_code < 300 + # event.completed = true + # event.completed_at = Time.now.utc + # end + event.save + raise Sidekiq::LargeStatusCode if event.status_code >= 300 +end + +class EventWorker + include Sidekiq::Worker + sidekiq_options retry: 0 - def perform(event, bridge, test = false) - @test = test - @event = event - # bridge = Bridge.find(@event.bridge_id) + def perform(event_id) + event = Event.find(event_id) + bridge = Bridge.find(event.bridge_id) method = bridge.method.capitalize + EventWorker.sidekiq_options(retry: bridge.retries) + + # Generate and save request uri = URI(make_http_uri(bridge.outbound_url)) http = Net::HTTP.new(uri.host, uri.port) http.use_ssl = (uri.scheme == 'https') - req = "Net::HTTP::#{method}".constantize.new(uri, 'Content-Type' => 'application/json') - request_body = bridge.payload # Done on the frontend / bridge controller? - req.body = request_body.to_json - save_request(request_body, req.body.length) + req.body = bridge.payload.to_json + save_request(event, req) - response = http.request(req) - save_response(response) + current_attempts = 0 + begin + response = http.request(req) + save_response(event, response) + rescue StandardError => e + rescue *HTTP_ERROS => e + rescue Sidekiq::LargeStatusCode + # RESCUE OTHER ERRORS + current_attempts += 1 + if current_attempts > bridge.retries + sleep bridge.delay * 60 + retry + end + end + event.completed = true + event.completed_at = Time.now.utc - # Send back test event from here? + # retry_in do |_count, exception| + # case exception + # when *HTTP_ERRORS, Sidekiq::LargeStatusCode + # handles += 1 + # bridge.delay + # end + # end + # rescue SocketError + # binding.pry + # attempts.times do + # sleep 5 if bridge.delay + # # sleep bridge.delay * 60 if bridge.delay + # response = http.request(req) + # save_response(event, response) + # break if event.completed + # end - # payload = { - # test: {}, - # production: {} - # } + # rescue *HTTP_ERRORS => e + # binding.pry + # rescue Errno::ECONNREFUSED => e + # response = http.request(req) + # save_response(event, response) + # end end end diff --git a/config/routes.rb b/config/routes.rb index 96000d3..4598b33 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,3 +1,4 @@ +require 'sidekiq/web' # frozen_string_literal: true Rails.application.routes.draw do @@ -10,4 +11,6 @@ post 'events', to: 'events#create' get 'events', to: 'events#index' get 'events/:id', to: 'events#show' + delete 'routes', to: 'events#destroy' + mount Sidekiq::Web => '/sidekiq' end diff --git a/db/migrate/20201113001336_add_test_to_events.rb b/db/migrate/20201113001336_add_test_to_events.rb new file mode 100644 index 0000000..37e8280 --- /dev/null +++ b/db/migrate/20201113001336_add_test_to_events.rb @@ -0,0 +1,5 @@ +class AddTestToEvents < ActiveRecord::Migration[6.0] + def change + add_column :events, :test, :boolean, default: false, null: false + end +end diff --git a/db/schema.rb b/db/schema.rb index 9a2d83f..5d9664e 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2020_11_11_195349) do +ActiveRecord::Schema.define(version: 2020_11_13_001336) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -49,6 +49,7 @@ t.bigint "bridge_id", null: false t.datetime "created_at", precision: 6, null: false t.datetime "updated_at", precision: 6, null: false + t.boolean "test", default: false, null: false t.index ["bridge_id"], name: "index_events_on_bridge_id" end diff --git a/db/seeds.rb b/db/seeds.rb index 16cdaa2..82eb59d 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -8,11 +8,11 @@ def test_url user2 = User.create(email: 'tester@bridge.io', password: 'password', notifications: false) bridge = Bridge.create( - user: user, + user_id: user.id, name: 'My First Bridge', - payload: '', - inbound_url: Bridge.generate_inbound_url, - outbound_url: test_url, + payload: "{\"FirstName\":\"Lee\",\"LastName\":\"Oswald\",\"UserName\":\"GrassyKnoll\",\"Password\":{\"nested\":\"magic bullet\"},\"Email\":\"kgb63@yandex.ru\"}", + inbound_url: test_url, + outbound_url: 'ip.jsontest.com', method: 'POST', retries: 5, delay: 15 @@ -24,9 +24,9 @@ def test_url bridge.headers << Header.create(key: 'Authentication', value: 'Bearer &&&&&&&&&&&&&&&&') bridge2 = Bridge.create( - user: user2, + user_id: user2.id, name: 'My Second Bridge', - payload: '', + payload: "{\"FirstName\":\"Booths\",\"LastName\":\"John\",\"UserName\":\"FordTheatre\",\"Password\":{\"nested\":\"sic temper tyrannis\"},\"Email\":\"mail@mail.com\"}", inbound_url: Bridge.generate_inbound_url, outbound_url: test_url, method: 'PATCH', diff --git a/dump.rdb b/dump.rdb index 0617018086f58b4e53891b6cbbd8f8ad250e7939..6b891ec1cd95b62d8fe77986099a94a2572251ab 100644 GIT binary patch literal 4829 zcmchbUyM{&9mi*8XLe!#@F&s|1u}<$h>f#%&!2n#jnK}tK-vcgDJ{f6yL11XyIk14 zce!(C0m{%9LU_3Z$H$iCKHMD#?s{E zxT*~Ad3ayJ#FP~vfy^-#b)aXZGsmmGiqmmaRcTvU@fyS+#il?+8k{ge({{CTs*z0l zffrsY{-pSB_H^z@ScwB`Lgfm%xgXEvbCuYNVwH7AtvFZc{1E4+X_y#7GoPFE-CS!4 z8J5q*))@HOz{8R6j;YeV$Bf7ZJNlh4aH>(H0_PNXa3jp;j#pJxRdR(xIahgBbv({J zmCub^wi++40|ULjtAf~%fzeDoTeep%*jcW2d*toI)t5Db#259#;=*JUI;x_{6)?nb z7@=W=8(`H?`%enpKS*9Ho-JNTU(iqWgPyE@y7=1ZT%_XYRIV_yBE9eK*ZcIYopGyD z0c-nrP3hGf45MQz0{5k?Xbh}daR^Dh;-kI`vg>V+yT~z-lnPN3W6Y_iFn2&c>*&La z)qHLO5@B#j8Pu>;A}CjzgV`P?Yo8@nE!<6GBGudBl9*WwATT+6jQmk^U#R9Z`}o(YT$!YL%C$mfzOC{Hy9K~iFn?T?}` z8lBJwVT~2f4m}c9$K7CC3_L$@LHS36;DjF^0o#0c+YU}Zlu%{dcz>v*)jy>kS56qY zg+jfrZ*Lgy4y*c+e2-`OVU6;D@j;0K6?Jq=l*h|bgd*y zN0xz;RxRN>2WyE&5D;x*9$W}@$O;OvyK!OUB%vfR| zo09NOQN8kir1*19F|&wb-;E6B+93JqC%Odxt_kLz{rF~wpjrC%@(8k7f)sM;S<1wm zd$x3#p0~uK65fR%QXl%OSwGNH91k69986h}Z|QUaVFV^rd?eIf zRp4T38}Pu|#9?9_wsR9FR1 zr>)>20E$nQHr8LdnoIB1qTN+|3;eZX=9M4!a)|^Z`RU1?mIT4BQ~}nmUx}`F0Ls!c z%L7Pf0eXrk1&L?7o+G(UkmI?WczUobF_kGdnIV{fSj++(06hpI0l$t`3DC+cxDFVndO0; z%N5UtO1O^cnaFd9X=+7c1=Z3qtGg63sHB21JmY8GS{UN;&SYsj`oBSGJJ6zP`6x5Z z41wGr2qcm^pTE@wxmkmJ>C@l2-_g->`{ud#rrL$={&vM_1^fKAMjG4azH--2M;d!F znReyry;cIH19KOOj0e^6@$SIby56#puw6$eOF$1|sf4nX zU=Bcw2L_gi69aK(a(zIp8uyK6D%)c+DvH@23{CTnb2`8JL=*;NAgE5*x<-Z`052q~ z6X5l9MTfl)e!2qSJWwYml>;g%fF1z#da3c7MCts9*6qO=-S*UmNXw9h$uJ>M3wLXW zckxOB*4HHbW)Ey!OzX~K);4eW)jMa}!JaD}TsADc5EiGVL-jbo$P&sST1gfW9E!WZ z(sCA>7@diy)6EnZFd5-`>T<%RxC7Vgi*cQ4;d-Ns>#cdX;x?{njqCbiTuB1Kn|%f& z5=y=Vt&zrs3>8v}+UnK9>)_g4dUDyg$UzOu}wu|XqwS^5#Bx8I| zTmAaPku-=9k`xz~?R%!J!5M8KQ?mT@maeoe&q!;eexcW-97i3JyOvLVx+-2nA%#6*zEZuWMD711|PLt;YlqG6RCc~W1> zDkE0wh0KV)m>t#kx{-PBKoWNGOUaQF<)5W0BdT~gF0)U-;x}K{*-|Xr`DL)u1uId}sFM|AFvRYbO9!ZQ|U7y%r-l=op9QaeG&-7dy zBzlbMYcit*>3V&%8^h^@tgmjxf^q)Sj6{0W)%H()@Q3edNhSv7g>hrw+*_5ULmD^M PefIq;n~G Date: Sat, 14 Nov 2020 13:28:16 -0600 Subject: [PATCH 19/64] updated create action and event_worker to work with delays, retries and errors --- app/controllers/events_controller.rb | 51 +-------------- app/workers/event_worker.rb | 91 ++++++++++++--------------- config/routes.rb | 7 ++- dump.rdb | Bin 4829 -> 15073 bytes 4 files changed, 46 insertions(+), 103 deletions(-) diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index 773473f..b1e1de2 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -11,7 +11,6 @@ class EventsController < ApplicationController # before_action :authorize_request def index - # binding.pry if event_params[:event_id] event = Event.find(event_params[:event_id]) events = Bridge.find(event.bridge_id).events @@ -62,20 +61,9 @@ def create outbound_url: bridge.outbound_url ) event.save! - # EventWorker.new.perform(event.id) EventWorker.perform_async(event.id) - # (bridge.retries + 1).times do - # if bridge.delay - # seconds = bridge.delay * 60 - # EventWorker.perform_in(seconds, event.id, event_params[:test_mode]) - # sleep seconds + 5 - # break if event.completed - # else - # EventWorker.perform_async(event.id, event_params[:test_mode]) - # end - # end - render json: {}, status: 201 # Created + render json: {}, status: 202 # Accepted (asynchronous processing) rescue ActiveRecord::RecordNotFound render json: { error: 'a bridge by that id was not found' }, status: 400 rescue ActiveRecord::RecordInvalid @@ -94,40 +82,3 @@ def event_params params.permit(:id, :bridge_id, :event_id, :test) end end - -# Step 2 -# payload: -# { -# test: 'user entered string from editor', -# production: 'user entered string from editor' -# } - -# Step 3 -# - -# NB: Need to run `bundle exec sidekiq` in separate terminal -# localhost:3000/sidekiq to monitor sidekiq while running -# after turning it on -# => mount Sidekiq::Web => '/sidekiq - -# events_data: { -# inbound: {payload}, -# outbound: [ # 0 - 5 -# { request: {payload}, -# response: {payload} -# }, -# { request: {payload}, -# response: {payload} -# }, -# { request: {payload}, -# response: {payload} -# }, -# { request: {payload}, -# response: {payload} -# }, -# { request: {payload}, -# response: {payload} -# } -# ] - -# } diff --git a/app/workers/event_worker.rb b/app/workers/event_worker.rb index ac3a7ba..0bb8a1c 100644 --- a/app/workers/event_worker.rb +++ b/app/workers/event_worker.rb @@ -2,15 +2,16 @@ require 'net/http' require_relative '../lib/exceptions/large_status_code' + SCHEME = 'http://' HTTP_ERRORS = [ - EOFError, - Errno::ECONNRESET, - Errno::EINVAL, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError, + EOFError, + Errno::ECONNRESET, + Errno::EINVAL, SocketError, Timeout::Error ].freeze @@ -22,12 +23,12 @@ def make_http_uri(uri) "#{SCHEME}#{minus_scheme}" end -def save_request(event, req) +def save_request(event, length, payload) request = { - payload: JSON.parse(req.body), + payload: JSON.parse(payload), date: DateTime.now.utc.to_s.split(' ').first, time: DateTime.now.utc.to_s.split(' ')[1], - content_length: req.length + content_length: length } event_data = JSON.parse(event.data) event_data['outbound'].push({ 'request' => request, 'response' => {} }) @@ -36,25 +37,39 @@ def save_request(event, req) end def save_response(event, resp) + resp_code = resp.code.to_i + payload = (resp_code >= 300 ? {} : JSON.parse(resp.body)) response = { date: DateTime.now.utc.to_s.split(' ').first, time: DateTime.now.utc.to_s.split(' ')[1], status_code: resp.code, message: resp.message, size: resp.size, - payload: resp.body + payload: payload } event_data = JSON.parse(event.data) event_data['outbound'].last['response'] = response event.data = event_data.to_json - event.status_code = resp.code - # if event.status_code < 300 - # event.completed = true - # event.completed_at = Time.now.utc - # end + event.status_code = resp_code + event.save + raise Sidekiq::LargeStatusCode if resp_code >= 300 +end + +def save_http_error(event, error) + response = { + date: '', + time: '', + status_code: '', + message: error.message, + size: '', + payload: {} + } + + event_data = JSON.parse(event.data) + event_data['outbound'].last['response'] = response + event.data = event_data.to_json event.save - raise Sidekiq::LargeStatusCode if event.status_code >= 300 end class EventWorker @@ -65,56 +80,32 @@ def perform(event_id) event = Event.find(event_id) bridge = Bridge.find(event.bridge_id) method = bridge.method.capitalize + retries = bridge.retries + current_attempts = 0 - EventWorker.sidekiq_options(retry: bridge.retries) - - # Generate and save request + # Generate request uri = URI(make_http_uri(bridge.outbound_url)) http = Net::HTTP.new(uri.host, uri.port) http.use_ssl = (uri.scheme == 'https') req = "Net::HTTP::#{method}".constantize.new(uri, 'Content-Type' => 'application/json') req.body = bridge.payload.to_json - save_request(event, req) - - current_attempts = 0 + begin + current_attempts += 1 + save_request(event, req.length, bridge.payload) response = http.request(req) save_response(event, response) - rescue StandardError => e - rescue *HTTP_ERROS => e - rescue Sidekiq::LargeStatusCode - # RESCUE OTHER ERRORS - current_attempts += 1 - if current_attempts > bridge.retries - sleep bridge.delay * 60 + rescue *HTTP_ERRORS, Sidekiq::LargeStatusCode => e + save_http_error(event, e) if HTTP_ERRORS.include?(e.class) + if current_attempts <= bridge.retries + sleep 1 # DEVELOPMENT + # sleep bridge.delay * 60 # PRODUCTION retry end end + event.completed = true event.completed_at = Time.now.utc - - # retry_in do |_count, exception| - # case exception - # when *HTTP_ERRORS, Sidekiq::LargeStatusCode - # handles += 1 - # bridge.delay - # end - # end - # rescue SocketError - # binding.pry - # attempts.times do - # sleep 5 if bridge.delay - # # sleep bridge.delay * 60 if bridge.delay - # response = http.request(req) - # save_response(event, response) - # break if event.completed - # end - - # rescue *HTTP_ERRORS => e - # binding.pry - # rescue Errno::ECONNREFUSED => e - # response = http.request(req) - # save_response(event, response) - # end + event.save end end diff --git a/config/routes.rb b/config/routes.rb index 4598b33..80a5666 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,6 +1,7 @@ -require 'sidekiq/web' # frozen_string_literal: true +require 'sidekiq/web' + Rails.application.routes.draw do post 'users', to: 'users#create' get 'users', to: 'users#show' @@ -10,7 +11,7 @@ post 'login', to: 'sessions#create' post 'events', to: 'events#create' get 'events', to: 'events#index' - get 'events/:id', to: 'events#show' - delete 'routes', to: 'events#destroy' + get 'events/id', to: 'events#show' + delete 'events', to: 'events#destroy' mount Sidekiq::Web => '/sidekiq' end diff --git a/dump.rdb b/dump.rdb index 6b891ec1cd95b62d8fe77986099a94a2572251ab..142cd0ae86950e089cb6ec66b3ac167f947388bf 100644 GIT binary patch literal 15073 zcmeHOeQ;dWb>CfSwbq9X1_wgG=xPQ2Li_aYz3;vI?n*Md{(wOE@L-dVBBFio-TS1) zt9|R;l|N!MkyG0=Z6|5c&JY6*ZbM+wxQUa{riB<}5(rH@aY&N@Nv2%`$&hI$q%_SW zE+IW{SJLiED~i3C&UE@OX7=g)?zz8n&hPy0Et@x8w^dbDe!4v`MY<5%Df3}NQ5Al= zqw*i@bg?j2dkQSLGK+PA{1#ND&*C<4+trdS2jkm*u&!TwYq^c1BJIUJv|N zS0vQ7V?s{ILNFqvYfWT|uk{}tp13C%2m~%F6s=<1wbB_W;zT8C4C5Fk(<=hY%Kwtf zdyXu?7fyUG5N;LH60xr*UQG6O?~UeVF~27oSNo#nUysJ4Gru0~i#nN*RVbk49h-N` zZ1H_we!I-0cZ96`C|a@PHb!sni{6nIsC$`YwvBb{T9$Bh!O+!VL$@$zsIS|BAc|6q zSjA`@8>&u;Zepe|lNy@d7nOl*YNFKH-NICY5$ee2z5K{H{H@k#;=cB+X(6|#cf{j2 zpo1N9D+D=F*yss~?wDFP{gu)MYg+u6OW}Mc=bEA#m>UM^+qb-R#PKGw=qo{W^6Jq3 zb>IKjXOnM1yks`si%slMY;$QdlVeAy(v=2gw$(UZ#+jm;lxQaIiw0#jwX@WTLX>K# z`z+(#(Qaa*FxB9kpif%0@t#h_e*3+0ye~9+zIZNPOLuIWSlqCMz{JtG$t}i-yGXoL zR}7-*)Fjn-Ti`FAf53-p5BRRS1Kth?{P5fXx8>V)xIvf`rqv(sicP^+D&fAEj5e3f z>Rh;{$)q7vYB(m9hGTP0)l^GgB%L*lD@3DQ*JjhXqq{|i{GNBb_tr*xpf1`pMEm8r zXu~_&7$pidIl4c{`}Unblg?j;baq~O=g%tfmXbY9#cS}ncvVfBj;Y#`a8)wlz+Ge} zv7N@ps}TyC8OaaNBmYQ$x{-rkS*F-+42^ehGq|y1VE8*<5b!a1C=V zSgPxg%GegFa2;E^s;jC@cV**XiK!_VPPocd{^6?ljSsiI%2kQ^XWI$Mq!7+kQ^)mZ z`;~PU-u)JZ%jUwxs!cgnZPO&oAqE!?hl}ZIjl;!^E7;J8R&%y36nN(BskwtNst}z4 zhV|k8Y--ESO1S@=M9rnLF&8f5Iu>wyRP4CSGK8xenr1laBH1 zfzRJuiT7WcWhregUgnsl0hL#in84)`7PpDb9ll7s6j;h&SZA~Gc6NsdKI?EJJC+!5 zZ84#GJxM+AxeEqnV{MY8<{GMtnJ&3y>Na*w&Z*E0S8|&8;?h?0gbD?pr zE4r>>X3WM4=St~W2iv-%Q&&}@=-hzDx&COA{ST5#xTPkW3!~agnb7wdh9+%Q6M`EQ zE~UvV=;In)TOduC8zAirC>AxnBO&t+_f7Z8kjEG}&(QF^&Z7O6_w<&7O`6GlP0_Jl ztv*z9GzU|iLW|4|ZfVrwl#9l97AAvoWyD&|orUUy=l6~G1);hBbay%{1`jmyo6&!I~y6G!`ZB1|Vx7a?Q3 z3eop02cp0DD)k4?_}zEXNPf6E5UVxKrqF##pgjlbxHL=)x>E~FyT{mIc;KZ0C zgU;?C+>iRIc8?F?eP;&YSE~>@4%djPG1rIicp1Xui$nN*fbjAK5PmzXsno<2y<&~e zKzJTNn11Qd$qIx`=V7&0sKeu>^Xppm(!l^T|I#&= zhmHeXw`)tRo_!vk3{$KVW}shKwR!j@`e#p8K%~v)VYL98oEo~6TsLjgGNf%v$F_)H zlrC8~RP+Gdt5E*aE*21*Z`x=g>b0LP1ho(+4vq~ z7y&uZzBxBmf=WURx_}Hjq0Nb-iLBh6lMZ^{WvJ+Fmsw;L(dxmTO*e1c_JJ*%(O7Xj z0|Ymih(u7v%8m}f|8d?5ga&~vE#!<@`GPDCO%&ajSqfdeJPUL;v@ZIRshnKL%0u(qvfITiSbhK{H|STQ5+i*a%b9+u~b)ZI6C$Iwqm-N zk?S`nz7aq@S6qVD+*l#!o4g`QW-{I`Dc1J%;KBZ>HK_+eh2kDW6CJGxt=l`^lU

&5ai7u_^H7L4UncLs@yRt6_? z)5n5P=dKE>DA^Vs@IW(kdv5EDkEMl>*~xUau=mI>yBT8xMY+2e6VmbWRx#~m<5@2& zCoPm7AKh!WdU+x9J$5894$7m23em?le6S@e(3(@hB|*k|GJ&X{RrPuB) znMVVp)N>S);kA%@Wxw5;Xblg<#=Q^4_8ktyY$iD3N=m8z`c9=+;|qIMN^JKXO$-K?f5frQojiOa|?eXnWS%RSJA4uzJ<8)E@^% zp88GO^;>VgDIVW)jT*?yCs9wiFZpTsOPwG@DIixt)5ZJl+S2<+K(VV{g}VCvPd)Kq zh0;!3n`(~Is=YflEohuAraD-&U}BLJyROuY#+0T)N#F*9(ooyGV>Yj>-doRT?>$$H z_RHoNZ6yLeKO;sP|3A%W8{ZyAJO2NS_P@qx?>Z-p_Qf)z{kk!uMUJ(ABuQ62zv|fN zO}&5Uv)M0z&Hn!5PY;*bEODyIHHTKy;F?$pn}X&dn1F4PAi~yENgJ}6$~73Y7&Aa5 zsp{ZNU7&+6-ICuEk6)L~O%!i{l9cEGrjG_Ea*4k}OVCxY#=3BAWvFZ|L@=dT(J@W^ zY28w%Z}ilC-zkrkT}izu?$)Y8gzjiAb3kE|l-oAfZORgu|bwnKd(! z&hR|)zOQ>YIy}64GZ~H|H}8$3KLT=;VxyOZ)~5iC&E~`iA~XBF1MMbJ6_vmU*0-;BMp%6k(%{F>276D=0K=LC zrn>%-pp#=uSSsxHFpN2%51E!jMWZwTU_gA+Ky}8|?OqoO8uNw5bAe0SfhC}F8mg*m z271uCat4@h0x&&$+$X+W24-o}XbKow-4k(3w}fFy0WvHoQ`~hmLnV5{DH?NX>IyM^ zDWZBmTHw+w*d?5cj5*i=0+#}d41T@ktS1x(LMC8QbC(#M-w2(XoarzL2t=TzGq_B(Ia7G#z+YF8S(a=P znW_SaRRi=E2Dt*Nl}|L5R9(1T4Uy3~=L++cwW{0+m&trPE>r7)FszrEVxavk+!se{ zp?T*s>3b{CbTxaOs=6NR9&N}L?s5wjZh#GND^#6^dH~ioe9$f z!SjLzLKD_ePlsY~y&`zdxJjs{k@+jjb3ij~2O}I-FDizwJy?g!&wu`JH&k%xXtH9{ zn*PKV#F7@J;KDF`H-z;4Ck$AovlbV0>2e)ZLB^>96PKy_YI;S03&EoBZGa_w5b8Co z1~IG~>+AKmwyhqjz;a%*=~APsxh5y#z3&)2%89~K8-0xhh+}~qA9~tb|1BO zhoOcpw5Oa4`VAlK0PlnXjslP!>kHHKCx=M|o9^Vc=1O*TKzG6Hz%|zeHPIoi>2gXW zNZSnubnF)-h4}t4C=D~hw@<$dD5ggDp53+-y^%W#Ue<<<3+!Ne2e%gUZDrZcU%I>l zmM*I9I4*XjS+i#usO(ve#L%nUi^79St(~JnjEY@o~W)6uwqBDE#n^289reDCo1TP%uEzubmX0{^YGcoW-TN z=7v-oV!?Ko{4JDY)Rub+5pZW5`s!-flGi5mWf1B^eWN?n6H8u&IF?6|=<3W_E4 zL8e~2(uaZbZAq9;SimbBtTP(873&6t(5t_Dyn;-#Js+vI!_d9K_7NMF0Mi3=ga(qn zk2`;Ym% z|2M@id7ob%t>c&H7w4Du{UOFR^o2m*zUWMy{sz?P?5S4=Dwr&5vKrwk>)I|8U<1RP zODMwtj&I)tcVh!~0nasf3}D#S4RwxPpc;Xp@!VIV(1E2C1U(HDJ&ox;z0?*Yc9C=P z>2eDbN-9mIOVx;fu?G5#YCu0M1qfKKfUg^{p=s2@fHY!)bVZr2))k~6Sdr?nxo_!4 zZz3bItBc4@!!j}TsV=lDT^vKJ(_-}yqbMJcDKh=&(j+`+*eP%EcatI>pNd3a&3lc% zgGGIncCK}@lMw^@CY=xibeYS7BM!O>R`=?p^39LF`wz-+@Z?xiaMXBxVeJUhwP`zG zPX4DC5bigX`ieVlcmt9kSXyMD5UJmx?+cs|TSL9g< z;d|M{Mcu+19rdhST1k6xQ$=e=VaF~+KAm;FIP8B#tMvElXh}GaU}3#8)zuE0z@6~c zLG&T?j`l1p$Nha@b}c$SHF=_~Z7ayevbZ_%33O?|O?|Y~`do-I1${55>*!hQy)*ai z*MQ>6m*4&3!7_?^(rpgKYR#f1_|zOO3=p_r4tE??U|Y&YIt8EzcTK@2Sh?nmreM$m zv&}pd?|mCl{2HJbUVvic*)VwTD6sgp83ca?2uAOE=D7-jTJj^!Ay_SLx(WgqShzr6 za>24{Fs4D5$Q!wmeFVX(YjCi0s&n!nkj1Kz{-v2-Ok0knH4TO)@L1=)F2Q(Id@I)EH6N^E(5+LlIgYcIX2qWKZXTH-+ zH3&GU^A!}Y@lo_H{N_JaP#jEtvMChxYT+afJay1C>X_It9qK^)31&h9ZTW&8i=b6Y zF;zfx!yxcin@^Ai=3V598FNM1j5#X?6e{nwMp93AjHI)(l0Yf+a1}eQJ0J;TY5D2* zNEy3F>an|{l76K~Iy_P~bD)=|9yk#ziv;(8Nbtz4NYGm9(t*7d7{=j(L9;qlxz9aO zxX&Md@8kPtf&4^sKvw4)P&1?{V1EX)umf5nWF+)yvT>Q!hz=b9xaRz~X)~${C;j^#o#+Zx_6k(R}0D1aA}J+xepGu6wc(xP^742>p`p?gsDLWN)i;`pqD7`rptB5R zYKl{CBzZVAB(>NeBX)tXS;!op0&sC<@ z>7005nL^VO6XMV+M^@;z6O0zn;J6X!2_eA|goIQiIC0~MkcN}CYO{7iLU7FG#qaaJ zzwhst_~$1U;K7@ts}q0r$2IPcP4nrm+jkc2M_(Jv{KY zoz{y?KA~gN#8|;PgjkklD!$Y?J7dUF-I)YW51x!F{v@~0yr4ubcxJ$>hF%;+JoIM4 zv#l?fa_H@{JJjd#1%mh20*%)+{gN$HPTAB`$>J zH`n`$X>1T`71KJz7!(Jhs2WX7C)aQ$t>xj%Q8k|{| z@KHa|Wx%h@a1Ye_fc6wuK0t6%o;JW2EryCyT%TV6WMdJ>F{}w_eUZ(u1M*VwePb1bLpWIh1?I zA>2XKg!Ca8L{n%YLLif+6c5W+*ZaUAHjrs3)I=tu4{+S*;g|!}gk!P8adVU7_70AE zmt#KR_3PEryrx=JqX6~V(wpTKQGDaK2!!+377_rawaHMu;1NCw^6gkjzA=0s{ z`2luK=K4B$Xf||3N6awH?Q9Gr52i5%QOc0@e`RC;RyH)u40-<8W;E_@MMG__WHB)z z(;MUV64=lvJQ$8$gQE3f5Z%!!50Z!(froIA42g+74)@9iG#SY`O;H1^3XN}$pw>cnMhXx1g)DW7r!}6< z_Y4Z`#c-hkJ1yb41~r)xBuQ~8)yUo$6lw~Dn!(UE#vq0iVMQkxqkpK4xz3#YyV__j zWp$(zP1_NREsPbx0Q*w3u^22gH!$9?-|LwW{tyh(i-7{{If#?wMt)Kj%s`Mp;y8O@ zAU$~+MZ+32Skj@}t${S6D@;d3-_;sL^<0fdHPz{?!#OqC5*N5vS9feG!L=Swsh7*2 z_SIBw-4!O9c~B5apx$2swb`)lTmj`*vNDLnq-w#b9QOeaeHk?KeU)(18p@4TgRx3D z6Wa&Nsxa}@Vc~4~R8qi8;J3{JUfYk7LNgSN83buap1-g$Wh0`%n!}v}R+j(BFfA#a l#@S0>{`5|gE@7~{)jAEHth6(Wm1FzvzkBbfb^Y2c@;7voZWaIl From ca94904bf2c3a784bc9616b8a50ac461ed377504 Mon Sep 17 00:00:00 2001 From: William Jackson Date: Sat, 14 Nov 2020 13:38:11 -0600 Subject: [PATCH 20/64] cleaned up actions --- app/controllers/events_controller.rb | 18 +++++++----------- app/workers/event_worker.rb | 8 ++++---- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index b1e1de2..80e1855 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -8,7 +8,7 @@ def date_format(time) end class EventsController < ApplicationController - # before_action :authorize_request + before_action :authorize_request def index if event_params[:event_id] @@ -28,21 +28,21 @@ def index date: date, status_code: event.status_code } end - render json: safe_events, status: 200 # OK + render json: safe_events, status: 200 end def show - event = Event.find(event_params[:id]) - render json: event, status: 200 # OK + event = Event.find(event_params[:event_id]) + render json: event, status: 200 end def destroy - event = Event.find(event_params[:id]) + event = Event.find(event_params[:event_id]) event.destroy end def create - bridge = Bridge.find(create_event_params[:id]) + bridge = Bridge.find(event_params[:bridge_id]) test_mode = create_event_params[:test] payload = JSON.parse(request.body.read) data = { 'inbound' => { @@ -74,11 +74,7 @@ def create private - def create_event_params - params.require(:bridge).permit(:id, :test) - end - def event_params - params.permit(:id, :bridge_id, :event_id, :test) + params.permit(:bridge_id, :event_id) end end diff --git a/app/workers/event_worker.rb b/app/workers/event_worker.rb index 0bb8a1c..d30933f 100644 --- a/app/workers/event_worker.rb +++ b/app/workers/event_worker.rb @@ -45,7 +45,7 @@ def save_response(event, resp) status_code: resp.code, message: resp.message, size: resp.size, - payload: payload + payload: payload } event_data = JSON.parse(event.data) event_data['outbound'].last['response'] = response @@ -89,7 +89,7 @@ def perform(event_id) http.use_ssl = (uri.scheme == 'https') req = "Net::HTTP::#{method}".constantize.new(uri, 'Content-Type' => 'application/json') req.body = bridge.payload.to_json - + begin current_attempts += 1 save_request(event, req.length, bridge.payload) @@ -97,8 +97,8 @@ def perform(event_id) save_response(event, response) rescue *HTTP_ERRORS, Sidekiq::LargeStatusCode => e save_http_error(event, e) if HTTP_ERRORS.include?(e.class) - if current_attempts <= bridge.retries - sleep 1 # DEVELOPMENT + if current_attempts <= retries + sleep 1 # DEVELOPMENT # sleep bridge.delay * 60 # PRODUCTION retry end From 7bbc6adc245982c71dc3e0321d15440a2465b587 Mon Sep 17 00:00:00 2001 From: William Jackson Date: Mon, 16 Nov 2020 20:14:06 -0600 Subject: [PATCH 21/64] updated event_worker --- app/controllers/events_controller.rb | 7 +++---- app/workers/event_worker.rb | 28 ++++++++++++++++++++------- db/dump.rdb | Bin 0 -> 10191 bytes db/schema.rb | 4 ---- 4 files changed, 24 insertions(+), 15 deletions(-) create mode 100644 db/dump.rdb diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index 80e1855..3b5cebe 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -8,7 +8,7 @@ def date_format(time) end class EventsController < ApplicationController - before_action :authorize_request + # before_action :authorize_request def index if event_params[:event_id] @@ -43,7 +43,7 @@ def destroy def create bridge = Bridge.find(event_params[:bridge_id]) - test_mode = create_event_params[:test] + test_mode = event_params[:test] payload = JSON.parse(request.body.read) data = { 'inbound' => { 'payload' => payload, @@ -62,7 +62,6 @@ def create ) event.save! EventWorker.perform_async(event.id) - render json: {}, status: 202 # Accepted (asynchronous processing) rescue ActiveRecord::RecordNotFound render json: { error: 'a bridge by that id was not found' }, status: 400 @@ -75,6 +74,6 @@ def create private def event_params - params.permit(:bridge_id, :event_id) + params.permit(:bridge_id, :event_id, :test) end end diff --git a/app/workers/event_worker.rb b/app/workers/event_worker.rb index d30933f..4f221a7 100644 --- a/app/workers/event_worker.rb +++ b/app/workers/event_worker.rb @@ -23,9 +23,13 @@ def make_http_uri(uri) "#{SCHEME}#{minus_scheme}" end +def set_headers(req, bridge) + bridge.headers.each { |header| req[header['key']] = header['value'] } +end + def save_request(event, length, payload) request = { - payload: JSON.parse(payload), + payload: payload, date: DateTime.now.utc.to_s.split(' ').first, time: DateTime.now.utc.to_s.split(' ')[1], content_length: length @@ -78,7 +82,13 @@ class EventWorker def perform(event_id) event = Event.find(event_id) + # payload = JSON.parse(event.data)['inbound']['payload'] bridge = Bridge.find(event.bridge_id) + payload = if event.test + JSON.parse(bridge.data)['test-payload'] + else + JSON.parse(bridge.data)['payload'] + end method = bridge.method.capitalize retries = bridge.retries current_attempts = 0 @@ -88,12 +98,16 @@ def perform(event_id) http = Net::HTTP.new(uri.host, uri.port) http.use_ssl = (uri.scheme == 'https') req = "Net::HTTP::#{method}".constantize.new(uri, 'Content-Type' => 'application/json') - req.body = bridge.payload.to_json + set_headers(req, bridge) + # binding.pry + req.body = payload.to_json + binding.pry begin current_attempts += 1 - save_request(event, req.length, bridge.payload) + save_request(event, req.length, payload) response = http.request(req) + binding.pry save_response(event, response) rescue *HTTP_ERRORS, Sidekiq::LargeStatusCode => e save_http_error(event, e) if HTTP_ERRORS.include?(e.class) @@ -102,10 +116,10 @@ def perform(event_id) # sleep bridge.delay * 60 # PRODUCTION retry end + ensure + event.completed = true + event.completed_at = Time.now.utc + event.save end - - event.completed = true - event.completed_at = Time.now.utc - event.save end end diff --git a/db/dump.rdb b/db/dump.rdb new file mode 100644 index 0000000000000000000000000000000000000000..bacf6c1e800ee888b63c5b51af620bc3c38c79f3 GIT binary patch literal 10191 zcmd5?ZH!!3d7hn}8Sl=njqL;$OC)m`NYj+vtMmP_Bw?1uB~d;ULFt!O#pm90t~0Qn zS!QOFx)3QwAlP-G{ZXlq#uOp`q=}j;O%Oj9QV7ugp(&v%qTMRVk0KiMH%40Tq^heAgp)V zJGD#2s8c`dzy7D>A0D_gx7qe?<*YyZb^M>lbC(uvGhW$fd3!co_dB^?=6*3Z`}*B0 zmtNeQ$>j<+tZi(p`}WGc@4bJOVaoL?5fRat8Ar7*6vEB+xt(0`Bb#;KDaLla?F;or z+l`n!~dwdTy1x3tK%!~0o$pr5{ZRSLQ@hbsWBJJ)s4EV z!tWh|A6%|>>=|_1!h2hFcgC;Xe^<~7(R<2Kvk`5!TD}pTL$$Zz;jnm{N+qm^%^2Nj+Yh4pe52DuALz8|jWg(DkKVfT&BE`V+wdoA_n{NB z2@<=-&9b39z*-Q_md{^bcpz$SHqdi3_>mu;E8h0)AAf)C`PyrlfxrVGxKlQYt&54k zv8R}6AtFu{S~)QR2xWKxcZqTZ3P?0gKfF7S3Jj`6jzRH-Yl7ktfZ~O5DCRe3ndStw z?I;S*T>RYIx+ttItgU9F12zK`qR_=6q$DPxkj`+!tZ@UF< z9Uh;7oBKkg-tZ2c^_>Ts4xQex=hmCn-HhU93%%pkqymd){3`wvjB<-A)^Y4Ti>cSQ zqd$A}-kq27_ciZ5{oA>dwFuqNW61ngffK?Adcyu@pB>)@cHHtu|Gx39gdJ<;wOcdd zqElQBTPld+)KVgh(rB?nt#&LXXyq*v*ny1-(9MSkp(#5S(Tp+U)JS0Y;^zI(mVgXPHDyEDR~QdpK86`n%Tc%~SR1hJe) zR0uqoE|{}G5F!m3p^M^b43~=uE`M{axOhjU!a$eL050FZ9GBAetfm1&&*ha^K7UIe zn(qKKpSt0%K9oXpTSjPP3Qdtlf`_J-JFEm&Rx%M1g)u`bM=v7`di!C<0>w1dDkaPU z$}@IAjB&=r`!J@oUDnVZDdSj><@4|AL-Wr7&A)uG(TC<}X4Z%lnk5=XBqAoJPKK6K z;w308;o(Hqa04Gl2-aARu*LwA2O%;pvZ0nxO;bdsur(ua?THfSnkfKhls7lub5|dn zW0@7055XZ&Lm~X05MJO=TL#^j%4y(86ELBT8Guu+%o?odfuQ3pc9iUg&Vqo82`E6Q z^z!*p9sb7e9q*&_rcCI}oib@rRhC?A!IEl?MQF9RU|tNT-Z->8p-FR!bs(rj@0{)S z=*_NK5#!MObs{$xuP8T#Zg)a|aTs^IlhayZ+3!x@yIgJxTXRtNCQyoN^n=_Gn?!N` z`|8XIZhnU00h7A;S3a?m+-K!7+BgO!^B2lLQw z83A(+s(CvR^RTbR6PkBnYq1LtRlrqchJ}^%s;{2Z`rRb?q8k{eixaM(-;sqQbuBOe`$W9HroP7|~#IZXWS||8eM%J~T(OYEEkXDoey+t7({Yt0+!7 zmC@V?HH|oe1zHNm2hc23W;vG{?H!Tcm?w5NR8X-!lQXHTwtL`!L~;RDl?rkqIzNB< zNgUuLaJCX~ zL?9%i5CrT2XZN~S9_%Ud;Wd@5sM2AovV_A>dSM(44`4E+p|KD$CT21?RIz}74N@#E zs)fpoQ0T@zv(#frw|e}F-D*%&{aMP1NG5`F0igNU3Ek?`MS;OeVD$71^zobeHT43} z=4UV5aVAA3>zp}^9pczRdjd-pQjJ5YL6$jF23*brdwe3#RdBl8AGN!ggB0Ux^9BnF zym80?n5q769)OXF5*s=;X_6yP#XV%QsSN}&IBnx-i3=aaN=qnxlaXl_q;Z8H~B1Uc^_he-eTaY`}>P3o04IcjgZD!O3R-5^z$vK}(S)Q4|^J zI46R~GIEUih)hnaxCFT&N>VW}$B4#jUcDEHUSVL4cJiae~bg zKYQt(E^%rvX9}>OgQr7PWu9P6lvNSj+hG-E;V!*+g)ZQ$jyfI{~ zOIRpYE&CWf|=)<*gu7j#Gn;W&K(Hwf$&U3rh-O|tNI`a}jR!B(6NW)s&`K`KjduUayKod_MshcFBV?#P3<9=~kBRt%whAQ&q5 zCV|fG!i{;*<1r#UHubbmbx-@u;IxJI0`DG|GD*OM9PJC{GjQ6+KV|!;eKphVCd?nk zlZgQd2?XR;$}l2WSWdi|DhrrGqlKy@{ri1*7aA?+YV5?5b&cx^80%#l*Mvc#-{p+m(^VQ3%LI_j~LNd+%W zs9@NcKzcV{WU4>RCnLvrpS#wPBd_8J6-tIf!<{~I%x~qnCj;Q+&>v3~pSMjYZYKmCSaDVB&PVcPyD zOk%~Y1qhl0rWrMCY}k0|&;Q_L51VYkAz(9{#F&EhAeB6V*bs6ZSX%*uz&TFFhH(iR zs3AEV`sq9+%;ZOQpoZ#&WDOtBPesY!PdyRScN+yUDOmu@GD z+Y1un?j&HLFeVRz!(@X0p?xji11DP+58w>*4MGgW@R3o>0bfEVyd;*n$?*?k6y_jE zR?Wz;G0QMawO1ZfuLd2ijPg)XpMCVcei*+8okJH&um=TyqR^9j!i|fcw9kICyS+J_ zae~*w)R0IJ7Oxyc%rQ{f#!xZFfgPBXj4=kBhIN~O^l<2nX4KWpsQV%VlUNwX5s#8| zZYnaR?Ggb;tr$mJHZ_gO`2LHxU-CjqIPm`QO3wu#&68cO9j^z!K*Hb=5*O*+0;mJGZMU^ZfL6e);YEO0PrnXFcD6#sN!}RiYdSf=6qOfjf&0hy!eTGdeAIp zEveze=3tox`8x@51a$Vv3I|?HNIw}G$tlnVhFCeEO&Ot;oE{rtCgnc+(gwis9AaDx z8T|APHNRaFFfT!Lu!l2uzw)KI9x$2v)6fX}n8VII3?nb12qzILjv@*#8zybFC_Enu z3{=m|$W{wl=@=~6tioMbAQ|;e_cGLJDh&HRNDc%U+4m(X{FA?3eC-?E&YjiNsG%EX zpemYTSkl5XAYvmNDgu%tCsRgX5*WjDvWpxpZ8(*+D=%%Xf-)|I{C^+H9NM0v5H54Z zMi#}1-}}d#zWizeO}^B7NH%k*|57VIztL(&@Mg-lXZC*$mpjsZ3YPwJw-XHCkDb{2 VSEEnJf?u8ZlKbhmzIOZn{u(pzEb;&V literal 0 HcmV?d00001 diff --git a/db/schema.rb b/db/schema.rb index 2983a39..540210e 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,11 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -<<<<<<< HEAD ActiveRecord::Schema.define(version: 2020_11_13_001336) do -======= -ActiveRecord::Schema.define(version: 2020_11_12_094842) do ->>>>>>> aac310c313a7cfb304f513a700375599ede024e3 # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" From 4049037a0d01fab200d3a43109a3284b04718e6b Mon Sep 17 00:00:00 2001 From: William Jackson Date: Mon, 16 Nov 2020 20:57:54 -0600 Subject: [PATCH 22/64] updated seed db --- app/controllers/events_controller.rb | 1 + app/workers/event_worker.rb | 9 ++++----- db/dump.rdb | Bin 10191 -> 13706 bytes db/seeds.rb | 17 ++++++++--------- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index 3b5cebe..3efc6dc 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -61,6 +61,7 @@ def create outbound_url: bridge.outbound_url ) event.save! + binding.pry EventWorker.perform_async(event.id) render json: {}, status: 202 # Accepted (asynchronous processing) rescue ActiveRecord::RecordNotFound diff --git a/app/workers/event_worker.rb b/app/workers/event_worker.rb index 4f221a7..5409373 100644 --- a/app/workers/event_worker.rb +++ b/app/workers/event_worker.rb @@ -82,12 +82,11 @@ class EventWorker def perform(event_id) event = Event.find(event_id) - # payload = JSON.parse(event.data)['inbound']['payload'] bridge = Bridge.find(event.bridge_id) payload = if event.test - JSON.parse(bridge.data)['test-payload'] + JSON.parse(bridge.data['test_payload']) else - JSON.parse(bridge.data)['payload'] + JSON.parse(bridge.data['payload']) end method = bridge.method.capitalize retries = bridge.retries @@ -99,9 +98,7 @@ def perform(event_id) http.use_ssl = (uri.scheme == 'https') req = "Net::HTTP::#{method}".constantize.new(uri, 'Content-Type' => 'application/json') set_headers(req, bridge) - # binding.pry req.body = payload.to_json - binding.pry begin current_attempts += 1 @@ -110,8 +107,10 @@ def perform(event_id) binding.pry save_response(event, response) rescue *HTTP_ERRORS, Sidekiq::LargeStatusCode => e + binding.pry save_http_error(event, e) if HTTP_ERRORS.include?(e.class) if current_attempts <= retries + binding.pry sleep 1 # DEVELOPMENT # sleep bridge.delay * 60 # PRODUCTION retry diff --git a/db/dump.rdb b/db/dump.rdb index bacf6c1e800ee888b63c5b51af620bc3c38c79f3..2bd324c898f775e8816904bed6ad3425dd283279 100644 GIT binary patch delta 2630 zcmbtWOKenC7@nC9eK0^NQnC2x=|Dl7>Gizsc}ywJsBB1hC?>>EV9vegT&zxK%FGlj z*Z>=)p|F_^e=<4#2vCJ2QpJ9VQmxT=Y9qj3R2)DZRD&ZFhwdwp~7{a*j~|DSI= z{>&?|t1@(Y@2~9>Wuphj%-DtPmrrE7k95SZU+a49_^D_l61lTnQI&L{M)O9eF~o2X zp+Ur_MI^D{$+Wn>6ZCyCVc&@NXUE6$raZXi;q7T6Bq3=?g@lMe1&K)?ES8=zrE;{R zc(78)Kkacjlld_7-Ri^1{S#)wB-6?2*YU+i#?@rLs5H1^>EV%N$y7>{Bgr&OjT{{S ziO9%wDmsz`-Elv!%H>@5;!Oukp|W%C%ekJID(%bM)v^n--C%v}=~12QU!|3H5m!OL zOdt`(#`wqxBXYRiFBw%adanYzy3&Y2*A<8%Mjb>2p$Ke>nnLcK*(H53N~XZsSZSgF zHqOR|qjYLfTpOhyoXHMk$E)X;4X?p;cPeTB2zplq$di%?;bEV8$QK@y9>v(N_I0kj zTOcB#<1zs;N+nICkeirI^u{o9Gu6jBS0Bk2eU%3XRB2Rsc>{_iFlH)G6*b6>x7UPO z4+5|~$M(ec`o)PtUnJTIryf0f>C|UGe3hNauFDo`+(?s~4kXb5 z=1^jasjskcNuU%Bx%F6XE^&lF60!!jCA|@rslMC)@0gvrkr{Ubtc7KEVUF2_IcA?& zX3K77)_S%>aED;-5}7(SQsV`Fi^8d;S0Z1&TI2OVwpinZo4l~X&=^LPZ#Xd!Yvl$| ziy-8++VUdyqm)6~}B|b>8zoEF%o5fQH2W*0R=fZ7y^OK@fqnYImL3?;)|b z*~u5@h-I#3*VKriCNYf}#u!;X#yaMEHjz*f_K)d%UfoUH@DOq4{>TljaNJbIt7DP|8;l%D@$KTVue2LeQa{Qa5a& z#zu4rxd~R-LW3I)TfIK68`{*Xt1rH40y~Ito9N^mn4>0 zr6p$Oq^2CuVP#24O-xbV?8Em&RUN|>418e41x5MEsl`CA9MA)qGdbQ=kxTEt Date: Mon, 16 Nov 2020 22:18:20 -0600 Subject: [PATCH 23/64] DRY events_controller --- app/controllers/events_controller.rb | 112 ++++++++++++++++----------- app/workers/event_worker.rb | 3 - config/routes.rb | 4 +- db/dump.rdb | Bin 13706 -> 13684 bytes db/seeds.rb | 10 +-- 5 files changed, 74 insertions(+), 55 deletions(-) diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index 3efc6dc..f67d9f4 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -1,67 +1,39 @@ # frozen_string_literal: true -def date_format(time) - year = time.split('-')[0] - month = time.split('-')[1] - day = time.split('-')[2] - "#{year}-#{month}-#{day}" -end - class EventsController < ApplicationController # before_action :authorize_request def index - if event_params[:event_id] - event = Event.find(event_params[:event_id]) - events = Bridge.find(event.bridge_id).events - elsif event_params[:bridge_id] - events = Bridge.find(event_params[:bridge_id]).events - else raise ActiveRecord::RecordNotFound - end + events = retrieve_events + sidebar_events = create_sidebar_events(events) - safe_events = events.map do |event| - updated_at = String(event.updated_at) - time = date_format(updated_at.split(' ')[1]) - date = updated_at.split(' ')[0] - { id: event.id, - time: time.slice(0..-3), - date: date, - status_code: event.status_code } - end - render json: safe_events, status: 200 + render json: sidebar_events, status: 200 + rescue ActiveRecord::RecordInvalid + render json: { error: 'neither event_id nor bridge_id were valid' }, status: 400 # Bad Request + rescue ActiveRecord::RecordNotFound + render json: { error: 'events matching that id were not found' }, status: 400 + rescue ActiveRecord::NotNullViolation + render json: { error: 'neither event_id nor bridge_id were not submitted' }, status: 400 # Bad Request end def show - event = Event.find(event_params[:event_id]) + event = Event.find(event_params[:id]) render json: event, status: 200 + rescue ActiveRecord::RecordNotFound + render json: { error: 'an event by that id was not found' }, status: 400 end def destroy - event = Event.find(event_params[:event_id]) + event = Event.find(event_params[:id]) event.destroy end def create bridge = Bridge.find(event_params[:bridge_id]) - test_mode = event_params[:test] - payload = JSON.parse(request.body.read) - data = { 'inbound' => { - 'payload' => payload, - 'date' => DateTime.now.utc.to_s.split(' ').first, - 'time' => DateTime.now.utc.to_s.split(' ')[1], - 'ip' => request.ip, - 'content_length' => request.content_length - }, - 'outbound' => [] } - event = Event.new( - data: data.to_json, - bridge_id: bridge.id, - test: test_mode, - inbound_url: bridge.inbound_url, - outbound_url: bridge.outbound_url - ) + data = create_data_object + event = create_event_object(data, bridge) + event.save! - binding.pry EventWorker.perform_async(event.id) render json: {}, status: 202 # Accepted (asynchronous processing) rescue ActiveRecord::RecordNotFound @@ -75,6 +47,56 @@ def create private def event_params - params.permit(:bridge_id, :event_id, :test) + params.permit(:id, :bridge_id, :event_id, :test) + end + + def date_format(time) + year = time.split('-')[0] + month = time.split('-')[1] + day = time.split('-')[2] + "#{year}-#{month}-#{day}" + end + + def create_data_object + { 'inbound' => { + 'payload' => JSON.parse(request.body.read), + 'date' => DateTime.now.utc.to_s.split(' ').first, + 'time' => DateTime.now.utc.to_s.split(' ')[1], + 'ip' => request.ip, + 'content_length' => request.content_length + }, + 'outbound' => [] } + end + + def create_event_object(data, bridge) + Event.new( + data: data.to_json, + bridge_id: bridge.id, + test: event_params[:test] || false, + inbound_url: bridge.inbound_url, + outbound_url: bridge.outbound_url + ) + end + + def create_sidebar_events(events) + events.map do |event| + updated_at = String(event.updated_at) + time = date_format(updated_at.split(' ')[1]) + date = updated_at.split(' ')[0] + { id: event.id, + time: time.slice(0..-3), + date: date, + status_code: event.status_code } + end + end + + def retrieve_events + if event_params[:bridge_id] + Bridge.find(event_params[:bridge_id]).events + elsif event_params[:event_id] + Event.find(event_params[:event_id]).bridge.events + else + raise ActiveRecord::NotNullViolation + end end end diff --git a/app/workers/event_worker.rb b/app/workers/event_worker.rb index 5409373..e3f1e41 100644 --- a/app/workers/event_worker.rb +++ b/app/workers/event_worker.rb @@ -104,13 +104,10 @@ def perform(event_id) current_attempts += 1 save_request(event, req.length, payload) response = http.request(req) - binding.pry save_response(event, response) rescue *HTTP_ERRORS, Sidekiq::LargeStatusCode => e - binding.pry save_http_error(event, e) if HTTP_ERRORS.include?(e.class) if current_attempts <= retries - binding.pry sleep 1 # DEVELOPMENT # sleep bridge.delay * 60 # PRODUCTION retry diff --git a/config/routes.rb b/config/routes.rb index 5d53755..c8ae1f3 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -10,7 +10,7 @@ post 'login', to: 'sessions#create' post 'events', to: 'events#create' get 'events', to: 'events#index' - get 'events/id', to: 'events#show' - delete 'events', to: 'events#destroy' + get 'events/:id', to: 'events#show' + delete 'events/:id', to: 'events#destroy' mount Sidekiq::Web => '/sidekiq' end diff --git a/db/dump.rdb b/db/dump.rdb index 2bd324c898f775e8816904bed6ad3425dd283279..5d95a9ce99a510d11eeab735ae9cb72b4783ec3e 100644 GIT binary patch delta 453 zcmYk2ziZS`0LNc!?xJ3+AmVS^XXH50+>yK_?~*{JQE;<}9bA;_%e@y(O)hEjo<;O% zU7aie2N(YV7wsfC9M-kC2s#!=>F87sYtG62`}y+yeqKEKwenKeelBkH_797K(RRX; zPuZK}V1JsVVLQtiNrIS-z#s5inHqmS`D63mY|}kdUlmNSLQ*ITiHJF96r6A`BvH)h z$ejeso_%S4Y23V=d=(M^@ZIbejJFtBdh4#wE*V5E0--Koix z5vp%59Qayk)UO@E_ce7TqHwNkOJ~c?`l)q{t&U+ire$a<1A}F;+nAFq6V4vZ^EB@z ztSHEXh1WATQW4WZi^DJ)P`LbR8m>kBA#6poHIFdNHqhw9jNj+cV|Iu2)0}$VhN3K{ z{AMaDK%1Hv+LfO*kZyK6E^>|P*7I|xrCuLig&!q-cwzmi{blc)zwKY~v&q`69mz%K a!LfgWvpnrrTPnv(2Yc_D_>atn2F(U}bD!X>4p| zU}9luU}$WRXkeUbRi0mzomy1P$eCYWl9!W}WNvxbVW-10sp`oKd4&B65-W4^6H^p~ zd8=cTic(98Dr1za40K`|v=q1*Vh-pq#waO>vLxpu78fUoa=DhJ=9Pql4NQ<^N-Rou zkYJ8BGK*DEXUfV*v#5awMABRxYi ib3;>020n1a!Canlz!n^*^A)*l|3{Y0`^o(}*b4yT%coob diff --git a/db/seeds.rb b/db/seeds.rb index 0cc6293..f2a9189 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -27,7 +27,7 @@ def test_url inbound_url: 'bridgeapi.com/746353', outbound_url: test_url, method: 'PATCH', - retries: 2, + retries: 3, delay: 0, data: { payload: "{\"FirstName\":\"Booths\",\"LastName\":\"John\",\"UserName\":\"FordTheatre\",\"Password\":{\"nested\":\"sic temper tyrannis\"},\"Email\":\"mail@mail.com\"}", test_payload: "{\"test_key_one\":{\"nested\":11},\"test_key_two\":888}" } ) @@ -37,7 +37,7 @@ def test_url bridge2.headers << Header.create(key: 'X_API_KEY', value: 'returntheslab') bridge2.headers << Header.create(key: 'Authentication', value: 'Bearer *************') -5.times do - bridge.events << Event.create(completed: false, outbound_url: bridge.outbound_url, inbound_url: 'bridgeapi.com/249634', data: '', status_code: 300) - bridge2.events << Event.create(completed: false, outbound_url: bridge2.outbound_url, inbound_url: 'bridgeapi.com/746353', data: '', status_code: 302) -end +# 5.times do +# bridge.events << Event.create(completed: false, outbound_url: bridge.outbound_url, inbound_url: 'bridgeapi.com/249634', data: '', status_code: 300) +# bridge2.events << Event.create(completed: false, outbound_url: bridge2.outbound_url, inbound_url: 'bridgeapi.com/746353', data: '', status_code: 302) +# end From 29e4b20422c4a2003d7ac5024cd1462c5e454bb0 Mon Sep 17 00:00:00 2001 From: William Jackson Date: Tue, 17 Nov 2020 19:24:19 -0600 Subject: [PATCH 24/64] finished events_controller and event worker --- app/controllers/events_controller.rb | 35 ++----- app/models/event.rb | 53 +++++++++- app/workers/event_worker.rb | 149 ++++++++++++++++----------- dump.rdb | Bin 15073 -> 15829 bytes spec/models/event_spec.rb | 4 - spec/requests/events_request.spec.rb | 0 spec/support/main_helper.rb | 8 ++ 7 files changed, 156 insertions(+), 93 deletions(-) create mode 100644 spec/requests/events_request.spec.rb diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index f67d9f4..52b88b7 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -29,10 +29,7 @@ def destroy end def create - bridge = Bridge.find(event_params[:bridge_id]) - data = create_data_object - event = create_event_object(data, bridge) - + event = create_event_object(create_data_object, find_bridge) event.save! EventWorker.perform_async(event.id) render json: {}, status: 202 # Accepted (asynchronous processing) @@ -50,18 +47,12 @@ def event_params params.permit(:id, :bridge_id, :event_id, :test) end - def date_format(time) - year = time.split('-')[0] - month = time.split('-')[1] - day = time.split('-')[2] - "#{year}-#{month}-#{day}" - end - def create_data_object + datetime = DateTime.now.utc.to_s.split(' ') { 'inbound' => { 'payload' => JSON.parse(request.body.read), - 'date' => DateTime.now.utc.to_s.split(' ').first, - 'time' => DateTime.now.utc.to_s.split(' ')[1], + 'date' => datetime.first, + 'time' => datetime[1], 'ip' => request.ip, 'content_length' => request.content_length }, @@ -72,22 +63,12 @@ def create_event_object(data, bridge) Event.new( data: data.to_json, bridge_id: bridge.id, - test: event_params[:test] || false, - inbound_url: bridge.inbound_url, - outbound_url: bridge.outbound_url + test: event_params[:test] || false ) end def create_sidebar_events(events) - events.map do |event| - updated_at = String(event.updated_at) - time = date_format(updated_at.split(' ')[1]) - date = updated_at.split(' ')[0] - { id: event.id, - time: time.slice(0..-3), - date: date, - status_code: event.status_code } - end + events.map(&:sidebar_format).sort_by { |event| event[:id] }.reverse end def retrieve_events @@ -99,4 +80,8 @@ def retrieve_events raise ActiveRecord::NotNullViolation end end + + def find_bridge + Bridge.find(event_params[:bridge_id]) + end end diff --git a/app/models/event.rb b/app/models/event.rb index 42759bc..a1b1a88 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -1,7 +1,58 @@ # frozen_string_literal: true +BOOLEAN = [true, false].freeze +VALID_CODES = [100..599].freeze + class Event < ApplicationRecord - validates :outbound_url, presence: true + before_create :set_urls + validates :test, inclusion: { in: BOOLEAN, message: '"test" field must be true or false' } + validates :completed, inclusion: { in: BOOLEAN, message: '"completed" field must be true or false' }, allow_nil: true + validates :status_code, inclusion: { in: VALID_CODES, message: 'Invalid status code' }, allow_nil: true + validate :completed_at_format + validate :data_json_object belongs_to :bridge + + def sidebar_format + updated_at = String(self.updated_at) + time = date_format(updated_at.split(' ')[1]) + date = updated_at.split(' ')[0] + { id: id, + time: time.slice(0..-3), + date: date, + status_code: status_code } + end + + private + + def date_format(time) + year = time.split('-')[0] + month = time.split('-')[1] + day = time.split('-')[2] + "#{year}-#{month}-#{day}" + end + + def set_urls + self.inbound_url = bridge.inbound_url + self.outbound_url = bridge.outbound_url + end + + def data_json_object + data = JSON.parse(self.data) + %w[inbound outbound].all? { |key| data.include?(key) } && + %w[payload date time ip content_length].all? { |key| data['inbound'].include?(key) } || + errors.add( + :data, + 'must include the keys: "inbound", "outbound", + while "inbound" must include the keys "payload", "date", "time", "ip", "content_length"' + ) + rescue JSON::ParserError, TypeError + errors.add(:data, 'object must be a valid json object') + end + + def completed_at_format + return if completed_at.nil? || completed_at.instance_of?(ActiveSupport::TimeWithZone) + + errors.add(:completed_at, '"completed_at" must be a Time instance if event is completed') + end end diff --git a/app/workers/event_worker.rb b/app/workers/event_worker.rb index e3f1e41..d17c6e7 100644 --- a/app/workers/event_worker.rb +++ b/app/workers/event_worker.rb @@ -16,66 +16,113 @@ Timeout::Error ].freeze -def make_http_uri(uri) - return uri if uri.starts_with?('https') - - minus_scheme = uri.split('//').last - "#{SCHEME}#{minus_scheme}" -end - def set_headers(req, bridge) bridge.headers.each { |header| req[header['key']] = header['value'] } end -def save_request(event, length, payload) - request = { - payload: payload, - date: DateTime.now.utc.to_s.split(' ').first, - time: DateTime.now.utc.to_s.split(' ')[1], - content_length: length +def save_event(event) + event.completed = true + event.completed_at = Time.now.utc + event.save! +end + +def create_error_response(error) + { + date: '', + time: '', + status_code: '', + message: error.message, + size: '', + payload: {} } +end + +def save_http_error(event, error) + response = create_error_response(error) event_data = JSON.parse(event.data) - event_data['outbound'].push({ 'request' => request, 'response' => {} }) + + event_data['outbound'].last['response'] = response event.data = event_data.to_json event.save end -def save_response(event, resp) - resp_code = resp.code.to_i - payload = (resp_code >= 300 ? {} : JSON.parse(resp.body)) - response = { +def create_response_object(payload, response) + { date: DateTime.now.utc.to_s.split(' ').first, time: DateTime.now.utc.to_s.split(' ')[1], - status_code: resp.code, - message: resp.message, - size: resp.size, + status_code: response.code, + message: response.message, + size: response.size, payload: payload } +end + +def save_response(event, resp) + resp_code = resp.code.to_i + payload = (resp_code >= 300 ? {} : JSON.parse(resp.body)) + response = create_response_object(payload, resp) event_data = JSON.parse(event.data) + event_data['outbound'].last['response'] = response event.data = event_data.to_json - - event.status_code = resp_code event.save raise Sidekiq::LargeStatusCode if resp_code >= 300 end -def save_http_error(event, error) - response = { - date: '', - time: '', - status_code: '', - message: error.message, - size: '', - payload: {} +def create_request_object(payload, length) + { + payload: payload, + date: DateTime.now.utc.to_s.split(' ').first, + time: DateTime.now.utc.to_s.split(' ')[1], + content_length: length } +end +def save_request(event, length, payload) + request = create_request_object(payload, length) event_data = JSON.parse(event.data) - event_data['outbound'].last['response'] = response + + event_data['outbound'].push({ 'request' => request, 'response' => {} }) event.data = event_data.to_json event.save end +def prepend_scheme(uri) + return uri if uri.starts_with?('https') + + minus_scheme = uri.split('//').last + "#{SCHEME}#{minus_scheme}" +end + +def generate_http_request(bridge, payload) + method = bridge.method.capitalize + uri = URI(prepend_scheme(bridge.outbound_url)) + http = Net::HTTP.new(uri.host, uri.port) + req = "Net::HTTP::#{method}".constantize.new(uri, 'Content-Type' => 'application/json') + + http.use_ssl = (uri.scheme == 'https') + set_headers(req, bridge) + req.body = payload.to_json + [http, req] +end + +def extract_payload(bridge, event) + if event.test + JSON.parse(bridge.data['test_payload']) + else + JSON.parse(bridge.data['payload']) + end +end + +def execute_request_response_cycle(event, bridge) + payload = extract_payload(bridge, event) + http, req = generate_http_request(bridge, payload) + + save_request(event, req.length, payload) + response = http.request(req) + save_response(event, response) +end + class EventWorker include Sidekiq::Worker sidekiq_options retry: 0 @@ -83,39 +130,15 @@ class EventWorker def perform(event_id) event = Event.find(event_id) bridge = Bridge.find(event.bridge_id) - payload = if event.test - JSON.parse(bridge.data['test_payload']) - else - JSON.parse(bridge.data['payload']) - end - method = bridge.method.capitalize - retries = bridge.retries - current_attempts = 0 - - # Generate request - uri = URI(make_http_uri(bridge.outbound_url)) - http = Net::HTTP.new(uri.host, uri.port) - http.use_ssl = (uri.scheme == 'https') - req = "Net::HTTP::#{method}".constantize.new(uri, 'Content-Type' => 'application/json') - set_headers(req, bridge) - req.body = payload.to_json - - begin - current_attempts += 1 - save_request(event, req.length, payload) - response = http.request(req) - save_response(event, response) + + (bridge.retries + 1).times do + execute_request_response_cycle(event, bridge) + break rescue *HTTP_ERRORS, Sidekiq::LargeStatusCode => e save_http_error(event, e) if HTTP_ERRORS.include?(e.class) - if current_attempts <= retries - sleep 1 # DEVELOPMENT - # sleep bridge.delay * 60 # PRODUCTION - retry - end - ensure - event.completed = true - event.completed_at = Time.now.utc - event.save + sleep bridge.delay * 60 end + save_event(event) + binding.pry end end diff --git a/dump.rdb b/dump.rdb index 142cd0ae86950e089cb6ec66b3ac167f947388bf..f28cbf40dd5d9340d2f610824d63559ba5efd578 100644 GIT binary patch delta 712 zcma*jziSg=90u^aO>2(?+rgy1Kkt`~_6m-QTZcyKo7{Vnl1tNGP6Vl_ zAc!C}+~A~tf*^F0!qvEnli1Bz7r|0=t+ARb3W9@;ULLY>|%3`GpADLfp92%ODAHI#hG=cW9CiAZ>LLnq+TiHmTQ(gIY?{; zLrQp+$?>5I#S}9HbY%N*B$HK;LUcGoD8g7PUEQoqN>aO#4u@8;%hbQ8lRSJ zIV0w0yO9@B1Ka?zkWy`%0K$R7A&}?fZLw7Drne*I+&THx?R(oP=@N7Jg}KM_Skz{< zdUtS3q6BKXyu}u!rRs{;kiUyyeew0~(}TTtX2(2bHbT08x|kTn0H_tjv7%apRw*MD z&Y@0WXdVyEI5ZOo&~I~MGJ+I(gm(Uch8Ct(@So_oiV@Y2R=89Cr9KSRd;1$l>bZ}8 zGkgB|CsAv=$D<@Dv}+{Z5qdjMHm|oE*)bfvO$3cRfAbo#;L}rZXwAx^wJ51B8Ze3 zgn$|j(X3gyem33@`)z#z{#nSa$5y=>Z^dj@cezyItThZ3KotcNqRn6obb|c%+O%|c zd3?ivXq+5SgOMX?cto{ez`t)y`6**6)(azje^d=3{xf4V`?v3h_9mIh_r5wm0hiF^ A0RR91 delta 314 zcmcaw{jhX`f#vqp_3^(rN{drdbaPX44{cb(&+v;UF+WYWpeQvlHz_Ce0K-3q-#qM0 zY=xz%rK!b?>?x^fiKRIulVz<TvZSOYrs!@y$mDD>`LvC!$qN>>qos@t3@oya z1qC^&#k#&u0anH)hUO+#DTXPDMuy3WX@+K|EUbm4nW-hLX^A<-sgoaB+e#T58JU_} z=oy-tTUr?FC>3O;C;|1c12s;zv Date: Wed, 18 Nov 2020 00:21:13 -0600 Subject: [PATCH 25/64] created event_spec test file --- app/models/event.rb | 7 ++-- spec/models/event_spec.rb | 76 +++++++++++++++++++++++++++++-------- spec/support/main_helper.rb | 33 ++++++++++++++-- 3 files changed, 92 insertions(+), 24 deletions(-) diff --git a/app/models/event.rb b/app/models/event.rb index a1b1a88..aaf477b 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -1,13 +1,12 @@ # frozen_string_literal: true BOOLEAN = [true, false].freeze -VALID_CODES = [100..599].freeze class Event < ApplicationRecord - before_create :set_urls + before_validation :set_urls validates :test, inclusion: { in: BOOLEAN, message: '"test" field must be true or false' } - validates :completed, inclusion: { in: BOOLEAN, message: '"completed" field must be true or false' }, allow_nil: true - validates :status_code, inclusion: { in: VALID_CODES, message: 'Invalid status code' }, allow_nil: true + validates :completed, inclusion: [true, false] + validates :status_code, numericality: { greater_than_or_equal_to: 100, less_than_or_equal_to: 599 }, allow_nil: true validate :completed_at_format validate :data_json_object diff --git a/spec/models/event_spec.rb b/spec/models/event_spec.rb index 9a737dd..83c7c57 100644 --- a/spec/models/event_spec.rb +++ b/spec/models/event_spec.rb @@ -3,29 +3,73 @@ require 'rails_helper' RSpec.describe Event, type: :model do - before do + before(:context) do create_user end subject do - create_bridge + create_event end - it 'belongs to bridge' do - event1 = Event.create( - completed: false, - data: '', - status_code: 300 - ) - event2 = Event.create( - completed: false, - data: '', - status_code: 300 + it 'is valid as long as long as it has a valid bridge_id and data object' do + event = Event.create({ bridge_id: subject.bridge_id, data: subject.data }) + expect(event).to be_valid + end + + it 'is invalid without a data object' do + subject.data = nil + expect(subject).to_not be_valid + end + + it 'can be converted to side_bar format' do + expect(subject.sidebar_format).to match( + id: subject.id, + time: a_string_matching(/\d{1,2}:\d{1,2}:\d{1,2}/), + date: a_string_matching(/\d{4}-\d{1,2}-\d{1,2}/), + status_code: an_instance_of(Integer) ) + end + + it 'belongs to a bridge' do + expect(subject.bridge.class).to eql Bridge + end + + it 'has its urls set to the url of the bridge it belongs to' do + bridge = subject.bridge + expect(bridge.inbound_url).to eql subject.inbound_url + expect(bridge.outbound_url).to eql subject.outbound_url + end + + it 'has a data attribute referencing a json object' do + expect(JSON.parse(subject.data)).to be_an_instance_of(Hash) + end + + it 'raises an error if data attribute does not reference a JSON object' do + subject.data = 'not a json object' + expect { subject.save! }.to raise_error('Validation failed: Data object must be a valid json object') + end + + it 'does not accept removing inbound or outbound key from json data object' do + data = JSON.parse(subject.data) + data.delete('outbound') + subject.data = data.to_json + expect { subject.save! }.to raise_error(ActiveRecord::RecordInvalid) + end + + it 'does not accept removing payload, date, time, ip or content length from inbound key in json data object' do + data = JSON.parse(subject.data) + data['inbound'].delete('payload') + subject.data = data.to_json + expect { subject.save! }.to raise_error(ActiveRecord::RecordInvalid) + end + + it 'does not accept status codes above 599' do + subject.status_code = 5555 + expect { subject.save! }.to raise_error('Validation failed: Status code must be less than or equal to 599') + end - subject.events << event1 - subject.events << event2 - expect(event1.bridge).to eq subject - expect(event2.bridge).to eq subject + it 'does not accept status codes below 100' do + subject.status_code = 10 + expect { subject.save! }.to raise_error('Validation failed: Status code must be greater than or equal to 100') end end diff --git a/spec/support/main_helper.rb b/spec/support/main_helper.rb index f375501..9aaec91 100644 --- a/spec/support/main_helper.rb +++ b/spec/support/main_helper.rb @@ -1,8 +1,15 @@ # frozen_string_literal: true +def random_email + local = [] + 8.times { |_| local.push(('a'..'z').to_a.sample) } + domain = ['aol.com', 'nsa.gov', 'jubii.dk', 'kreml.ru', 'hotmail.com', 'netscape.com'][rand(6)] + "#{local.join('')}@#{domain}" +end + module MainHelper def create_user - @current_user = User.create(email: 'admin@bridge.io', password: 'password', notifications: false) + @current_user = User.create(email: random_email, password: 'password', notifications: false) @token = JsonWebToken.encode(user_id: @current_user.id) end @@ -32,11 +39,29 @@ def create_bridge ) end - def event_data; end + def event_data + @data = { 'inbound' => { 'payload' => { 'FirstName' => 'Lee', 'LastName' => 'Oswald', 'UserName' => 'GrassyKnoll', 'Password' => { 'nested' => 'magic bullet' }, 'Email' => 'kgb63@yandex.ru' }, + 'date' => '2020-11-17', + 'time' => '03:23:35', + 'ip' => '::1', + 'content_length' => 152 }, + 'outbound' => [{ 'request' => { 'payload' => { 'FirstName' => 'Lee', 'LastName' => 'Oswald', 'UserName' => 'GrassyKnoll', 'Password' => { 'nested' => 'magic bullet' }, 'Email' => 'kgb63@yandex.ru' }, 'date' => '2020-11-17', 'time' => '03:23:35', 'content_length' => 7 }, 'response' => { 'date' => '2020-11-17', 'time' => '03:23:35', 'status_code' => '200', 'message' => 'OK', 'size' => 7, 'payload' => { 'ip' => '153.33.111.24' } } }] }.to_json + @data + end def create_event - @event = Event.create({ bridge_id: 1, data: event_data }) + @bridge = create_bridge + @bridge.save + @event = Event.create({ + bridge_id: @bridge.id, + data: event_data, + status_code: 200, + completed: true, + completed_at: Time.now.utc + 30 + }) end - def destroy_event; end + def destroy_event + @event.destroy! + end end From 3d348ca9da27cdfbb56728ca3a02fba1c05b92ea Mon Sep 17 00:00:00 2001 From: William Jackson Date: Wed, 18 Nov 2020 00:40:47 -0600 Subject: [PATCH 26/64] addressed rubocop complaints --- app/workers/event_worker.rb | 1 - spec/requests/events_request.spec.rb | 11 +++++++ spec/support/main_helper.rb | 43 ++++++++++++++++++++++++---- 3 files changed, 48 insertions(+), 7 deletions(-) diff --git a/app/workers/event_worker.rb b/app/workers/event_worker.rb index d17c6e7..4718b5a 100644 --- a/app/workers/event_worker.rb +++ b/app/workers/event_worker.rb @@ -139,6 +139,5 @@ def perform(event_id) sleep bridge.delay * 60 end save_event(event) - binding.pry end end diff --git a/spec/requests/events_request.spec.rb b/spec/requests/events_request.spec.rb index e69de29..38f5cd6 100644 --- a/spec/requests/events_request.spec.rb +++ b/spec/requests/events_request.spec.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +RSpec.describe 'Events', type: :request do + before(:context) do + create_event + end + + after(:context) do + destroy_event + end +end diff --git a/spec/support/main_helper.rb b/spec/support/main_helper.rb index 9aaec91..2fe68c6 100644 --- a/spec/support/main_helper.rb +++ b/spec/support/main_helper.rb @@ -40,12 +40,43 @@ def create_bridge end def event_data - @data = { 'inbound' => { 'payload' => { 'FirstName' => 'Lee', 'LastName' => 'Oswald', 'UserName' => 'GrassyKnoll', 'Password' => { 'nested' => 'magic bullet' }, 'Email' => 'kgb63@yandex.ru' }, - 'date' => '2020-11-17', - 'time' => '03:23:35', - 'ip' => '::1', - 'content_length' => 152 }, - 'outbound' => [{ 'request' => { 'payload' => { 'FirstName' => 'Lee', 'LastName' => 'Oswald', 'UserName' => 'GrassyKnoll', 'Password' => { 'nested' => 'magic bullet' }, 'Email' => 'kgb63@yandex.ru' }, 'date' => '2020-11-17', 'time' => '03:23:35', 'content_length' => 7 }, 'response' => { 'date' => '2020-11-17', 'time' => '03:23:35', 'status_code' => '200', 'message' => 'OK', 'size' => 7, 'payload' => { 'ip' => '153.33.111.24' } } }] }.to_json + @data = { + 'inbound' => { + 'payload' => { + 'FirstName' => 'Lee', + 'LastName' => 'Oswald', + 'UserName' => 'GrassyKnoll', + 'Password' => { 'nested' => 'magic bullet' }, + 'Email' => 'kgb63@yandex.ru' + }, + 'date' => '2020-11-17', + 'time' => '03:23:35', + 'ip' => '::1', + 'content_length' => 152 + }, + 'outbound' => [ + { 'request' => { + 'payload' => { + 'FirstName' => 'Lee', + 'LastName' => 'Oswald', + 'UserName' => 'GrassyKnoll', + 'Password' => { 'nested' => 'magic bullet' }, + 'Email' => 'kgb63@yandex.ru' + }, + 'date' => '2020-11-17', + 'time' => '03:23:35', + 'content_length' => 7 + }, + 'response' => { + 'date' => '2020-11-17', + 'time' => '03:23:35', + 'status_code' => '200', + 'message' => 'OK', + 'size' => 7, + 'payload' => { 'ip' => '153.33.111.24' } + } } + ] + }.to_json @data end From d8f61debb35c7c95a8cb52fd858e61f20fe16490 Mon Sep 17 00:00:00 2001 From: William Jackson Date: Thu, 19 Nov 2020 00:01:44 -0600 Subject: [PATCH 27/64] editet data formatting and added sidekiq retry middleware --- app/controllers/events_controller.rb | 43 +++-- app/models/event.rb | 21 +-- app/workers/event_worker.rb | 224 +++++++++++++-------------- config/initializers/sidekiq.rb | 12 ++ config/routes.rb | 4 +- dump.rdb | Bin 15829 -> 18158 bytes spec/support/main_helper.rb | 3 +- 7 files changed, 147 insertions(+), 160 deletions(-) create mode 100644 config/initializers/sidekiq.rb diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index 52b88b7..cef125b 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -2,12 +2,11 @@ class EventsController < ApplicationController # before_action :authorize_request + before_action :set_events, only: :index + before_action :set_event, only: %i[show destroy] def index - events = retrieve_events - sidebar_events = create_sidebar_events(events) - - render json: sidebar_events, status: 200 + render json: @events, status: 200 rescue ActiveRecord::RecordInvalid render json: { error: 'neither event_id nor bridge_id were valid' }, status: 400 # Bad Request rescue ActiveRecord::RecordNotFound @@ -17,15 +16,13 @@ def index end def show - event = Event.find(event_params[:id]) - render json: event, status: 200 + render json: @event, status: 200 rescue ActiveRecord::RecordNotFound render json: { error: 'an event by that id was not found' }, status: 400 end def destroy - event = Event.find(event_params[:id]) - event.destroy + @event.destroy end def create @@ -48,13 +45,11 @@ def event_params end def create_data_object - datetime = DateTime.now.utc.to_s.split(' ') { 'inbound' => { 'payload' => JSON.parse(request.body.read), - 'date' => datetime.first, - 'time' => datetime[1], + 'dateTime' => DateTime.now.utc, 'ip' => request.ip, - 'content_length' => request.content_length + 'contentLength' => request.content_length }, 'outbound' => [] } end @@ -67,18 +62,22 @@ def create_event_object(data, bridge) ) end - def create_sidebar_events(events) - events.map(&:sidebar_format).sort_by { |event| event[:id] }.reverse + def set_events + @events = if event_params[:bridge_id] + Event.where(bridge_id: event_params[:bridge_id]).order(completed_at: :desc).limit(100) + elsif event_params[:event_id] + Event.where(bridge_id: find_event.bridge_id).order(completed_at: :desc).limit(100) + else + raise ActiveRecord::NotNullViolation + end + end + + def set_event + @event = find_event end - def retrieve_events - if event_params[:bridge_id] - Bridge.find(event_params[:bridge_id]).events - elsif event_params[:event_id] - Event.find(event_params[:event_id]).bridge.events - else - raise ActiveRecord::NotNullViolation - end + def find_event + Event.find(event_params[:event_id]) end def find_bridge diff --git a/app/models/event.rb b/app/models/event.rb index aaf477b..c347d82 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -12,25 +12,8 @@ class Event < ApplicationRecord belongs_to :bridge - def sidebar_format - updated_at = String(self.updated_at) - time = date_format(updated_at.split(' ')[1]) - date = updated_at.split(' ')[0] - { id: id, - time: time.slice(0..-3), - date: date, - status_code: status_code } - end - private - def date_format(time) - year = time.split('-')[0] - month = time.split('-')[1] - day = time.split('-')[2] - "#{year}-#{month}-#{day}" - end - def set_urls self.inbound_url = bridge.inbound_url self.outbound_url = bridge.outbound_url @@ -39,11 +22,11 @@ def set_urls def data_json_object data = JSON.parse(self.data) %w[inbound outbound].all? { |key| data.include?(key) } && - %w[payload date time ip content_length].all? { |key| data['inbound'].include?(key) } || + %w[payload dateTime ip contentLength].all? { |key| data['inbound'].include?(key) } || errors.add( :data, 'must include the keys: "inbound", "outbound", - while "inbound" must include the keys "payload", "date", "time", "ip", "content_length"' + while "inbound" must include the keys "payload", "dateTime", "ip", "contentLength"' ) rescue JSON::ParserError, TypeError errors.add(:data, 'object must be a valid json object') diff --git a/app/workers/event_worker.rb b/app/workers/event_worker.rb index 4718b5a..117f872 100644 --- a/app/workers/event_worker.rb +++ b/app/workers/event_worker.rb @@ -3,141 +3,135 @@ require 'net/http' require_relative '../lib/exceptions/large_status_code' -SCHEME = 'http://' - -HTTP_ERRORS = [ - Net::HTTPBadResponse, - Net::HTTPHeaderSyntaxError, - Net::ProtocolError, - EOFError, - Errno::ECONNRESET, - Errno::EINVAL, - SocketError, - Timeout::Error -].freeze - -def set_headers(req, bridge) - bridge.headers.each { |header| req[header['key']] = header['value'] } -end - -def save_event(event) - event.completed = true - event.completed_at = Time.now.utc - event.save! -end +class EventWorker + include Sidekiq::Worker + attr_accessor :retry_count + + sidekiq_retry_in { 5 } # TODO: SET RETRY_IN DYNAMICALLY + + SCHEME = 'http://' + HTTP_ERRORS = [ + Net::HTTPBadResponse, + Net::HTTPHeaderSyntaxError, + Net::ProtocolError, + EOFError, + Errno::ECONNRESET, + Errno::EINVAL, + SocketError, + Timeout::Error + ].freeze -def create_error_response(error) - { - date: '', - time: '', - status_code: '', - message: error.message, - size: '', - payload: {} - } -end + def perform(event_id) + event = Event.find(event_id) + bridge = Bridge.find(event.bridge_id) + execute_request_response_cycle(event, bridge) + save_event(event) + rescue *HTTP_ERRORS, Sidekiq::LargeStatusCode => e + save_http_error(event, e) if HTTP_ERRORS.include?(e.class) + save_event(event) && return if retry_count&.>= bridge.retries -def save_http_error(event, error) - response = create_error_response(error) - event_data = JSON.parse(event.data) + raise 'Error or high status code' # activates sidekiq retry + end - event_data['outbound'].last['response'] = response - event.data = event_data.to_json - event.save -end + private -def create_response_object(payload, response) - { - date: DateTime.now.utc.to_s.split(' ').first, - time: DateTime.now.utc.to_s.split(' ')[1], - status_code: response.code, - message: response.message, - size: response.size, - payload: payload - } -end + def set_headers(req, bridge) + bridge.headers.each { |header| req[header['key']] = header['value'] } + end -def save_response(event, resp) - resp_code = resp.code.to_i - payload = (resp_code >= 300 ? {} : JSON.parse(resp.body)) - response = create_response_object(payload, resp) - event_data = JSON.parse(event.data) + def save_event(event) + event.completed = true + event.completed_at = Time.now.utc + event.save! + end - event_data['outbound'].last['response'] = response - event.data = event_data.to_json - event.save - raise Sidekiq::LargeStatusCode if resp_code >= 300 -end + def create_error_response(error) + { + message: error.message + } + end -def create_request_object(payload, length) - { - payload: payload, - date: DateTime.now.utc.to_s.split(' ').first, - time: DateTime.now.utc.to_s.split(' ')[1], - content_length: length - } -end + def save_http_error(event, error) + response = create_error_response(error) + event_data = JSON.parse(event.data) -def save_request(event, length, payload) - request = create_request_object(payload, length) - event_data = JSON.parse(event.data) + event_data['outbound'].last['response'] = response + event.data = event_data.to_json + event.save + end - event_data['outbound'].push({ 'request' => request, 'response' => {} }) - event.data = event_data.to_json - event.save -end + def create_response_object(payload, response) + { + dateTime: DateTime.now.utc, + statusCode: response.code, + message: response.message, + size: response.size, + payload: payload + } + end -def prepend_scheme(uri) - return uri if uri.starts_with?('https') + def save_response(event, resp) + resp_code = resp.code.to_i + payload = (resp_code >= 300 ? {} : JSON.parse(resp.body)) + response = create_response_object(payload, resp) + event_data = JSON.parse(event.data) - minus_scheme = uri.split('//').last - "#{SCHEME}#{minus_scheme}" -end + event_data['outbound'].last['response'] = response + event.data = event_data.to_json + event.save + raise Sidekiq::LargeStatusCode if resp_code >= 300 + end -def generate_http_request(bridge, payload) - method = bridge.method.capitalize - uri = URI(prepend_scheme(bridge.outbound_url)) - http = Net::HTTP.new(uri.host, uri.port) - req = "Net::HTTP::#{method}".constantize.new(uri, 'Content-Type' => 'application/json') + def create_request_object(payload, length) + { + payload: payload, + dateTime: DateTime.now.utc, + contentLength: length + } + end - http.use_ssl = (uri.scheme == 'https') - set_headers(req, bridge) - req.body = payload.to_json - [http, req] -end + def save_request(event, length, payload) + request = create_request_object(payload, length) + event_data = JSON.parse(event.data) -def extract_payload(bridge, event) - if event.test - JSON.parse(bridge.data['test_payload']) - else - JSON.parse(bridge.data['payload']) + event_data['outbound'].push({ 'request' => request, 'response' => {} }) + event.data = event_data.to_json + event.save end -end -def execute_request_response_cycle(event, bridge) - payload = extract_payload(bridge, event) - http, req = generate_http_request(bridge, payload) + def prepend_scheme(uri) + return uri if uri.starts_with?('https') - save_request(event, req.length, payload) - response = http.request(req) - save_response(event, response) -end + minus_scheme = uri.split('//').last + "#{SCHEME}#{minus_scheme}" + end -class EventWorker - include Sidekiq::Worker - sidekiq_options retry: 0 + def generate_http_request(bridge, payload) + method = bridge.method.capitalize + uri = URI(prepend_scheme(bridge.outbound_url)) + http = Net::HTTP.new(uri.host, uri.port) + req = "Net::HTTP::#{method}".constantize.new(uri, 'Content-Type' => 'application/json') - def perform(event_id) - event = Event.find(event_id) - bridge = Bridge.find(event.bridge_id) + http.use_ssl = (uri.scheme == 'https') + set_headers(req, bridge) + req.body = payload.to_json + [http, req] + end - (bridge.retries + 1).times do - execute_request_response_cycle(event, bridge) - break - rescue *HTTP_ERRORS, Sidekiq::LargeStatusCode => e - save_http_error(event, e) if HTTP_ERRORS.include?(e.class) - sleep bridge.delay * 60 + def extract_payload(bridge, event) + if event.test + JSON.parse(bridge.data['test_payload']) + else + JSON.parse(bridge.data['payload']) end - save_event(event) + end + + def execute_request_response_cycle(event, bridge) + payload = extract_payload(bridge, event) + http, req = generate_http_request(bridge, payload) + + save_request(event, req.length, payload) + response = http.request(req) + save_response(event, response) end end diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb new file mode 100644 index 0000000..eb1f6b4 --- /dev/null +++ b/config/initializers/sidekiq.rb @@ -0,0 +1,12 @@ +class SidekiqMiddleware + def call(worker, job, _queue) + worker.retry_count = job['retry_count'] if worker.respond_to?(:retry_count) + yield + end +end + +Sidekiq.configure_server do |config| + config.server_middleware do |chain| + chain.add SidekiqMiddleware + end +end diff --git a/config/routes.rb b/config/routes.rb index c8ae1f3..26b49bf 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -10,7 +10,7 @@ post 'login', to: 'sessions#create' post 'events', to: 'events#create' get 'events', to: 'events#index' - get 'events/:id', to: 'events#show' - delete 'events/:id', to: 'events#destroy' + get 'events/:event_id', to: 'events#show' + delete 'events/:event_id', to: 'events#destroy' mount Sidekiq::Web => '/sidekiq' end diff --git a/dump.rdb b/dump.rdb index f28cbf40dd5d9340d2f610824d63559ba5efd578..ceb36e8b2a1d578806976ac6bc5039b9de3ba506 100644 GIT binary patch delta 2385 zcmcJQTWl0n9L8t2v)gWqr5BNlKu=eJXxizSGv}O{vvkXBu_Qh~h>0PFLU*q1!rqwO zt=76}@kL|2KqfvBBH)`*6BYEqOSav>i%&injhC~4Wp5nt^4WO9Dr z_x-=`%h_LF37mq1*CQWJ+-NHm9D6Y1WERhT)+5|lXXM<$yyqAhGwoD_KZQSfJC|O< zhhFn<#va_ORfJ$9@8v9~P;d%?ZHat7?Gy(0j~EyAhenr2wW!66*>Yd9QGwGaRgWj&$@u=q%y_xZ7tmu`Tf${l4&?$^ph63YJutBua zMxl`0&^|WpWQ&Iy|4eQU8s4P7spT-GN5HP&v6P+Mv60Cxlx>qT!(a}gief3qHe@r2 z+AOYX(Xl5CaJV}rQ%Kak6eC6@j1;6WFcxsK$q#Cweo=u$!r+l0lgnT+=#{cyPfZwU zA&{1rIV{($>Q()xHRC`?Q8bKI17d;=N2S{k(v-+38Ci9Y9gA*t`$TN!;COP#kQqTS+*T!A`l2@a5coT8FLgX zRFOD{M)4gpA_PH+X^5yy?{D|KoHvo-`8Ou#-x0?2ZUKN`=TvHP3e>Y)DgY~IgKafo zs1~>!exj5urZP?vCqYX+<+X4gyxPij!h}%-62{Kpgp~stxDpV7Cdkp zL(OupK_atOUT-8OPnu0k!c5A2yJV_PMAVH zZ|rC|t%8f(>He|p4W}1t!UA~XwnVhxRTCs}YRx1<7QwC|tD0gX*)TD~8lg6#t0n4b zb`w7bo(74@SVKqy`|4MHu@;I(6pAGbnM(hD5AXiRJzTllJw%!XNEniVzI+d=e(79y zOi0zjs6%Pmq;WUP{ZkFueyj;UG-^cXZ0yIaMY0!$0?m;*8sAP?C4FZk?mbnan991j zs{WRK$^U)#K%AEyPqG9d5BWeepR%L%eG6;KK(uH~g5i!)FJ(_UiGzGRHVro27P{hz2YAVb|3wuB?g~?Kxlxu1R}?_;Rtw>gI)h8SUVz2A~M%s9XJ_Z jX8Ud>o2Ve9mQT0ko~@I;H*qEwxq1HZ_xsn4&wctAgE-p9 delta 437 zcmaFY%XqbVf`JKF@s{{s9HqsnDZ06-xraVD3N!rTNz6~vEhtJ&%uUKkJ;3mf;kN)c zD@#ghVv7D@M`g#{$pRER-B=ezIrBusWSelty!kU(tQ=H0@ zl$ux~XlQ0&YG!6=Vq&3ZY;0g@V)Uo|+pOxz3pwRDOb*!mY08r62XUCq4+u@3#~3zw f8K=x-enw?uCQeu|6*J=Z;D3wrJi^L { 'payload' => { 'FirstName' => 'Lee', @@ -77,7 +77,6 @@ def event_data } } ] }.to_json - @data end def create_event From 918d0c10653c95388c1ed98c097a5bf070602127 Mon Sep 17 00:00:00 2001 From: William Jackson Date: Thu, 19 Nov 2020 00:27:51 -0600 Subject: [PATCH 28/64] added dynamic delay to event_worker --- app/workers/event_worker.rb | 10 +++++----- dump.rdb | Bin 18158 -> 16545 bytes 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/workers/event_worker.rb b/app/workers/event_worker.rb index 117f872..67f937f 100644 --- a/app/workers/event_worker.rb +++ b/app/workers/event_worker.rb @@ -5,9 +5,9 @@ class EventWorker include Sidekiq::Worker - attr_accessor :retry_count + # attr_accessor :retry_count - sidekiq_retry_in { 5 } # TODO: SET RETRY_IN DYNAMICALLY + # sidekiq_retry_in { 5 } # TODO: SET RETRY_IN DYNAMICALLY SCHEME = 'http://' HTTP_ERRORS = [ @@ -21,16 +21,16 @@ class EventWorker Timeout::Error ].freeze - def perform(event_id) + def perform(event_id, retries = 0) event = Event.find(event_id) bridge = Bridge.find(event.bridge_id) execute_request_response_cycle(event, bridge) save_event(event) rescue *HTTP_ERRORS, Sidekiq::LargeStatusCode => e save_http_error(event, e) if HTTP_ERRORS.include?(e.class) - save_event(event) && return if retry_count&.>= bridge.retries + save_event(event) && return if retries >= bridge.retries - raise 'Error or high status code' # activates sidekiq retry + self.class.perform_in(bridge.delay.minutes, event_id, retries + 1) end private diff --git a/dump.rdb b/dump.rdb index ceb36e8b2a1d578806976ac6bc5039b9de3ba506..742553dda185a3ec70ad2801397551e954ac6f62 100644 GIT binary patch delta 463 zcmaFY%eb(Sae_g(zQDHlUmT^ysVTa-skw&&*n}B=@g(M_=@t~FCgvvPq#j`S$M9Q- zmx;5WC_g#1xHz?#QP#1bASbn0*Vie)%F@Kr!o(^i)yT}k)HKD&&@}l^`?p!ulNWNz zNtzt6VPY#RO)UlLVNXd-ODxSP`J=xkrgU-xqcn%n0g*pVSu*_~r5xr5g#L7yg!hAF zm@E!V=3`V5W#BF@Ni4BSOU%qkO*wd<;m?h^TgpHR5{wVn{P`ncaKDt1fq@0TYgkwd zOEXhTSkn@7ic?vVQWHxA4b2Qp&CSe>42|^6EP=p;C8@Ny@&E%%W?owUVTUyi+b5so z(6co&GBVV&G&L|cFf!6nD#%Pxva;j=`Bq1%Br#pVj@zjyGbKIMF~C!=C`rMAD>*+e zxwNP#H7~hR!Ev$?r=CPtLoy#q&?wdZna#CyD#*vomIq2E&*4;Kg@}Lx#PUEQ#ET$l r;{(2%4{<8n3N!G5Lje~0DF@9NC(mP4pIpPJ#QA@cy}3?OVniVTraYbT delta 1997 zcmcJQ&rcjx0LN!`hlPcP2&B@gC684kCNRvrdGlu8O6e>}6A#4dr3coqZ)V;uOZQi2 zb})nx(2K_Sql9=dX{1+EJy_6quvyr&e?wz5ohEwF7~{!9-KcLDO2O26k=t%&c6Pqs z&-eSj*WMI=eP6tyjQ$e;a_)Xd;4^o$$cnq0H+zNqCuli4TJaby+67h<{uKV`>u$V@ zE`GRv^XVzGCPd;Duk0}2XTEqQRjCx1KRPpcA!!m~YDt^8ib74-(N!lByBx59-`4K3 zEDZ|P#nuc$V%@rpGpwJzTq;Fo9CzkV*2yxxB|5CuURdhDwu-za;R znCHjhDe`r4N*KhN z=4jBRs-1xy4)?ms+&Kzndy*q zeBhK_a3&PSLh)AYwV+hZ7g+{nK(rb9FjfH{v~!=3PA~*1`g~;0DFYL1i^>WZJl}Gu z9tyi)Q+Plc7fW^P)-fg)8dS|XSyMxt9@aT$$5o;FXp<(JiG?ykh#UIQq)tsyWe9md z^}~q;PaeoHN}M%F6_1N`zi#uSxw@?r8*`5QEs3c*I{8=D2+0r;#W+;Febx$x^S`kE zs*T49Va2M@p%!TaY;n?q(*rHiyP>cHJ~^-m`(8bzr>-2EMbObvma2xSxlpAxBFNOS z3-M8ldiTyIUI(5AnW)Hw&;;k3t=`4mwFb2D{JxnI2xiUE4vhBxE_@Q--2;HqYc`#G%&X~BjJog4LsTCRr8}a!0 zlB9eoTdrH5TH70UMEOO;V=OBQPxwfpl6Mo$#fzERNTNz}V7zP6%ey(2y702>*ishz(_V9gPzt3V4ZhKCIn#v fHpZ@}1HP6~SOYgsN1r_#|LfZ9_cJHbtKa+$l=y4H From 17dfdf4aa1020c7478987a18b46cea6b09ccebc6 Mon Sep 17 00:00:00 2001 From: = Date: Thu, 19 Nov 2020 09:42:01 -0500 Subject: [PATCH 29/64] Syntax Parser --- app/lib/exceptions.rb | 12 +++ app/lib/exceptions/jwt_expired_token.rb | 6 -- app/lib/json_web_token.rb | 2 +- app/lib/syntax_parser.rb | 135 ++++++++++++++++++++++++ 4 files changed, 148 insertions(+), 7 deletions(-) create mode 100644 app/lib/exceptions.rb delete mode 100644 app/lib/exceptions/jwt_expired_token.rb create mode 100644 app/lib/syntax_parser.rb diff --git a/app/lib/exceptions.rb b/app/lib/exceptions.rb new file mode 100644 index 0000000..c1ba73e --- /dev/null +++ b/app/lib/exceptions.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +class InvalidPayloadKey < StandardError +end + +class InvalidEnvironmentVariable < StandardError +end + +module JWT + class ExpiredWebToken < StandardError + end +end diff --git a/app/lib/exceptions/jwt_expired_token.rb b/app/lib/exceptions/jwt_expired_token.rb deleted file mode 100644 index b80506e..0000000 --- a/app/lib/exceptions/jwt_expired_token.rb +++ /dev/null @@ -1,6 +0,0 @@ -# frozen_string_literal: true - -module JWT - class ExpiredWebToken < StandardError - end -end diff --git a/app/lib/json_web_token.rb b/app/lib/json_web_token.rb index bc6da71..1e9ffb8 100644 --- a/app/lib/json_web_token.rb +++ b/app/lib/json_web_token.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative './exceptions/jwt_expired_token' +require_relative './exceptions' class JsonWebToken SECRET_KEY = Rails.application.secrets.secret_key_base.to_s diff --git a/app/lib/syntax_parser.rb b/app/lib/syntax_parser.rb new file mode 100644 index 0000000..b631250 --- /dev/null +++ b/app/lib/syntax_parser.rb @@ -0,0 +1,135 @@ +# frozen_string_literal: true + +require_relative './exceptions' + +# This class parses user defined headers & payloads into the values we expect +# to send to the outbound service. +# +# Example: +# +# ```ruby +# custom_user_payload = { +# 'hello' => '$payload.top_level_key', +# 'environment_variable' => '$env.API_KEY', +# 'did_you' => '$payload.nested_key_1.nested_key_2.nested_key_3' +# } +# +# incoming_request_from_service_a = { +# 'top_level_key' => 'world', +# 'nested_key_1' => { +# 'nested_key_2' => { +# 'nested_key_3' => 'make it!' +# } +# } +# } +# +# bridge = Bridge.where(user_id: @current_user.id) +# +# syntax_parser = SyntaxParser.new(bridge) +# +# syntax_parser.parse_payload( +# incoming_request_from_service_a, +# custom_user_payload +# ) # => Hash(String, String) where $payload & $env are replaced with their respective values +# +# syntax_parser.parse_headers # => Array(Hash(String, String)) where $env is +# replaced with decrypted environment variable value +# ``` +class SyntaxParser + # Set bridge headers & env's. + # + # @param [Bridge] bridge + def initialize(bridge) + @headers = bridge.headers + @environment_variables = bridge.environment_variables + end + + # Parses the user's custom payload replacing any values containing `$env` + # or `$payload` with the respective value + # + # @param [Hash(String, String)] incoming_request + # @param [Hash(String, String)] user_data + # + # @return [Hash(String, String)] + def parse_payload(incoming_request, user_data) + @incoming_request = incoming_request + @user_data = user_data + + parse_payload! + + outbound_request + end + + # Parses the user's headers replacing any values containing `$env` + # with the decrypted environment variable value + # + # @return [Array(Hash(String, String))] + def parse_headers + parse_headers! + + outbound_headers + end + + private + + attr_reader :user_data, # User defined payload (bridge.data['payload']) + :outbound_request, # Parsed request returned from `parse_payload` + :incoming_request, # Inbound request from service A + :outbound_headers, # Parsed headers returned from `parse_headers` + :environment_variables, + :headers + + # Iterates through user defined payload and parse values containing `$env` or `$payload` + # Reinitializes & mutates `outbound_request` + def parse_payload! + @outbound_request = {} # Reset + + user_data.each do |key, val| + outbound_request[key] = if val.include?('$env') + fetch_environment_variable(val) + elsif val.include?('$payload') + fetch_payload_data(val) + else + val + end + end + end + + # Iterates through user defined payload and parse values containing `$env` + # Reinitializes & mutates `outbound_headers` + def parse_headers! + @outbound_headers = [] # Reset + headers.each do |header| + parsed_header = {} + parsed_header[header.key] = if header.value.include?('$env') + fetch_environment_variable(header.value) + else + header.value + end + + outbound_headers << parsed_header + end + end + + def fetch_environment_variable(value) + key = value.split('.').last + environment_variable = environment_variables.where(key: key) + raise InvalidEnvironmentVariable unless environment_variable + + environment_variable.decrypt + end + + def fetch_payload_data(value) + values = value.split('.') + data = incoming_request # Set data to the incoming request + + values.each_with_index do |val, idx| + next if idx.zero? # skip the $payload + + data = data[val] # dig deeper into the request on each iteration + raise InvalidPayloadKey if data.nil? + end + + data + end +end From 81cf04b9e9c388e81ef34f530d9c222d133bfe3f Mon Sep 17 00:00:00 2001 From: = Date: Thu, 19 Nov 2020 10:01:00 -0500 Subject: [PATCH 30/64] Skeleton classes --- app/lib/request_deconstructor.rb | 11 +++++++++++ app/lib/request_handler.rb | 11 +++++++++++ 2 files changed, 22 insertions(+) create mode 100644 app/lib/request_deconstructor.rb create mode 100644 app/lib/request_handler.rb diff --git a/app/lib/request_deconstructor.rb b/app/lib/request_deconstructor.rb new file mode 100644 index 0000000..1dc3304 --- /dev/null +++ b/app/lib/request_deconstructor.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +require_relative './exceptions' + +# TODO: Build +# +# Deconstructs the request & response from us & service B. Replaces +# any headers that contain environment variables with placeholder values +# to prevent data leakage. +class RequestDeconstructor +end diff --git a/app/lib/request_handler.rb b/app/lib/request_handler.rb new file mode 100644 index 0000000..6a1bed5 --- /dev/null +++ b/app/lib/request_handler.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +require_relative './exceptions' + +# TODO: Build or delete +# +# Thinking some EventWorker functionality will be moved into here. +# SyntaxParser & RequestDeconstructor would be attributes and aid +# in setting up & properly saving requests/responses. +class RequestHandler +end From ae590efe3fe30fba1172a60f78da80aea790606a Mon Sep 17 00:00:00 2001 From: = Date: Thu, 19 Nov 2020 10:04:26 -0500 Subject: [PATCH 31/64] Fix Zeitwerk::NameError --- app/lib/exceptions.rb | 12 ------------ app/lib/exceptions/invalid_environment_variable.rb | 4 ++++ app/lib/exceptions/invalid_payload_key.rb | 4 ++++ app/lib/exceptions/jwt/expired_web_token.rb | 6 ++++++ app/lib/json_web_token.rb | 2 +- app/lib/syntax_parser.rb | 3 ++- 6 files changed, 17 insertions(+), 14 deletions(-) delete mode 100644 app/lib/exceptions.rb create mode 100644 app/lib/exceptions/invalid_environment_variable.rb create mode 100644 app/lib/exceptions/invalid_payload_key.rb create mode 100644 app/lib/exceptions/jwt/expired_web_token.rb diff --git a/app/lib/exceptions.rb b/app/lib/exceptions.rb deleted file mode 100644 index c1ba73e..0000000 --- a/app/lib/exceptions.rb +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -class InvalidPayloadKey < StandardError -end - -class InvalidEnvironmentVariable < StandardError -end - -module JWT - class ExpiredWebToken < StandardError - end -end diff --git a/app/lib/exceptions/invalid_environment_variable.rb b/app/lib/exceptions/invalid_environment_variable.rb new file mode 100644 index 0000000..750487e --- /dev/null +++ b/app/lib/exceptions/invalid_environment_variable.rb @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +class InvalidEnvironmentVariable < StandardError +end diff --git a/app/lib/exceptions/invalid_payload_key.rb b/app/lib/exceptions/invalid_payload_key.rb new file mode 100644 index 0000000..af828fe --- /dev/null +++ b/app/lib/exceptions/invalid_payload_key.rb @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +class InvalidPayloadKey < StandardError +end \ No newline at end of file diff --git a/app/lib/exceptions/jwt/expired_web_token.rb b/app/lib/exceptions/jwt/expired_web_token.rb new file mode 100644 index 0000000..b80506e --- /dev/null +++ b/app/lib/exceptions/jwt/expired_web_token.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +module JWT + class ExpiredWebToken < StandardError + end +end diff --git a/app/lib/json_web_token.rb b/app/lib/json_web_token.rb index 1e9ffb8..b8e3ded 100644 --- a/app/lib/json_web_token.rb +++ b/app/lib/json_web_token.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative './exceptions' +require_relative './exceptions/jwt/expired_web_token' class JsonWebToken SECRET_KEY = Rails.application.secrets.secret_key_base.to_s diff --git a/app/lib/syntax_parser.rb b/app/lib/syntax_parser.rb index b631250..db2563a 100644 --- a/app/lib/syntax_parser.rb +++ b/app/lib/syntax_parser.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true -require_relative './exceptions' +require_relative './exceptions/invalid_environment_variable' +require_relative './exceptions/invalid_payload_key' # This class parses user defined headers & payloads into the values we expect # to send to the outbound service. From 257af7e611eafecc234cd742a0a2008b4286638b Mon Sep 17 00:00:00 2001 From: William Jackson Date: Thu, 19 Nov 2020 11:10:49 -0600 Subject: [PATCH 32/64] small cleanup --- app/models/event.rb | 2 +- app/workers/event_worker.rb | 24 +++++++----------------- dump.rdb | Bin 16545 -> 16544 bytes 3 files changed, 8 insertions(+), 18 deletions(-) diff --git a/app/models/event.rb b/app/models/event.rb index c347d82..c1bd581 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -4,7 +4,7 @@ class Event < ApplicationRecord before_validation :set_urls - validates :test, inclusion: { in: BOOLEAN, message: '"test" field must be true or false' } + validates :test, inclusion: [true, false] validates :completed, inclusion: [true, false] validates :status_code, numericality: { greater_than_or_equal_to: 100, less_than_or_equal_to: 599 }, allow_nil: true validate :completed_at_format diff --git a/app/workers/event_worker.rb b/app/workers/event_worker.rb index 67f937f..ffe3340 100644 --- a/app/workers/event_worker.rb +++ b/app/workers/event_worker.rb @@ -10,27 +10,18 @@ class EventWorker # sidekiq_retry_in { 5 } # TODO: SET RETRY_IN DYNAMICALLY SCHEME = 'http://' - HTTP_ERRORS = [ - Net::HTTPBadResponse, - Net::HTTPHeaderSyntaxError, - Net::ProtocolError, - EOFError, - Errno::ECONNRESET, - Errno::EINVAL, - SocketError, - Timeout::Error - ].freeze def perform(event_id, retries = 0) event = Event.find(event_id) bridge = Bridge.find(event.bridge_id) execute_request_response_cycle(event, bridge) - save_event(event) - rescue *HTTP_ERRORS, Sidekiq::LargeStatusCode => e - save_http_error(event, e) if HTTP_ERRORS.include?(e.class) - save_event(event) && return if retries >= bridge.retries + complete_event(event) + rescue StandardError => e + binding.pry + save_http_error(event, e) unless e.instance_of?(Sidekiq::LargeStatusCode) + complete_event(event) && return if retries >= bridge.retries - self.class.perform_in(bridge.delay.minutes, event_id, retries + 1) + EventWorker.perform_in(bridge.delay.minutes, event_id, retries + 1) end private @@ -39,7 +30,7 @@ def set_headers(req, bridge) bridge.headers.each { |header| req[header['key']] = header['value'] } end - def save_event(event) + def complete_event(event) event.completed = true event.completed_at = Time.now.utc event.save! @@ -129,7 +120,6 @@ def extract_payload(bridge, event) def execute_request_response_cycle(event, bridge) payload = extract_payload(bridge, event) http, req = generate_http_request(bridge, payload) - save_request(event, req.length, payload) response = http.request(req) save_response(event, response) diff --git a/dump.rdb b/dump.rdb index 742553dda185a3ec70ad2801397551e954ac6f62..7f3083b766853283766f8f22c9d6fe0520ea56c2 100644 GIT binary patch delta 460 zcmZ43$he@9ae{#^@A7T&zc@;ZQ&V(vQ*#dm>=k18#gmwyrdv>ynwXoElX`&RAH#1U z-aqZ%W>rtzC(UefKxeYQgZji>$}C`^hjJVy2W-stcSectznpmYKCYvXkBpIir8m0cZF?UNDP?Lo50UHMH;*!J?tF*+- zoYa(qZy5eOO!zc?;zPMa%Y%0r{y65>J}hNqU|_*(9t%rSYGR3yp_zfH1rQmV>X}$r znp&6`u_ToiR~}$sEiBDUEn!Vd%qdP~$;?a3KkTsDVe8}@j5>A}1||k37J3$zCWht~ zMmkCbnJG$E=IlTh>nN2ZrYqQTI~8T7q^CLtccG7$30t^GCwqekmgZ0}Fm@Sy&58GgC`g(-L!vQ(2Ny6H5dQ z%?wP<&CHDqjr7bcfxv_%skFHA00T>AURwTPhcyn{C%AR{+Z3SbSf|$m@N;KOfGa#V}%I3m*%iMkO=W1P}2B-?`BbEWm{neK5!Vo gLOJE2Ipf4?brx`7zL(}PIiT}@lD)Z3Qes3Q0C=IAF#rGn From a2bb448cea76537bed5234d93c212ba684ac6c87 Mon Sep 17 00:00:00 2001 From: = Date: Fri, 20 Nov 2020 08:02:47 -0500 Subject: [PATCH 33/64] Broke parser into smaller classes, rubocop seeds --- app/lib/parser.rb | 8 ++ app/lib/parser/environment_variables.rb | 14 +++ app/lib/parser/headers.rb | 88 +++++++++++++++ app/lib/parser/payload.rb | 97 +++++++++++++++++ app/lib/request.rb | 6 ++ app/lib/request/deconstructor.rb | 9 ++ app/lib/request/handler.rb | 6 ++ app/lib/syntax_parser.rb | 136 ------------------------ app/workers/event_worker.rb | 18 +++- config/database.yml | 6 +- db/seeds.rb | 32 +++--- 11 files changed, 265 insertions(+), 155 deletions(-) create mode 100644 app/lib/parser.rb create mode 100644 app/lib/parser/environment_variables.rb create mode 100644 app/lib/parser/headers.rb create mode 100644 app/lib/parser/payload.rb create mode 100644 app/lib/request.rb create mode 100644 app/lib/request/deconstructor.rb create mode 100644 app/lib/request/handler.rb delete mode 100644 app/lib/syntax_parser.rb diff --git a/app/lib/parser.rb b/app/lib/parser.rb new file mode 100644 index 0000000..7b4a55e --- /dev/null +++ b/app/lib/parser.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +require_relative './exceptions/invalid_environment_variable' +require_relative './exceptions/invalid_payload_key' +require_relative './parser/**' + +module Parser +end diff --git a/app/lib/parser/environment_variables.rb b/app/lib/parser/environment_variables.rb new file mode 100644 index 0000000..e1ccd77 --- /dev/null +++ b/app/lib/parser/environment_variables.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Parser + module EnvironmentVariables + # TODO + def fetch_environment_variable(value) + key = value.split('.').last + environment_variable = environment_variables.where(key: key) + raise InvalidEnvironmentVariable unless environment_variable + + environment_variable.decrypt + end + end +end diff --git a/app/lib/parser/headers.rb b/app/lib/parser/headers.rb new file mode 100644 index 0000000..8d321ae --- /dev/null +++ b/app/lib/parser/headers.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +# This class parses user defined headers & payloads into the values we expect +# to send to the outbound service. +# +# Example: +# +# ```ruby +# custom_user_payload = { +# 'hello' => '$payload.top_level_key', +# 'environment_variable' => '$env.API_KEY', +# 'did_you' => '$payload.nested_key_1.nested_key_2.nested_key_3' +# } +# +# incoming_request_from_service_a = { +# 'top_level_key' => 'world', +# 'nested_key_1' => { +# 'nested_key_2' => { +# 'nested_key_3' => 'make it!' +# } +# } +# } +# +# bridge = Bridge.where(user_id: @current_user.id) +# +# syntax_parser = SyntaxParser.new(bridge) +# +# syntax_parser.parse_payload( +# incoming_request_from_service_a, +# custom_user_payload +# ) # => Hash(String, String) where $payload & $env are replaced with their respective values +# +# syntax_parser.parse_headers # => Array(Hash(String, String)) where $env is +# replaced with decrypted environment variable value +# ``` +module Parser + class Headers + include EnvironmentVariables + # Set headers + # + # @param [ActiveRecord::Relation(Header)] environment_variables + # @param [ActiveRecord::Relation(EnvironmentVariable)] environment_variables + def initialize(headers, environment_variables) + @headers = headers + @environment_variables = environment_variables + end + + # Parses the user's headers replacing any values containing `$env` + # with the decrypted environment variable value + # + # @return [Array(Hash(String, String))] + def parse_headers + parse_headers! + + outbound_headers + end + + private + + attr_reader :outbound_headers, # Parsed headers returned from `parse_headers` + :environment_variables, + :headers + + # Iterates through user defined payload and parse values containing `$env` + # Reinitializes & mutates `outbound_headers` + def parse_headers! + @outbound_headers = [] # Reset + headers.each do |header| + parsed_header = {} + parsed_header[header.key] = if header.value.include?('$env') + fetch_environment_variable(header.value) + else + header.value + end + + outbound_headers << parsed_header + end + end + + def fetch_environment_variable(value) + key = value.split('.').last + environment_variable = environment_variables.where(key: key) + raise InvalidEnvironmentVariable unless environment_variable + + environment_variable.decrypt + end + end +end diff --git a/app/lib/parser/payload.rb b/app/lib/parser/payload.rb new file mode 100644 index 0000000..610885b --- /dev/null +++ b/app/lib/parser/payload.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +# This class parses user defined headers & payloads into the values we expect +# to send to the outbound service. +# +# Example: +# +# ```ruby +# custom_user_payload = { +# 'hello' => '$payload.top_level_key', +# 'environment_variable' => '$env.API_KEY', +# 'did_you' => '$payload.nested_key_1.nested_key_2.nested_key_3' +# } +# +# incoming_request_from_service_a = { +# 'top_level_key' => 'world', +# 'nested_key_1' => { +# 'nested_key_2' => { +# 'nested_key_3' => 'make it!' +# } +# } +# } +# +# bridge = Bridge.where(user_id: @current_user.id) +# +# syntax_parser = SyntaxParser.new(bridge) +# +# syntax_parser.parse_payload( +# incoming_request_from_service_a, +# custom_user_payload +# ) # => Hash(String, String) where $payload & $env are replaced with their respective values +# +# syntax_parser.parse_headers # => Array(Hash(String, String)) where $env is +# replaced with decrypted environment variable value +# ``` +module Parser + class Payload + include EnvironmentVariables + # Set bridge headers & env's. + # + # @param [Hash(String, String)] incoming_request + # @param [Hash(String, String)] user_data + # @param [ActiveRecord::Relation(EnvironmentVariable)] environment_variables + def initialize(incoming_request, user_data, environment_variables) + @incoming_request = incoming_request + @user_data = user_data + @environment_variables = environment_variables + end + + # Parses the user's custom payload replacing any values containing `$env` + # or `$payload` with the respective value + # + # @return [Hash(String, String)] + def parse_payload + parse_payload! + + outbound_request + end + + private + + attr_reader :user_data, # User defined payload (bridge.data['payload']) + :outbound_request, # Parsed request returned from `parse_payload` + :incoming_request, # Inbound request from service A + :environment_variables + + # Iterates through user defined payload and parse values containing `$env` or `$payload` + # Reinitializes & mutates `outbound_request` + def parse_payload! + @outbound_request = {} # Reset + + user_data.each do |key, val| + outbound_request[key] = if val.include?('$env') + fetch_environment_variable(val) + elsif val.include?('$payload') + fetch_payload_data(val) + else + val + end + end + end + + def fetch_payload_data(value) + values = value.split('.') + data = incoming_request # Set data to the incoming request + + values.each_with_index do |val, idx| + next if idx.zero? # skip the $payload + + data = data[val] # dig deeper into the request on each iteration + raise InvalidPayloadKey if data.nil? + end + + data + end + end +end diff --git a/app/lib/request.rb b/app/lib/request.rb new file mode 100644 index 0000000..2477a10 --- /dev/null +++ b/app/lib/request.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +require_relative './request/**' + +module Request +end diff --git a/app/lib/request/deconstructor.rb b/app/lib/request/deconstructor.rb new file mode 100644 index 0000000..703d77a --- /dev/null +++ b/app/lib/request/deconstructor.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Request + # Deconstructs the request & response from us & service B. Replaces + # any headers that contain environment variables with placeholder values + # to prevent data leakage. + class Deconstructor + end +end diff --git a/app/lib/request/handler.rb b/app/lib/request/handler.rb new file mode 100644 index 0000000..9cb3483 --- /dev/null +++ b/app/lib/request/handler.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +module Request + class Handler + end +end diff --git a/app/lib/syntax_parser.rb b/app/lib/syntax_parser.rb deleted file mode 100644 index db2563a..0000000 --- a/app/lib/syntax_parser.rb +++ /dev/null @@ -1,136 +0,0 @@ -# frozen_string_literal: true - -require_relative './exceptions/invalid_environment_variable' -require_relative './exceptions/invalid_payload_key' - -# This class parses user defined headers & payloads into the values we expect -# to send to the outbound service. -# -# Example: -# -# ```ruby -# custom_user_payload = { -# 'hello' => '$payload.top_level_key', -# 'environment_variable' => '$env.API_KEY', -# 'did_you' => '$payload.nested_key_1.nested_key_2.nested_key_3' -# } -# -# incoming_request_from_service_a = { -# 'top_level_key' => 'world', -# 'nested_key_1' => { -# 'nested_key_2' => { -# 'nested_key_3' => 'make it!' -# } -# } -# } -# -# bridge = Bridge.where(user_id: @current_user.id) -# -# syntax_parser = SyntaxParser.new(bridge) -# -# syntax_parser.parse_payload( -# incoming_request_from_service_a, -# custom_user_payload -# ) # => Hash(String, String) where $payload & $env are replaced with their respective values -# -# syntax_parser.parse_headers # => Array(Hash(String, String)) where $env is -# replaced with decrypted environment variable value -# ``` -class SyntaxParser - # Set bridge headers & env's. - # - # @param [Bridge] bridge - def initialize(bridge) - @headers = bridge.headers - @environment_variables = bridge.environment_variables - end - - # Parses the user's custom payload replacing any values containing `$env` - # or `$payload` with the respective value - # - # @param [Hash(String, String)] incoming_request - # @param [Hash(String, String)] user_data - # - # @return [Hash(String, String)] - def parse_payload(incoming_request, user_data) - @incoming_request = incoming_request - @user_data = user_data - - parse_payload! - - outbound_request - end - - # Parses the user's headers replacing any values containing `$env` - # with the decrypted environment variable value - # - # @return [Array(Hash(String, String))] - def parse_headers - parse_headers! - - outbound_headers - end - - private - - attr_reader :user_data, # User defined payload (bridge.data['payload']) - :outbound_request, # Parsed request returned from `parse_payload` - :incoming_request, # Inbound request from service A - :outbound_headers, # Parsed headers returned from `parse_headers` - :environment_variables, - :headers - - # Iterates through user defined payload and parse values containing `$env` or `$payload` - # Reinitializes & mutates `outbound_request` - def parse_payload! - @outbound_request = {} # Reset - - user_data.each do |key, val| - outbound_request[key] = if val.include?('$env') - fetch_environment_variable(val) - elsif val.include?('$payload') - fetch_payload_data(val) - else - val - end - end - end - - # Iterates through user defined payload and parse values containing `$env` - # Reinitializes & mutates `outbound_headers` - def parse_headers! - @outbound_headers = [] # Reset - headers.each do |header| - parsed_header = {} - parsed_header[header.key] = if header.value.include?('$env') - fetch_environment_variable(header.value) - else - header.value - end - - outbound_headers << parsed_header - end - end - - def fetch_environment_variable(value) - key = value.split('.').last - environment_variable = environment_variables.where(key: key) - raise InvalidEnvironmentVariable unless environment_variable - - environment_variable.decrypt - end - - def fetch_payload_data(value) - values = value.split('.') - data = incoming_request # Set data to the incoming request - - values.each_with_index do |val, idx| - next if idx.zero? # skip the $payload - - data = data[val] # dig deeper into the request on each iteration - raise InvalidPayloadKey if data.nil? - end - - data - end -end diff --git a/app/workers/event_worker.rb b/app/workers/event_worker.rb index ffe3340..27abf09 100644 --- a/app/workers/event_worker.rb +++ b/app/workers/event_worker.rb @@ -11,18 +11,26 @@ class EventWorker SCHEME = 'http://' + # rubocop:disable Metrics/MethodLength + # rubocop:disable Lint/RescueException def perform(event_id, retries = 0) - event = Event.find(event_id) - bridge = Bridge.find(event.bridge_id) + event = Event.find(event_id).includes(:bridge) + bridge = event.bridge + execute_request_response_cycle(event, bridge) complete_event(event) - rescue StandardError => e - binding.pry + rescue Exception => e save_http_error(event, e) unless e.instance_of?(Sidekiq::LargeStatusCode) - complete_event(event) && return if retries >= bridge.retries + if retries >= bridge.retries.to_i + complete_event(event) + return + end + # TODO EventWorker.perform_in(bridge.delay.minutes, event_id, retries + 1) end + # rubocop:enable Lint/RescueException + # rubocop:enable Metrics/MethodLength private diff --git a/config/database.yml b/config/database.yml index 1cab17a..f5da28b 100644 --- a/config/database.yml +++ b/config/database.yml @@ -24,6 +24,8 @@ default: &default development: <<: *default database: bridgeapi_rb_development + username: andrew + password: # The specified database role being used to connect to postgres. # To create additional roles in postgres see `$ createuser --help`. @@ -58,8 +60,8 @@ development: test: <<: *default database: bridgeapi_rb_test - username: ruby - password: ruby + username: andrew + password: host: localhost port: 5432 diff --git a/db/seeds.rb b/db/seeds.rb index f2a9189..f71bfa9 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -1,5 +1,7 @@ +# frozen_string_literal: true + def test_url - 'doggoapi.io/' + String(rand).split('.')[1] + "doggoapi.io/#{String(rand).split('.')[1]}" end user = User.create(email: 'admin@bridge.io', password: 'password', notifications: false) @@ -7,13 +9,16 @@ def test_url bridge = Bridge.create( user_id: user.id, - title: 'My First Bridge', - inbound_url: 'bridgeapi.com/249634', + title: 'My First Bridge', + inbound_url: 'bridgeapi.com/249634', outbound_url: 'ip.jsontest.com', - method: 'POST', - retries: 5, + method: 'POST', + retries: 5, delay: 15, - data: { payload: "{\"FirstName\":\"Lee\",\"LastName\":\"Oswald\",\"UserName\":\"GrassyKnoll\",\"Password\":{\"nested\":\"magic bullet\"},\"Email\":\"kgb63@yandex.ru\"}", test_payload: "{\"test_key_one\":\"testerstring\",\"test_key_two\":[\"stringinarray\"]}" } + data: { + payload: '{"FirstName":"Lee","LastName":"Oswald","UserName":"GrassyKnoll","Password":{"nested":"magic bullet"},"Email":"kgb63@yandex.ru"}', + test_payload: '{"test_key_one":"testerstring","test_key_two":["stringinarray"]}' + } ) bridge.environment_variables << EnvironmentVariable.create(key: 'database', value: 'a102345ij2') @@ -23,13 +28,16 @@ def test_url bridge2 = Bridge.create( user_id: user2.id, - title: 'My Second Bridge', - inbound_url: 'bridgeapi.com/746353', - outbound_url: test_url, - method: 'PATCH', - retries: 3, + title: 'My Second Bridge', + inbound_url: 'bridgeapi.com/746353', + outbound_url: test_url, + method: 'PATCH', + retries: 3, delay: 0, - data: { payload: "{\"FirstName\":\"Booths\",\"LastName\":\"John\",\"UserName\":\"FordTheatre\",\"Password\":{\"nested\":\"sic temper tyrannis\"},\"Email\":\"mail@mail.com\"}", test_payload: "{\"test_key_one\":{\"nested\":11},\"test_key_two\":888}" } + data: { + payload: '{"FirstName":"Booths","LastName":"John","UserName":"FordTheatre","Password":{"nested":"sic temper tyrannis"},"Email":"mail@mail.com"}', + test_payload: '{"test_key_one":{"nested":11},"test_key_two":888}' + } ) bridge2.environment_variables << EnvironmentVariable.create(key: 'database', value: 'z9992374623') From f694933c88534d34ada88d7d9ab77049732a8d3c Mon Sep 17 00:00:00 2001 From: = Date: Fri, 20 Nov 2020 14:04:11 -0500 Subject: [PATCH 34/64] Broken EventWorker into smaller classes --- app/controllers/events_controller.rb | 7 +- app/lib/bridge_api.rb | 4 + .../handler.rb => bridge_api/http.rb} | 4 +- app/lib/bridge_api/http/builder.rb | 125 +++++++++++++++++ app/lib/bridge_api/http/deconstructor.rb | 11 ++ app/lib/bridge_api/http/formatter.rb | 106 ++++++++++++++ app/lib/bridge_api/http/handler.rb | 49 +++++++ app/lib/bridge_api/http/interfaces.rb | 8 ++ app/lib/bridge_api/http/interfaces/builder.rb | 18 +++ .../bridge_api/http/interfaces/formatter.rb | 15 ++ app/lib/request.rb | 6 - app/lib/request/deconstructor.rb | 9 -- app/lib/request_deconstructor.rb | 11 -- app/lib/request_handler.rb | 11 -- app/models/event.rb | 7 + app/workers/event_worker.rb | 131 ++---------------- 16 files changed, 358 insertions(+), 164 deletions(-) create mode 100644 app/lib/bridge_api.rb rename app/lib/{request/handler.rb => bridge_api/http.rb} (56%) create mode 100644 app/lib/bridge_api/http/builder.rb create mode 100644 app/lib/bridge_api/http/deconstructor.rb create mode 100644 app/lib/bridge_api/http/formatter.rb create mode 100644 app/lib/bridge_api/http/handler.rb create mode 100644 app/lib/bridge_api/http/interfaces.rb create mode 100644 app/lib/bridge_api/http/interfaces/builder.rb create mode 100644 app/lib/bridge_api/http/interfaces/formatter.rb delete mode 100644 app/lib/request.rb delete mode 100644 app/lib/request/deconstructor.rb delete mode 100644 app/lib/request_deconstructor.rb delete mode 100644 app/lib/request_handler.rb diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index cef125b..e64a9bb 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -6,7 +6,7 @@ class EventsController < ApplicationController before_action :set_event, only: %i[show destroy] def index - render json: @events, status: 200 + render json: { events: @events }, status: 200 rescue ActiveRecord::RecordInvalid render json: { error: 'neither event_id nor bridge_id were valid' }, status: 400 # Bad Request rescue ActiveRecord::RecordNotFound @@ -28,7 +28,8 @@ def destroy def create event = create_event_object(create_data_object, find_bridge) event.save! - EventWorker.perform_async(event.id) + # EventWorker.perform_async(event.id) + EventWorker.new.perform(event.id) render json: {}, status: 202 # Accepted (asynchronous processing) rescue ActiveRecord::RecordNotFound render json: { error: 'a bridge by that id was not found' }, status: 400 @@ -41,7 +42,7 @@ def create private def event_params - params.permit(:id, :bridge_id, :event_id, :test) + params.require(:event).permit(:id, :bridge_id, :event_id, :test) end def create_data_object diff --git a/app/lib/bridge_api.rb b/app/lib/bridge_api.rb new file mode 100644 index 0000000..5a2f520 --- /dev/null +++ b/app/lib/bridge_api.rb @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +module BridgeApi +end diff --git a/app/lib/request/handler.rb b/app/lib/bridge_api/http.rb similarity index 56% rename from app/lib/request/handler.rb rename to app/lib/bridge_api/http.rb index 9cb3483..718d919 100644 --- a/app/lib/request/handler.rb +++ b/app/lib/bridge_api/http.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Request - class Handler +module BridgeApi + module Http end end diff --git a/app/lib/bridge_api/http/builder.rb b/app/lib/bridge_api/http/builder.rb new file mode 100644 index 0000000..8ae661e --- /dev/null +++ b/app/lib/bridge_api/http/builder.rb @@ -0,0 +1,125 @@ +# frozen_string_literal: true + +module BridgeApi + module Http + # TODO: Parser + + # Handles building a HTTP Request object + class Builder + SCHEME = 'http://' + + include Interfaces::Builder + + # @param [Bridge] bridge - The bridge that event belongs to + def initialize(bridge, test_env) + @bridge = bridge + @test_env = test_env + end + + # Generate & return `Net::HTTP`(http) & `Net::HTTP::{http_method}`(request) objects + # + # @return [Tuple(Net::HTTP, Net::HTTP::{http_method})] + def generate + [net_http, http_request] + end + + # TODO + # + # @return [Hash(String, String)] + # def payload + # parsed_payload + # end + + private + + attr_reader :request, + :bridge + + def net_http + http = Net::HTTP.new(uri.host, uri.port) + # http.use_ssl = true # TODO: (uri.scheme == 'https') + puts 'use ssl' + http + end + + def http_request + @request = net_http_request(uri) + generate_headers + request.body = parsed_payload.to_json + request + end + + # Convert user defined outbound_url into a valid URI + # + # @return [String] + def uri + @uri ||= URI(scheme) + end + + # Sets the user defined headers into the `request` object + def generate_headers + headers.each { |header| request[header['key']] = header['value'] } + end + + # Parse our custom syntax into the expected values + # + # @return [Hash(String, String)] + def parsed_payload + # TODO: Parser + @parsed_payload ||= unparsed_payload # payload_parser.parse unparsed_payload + end + + # Returns either payload or test_payload depending + # on the event environment + # + # @return [JSON] + def unparsed_payload + @unparsed_payload ||= JSON.parse data[test_env? ? 'test_payload' : 'payload'] + end + + # Ensures the scheme used is using TSL + # + # @return [String] + def scheme + return outbound_url if outbound_url.starts_with?('https') + + "#{SCHEME}#{outbound_url.split('//').last}" + end + + # Create a HTTP request object based on the user defined + # HTTP method. + # + # @return [Net::HTTP::{http_method}] + def net_http_request(uri) + "Net::HTTP::#{method}".constantize.new(uri, 'Content-Type' => 'application/json') + end + + # @return [String] + def method + bridge.method.capitalize + end + + # @return [Hash(String, JSON)] + def data + bridge.data + end + + # @return [String] + def outbound_url + bridge.outbound_url + end + + # @return [ActiveRecord::Relation(Header)] + def headers + bridge.headers + end + + # Checks if this request is a test event + # + # @return [Bool] + def test_env? + @test_env + end + end + end +end diff --git a/app/lib/bridge_api/http/deconstructor.rb b/app/lib/bridge_api/http/deconstructor.rb new file mode 100644 index 0000000..72220fb --- /dev/null +++ b/app/lib/bridge_api/http/deconstructor.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module BridgeApi + module Http + # Deconstructs the request & response from us & service B. Replaces + # any headers that contain environment variables with placeholder values + # to prevent data leakage. + class Deconstructor + end + end +end diff --git a/app/lib/bridge_api/http/formatter.rb b/app/lib/bridge_api/http/formatter.rb new file mode 100644 index 0000000..cd6dcca --- /dev/null +++ b/app/lib/bridge_api/http/formatter.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +module BridgeApi + module Http + # Handles formatting requests & responses + class Formatter + include Interfaces::Formatter + + def initialize; end + + # Mutate the event object by storing a formatted request inside + # data. + # + # @param [Event] event + # @param [Net::HTTP] req + # @param [Net::HTTP::Response] res + def format!(event, req, res) + data = JSON.parse(event.data) + + data['outbound'].push({ + 'request' => formatted_request(req), + 'response' => formatted_response(res) + }) + event.data = data.to_json + end + + # Mutate the event object by storing a formatted request inside + # data. + # + # @param [Event] event + # @param [Net::HTTP] req + # @param [Net::HTTP::Response] res + def format_error!(event, req, error) + data = JSON.parse(event.data) + + data['outbound'].push({ + 'request' => formatted_request(req), + 'response' => formatted_error(error) + }) + event.data = data.to_json + end + + # Mutate the event object by storing a formatted request inside + # data. + # + # @param [Event] event + # @param [Net::HTTP] req + # def format_request!(event, req) + # data = JSON.parse(event.data) + + # data['outbound'].push({ 'request' => formatted_request(req), 'response' => {} }) + # event.data = data.to_json + # end + + # Mutate the event object by storing a formatted request inside + # data. + # + # @param [Event] event + # @param [Net::HTTP::Response] res + # def format_response!(event, res) + # data = JSON.parse(event.data) + + # data['outbound'].last['response'] = formatted_response res + # event.data = data.to_json + # end + + # Mutate the event object by storing a formatted error message inside + # data. + # + # @param [Event] event + # @param [Net::HTTP::TODO] error + # def format_error!(event, error) + # event_data = JSON.parse(event.data) + + # event_data['outbound'].last['response'] = formatted_error error + # event.data = event_data.to_json + # # event.save + # end + + private + + def formatted_response(response) + { + dateTime: DateTime.now.utc, + statusCode: response.code, + message: response.message, + size: response.size, + payload: response.code.to_i >= 300 ? {} : JSON.parse(response.body) + } + end + + def formatted_request(request) + { + payload: request.body, + dateTime: DateTime.now.utc, + contentLength: request.length, + uri: request.uri.to_s + } + end + + def formatted_error(error) + { message: error.message } + end + end + end +end diff --git a/app/lib/bridge_api/http/handler.rb b/app/lib/bridge_api/http/handler.rb new file mode 100644 index 0000000..4cf238b --- /dev/null +++ b/app/lib/bridge_api/http/handler.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module BridgeApi + module Http + class Handler + # Inject a different builder or formatter + # @param [BridgeApi::Http::Interfaces::Builder] http_builder + attr_writer :http_builder, + # @param [BridgeApi::Http::Interfaces::Formatter] formatter + :formatter + + # @param [Integer] event_id + def initialize(event) + @event = event + @bridge = @event.bridge + end + + def execute + http, @request = http_builder.generate + response = http.request request + # formatter.format! event, request, response + formatter.format_request! event, request + formatter.format_response! event, response + + raise Sidekiq::LargeStatusCode if response.code.to_i >= 300 + end + + def cleanup(error) + formatter.format_error! event, request, error + end + + private + + attr_accessor :event, + :bridge, + :request + + # @return [BridgeApi::Http::Interfaces::Builder] + def http_builder + @http_builder ||= Builder.new bridge, event.test + end + + # @return [BridgeApi::Http::Interfaces::Formatter] + def formatter + @formatter ||= Formatter.new + end + end + end +end diff --git a/app/lib/bridge_api/http/interfaces.rb b/app/lib/bridge_api/http/interfaces.rb new file mode 100644 index 0000000..f0e8962 --- /dev/null +++ b/app/lib/bridge_api/http/interfaces.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module BridgeApi + module Http + module Interfaces + end + end +end diff --git a/app/lib/bridge_api/http/interfaces/builder.rb b/app/lib/bridge_api/http/interfaces/builder.rb new file mode 100644 index 0000000..d146ecf --- /dev/null +++ b/app/lib/bridge_api/http/interfaces/builder.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module BridgeApi + module Http + module Interfaces + # Abstract "class" + module Builder + def generate + raise NotImplementedError, 'A request builder class must implement #generate' + end + + # def payload + # raise NotImplementedError, 'A request builder class must implement #payload' + # end + end + end + end +end diff --git a/app/lib/bridge_api/http/interfaces/formatter.rb b/app/lib/bridge_api/http/interfaces/formatter.rb new file mode 100644 index 0000000..f5be858 --- /dev/null +++ b/app/lib/bridge_api/http/interfaces/formatter.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BridgeApi + module Http + module Interfaces + # Abstract "class" + module Formatter + # TODO + # def generate + # raise NotImplementedError, 'A request builder class must implement #generate' + # end + end + end + end +end diff --git a/app/lib/request.rb b/app/lib/request.rb deleted file mode 100644 index 2477a10..0000000 --- a/app/lib/request.rb +++ /dev/null @@ -1,6 +0,0 @@ -# frozen_string_literal: true - -require_relative './request/**' - -module Request -end diff --git a/app/lib/request/deconstructor.rb b/app/lib/request/deconstructor.rb deleted file mode 100644 index 703d77a..0000000 --- a/app/lib/request/deconstructor.rb +++ /dev/null @@ -1,9 +0,0 @@ -# frozen_string_literal: true - -module Request - # Deconstructs the request & response from us & service B. Replaces - # any headers that contain environment variables with placeholder values - # to prevent data leakage. - class Deconstructor - end -end diff --git a/app/lib/request_deconstructor.rb b/app/lib/request_deconstructor.rb deleted file mode 100644 index 1dc3304..0000000 --- a/app/lib/request_deconstructor.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -require_relative './exceptions' - -# TODO: Build -# -# Deconstructs the request & response from us & service B. Replaces -# any headers that contain environment variables with placeholder values -# to prevent data leakage. -class RequestDeconstructor -end diff --git a/app/lib/request_handler.rb b/app/lib/request_handler.rb deleted file mode 100644 index 6a1bed5..0000000 --- a/app/lib/request_handler.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -require_relative './exceptions' - -# TODO: Build or delete -# -# Thinking some EventWorker functionality will be moved into here. -# SyntaxParser & RequestDeconstructor would be attributes and aid -# in setting up & properly saving requests/responses. -class RequestHandler -end diff --git a/app/models/event.rb b/app/models/event.rb index c1bd581..5a941d4 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -12,8 +12,15 @@ class Event < ApplicationRecord belongs_to :bridge + def complete! + self.completed = true + self.completed_at = Time.now.utc + save! # TODO: With a bang? + end + private + # TODO: Pass in bridge (or urls) to prevent db hit def set_urls self.inbound_url = bridge.inbound_url self.outbound_url = bridge.outbound_url diff --git a/app/workers/event_worker.rb b/app/workers/event_worker.rb index 27abf09..26ea2fc 100644 --- a/app/workers/event_worker.rb +++ b/app/workers/event_worker.rb @@ -5,131 +5,18 @@ class EventWorker include Sidekiq::Worker - # attr_accessor :retry_count + attr_accessor :retry_count - # sidekiq_retry_in { 5 } # TODO: SET RETRY_IN DYNAMICALLY - - SCHEME = 'http://' - - # rubocop:disable Metrics/MethodLength - # rubocop:disable Lint/RescueException def perform(event_id, retries = 0) - event = Event.find(event_id).includes(:bridge) - bridge = event.bridge - - execute_request_response_cycle(event, bridge) - complete_event(event) - rescue Exception => e - save_http_error(event, e) unless e.instance_of?(Sidekiq::LargeStatusCode) - if retries >= bridge.retries.to_i - complete_event(event) - return - end - - # TODO - EventWorker.perform_in(bridge.delay.minutes, event_id, retries + 1) - end - # rubocop:enable Lint/RescueException - # rubocop:enable Metrics/MethodLength - - private - - def set_headers(req, bridge) - bridge.headers.each { |header| req[header['key']] = header['value'] } - end - - def complete_event(event) - event.completed = true - event.completed_at = Time.now.utc - event.save! - end - - def create_error_response(error) - { - message: error.message - } - end - - def save_http_error(event, error) - response = create_error_response(error) - event_data = JSON.parse(event.data) - - event_data['outbound'].last['response'] = response - event.data = event_data.to_json - event.save - end - - def create_response_object(payload, response) - { - dateTime: DateTime.now.utc, - statusCode: response.code, - message: response.message, - size: response.size, - payload: payload - } - end - - def save_response(event, resp) - resp_code = resp.code.to_i - payload = (resp_code >= 300 ? {} : JSON.parse(resp.body)) - response = create_response_object(payload, resp) - event_data = JSON.parse(event.data) - - event_data['outbound'].last['response'] = response - event.data = event_data.to_json - event.save - raise Sidekiq::LargeStatusCode if resp_code >= 300 - end - - def create_request_object(payload, length) - { - payload: payload, - dateTime: DateTime.now.utc, - contentLength: length - } - end - - def save_request(event, length, payload) - request = create_request_object(payload, length) - event_data = JSON.parse(event.data) - - event_data['outbound'].push({ 'request' => request, 'response' => {} }) - event.data = event_data.to_json - event.save - end - - def prepend_scheme(uri) - return uri if uri.starts_with?('https') - - minus_scheme = uri.split('//').last - "#{SCHEME}#{minus_scheme}" - end + event = Event.includes(:bridge).find(event_id) + request_handler = ::BridgeApi::Http::Handler.new(event) + request_handler.execute + event.complete! + rescue StandardError => e + request_handler.cleanup(e) unless e.instance_of? Sidekiq::LargeStatusCode - def generate_http_request(bridge, payload) - method = bridge.method.capitalize - uri = URI(prepend_scheme(bridge.outbound_url)) - http = Net::HTTP.new(uri.host, uri.port) - req = "Net::HTTP::#{method}".constantize.new(uri, 'Content-Type' => 'application/json') - - http.use_ssl = (uri.scheme == 'https') - set_headers(req, bridge) - req.body = payload.to_json - [http, req] - end - - def extract_payload(bridge, event) - if event.test - JSON.parse(bridge.data['test_payload']) - else - JSON.parse(bridge.data['payload']) - end - end + return event.complete! if retries >= event.bridge.retries.to_i - def execute_request_response_cycle(event, bridge) - payload = extract_payload(bridge, event) - http, req = generate_http_request(bridge, payload) - save_request(event, req.length, payload) - response = http.request(req) - save_response(event, response) + raise StandardError end end From 583cbdffe9d532b16d68494dc08d0815fc90946c Mon Sep 17 00:00:00 2001 From: = Date: Fri, 20 Nov 2020 16:09:02 -0500 Subject: [PATCH 35/64] Deconstructor is working --- app/controllers/events_controller.rb | 1 + app/lib/bridge_api/http/builder.rb | 21 ++--- app/lib/bridge_api/http/deconstructor.rb | 36 +++++++- app/lib/bridge_api/http/formatter.rb | 82 ++++++++----------- app/lib/bridge_api/http/handler.rb | 21 +++-- .../http/interfaces/deconstructor.rb | 14 ++++ .../bridge_api/http/interfaces/formatter.rb | 11 ++- app/models/event.rb | 1 + app/models/header.rb | 11 +++ db/seeds.rb | 8 +- 10 files changed, 128 insertions(+), 78 deletions(-) create mode 100644 app/lib/bridge_api/http/interfaces/deconstructor.rb diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index e64a9bb..7b93d00 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -29,6 +29,7 @@ def create event = create_event_object(create_data_object, find_bridge) event.save! # EventWorker.perform_async(event.id) + # TODO: Remove EventWorker.new.perform(event.id) render json: {}, status: 202 # Accepted (asynchronous processing) rescue ActiveRecord::RecordNotFound diff --git a/app/lib/bridge_api/http/builder.rb b/app/lib/bridge_api/http/builder.rb index 8ae661e..9c73ee7 100644 --- a/app/lib/bridge_api/http/builder.rb +++ b/app/lib/bridge_api/http/builder.rb @@ -6,7 +6,7 @@ module Http # Handles building a HTTP Request object class Builder - SCHEME = 'http://' + SCHEME = 'https://' include Interfaces::Builder @@ -16,20 +16,13 @@ def initialize(bridge, test_env) @test_env = test_env end - # Generate & return `Net::HTTP`(http) & `Net::HTTP::{http_method}`(request) objects + # Generate & return `Net::HTTP`(net_http) & `Net::HTTP::{http_method}`(http_request) objects # # @return [Tuple(Net::HTTP, Net::HTTP::{http_method})] def generate [net_http, http_request] end - # TODO - # - # @return [Hash(String, String)] - # def payload - # parsed_payload - # end - private attr_reader :request, @@ -37,8 +30,7 @@ def generate def net_http http = Net::HTTP.new(uri.host, uri.port) - # http.use_ssl = true # TODO: (uri.scheme == 'https') - puts 'use ssl' + http.use_ssl = true http end @@ -58,6 +50,7 @@ def uri # Sets the user defined headers into the `request` object def generate_headers + # TODO: Parser headers.each { |header| request[header['key']] = header['value'] } end @@ -66,7 +59,7 @@ def generate_headers # @return [Hash(String, String)] def parsed_payload # TODO: Parser - @parsed_payload ||= unparsed_payload # payload_parser.parse unparsed_payload + @parsed_payload ||= JSON.parse unparsed_payload # payload_parser.parse unparsed_payload end # Returns either payload or test_payload depending @@ -74,7 +67,7 @@ def parsed_payload # # @return [JSON] def unparsed_payload - @unparsed_payload ||= JSON.parse data[test_env? ? 'test_payload' : 'payload'] + @unparsed_payload ||= data[test_env? ? 'test_payload' : 'payload'] end # Ensures the scheme used is using TSL @@ -111,7 +104,7 @@ def outbound_url # @return [ActiveRecord::Relation(Header)] def headers - bridge.headers + @headers ||= bridge.headers end # Checks if this request is a test event diff --git a/app/lib/bridge_api/http/deconstructor.rb b/app/lib/bridge_api/http/deconstructor.rb index 72220fb..6722e59 100644 --- a/app/lib/bridge_api/http/deconstructor.rb +++ b/app/lib/bridge_api/http/deconstructor.rb @@ -2,10 +2,40 @@ module BridgeApi module Http - # Deconstructs the request & response from us & service B. Replaces - # any headers that contain environment variables with placeholder values - # to prevent data leakage. + # Deconstructs the request headers. Replaces + # any headers that contain environment variables with + # placeholder values to prevent data leakage. class Deconstructor + include Interfaces::Deconstructor + + # @param [ActiveRecord::Relation(Header)] user_headers + def initialize(headers) + @headers = headers + @header_keys = headers.pluck(:key) + end + + # Prevents sensitive headers from being stored in cleared text. + # If the header value can be found & the value contains `$env` + # we return a safe value otherwise the real value is returned. + # + # @param [String] key + # @param [String] value + # + # @return [String] + def deconstruct(key, value) + # Don't waste time on querying unless we know the header exists + return value unless header_keys.include?(key) + + if headers.find_by(key: key).value.include?('$env') + 'FILTERED' + else + value + end + end + + private + + attr_reader :headers, :header_keys end end end diff --git a/app/lib/bridge_api/http/formatter.rb b/app/lib/bridge_api/http/formatter.rb index cd6dcca..ab60932 100644 --- a/app/lib/bridge_api/http/formatter.rb +++ b/app/lib/bridge_api/http/formatter.rb @@ -2,18 +2,22 @@ module BridgeApi module Http - # Handles formatting requests & responses + # Handles formatting requests & responses. Accepts a deconstructor + # to aid in formatting. class Formatter include Interfaces::Formatter - def initialize; end + # @param [BridgeApi::Http::Interfaces::Deconstructor] deconstructor + def initialize(deconstructor) + @deconstructor = deconstructor + end - # Mutate the event object by storing a formatted request inside + # Mutates the event object by storing a formatted request inside # data. # # @param [Event] event # @param [Net::HTTP] req - # @param [Net::HTTP::Response] res + # @param [Net::HTTPResponse] res def format!(event, req, res) data = JSON.parse(event.data) @@ -24,12 +28,12 @@ def format!(event, req, res) event.data = data.to_json end - # Mutate the event object by storing a formatted request inside + # Mutates the event object by storing a formatted request inside # data. # # @param [Event] event # @param [Net::HTTP] req - # @param [Net::HTTP::Response] res + # @param [StandardError] error def format_error!(event, req, error) data = JSON.parse(event.data) @@ -40,45 +44,11 @@ def format_error!(event, req, error) event.data = data.to_json end - # Mutate the event object by storing a formatted request inside - # data. - # - # @param [Event] event - # @param [Net::HTTP] req - # def format_request!(event, req) - # data = JSON.parse(event.data) - - # data['outbound'].push({ 'request' => formatted_request(req), 'response' => {} }) - # event.data = data.to_json - # end - - # Mutate the event object by storing a formatted request inside - # data. - # - # @param [Event] event - # @param [Net::HTTP::Response] res - # def format_response!(event, res) - # data = JSON.parse(event.data) - - # data['outbound'].last['response'] = formatted_response res - # event.data = data.to_json - # end - - # Mutate the event object by storing a formatted error message inside - # data. - # - # @param [Event] event - # @param [Net::HTTP::TODO] error - # def format_error!(event, error) - # event_data = JSON.parse(event.data) - - # event_data['outbound'].last['response'] = formatted_error error - # event.data = event_data.to_json - # # event.save - # end - private + attr_reader :deconstructor + + # @param [Net::HTTPResponse] res def formatted_response(response) { dateTime: DateTime.now.utc, @@ -89,18 +59,38 @@ def formatted_response(response) } end + # @param [Net::HTTP] req def formatted_request(request) { - payload: request.body, + payload: JSON.parse(request.body), dateTime: DateTime.now.utc, - contentLength: request.length, - uri: request.uri.to_s + contentLength: request['content-length'], + uri: request.uri.to_s, + headers: safe_headers(request) } end + # @param [StandardError] error def formatted_error(error) { message: error.message } end + + # Handles creating an Array of all request headers + # + # @param [Net::HTTP] req + # + # @return [Array(Hash(String, String))] + def safe_headers(req) + headers = [] + req.each_header do |key, value| + headers << { + key: key, + value: deconstructor.deconstruct(key, value) + } + end + + headers + end end end end diff --git a/app/lib/bridge_api/http/handler.rb b/app/lib/bridge_api/http/handler.rb index 4cf238b..f0465b5 100644 --- a/app/lib/bridge_api/http/handler.rb +++ b/app/lib/bridge_api/http/handler.rb @@ -4,27 +4,29 @@ module BridgeApi module Http class Handler # Inject a different builder or formatter + # # @param [BridgeApi::Http::Interfaces::Builder] http_builder attr_writer :http_builder, # @param [BridgeApi::Http::Interfaces::Formatter] formatter :formatter - # @param [Integer] event_id + # @param [Event] event def initialize(event) @event = event @bridge = @event.bridge end + # Handles building and sending a HTTP request and then + # formats and stores the data into event object. def execute - http, @request = http_builder.generate - response = http.request request - # formatter.format! event, request, response - formatter.format_request! event, request - formatter.format_response! event, response + net_http, @request = http_builder.generate + response = net_http.request request + formatter.format! event, request, response raise Sidekiq::LargeStatusCode if response.code.to_i >= 300 end + # Handles storing the request & error into the event data def cleanup(error) formatter.format_error! event, request, error end @@ -42,7 +44,12 @@ def http_builder # @return [BridgeApi::Http::Interfaces::Formatter] def formatter - @formatter ||= Formatter.new + @formatter ||= Formatter.new deconstructor + end + + # @return [BridgeApi::Http::Interfaces::Deconstructor] + def deconstructor + @deconstructor ||= Deconstructor.new bridge.headers end end end diff --git a/app/lib/bridge_api/http/interfaces/deconstructor.rb b/app/lib/bridge_api/http/interfaces/deconstructor.rb new file mode 100644 index 0000000..149b7a3 --- /dev/null +++ b/app/lib/bridge_api/http/interfaces/deconstructor.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module BridgeApi + module Http + module Interfaces + # Abstract "class" + module Deconstructor + def deconstruct(_key, _value) + raise NotImplementedError, 'A request builder class must implement #deconstruct(key, value)' + end + end + end + end +end diff --git a/app/lib/bridge_api/http/interfaces/formatter.rb b/app/lib/bridge_api/http/interfaces/formatter.rb index f5be858..d6201e5 100644 --- a/app/lib/bridge_api/http/interfaces/formatter.rb +++ b/app/lib/bridge_api/http/interfaces/formatter.rb @@ -5,10 +5,13 @@ module Http module Interfaces # Abstract "class" module Formatter - # TODO - # def generate - # raise NotImplementedError, 'A request builder class must implement #generate' - # end + def format!(_event, _req, _res) + raise NotImplementedError, 'A request builder class must implement #format!(event, req, res)' + end + + def format_error!(_event, _req, _error) + raise NotImplementedError, 'A request builder class must implement #format_error!(event, req, error)' + end end end end diff --git a/app/models/event.rb b/app/models/event.rb index 5a941d4..d162268 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -12,6 +12,7 @@ class Event < ApplicationRecord belongs_to :bridge + # Marks an event as complete & saves def complete! self.completed = true self.completed_at = Time.now.utc diff --git a/app/models/header.rb b/app/models/header.rb index 882b59f..482f569 100644 --- a/app/models/header.rb +++ b/app/models/header.rb @@ -5,4 +5,15 @@ class Header < ApplicationRecord validates :value, presence: true belongs_to :bridge + + before_validation :downcase_key + + private + + # Downcase all keys since requests do that anyway. + # With a non-normalized case, we can't match headers + # when deconstructing requests to save. + def downcase_key + key.downcase! + end end diff --git a/db/seeds.rb b/db/seeds.rb index f71bfa9..062d70e 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -11,7 +11,7 @@ def test_url user_id: user.id, title: 'My First Bridge', inbound_url: 'bridgeapi.com/249634', - outbound_url: 'ip.jsontest.com', + outbound_url: 'c41a7126-a18c-4af6-880e-6857771a35c8.mock.pstmn.io/success_event', method: 'POST', retries: 5, delay: 15, @@ -21,10 +21,10 @@ def test_url } ) -bridge.environment_variables << EnvironmentVariable.create(key: 'database', value: 'a102345ij2') +bridge.environment_variables << EnvironmentVariable.create(key: 'DATABASE_URL', value: 'a102345ij2') bridge.environment_variables << EnvironmentVariable.create(key: 'database_password', value: 'supersecretpasswordwow') -bridge.headers << Header.create(key: 'X_API_KEY', value: 'ooosecrets') -bridge.headers << Header.create(key: 'Authentication', value: 'Bearer &&&&&&&&&&&&&&&&') +bridge.headers << Header.create(key: 'SHOULD_BE_FILTERED', value: '$env.DATABASE_URL') +bridge.headers << Header.create(key: 'not_filtered', value: 'bridge api') bridge2 = Bridge.create( user_id: user2.id, From 94d90e3744b7ee2b1c9f86b1ccab1b807b8f5543 Mon Sep 17 00:00:00 2001 From: = Date: Fri, 20 Nov 2020 16:26:44 -0500 Subject: [PATCH 36/64] Remove downcasing headers, better deconstructing for non-downcased headers --- app/lib/bridge_api/http/deconstructor.rb | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/app/lib/bridge_api/http/deconstructor.rb b/app/lib/bridge_api/http/deconstructor.rb index 6722e59..549cf26 100644 --- a/app/lib/bridge_api/http/deconstructor.rb +++ b/app/lib/bridge_api/http/deconstructor.rb @@ -10,8 +10,10 @@ class Deconstructor # @param [ActiveRecord::Relation(Header)] user_headers def initialize(headers) - @headers = headers - @header_keys = headers.pluck(:key) + # Remove any headers that don't contain `$env` as they + # don't need to be filtered. Keys are downcased because + # `request.each_header` returns headers that are downcased. + @header_keys = headers.where('value like ?', '$env%').pluck(:key).map(&:downcase) end # Prevents sensitive headers from being stored in cleared text. @@ -23,14 +25,7 @@ def initialize(headers) # # @return [String] def deconstruct(key, value) - # Don't waste time on querying unless we know the header exists - return value unless header_keys.include?(key) - - if headers.find_by(key: key).value.include?('$env') - 'FILTERED' - else - value - end + header_keys.include?(key) ? 'FILTERED' : value end private From be8ed1ed84484ab3319d913c605f79cf15f21d56 Mon Sep 17 00:00:00 2001 From: = Date: Fri, 20 Nov 2020 16:29:09 -0500 Subject: [PATCH 37/64] Support DI for Deconstructor --- app/lib/bridge_api/http/handler.rb | 6 ++++-- app/models/header.rb | 11 ----------- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/app/lib/bridge_api/http/handler.rb b/app/lib/bridge_api/http/handler.rb index f0465b5..19d46b3 100644 --- a/app/lib/bridge_api/http/handler.rb +++ b/app/lib/bridge_api/http/handler.rb @@ -3,12 +3,14 @@ module BridgeApi module Http class Handler - # Inject a different builder or formatter + # Inject a different builder, formatter or deconstructor # # @param [BridgeApi::Http::Interfaces::Builder] http_builder attr_writer :http_builder, # @param [BridgeApi::Http::Interfaces::Formatter] formatter - :formatter + :formatter, + # @param [BridgeApi::Http::Interfaces::Deconstructor] formatter + :deconstructor # @param [Event] event def initialize(event) diff --git a/app/models/header.rb b/app/models/header.rb index 482f569..882b59f 100644 --- a/app/models/header.rb +++ b/app/models/header.rb @@ -5,15 +5,4 @@ class Header < ApplicationRecord validates :value, presence: true belongs_to :bridge - - before_validation :downcase_key - - private - - # Downcase all keys since requests do that anyway. - # With a non-normalized case, we can't match headers - # when deconstructing requests to save. - def downcase_key - key.downcase! - end end From 7b6f2eabb034e97132c3a9512339239cbbac203f Mon Sep 17 00:00:00 2001 From: = Date: Fri, 20 Nov 2020 18:24:55 -0500 Subject: [PATCH 38/64] Integrating SyntaxParser into RequestHandler --- app/lib/bridge_api/http/builder.rb | 25 +++-- app/lib/bridge_api/http/handler.rb | 13 ++- app/lib/bridge_api/http/interfaces/builder.rb | 4 - app/lib/bridge_api/syntax_parser.rb | 6 ++ .../syntax_parser/environment_variables.rb | 16 +++ .../syntax_parser/headers_parser.rb | 81 ++++++++++++++ .../bridge_api/syntax_parser/interfaces.rb | 8 ++ .../interfaces/headers_parser.rb | 14 +++ .../interfaces/payload_parser.rb | 14 +++ .../syntax_parser/payload_parser.rb | 101 ++++++++++++++++++ app/lib/parser.rb | 8 -- app/lib/parser/environment_variables.rb | 14 --- app/lib/parser/headers.rb | 88 --------------- app/lib/parser/payload.rb | 97 ----------------- app/models/event.rb | 8 ++ app/workers/event_worker.rb | 10 +- 16 files changed, 282 insertions(+), 225 deletions(-) create mode 100644 app/lib/bridge_api/syntax_parser.rb create mode 100644 app/lib/bridge_api/syntax_parser/environment_variables.rb create mode 100644 app/lib/bridge_api/syntax_parser/headers_parser.rb create mode 100644 app/lib/bridge_api/syntax_parser/interfaces.rb create mode 100644 app/lib/bridge_api/syntax_parser/interfaces/headers_parser.rb create mode 100644 app/lib/bridge_api/syntax_parser/interfaces/payload_parser.rb create mode 100644 app/lib/bridge_api/syntax_parser/payload_parser.rb delete mode 100644 app/lib/parser.rb delete mode 100644 app/lib/parser/environment_variables.rb delete mode 100644 app/lib/parser/headers.rb delete mode 100644 app/lib/parser/payload.rb diff --git a/app/lib/bridge_api/http/builder.rb b/app/lib/bridge_api/http/builder.rb index 9c73ee7..81708dd 100644 --- a/app/lib/bridge_api/http/builder.rb +++ b/app/lib/bridge_api/http/builder.rb @@ -11,9 +11,11 @@ class Builder include Interfaces::Builder # @param [Bridge] bridge - The bridge that event belongs to - def initialize(bridge, test_env) + def initialize(bridge, test_env, payload_parser, headers_parser) @bridge = bridge @test_env = test_env + @payload_parser = payload_parser + @headers_parser = headers_parser end # Generate & return `Net::HTTP`(net_http) & `Net::HTTP::{http_method}`(http_request) objects @@ -26,18 +28,22 @@ def generate private attr_reader :request, - :bridge + :bridge, + :payload_parser, + :headers_parser def net_http http = Net::HTTP.new(uri.host, uri.port) http.use_ssl = true + http end def http_request @request = net_http_request(uri) - generate_headers + parse_headers request.body = parsed_payload.to_json + request end @@ -49,17 +55,20 @@ def uri end # Sets the user defined headers into the `request` object - def generate_headers - # TODO: Parser - headers.each { |header| request[header['key']] = header['value'] } + def parse_headers + headers_parser.parse(headers) do |key, value| + request[key] = value + end end # Parse our custom syntax into the expected values # # @return [Hash(String, String)] def parsed_payload - # TODO: Parser - @parsed_payload ||= JSON.parse unparsed_payload # payload_parser.parse unparsed_payload + @parsed_payload ||= payload_parser.parse( + event.inbound_payload, + JSON.parse(unparsed_payload) + ) end # Returns either payload or test_payload depending diff --git a/app/lib/bridge_api/http/handler.rb b/app/lib/bridge_api/http/handler.rb index 19d46b3..00bce7a 100644 --- a/app/lib/bridge_api/http/handler.rb +++ b/app/lib/bridge_api/http/handler.rb @@ -2,6 +2,7 @@ module BridgeApi module Http + # TODO class Handler # Inject a different builder, formatter or deconstructor # @@ -41,7 +42,7 @@ def cleanup(error) # @return [BridgeApi::Http::Interfaces::Builder] def http_builder - @http_builder ||= Builder.new bridge, event.test + @http_builder ||= Builder.new bridge, event.test, payload_parser, headers_parser end # @return [BridgeApi::Http::Interfaces::Formatter] @@ -53,6 +54,16 @@ def formatter def deconstructor @deconstructor ||= Deconstructor.new bridge.headers end + + # @return [BridgeApi::SyntaxParser::Interfaces::HeadersParser] + def headers_parser + @headers_parser ||= SyntaxParser::HeadersParser.new bridge.environment_variables + end + + # @return [BridgeApi::SyntaxParser::Interfaces::PayloadParser] + def payload_parser + @payload_parser ||= SyntaxParser::PayloadParser.new bridge.environment_variables + end end end end diff --git a/app/lib/bridge_api/http/interfaces/builder.rb b/app/lib/bridge_api/http/interfaces/builder.rb index d146ecf..4d820e7 100644 --- a/app/lib/bridge_api/http/interfaces/builder.rb +++ b/app/lib/bridge_api/http/interfaces/builder.rb @@ -8,10 +8,6 @@ module Builder def generate raise NotImplementedError, 'A request builder class must implement #generate' end - - # def payload - # raise NotImplementedError, 'A request builder class must implement #payload' - # end end end end diff --git a/app/lib/bridge_api/syntax_parser.rb b/app/lib/bridge_api/syntax_parser.rb new file mode 100644 index 0000000..32fc300 --- /dev/null +++ b/app/lib/bridge_api/syntax_parser.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +module BridgeApi + module SyntaxParser + end +end diff --git a/app/lib/bridge_api/syntax_parser/environment_variables.rb b/app/lib/bridge_api/syntax_parser/environment_variables.rb new file mode 100644 index 0000000..3c9cce8 --- /dev/null +++ b/app/lib/bridge_api/syntax_parser/environment_variables.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module BridgeApi + module SyntaxParser + module EnvironmentVariables + # TODO + def fetch_environment_variable(value) + key = value.split('.').last + environment_variable = environment_variables.find_by(key: key) + raise ::InvalidEnvironmentVariable unless environment_variable + + environment_variable.decrypt + end + end + end +end diff --git a/app/lib/bridge_api/syntax_parser/headers_parser.rb b/app/lib/bridge_api/syntax_parser/headers_parser.rb new file mode 100644 index 0000000..712234c --- /dev/null +++ b/app/lib/bridge_api/syntax_parser/headers_parser.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +# This class parses user defined headers & payloads into the values we expect +# to send to the outbound service. +# +# Example: +# +# ```ruby +# custom_user_payload = { +# 'hello' => '$payload.top_level_key', +# 'environment_variable' => '$env.API_KEY', +# 'did_you' => '$payload.nested_key_1.nested_key_2.nested_key_3' +# } +# +# incoming_request_from_service_a = { +# 'top_level_key' => 'world', +# 'nested_key_1' => { +# 'nested_key_2' => { +# 'nested_key_3' => 'make it!' +# } +# } +# } +# +# bridge = Bridge.where(user_id: @current_user.id) +# +# syntax_parser = SyntaxParser.new(bridge) +# +# syntax_parser.parse_payload( +# incoming_request_from_service_a, +# custom_user_payload +# ) # => Hash(String, String) where $payload & $env are replaced with their respective values +# +# syntax_parser.parse_headers # => Array(Hash(String, String)) where $env is +# replaced with decrypted environment variable value +# ``` +module BridgeApi + module SyntaxParser + # TODO: Doc + class HeadersParser + include EnvironmentVariables + include Interfaces::HeadersParser + + # @param [ActiveRecord::Relation(EnvironmentVariable)] environment_variables + def initialize(environment_variables) + @environment_variables = environment_variables + end + + # Handles parsing the user's headers replacing any values containing `$env` + # with the decrypted environment variable value. Expected to be invoked + # with a block. The block will be called on each iteration of a header passing + # in `header.key` & a parsed version of `header.value` as arguments. + # + # @param [ActiveRecord::Relation(Header)] environment_variables + # + # @return [Array(Hash(String, String))] + def parse(headers, &block) + @headers = headers + parse_headers!(block) + end + + private + + attr_reader :environment_variables, + :headers + + # TODO: Doc + def parse_headers!(block) + headers.each { |header| block.call(header.key, safe_value(header.value)) } + end + + # TODO: Doc + def safe_value(value) + if value.include?('$env') + fetch_environment_variable(value) + else + value + end + end + end + end +end diff --git a/app/lib/bridge_api/syntax_parser/interfaces.rb b/app/lib/bridge_api/syntax_parser/interfaces.rb new file mode 100644 index 0000000..c529498 --- /dev/null +++ b/app/lib/bridge_api/syntax_parser/interfaces.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module BridgeApi + module SyntaxParser + module Interfaces + end + end +end diff --git a/app/lib/bridge_api/syntax_parser/interfaces/headers_parser.rb b/app/lib/bridge_api/syntax_parser/interfaces/headers_parser.rb new file mode 100644 index 0000000..b622d40 --- /dev/null +++ b/app/lib/bridge_api/syntax_parser/interfaces/headers_parser.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module BridgeApi + module SyntaxParser + module Interfaces + # Abstract "class" + module HeadersParser + def parse + raise NotImplementedError, 'A header parser class must implement #parse(headers)' + end + end + end + end +end diff --git a/app/lib/bridge_api/syntax_parser/interfaces/payload_parser.rb b/app/lib/bridge_api/syntax_parser/interfaces/payload_parser.rb new file mode 100644 index 0000000..404aad0 --- /dev/null +++ b/app/lib/bridge_api/syntax_parser/interfaces/payload_parser.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module BridgeApi + module SyntaxParser + module Interfaces + # Abstract "class" + module PayloadParser + def parse + raise NotImplementedError, 'A payload parser class must implement #parse(incoming_request, user_data)' + end + end + end + end +end diff --git a/app/lib/bridge_api/syntax_parser/payload_parser.rb b/app/lib/bridge_api/syntax_parser/payload_parser.rb new file mode 100644 index 0000000..74f512c --- /dev/null +++ b/app/lib/bridge_api/syntax_parser/payload_parser.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +# This class parses user defined headers & payloads into the values we expect +# to send to the outbound service. +# +# Example: +# +# ```ruby +# custom_user_payload = { +# 'hello' => '$payload.top_level_key', +# 'environment_variable' => '$env.API_KEY', +# 'did_you' => '$payload.nested_key_1.nested_key_2.nested_key_3' +# } +# +# incoming_payload_from_service_a = { +# 'top_level_key' => 'world', +# 'nested_key_1' => { +# 'nested_key_2' => { +# 'nested_key_3' => 'make it!' +# } +# } +# } +# +# bridge = Bridge.where(user_id: @current_user.id) +# +# syntax_parser = SyntaxParser.new(bridge) +# +# syntax_parser.parse_payload( +# incoming_payload_from_service_a, +# custom_user_payload +# ) # => Hash(String, String) where $payload & $env are replaced with their respective values +# +# syntax_parser.parse_headers # => Array(Hash(String, String)) where $env is +# replaced with decrypted environment variable value +# ``` +module BridgeApi + module SyntaxParser + # TODO: Doc + class PayloadParser + include EnvironmentVariables + include Interfaces::PayloadParser + + # @param [ActiveRecord::Relation(EnvironmentVariable)] environment_variables + def initialize(environment_variables) + @environment_variables = environment_variables + end + + # Parses the user's custom payload replacing any values containing `$env` + # or `$payload` with the respective value + # + # @param [Hash(String, String)] incoming_payload + # @param [Hash(String, String)] user_data + # + # @return [Hash(String, String)] + def parse(incoming_payload, user_data) + @incoming_payload = incoming_payload + @user_data = user_data + parse_payload! + + outbound_request + end + + private + + attr_reader :user_data, # User defined payload (bridge.data['payload']) + :outbound_request, # Parsed request returned from `parse_payload` + :incoming_payload, # Inbound request from service A + :environment_variables + + # Iterates through user defined payload and parse values containing `$env` or `$payload` + # Reinitializes & mutates `outbound_request` + def parse_payload! + @outbound_request = {} # Reset + + user_data.each do |key, val| + outbound_request[key] = if val.include?('$env') + fetch_environment_variable(val) + elsif val.include?('$payload') + fetch_payload_data(val) + else + val + end + end + end + + def fetch_payload_data(value) + values = value.split('.') + data = incoming_payload # Set data to the incoming request + + values.each_with_index do |val, idx| + next if idx.zero? # skip the $payload + + data = data[val] # dig deeper into the request on each iteration + raise ::InvalidPayloadKey if data.nil? + end + + data + end + end + end +end diff --git a/app/lib/parser.rb b/app/lib/parser.rb deleted file mode 100644 index 7b4a55e..0000000 --- a/app/lib/parser.rb +++ /dev/null @@ -1,8 +0,0 @@ -# frozen_string_literal: true - -require_relative './exceptions/invalid_environment_variable' -require_relative './exceptions/invalid_payload_key' -require_relative './parser/**' - -module Parser -end diff --git a/app/lib/parser/environment_variables.rb b/app/lib/parser/environment_variables.rb deleted file mode 100644 index e1ccd77..0000000 --- a/app/lib/parser/environment_variables.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Parser - module EnvironmentVariables - # TODO - def fetch_environment_variable(value) - key = value.split('.').last - environment_variable = environment_variables.where(key: key) - raise InvalidEnvironmentVariable unless environment_variable - - environment_variable.decrypt - end - end -end diff --git a/app/lib/parser/headers.rb b/app/lib/parser/headers.rb deleted file mode 100644 index 8d321ae..0000000 --- a/app/lib/parser/headers.rb +++ /dev/null @@ -1,88 +0,0 @@ -# frozen_string_literal: true - -# This class parses user defined headers & payloads into the values we expect -# to send to the outbound service. -# -# Example: -# -# ```ruby -# custom_user_payload = { -# 'hello' => '$payload.top_level_key', -# 'environment_variable' => '$env.API_KEY', -# 'did_you' => '$payload.nested_key_1.nested_key_2.nested_key_3' -# } -# -# incoming_request_from_service_a = { -# 'top_level_key' => 'world', -# 'nested_key_1' => { -# 'nested_key_2' => { -# 'nested_key_3' => 'make it!' -# } -# } -# } -# -# bridge = Bridge.where(user_id: @current_user.id) -# -# syntax_parser = SyntaxParser.new(bridge) -# -# syntax_parser.parse_payload( -# incoming_request_from_service_a, -# custom_user_payload -# ) # => Hash(String, String) where $payload & $env are replaced with their respective values -# -# syntax_parser.parse_headers # => Array(Hash(String, String)) where $env is -# replaced with decrypted environment variable value -# ``` -module Parser - class Headers - include EnvironmentVariables - # Set headers - # - # @param [ActiveRecord::Relation(Header)] environment_variables - # @param [ActiveRecord::Relation(EnvironmentVariable)] environment_variables - def initialize(headers, environment_variables) - @headers = headers - @environment_variables = environment_variables - end - - # Parses the user's headers replacing any values containing `$env` - # with the decrypted environment variable value - # - # @return [Array(Hash(String, String))] - def parse_headers - parse_headers! - - outbound_headers - end - - private - - attr_reader :outbound_headers, # Parsed headers returned from `parse_headers` - :environment_variables, - :headers - - # Iterates through user defined payload and parse values containing `$env` - # Reinitializes & mutates `outbound_headers` - def parse_headers! - @outbound_headers = [] # Reset - headers.each do |header| - parsed_header = {} - parsed_header[header.key] = if header.value.include?('$env') - fetch_environment_variable(header.value) - else - header.value - end - - outbound_headers << parsed_header - end - end - - def fetch_environment_variable(value) - key = value.split('.').last - environment_variable = environment_variables.where(key: key) - raise InvalidEnvironmentVariable unless environment_variable - - environment_variable.decrypt - end - end -end diff --git a/app/lib/parser/payload.rb b/app/lib/parser/payload.rb deleted file mode 100644 index 610885b..0000000 --- a/app/lib/parser/payload.rb +++ /dev/null @@ -1,97 +0,0 @@ -# frozen_string_literal: true - -# This class parses user defined headers & payloads into the values we expect -# to send to the outbound service. -# -# Example: -# -# ```ruby -# custom_user_payload = { -# 'hello' => '$payload.top_level_key', -# 'environment_variable' => '$env.API_KEY', -# 'did_you' => '$payload.nested_key_1.nested_key_2.nested_key_3' -# } -# -# incoming_request_from_service_a = { -# 'top_level_key' => 'world', -# 'nested_key_1' => { -# 'nested_key_2' => { -# 'nested_key_3' => 'make it!' -# } -# } -# } -# -# bridge = Bridge.where(user_id: @current_user.id) -# -# syntax_parser = SyntaxParser.new(bridge) -# -# syntax_parser.parse_payload( -# incoming_request_from_service_a, -# custom_user_payload -# ) # => Hash(String, String) where $payload & $env are replaced with their respective values -# -# syntax_parser.parse_headers # => Array(Hash(String, String)) where $env is -# replaced with decrypted environment variable value -# ``` -module Parser - class Payload - include EnvironmentVariables - # Set bridge headers & env's. - # - # @param [Hash(String, String)] incoming_request - # @param [Hash(String, String)] user_data - # @param [ActiveRecord::Relation(EnvironmentVariable)] environment_variables - def initialize(incoming_request, user_data, environment_variables) - @incoming_request = incoming_request - @user_data = user_data - @environment_variables = environment_variables - end - - # Parses the user's custom payload replacing any values containing `$env` - # or `$payload` with the respective value - # - # @return [Hash(String, String)] - def parse_payload - parse_payload! - - outbound_request - end - - private - - attr_reader :user_data, # User defined payload (bridge.data['payload']) - :outbound_request, # Parsed request returned from `parse_payload` - :incoming_request, # Inbound request from service A - :environment_variables - - # Iterates through user defined payload and parse values containing `$env` or `$payload` - # Reinitializes & mutates `outbound_request` - def parse_payload! - @outbound_request = {} # Reset - - user_data.each do |key, val| - outbound_request[key] = if val.include?('$env') - fetch_environment_variable(val) - elsif val.include?('$payload') - fetch_payload_data(val) - else - val - end - end - end - - def fetch_payload_data(value) - values = value.split('.') - data = incoming_request # Set data to the incoming request - - values.each_with_index do |val, idx| - next if idx.zero? # skip the $payload - - data = data[val] # dig deeper into the request on each iteration - raise InvalidPayloadKey if data.nil? - end - - data - end - end -end diff --git a/app/models/event.rb b/app/models/event.rb index d162268..8b09772 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -19,6 +19,14 @@ def complete! save! # TODO: With a bang? end + # Parses `data` and fetches the payload from the + # inbound request. + # + # @return [Hash(String, String)] + def inbound_payload + JSON.parse(data)['inbound']['payload'] + end + private # TODO: Pass in bridge (or urls) to prevent db hit diff --git a/app/workers/event_worker.rb b/app/workers/event_worker.rb index 26ea2fc..98589bf 100644 --- a/app/workers/event_worker.rb +++ b/app/workers/event_worker.rb @@ -7,16 +7,16 @@ class EventWorker include Sidekiq::Worker attr_accessor :retry_count - def perform(event_id, retries = 0) + def perform(event_id, _retries = 0) event = Event.includes(:bridge).find(event_id) request_handler = ::BridgeApi::Http::Handler.new(event) request_handler.execute event.complete! - rescue StandardError => e - request_handler.cleanup(e) unless e.instance_of? Sidekiq::LargeStatusCode + # rescue StandardError => e + # request_handler.cleanup(e) unless e.instance_of? Sidekiq::LargeStatusCode - return event.complete! if retries >= event.bridge.retries.to_i + # return event.complete! if retry_count&.>= bridge.retries # TODO: Off by one? - raise StandardError + # raise StandardError end end From f203db146025a8c61ef343aefecf999a56606cd9 Mon Sep 17 00:00:00 2001 From: = Date: Fri, 20 Nov 2020 18:46:33 -0500 Subject: [PATCH 39/64] It works! Header & payload parsing works, deconstructing & saving works, everything works! --- app/controllers/events_controller.rb | 2 +- app/lib/bridge_api.rb | 3 +++ app/lib/bridge_api/http/builder.rb | 13 ++++++++----- app/lib/bridge_api/http/handler.rb | 18 +++++++++++------- .../bridge_api/syntax_parser/payload_parser.rb | 13 +++++++------ app/workers/event_worker.rb | 2 ++ db/seeds.rb | 9 ++++++++- 7 files changed, 40 insertions(+), 20 deletions(-) diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index 7b93d00..83d9745 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -43,7 +43,7 @@ def create private def event_params - params.require(:event).permit(:id, :bridge_id, :event_id, :test) + params.permit(:id, :bridge_id, :event_id, :test) end def create_data_object diff --git a/app/lib/bridge_api.rb b/app/lib/bridge_api.rb index 5a2f520..5a077ab 100644 --- a/app/lib/bridge_api.rb +++ b/app/lib/bridge_api.rb @@ -1,4 +1,7 @@ # frozen_string_literal: true +require_relative './exceptions/invalid_payload_key' +require_relative './exceptions/invalid_environment_variable' + module BridgeApi end diff --git a/app/lib/bridge_api/http/builder.rb b/app/lib/bridge_api/http/builder.rb index 81708dd..bf83bbc 100644 --- a/app/lib/bridge_api/http/builder.rb +++ b/app/lib/bridge_api/http/builder.rb @@ -10,10 +10,10 @@ class Builder include Interfaces::Builder + # TODO # @param [Bridge] bridge - The bridge that event belongs to - def initialize(bridge, test_env, payload_parser, headers_parser) - @bridge = bridge - @test_env = test_env + def initialize(request_handler, payload_parser, headers_parser) + @request_handler = request_handler @payload_parser = payload_parser @headers_parser = headers_parser end @@ -27,8 +27,11 @@ def generate private + delegate :bridge, to: :request_handler + delegate :event, to: :request_handler + attr_reader :request, - :bridge, + :request_handler, :payload_parser, :headers_parser @@ -120,7 +123,7 @@ def headers # # @return [Bool] def test_env? - @test_env + event.test end end end diff --git a/app/lib/bridge_api/http/handler.rb b/app/lib/bridge_api/http/handler.rb index 00bce7a..32fc8e6 100644 --- a/app/lib/bridge_api/http/handler.rb +++ b/app/lib/bridge_api/http/handler.rb @@ -7,11 +7,17 @@ class Handler # Inject a different builder, formatter or deconstructor # # @param [BridgeApi::Http::Interfaces::Builder] http_builder + # @param [BridgeApi::Http::Interfaces::Formatter] formatter + # @param [BridgeApi::Http::Interfaces::Deconstructor] formatter + # @param [BridgeApi::SyntaxParser::Interfaces::HeadersParser] headers_parser + # @param [BridgeApi::SyntaxParser::Interfaces::PayloadParser] payload_parser attr_writer :http_builder, - # @param [BridgeApi::Http::Interfaces::Formatter] formatter :formatter, - # @param [BridgeApi::Http::Interfaces::Deconstructor] formatter - :deconstructor + :deconstructor, + :headers_parser, + :payload_parser + + attr_reader :event, :bridge # @param [Event] event def initialize(event) @@ -36,13 +42,11 @@ def cleanup(error) private - attr_accessor :event, - :bridge, - :request + attr_reader :request # @return [BridgeApi::Http::Interfaces::Builder] def http_builder - @http_builder ||= Builder.new bridge, event.test, payload_parser, headers_parser + @http_builder ||= Builder.new self, payload_parser, headers_parser end # @return [BridgeApi::Http::Interfaces::Formatter] diff --git a/app/lib/bridge_api/syntax_parser/payload_parser.rb b/app/lib/bridge_api/syntax_parser/payload_parser.rb index 74f512c..6cad007 100644 --- a/app/lib/bridge_api/syntax_parser/payload_parser.rb +++ b/app/lib/bridge_api/syntax_parser/payload_parser.rb @@ -57,23 +57,23 @@ def parse(incoming_payload, user_data) @user_data = user_data parse_payload! - outbound_request + outbound_payload end private attr_reader :user_data, # User defined payload (bridge.data['payload']) - :outbound_request, # Parsed request returned from `parse_payload` - :incoming_payload, # Inbound request from service A + :outbound_payload, # Parsed request returned from `parse_payload` + :incoming_payload, # Inbound payload from service A :environment_variables # Iterates through user defined payload and parse values containing `$env` or `$payload` - # Reinitializes & mutates `outbound_request` + # Reinitializes & mutates `outbound_payload` def parse_payload! - @outbound_request = {} # Reset + @outbound_payload = {} # Reset user_data.each do |key, val| - outbound_request[key] = if val.include?('$env') + outbound_payload[key] = if val.include?('$env') fetch_environment_variable(val) elsif val.include?('$payload') fetch_payload_data(val) @@ -86,6 +86,7 @@ def parse_payload! def fetch_payload_data(value) values = value.split('.') data = incoming_payload # Set data to the incoming request + binding.pry values.each_with_index do |val, idx| next if idx.zero? # skip the $payload diff --git a/app/workers/event_worker.rb b/app/workers/event_worker.rb index 98589bf..2114254 100644 --- a/app/workers/event_worker.rb +++ b/app/workers/event_worker.rb @@ -17,6 +17,8 @@ def perform(event_id, _retries = 0) # return event.complete! if retry_count&.>= bridge.retries # TODO: Off by one? + # # TODO: We need filter error messages. ArgumentError and our stuff can be ignored but + # # should we really tell users "StandardError" if their service replied with 404? # raise StandardError end end diff --git a/db/seeds.rb b/db/seeds.rb index 062d70e..eb50d3f 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -16,7 +16,14 @@ def test_url retries: 5, delay: 15, data: { - payload: '{"FirstName":"Lee","LastName":"Oswald","UserName":"GrassyKnoll","Password":{"nested":"magic bullet"},"Email":"kgb63@yandex.ru"}', + payload: { + first_name: 'Lee', + last_name: 'Oswald', + username: 'GrassyKnoll', + email: 'kgb63@yandex.ru', + top_level_key: '$payload.top_level_key', + nested_key: '$payload.nested_key_1.nested_key_2' + }.to_json, test_payload: '{"test_key_one":"testerstring","test_key_two":["stringinarray"]}' } ) From 57cfc623899ae77cf9c9bddb206d1ddf7e6e75b8 Mon Sep 17 00:00:00 2001 From: = Date: Fri, 20 Nov 2020 19:47:22 -0500 Subject: [PATCH 40/64] Documentation & Clean up --- app/controllers/events_controller.rb | 4 +- app/lib/bridge_api/http/builder.rb | 25 +++++-- app/lib/bridge_api/http/deconstructor.rb | 21 ++++++ app/lib/bridge_api/http/formatter.rb | 39 +++++++++-- app/lib/bridge_api/http/handler.rb | 62 ++++++++++++----- .../syntax_parser/environment_variables.rb | 7 +- .../syntax_parser/headers_parser.rb | 66 ++++++++----------- .../syntax_parser/payload_parser.rb | 64 +++++++++--------- app/workers/event_worker.rb | 13 ++-- 9 files changed, 191 insertions(+), 110 deletions(-) diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index 83d9745..400f61a 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -28,9 +28,7 @@ def destroy def create event = create_event_object(create_data_object, find_bridge) event.save! - # EventWorker.perform_async(event.id) - # TODO: Remove - EventWorker.new.perform(event.id) + EventWorker.perform_async(event.id) render json: {}, status: 202 # Accepted (asynchronous processing) rescue ActiveRecord::RecordNotFound render json: { error: 'a bridge by that id was not found' }, status: 400 diff --git a/app/lib/bridge_api/http/builder.rb b/app/lib/bridge_api/http/builder.rb index bf83bbc..acea3f0 100644 --- a/app/lib/bridge_api/http/builder.rb +++ b/app/lib/bridge_api/http/builder.rb @@ -2,16 +2,27 @@ module BridgeApi module Http - # TODO: Parser - - # Handles building a HTTP Request object + # Handles building a HTTP Request object. Will parse user defined payloads + # and headers into expected values. Accepts custom parsers for extending. + # + # Example: + # + # ```ruby + # event = Event.find(1) + # + # handler = BridgeApi::Http::Handler.new event + # builder = BridgeApi::Http::Builder.new handler, handler.payload_parser, handler.headers_parser + # + # builder.generate # => returns an Tuple containing Net::Http & Net::Http::{user_request_type} objects + # ``` class Builder SCHEME = 'https://' include Interfaces::Builder - # TODO - # @param [Bridge] bridge - The bridge that event belongs to + # @param [BridgeApi::Http::Handler] request_handler - Used for delegation + # @param [BridgeApi::SyntaxParser::Interfaces::PayloadParser] payload_parser + # @param [BridgeApi::SyntaxParser::Interfaces::HeadersParser] headers_parser def initialize(request_handler, payload_parser, headers_parser) @request_handler = request_handler @payload_parser = payload_parser @@ -44,7 +55,7 @@ def net_http def http_request @request = net_http_request(uri) - parse_headers + parse_headers! request.body = parsed_payload.to_json request @@ -58,7 +69,7 @@ def uri end # Sets the user defined headers into the `request` object - def parse_headers + def parse_headers! headers_parser.parse(headers) do |key, value| request[key] = value end diff --git a/app/lib/bridge_api/http/deconstructor.rb b/app/lib/bridge_api/http/deconstructor.rb index 549cf26..c9a7c85 100644 --- a/app/lib/bridge_api/http/deconstructor.rb +++ b/app/lib/bridge_api/http/deconstructor.rb @@ -5,6 +5,27 @@ module Http # Deconstructs the request headers. Replaces # any headers that contain environment variables with # placeholder values to prevent data leakage. + # + # Example: + # + # ```ruby + # event = Event.find 1 + # deconstructor = BridgeApi::Http::Deconstructor.new event.bridge.headers + # safe_headers = [] + # + # request = Net::HTTP.start(uri.host, uri.port) do |http| + # request = Net::HTTP::Get.new uri + # set_your_decrypted_headers(request) + # request + # end + # + # req.each_header do |key, value| + # save_headers << { + # key: key, + # value: deconstructor.deconstruct(key, value) + # } + # end + # ``` class Deconstructor include Interfaces::Deconstructor diff --git a/app/lib/bridge_api/http/formatter.rb b/app/lib/bridge_api/http/formatter.rb index ab60932..99435f2 100644 --- a/app/lib/bridge_api/http/formatter.rb +++ b/app/lib/bridge_api/http/formatter.rb @@ -3,7 +3,24 @@ module BridgeApi module Http # Handles formatting requests & responses. Accepts a deconstructor - # to aid in formatting. + # to remove sensitive headers. + # + # Example: + # + # ```ruby + # event = Event.find 1 + # deconstructor = BridgeApi::Http::Deconstructor.new event.bridge.headers + # formatter = BridgeApi::Http::Formatter.new deconstructor + # uri = URI('http://example.com/some_path?query=string') + # + # Net::HTTP.start(uri.host, uri.port) do |http| + # request = Net::HTTP::Get.new uri + # response = http.request request # Net::HTTPResponse object + # formatter.format! event, request, response + # resuce StandardError => e + # formatter.format_error! event, request, e + # end + # ``` class Formatter include Interfaces::Formatter @@ -12,8 +29,8 @@ def initialize(deconstructor) @deconstructor = deconstructor end - # Mutates the event object by storing a formatted request inside - # data. + # Mutates the event object by storing a formatted request + # & response inside `event.data` # # @param [Event] event # @param [Net::HTTP] req @@ -28,8 +45,8 @@ def format!(event, req, res) event.data = data.to_json end - # Mutates the event object by storing a formatted request inside - # data. + # Mutates the event object by storing a formatted request + # & error inside `event.data`. # # @param [Event] event # @param [Net::HTTP] req @@ -48,7 +65,11 @@ def format_error!(event, req, error) attr_reader :deconstructor + # Formats the response without saving the entire object. + # # @param [Net::HTTPResponse] res + # + # @return [Hash(String, To many unions)] def formatted_response(response) { dateTime: DateTime.now.utc, @@ -59,7 +80,11 @@ def formatted_response(response) } end - # @param [Net::HTTP] req + # Formats the request without saving the entire object. + # + # @param [Net::HTTPResponse] req + # + # @return [Hash(Symbol, To many unions)] def formatted_request(request) { payload: JSON.parse(request.body), @@ -71,6 +96,8 @@ def formatted_request(request) end # @param [StandardError] error + # + # @return [Hash(Symbol, String)] def formatted_error(error) { message: error.message } end diff --git a/app/lib/bridge_api/http/handler.rb b/app/lib/bridge_api/http/handler.rb index 32fc8e6..7db5e7f 100644 --- a/app/lib/bridge_api/http/handler.rb +++ b/app/lib/bridge_api/http/handler.rb @@ -2,9 +2,36 @@ module BridgeApi module Http - # TODO + # Request handler is the main entry for our briding system. RequestHandler is the orchestrator + # managing flow & all the different pieces. + # + # Example: + # + # ```ruby + # event = Event.find(1) + # + # handler = BridgeApi::Http::Handler.new event + # handler.execute + # + # event.complete! # => events `data` attribute was mutated setting data['outbound'] with + # # {"request" => {request_data}, "response" => {response_from_server} } + # ``` + # + # ```ruby + # # Setting custom `http_builder` + # event = Event.find(1) + # + # handler = BridgeApi::Http::Handler.new event + # handler.http_builder = + # XmlHttpBuilder.new handler, handler.payload_parser, handler.headers_parser + # + # handler.execute # => Will now use your XmlHttpBuilder class + # + # event.complete! # => events `data` attribute was mutated setting data['outbound'] with + # # {"request" => {request_data}, "response" => {response_from_server} } + # ``` class Handler - # Inject a different builder, formatter or deconstructor + # Inject any piece of the request handling system for extending or testing. # # @param [BridgeApi::Http::Interfaces::Builder] http_builder # @param [BridgeApi::Http::Interfaces::Formatter] formatter @@ -40,19 +67,10 @@ def cleanup(error) formatter.format_error! event, request, error end - private - - attr_reader :request - - # @return [BridgeApi::Http::Interfaces::Builder] - def http_builder - @http_builder ||= Builder.new self, payload_parser, headers_parser - end - - # @return [BridgeApi::Http::Interfaces::Formatter] - def formatter - @formatter ||= Formatter.new deconstructor - end + # These next few methods are public for easy injection. + # If you wanted to inject your own http_builder but keep everything + # else as the default, that would be rather difficult without these + # being public. # @return [BridgeApi::Http::Interfaces::Deconstructor] def deconstructor @@ -68,6 +86,20 @@ def headers_parser def payload_parser @payload_parser ||= SyntaxParser::PayloadParser.new bridge.environment_variables end + + private + + attr_reader :request + + # @return [BridgeApi::Http::Interfaces::Builder] + def http_builder + @http_builder ||= Builder.new self, payload_parser, headers_parser + end + + # @return [BridgeApi::Http::Interfaces::Formatter] + def formatter + @formatter ||= Formatter.new deconstructor + end end end end diff --git a/app/lib/bridge_api/syntax_parser/environment_variables.rb b/app/lib/bridge_api/syntax_parser/environment_variables.rb index 3c9cce8..ea143cb 100644 --- a/app/lib/bridge_api/syntax_parser/environment_variables.rb +++ b/app/lib/bridge_api/syntax_parser/environment_variables.rb @@ -3,7 +3,12 @@ module BridgeApi module SyntaxParser module EnvironmentVariables - # TODO + # Takes a value such as `$env.API_KEY`, finds the corresponding + # EnvironmentVariable & returns the decrypted value. + # + # @param [String] value + # + # @return [String] def fetch_environment_variable(value) key = value.split('.').last environment_variable = environment_variables.find_by(key: key) diff --git a/app/lib/bridge_api/syntax_parser/headers_parser.rb b/app/lib/bridge_api/syntax_parser/headers_parser.rb index 712234c..80de767 100644 --- a/app/lib/bridge_api/syntax_parser/headers_parser.rb +++ b/app/lib/bridge_api/syntax_parser/headers_parser.rb @@ -1,41 +1,23 @@ # frozen_string_literal: true -# This class parses user defined headers & payloads into the values we expect -# to send to the outbound service. -# -# Example: -# -# ```ruby -# custom_user_payload = { -# 'hello' => '$payload.top_level_key', -# 'environment_variable' => '$env.API_KEY', -# 'did_you' => '$payload.nested_key_1.nested_key_2.nested_key_3' -# } -# -# incoming_request_from_service_a = { -# 'top_level_key' => 'world', -# 'nested_key_1' => { -# 'nested_key_2' => { -# 'nested_key_3' => 'make it!' -# } -# } -# } -# -# bridge = Bridge.where(user_id: @current_user.id) -# -# syntax_parser = SyntaxParser.new(bridge) -# -# syntax_parser.parse_payload( -# incoming_request_from_service_a, -# custom_user_payload -# ) # => Hash(String, String) where $payload & $env are replaced with their respective values -# -# syntax_parser.parse_headers # => Array(Hash(String, String)) where $env is -# replaced with decrypted environment variable value -# ``` module BridgeApi module SyntaxParser - # TODO: Doc + # This class parses user defined headers & payloads into the values we expect + # to send to the outbound service. + # + # Example: + # + # ```ruby + # request = Net::Http::Post.new URI('http://example.com/index.html?count=10') + # + # bridge = Bridge.where(user_id: @current_user.id) + # + # headers_parser = BridgeApi::SyntaxParser::HeadersParser.new bridge.environment_variables + # + # headers_parser.parse(bridge.headers) do |key, value| + # request[key] = value + # end + # ``` class HeadersParser include EnvironmentVariables include Interfaces::HeadersParser @@ -63,13 +45,21 @@ def parse(headers, &block) attr_reader :environment_variables, :headers - # TODO: Doc + # Iterates over headers, yields to a block passing in the header key & + # parsed value if parsing is required. + # + # @param [Proc] block def parse_headers!(block) - headers.each { |header| block.call(header.key, safe_value(header.value)) } + headers.each { |header| block.call(header.key, parse_value(header.value)) } end - # TODO: Doc - def safe_value(value) + # Replaces a `$env.API_KEY` string with the decrypted + # EnvironmentVariable value. + # + # @param [String] value + # + # @return [String] + def parse_value(value) if value.include?('$env') fetch_environment_variable(value) else diff --git a/app/lib/bridge_api/syntax_parser/payload_parser.rb b/app/lib/bridge_api/syntax_parser/payload_parser.rb index 6cad007..5dd05f7 100644 --- a/app/lib/bridge_api/syntax_parser/payload_parser.rb +++ b/app/lib/bridge_api/syntax_parser/payload_parser.rb @@ -1,41 +1,37 @@ # frozen_string_literal: true -# This class parses user defined headers & payloads into the values we expect -# to send to the outbound service. -# -# Example: -# -# ```ruby -# custom_user_payload = { -# 'hello' => '$payload.top_level_key', -# 'environment_variable' => '$env.API_KEY', -# 'did_you' => '$payload.nested_key_1.nested_key_2.nested_key_3' -# } -# -# incoming_payload_from_service_a = { -# 'top_level_key' => 'world', -# 'nested_key_1' => { -# 'nested_key_2' => { -# 'nested_key_3' => 'make it!' -# } -# } -# } -# -# bridge = Bridge.where(user_id: @current_user.id) -# -# syntax_parser = SyntaxParser.new(bridge) -# -# syntax_parser.parse_payload( -# incoming_payload_from_service_a, -# custom_user_payload -# ) # => Hash(String, String) where $payload & $env are replaced with their respective values -# -# syntax_parser.parse_headers # => Array(Hash(String, String)) where $env is -# replaced with decrypted environment variable value -# ``` module BridgeApi module SyntaxParser - # TODO: Doc + # This class parses user defined headers & payloads into the values we expect + # to send to the outbound service. + # + # Example: + # + # ```ruby + # custom_user_payload = { + # 'hello' => '$payload.top_level_key', + # 'environment_variable' => '$env.API_KEY', + # 'did_you' => '$payload.nested_key_1.nested_key_2.nested_key_3' + # } + # + # incoming_payload_from_service_a = { + # 'top_level_key' => 'world', + # 'nested_key_1' => { + # 'nested_key_2' => { + # 'nested_key_3' => 'make it!' + # } + # } + # } + # + # bridge = Bridge.where(user_id: @current_user.id) + # + # payload_parser = BridgeApi::SyntaxParser::PayloadParser.new bridge.environment_variables + # + # payload_parser.parse( + # incoming_payload_from_service_a, + # custom_user_payload + # ) # => Hash(String, String) where $payload & $env are replaced with their respective values + # ``` class PayloadParser include EnvironmentVariables include Interfaces::PayloadParser diff --git a/app/workers/event_worker.rb b/app/workers/event_worker.rb index 2114254..3a754eb 100644 --- a/app/workers/event_worker.rb +++ b/app/workers/event_worker.rb @@ -5,6 +5,7 @@ class EventWorker include Sidekiq::Worker + attr_accessor :retry_count def perform(event_id, _retries = 0) @@ -12,13 +13,13 @@ def perform(event_id, _retries = 0) request_handler = ::BridgeApi::Http::Handler.new(event) request_handler.execute event.complete! - # rescue StandardError => e - # request_handler.cleanup(e) unless e.instance_of? Sidekiq::LargeStatusCode + rescue StandardError => e + request_handler.cleanup(e) unless e.instance_of? Sidekiq::LargeStatusCode - # return event.complete! if retry_count&.>= bridge.retries # TODO: Off by one? + return event.complete! if retry_count&.>= bridge.retries # TODO: Off by one? - # # TODO: We need filter error messages. ArgumentError and our stuff can be ignored but - # # should we really tell users "StandardError" if their service replied with 404? - # raise StandardError + # TODO: We need filter error messages. ArgumentError and our stuff can be ignored but + # should we really tell users "StandardError" if their service replied with 404? + raise StandardError end end From c72d000995bfbbcafd683f12a04f23040fc8be95 Mon Sep 17 00:00:00 2001 From: = Date: Fri, 20 Nov 2020 19:56:26 -0500 Subject: [PATCH 41/64] Rubocop --- app/lib/bridge_api/syntax_parser/payload_parser.rb | 1 - app/lib/exceptions/invalid_payload_key.rb | 2 +- config/initializers/sidekiq.rb | 2 ++ spec/support/main_helper.rb | 2 ++ 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/app/lib/bridge_api/syntax_parser/payload_parser.rb b/app/lib/bridge_api/syntax_parser/payload_parser.rb index 5dd05f7..8e10a7a 100644 --- a/app/lib/bridge_api/syntax_parser/payload_parser.rb +++ b/app/lib/bridge_api/syntax_parser/payload_parser.rb @@ -82,7 +82,6 @@ def parse_payload! def fetch_payload_data(value) values = value.split('.') data = incoming_payload # Set data to the incoming request - binding.pry values.each_with_index do |val, idx| next if idx.zero? # skip the $payload diff --git a/app/lib/exceptions/invalid_payload_key.rb b/app/lib/exceptions/invalid_payload_key.rb index af828fe..f7855ad 100644 --- a/app/lib/exceptions/invalid_payload_key.rb +++ b/app/lib/exceptions/invalid_payload_key.rb @@ -1,4 +1,4 @@ # frozen_string_literal: true class InvalidPayloadKey < StandardError -end \ No newline at end of file +end diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index eb1f6b4..c81fc68 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class SidekiqMiddleware def call(worker, job, _queue) worker.retry_count = job['retry_count'] if worker.respond_to?(:retry_count) diff --git a/spec/support/main_helper.rb b/spec/support/main_helper.rb index 5efb8d2..1ec4056 100644 --- a/spec/support/main_helper.rb +++ b/spec/support/main_helper.rb @@ -39,6 +39,7 @@ def create_bridge ) end + # rubocop:disable Metrics/MethodLength def event_data { 'inbound' => { @@ -78,6 +79,7 @@ def event_data ] }.to_json end + # rubocop:enable Metrics/MethodLength def create_event @bridge = create_bridge From 7e499f8adb34f80110b2437bcaa59c57e4c6b817 Mon Sep 17 00:00:00 2001 From: = Date: Fri, 20 Nov 2020 20:01:30 -0500 Subject: [PATCH 42/64] Fix broken event model specs --- app/models/event.rb | 4 ++-- spec/models/event_spec.rb | 9 --------- spec/support/main_helper.rb | 13 ++++++------- 3 files changed, 8 insertions(+), 18 deletions(-) diff --git a/app/models/event.rb b/app/models/event.rb index 8b09772..3480f9b 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -38,11 +38,11 @@ def set_urls def data_json_object data = JSON.parse(self.data) %w[inbound outbound].all? { |key| data.include?(key) } && - %w[payload dateTime ip contentLength].all? { |key| data['inbound'].include?(key) } || + %w[payload dateTime ip contentLength headers].all? { |key| data['inbound'].include?(key) } || errors.add( :data, 'must include the keys: "inbound", "outbound", - while "inbound" must include the keys "payload", "dateTime", "ip", "contentLength"' + while "inbound" must include the keys "payload", "dateTime", "ip", "contentLength", "headers"' ) rescue JSON::ParserError, TypeError errors.add(:data, 'object must be a valid json object') diff --git a/spec/models/event_spec.rb b/spec/models/event_spec.rb index 83c7c57..231e8dc 100644 --- a/spec/models/event_spec.rb +++ b/spec/models/event_spec.rb @@ -21,15 +21,6 @@ expect(subject).to_not be_valid end - it 'can be converted to side_bar format' do - expect(subject.sidebar_format).to match( - id: subject.id, - time: a_string_matching(/\d{1,2}:\d{1,2}:\d{1,2}/), - date: a_string_matching(/\d{4}-\d{1,2}-\d{1,2}/), - status_code: an_instance_of(Integer) - ) - end - it 'belongs to a bridge' do expect(subject.bridge.class).to eql Bridge end diff --git a/spec/support/main_helper.rb b/spec/support/main_helper.rb index 1ec4056..6434da7 100644 --- a/spec/support/main_helper.rb +++ b/spec/support/main_helper.rb @@ -50,10 +50,10 @@ def event_data 'Password' => { 'nested' => 'magic bullet' }, 'Email' => 'kgb63@yandex.ru' }, - 'date' => '2020-11-17', - 'time' => '03:23:35', + 'dateTime' => '2020-11-17', 'ip' => '::1', - 'content_length' => 152 + 'contentLength' => '152', + 'headers' => [] }, 'outbound' => [ { 'request' => { @@ -64,16 +64,15 @@ def event_data 'Password' => { 'nested' => 'magic bullet' }, 'Email' => 'kgb63@yandex.ru' }, - 'date' => '2020-11-17', - 'time' => '03:23:35', - 'content_length' => 7 + 'dateTime' => '2020-11-17', + 'contentLength' => '7' }, 'response' => { 'date' => '2020-11-17', 'time' => '03:23:35', 'status_code' => '200', 'message' => 'OK', - 'size' => 7, + 'size' => '7', 'payload' => { 'ip' => '153.33.111.24' } } } ] From 9632eee1a246c558768bd779ada3f9a9471e9000 Mon Sep 17 00:00:00 2001 From: = Date: Sat, 21 Nov 2020 08:25:52 -0500 Subject: [PATCH 43/64] Fix database.yml & add pending spec --- config/database.yml | 8 ++++---- spec/models/event_spec.rb | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/config/database.yml b/config/database.yml index f5da28b..cedb472 100644 --- a/config/database.yml +++ b/config/database.yml @@ -24,8 +24,8 @@ default: &default development: <<: *default database: bridgeapi_rb_development - username: andrew - password: + username: ruby + password: ruby # The specified database role being used to connect to postgres. # To create additional roles in postgres see `$ createuser --help`. @@ -60,8 +60,8 @@ development: test: <<: *default database: bridgeapi_rb_test - username: andrew - password: + username: ruby + password: ruby host: localhost port: 5432 diff --git a/spec/models/event_spec.rb b/spec/models/event_spec.rb index 231e8dc..00744b2 100644 --- a/spec/models/event_spec.rb +++ b/spec/models/event_spec.rb @@ -63,4 +63,6 @@ subject.status_code = 10 expect { subject.save! }.to raise_error('Validation failed: Status code must be greater than or equal to 100') end + + pending '#inbound_payload returns the proper data' end From 726aee428a42a5397923df01a287b91a3bd191b8 Mon Sep 17 00:00:00 2001 From: = Date: Sat, 21 Nov 2020 08:35:55 -0500 Subject: [PATCH 44/64] Handler => RequestHandler, setting up tests --- app/lib/bridge_api/http/builder.rb | 2 +- app/lib/bridge_api/http/{handler.rb => request_handler.rb} | 4 ++-- spec/support/event_helper.rb | 0 spec/support/main_helper.rb | 4 ++++ 4 files changed, 7 insertions(+), 3 deletions(-) rename app/lib/bridge_api/http/{handler.rb => request_handler.rb} (97%) create mode 100644 spec/support/event_helper.rb diff --git a/app/lib/bridge_api/http/builder.rb b/app/lib/bridge_api/http/builder.rb index acea3f0..10d492f 100644 --- a/app/lib/bridge_api/http/builder.rb +++ b/app/lib/bridge_api/http/builder.rb @@ -10,7 +10,7 @@ module Http # ```ruby # event = Event.find(1) # - # handler = BridgeApi::Http::Handler.new event + # handler = BridgeApi::Http::RequestHandler.new event # builder = BridgeApi::Http::Builder.new handler, handler.payload_parser, handler.headers_parser # # builder.generate # => returns an Tuple containing Net::Http & Net::Http::{user_request_type} objects diff --git a/app/lib/bridge_api/http/handler.rb b/app/lib/bridge_api/http/request_handler.rb similarity index 97% rename from app/lib/bridge_api/http/handler.rb rename to app/lib/bridge_api/http/request_handler.rb index 7db5e7f..cd21161 100644 --- a/app/lib/bridge_api/http/handler.rb +++ b/app/lib/bridge_api/http/request_handler.rb @@ -21,7 +21,7 @@ module Http # # Setting custom `http_builder` # event = Event.find(1) # - # handler = BridgeApi::Http::Handler.new event + # handler = BridgeApi::Http::RequestHandler.new event # handler.http_builder = # XmlHttpBuilder.new handler, handler.payload_parser, handler.headers_parser # @@ -30,7 +30,7 @@ module Http # event.complete! # => events `data` attribute was mutated setting data['outbound'] with # # {"request" => {request_data}, "response" => {response_from_server} } # ``` - class Handler + class RequestHandler # Inject any piece of the request handling system for extending or testing. # # @param [BridgeApi::Http::Interfaces::Builder] http_builder diff --git a/spec/support/event_helper.rb b/spec/support/event_helper.rb new file mode 100644 index 0000000..e69de29 diff --git a/spec/support/main_helper.rb b/spec/support/main_helper.rb index 6434da7..0b53688 100644 --- a/spec/support/main_helper.rb +++ b/spec/support/main_helper.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require_relative './event_helper' + def random_email local = [] 8.times { |_| local.push(('a'..'z').to_a.sample) } @@ -8,6 +10,8 @@ def random_email end module MainHelper + include EventHelper + def create_user @current_user = User.create(email: random_email, password: 'password', notifications: false) @token = JsonWebToken.encode(user_id: @current_user.id) From 9c889645cd08c53d27169ceea8d37c49218b1812 Mon Sep 17 00:00:00 2001 From: = Date: Sat, 21 Nov 2020 10:14:04 -0500 Subject: [PATCH 45/64] Added webmock & factory bot for testing, created factories, created mock files, created spec files, payload parser & header parser now have tests --- .rubocop.yml | 2 +- Gemfile | 2 + Gemfile.lock | 16 +++++ app/controllers/bridges_controller.rb | 2 +- app/controllers/events_controller.rb | 31 +++++----- app/lib/bridge_api/http/builder.rb | 6 +- .../syntax_parser/headers_parser.rb | 1 + .../syntax_parser/payload_parser.rb | 2 +- app/models/bridge.rb | 7 ++- app/models/event.rb | 61 +++++++++++++++++-- app/workers/event_worker.rb | 2 +- .../20201121142638_change_bridge_method.rb | 5 ++ db/schema.rb | 4 +- db/seeds.rb | 4 +- spec/factories/bridge_factory.rb | 46 ++++++++++++++ .../factories/environment_variable_factory.rb | 8 +++ spec/factories/event_factory.rb | 14 +++++ spec/factories/header_factory.rb | 8 +++ spec/factories/user_factory.rb | 8 +++ spec/lib/bridge_api/http/builder_spec.rb | 19 ++++++ .../lib/bridge_api/http/deconstructor_spec.rb | 0 spec/lib/bridge_api/http/formatter_spec.rb | 0 .../bridge_api/http/request_handler_spec.rb | 0 .../syntax_parser/headers_parser_spec.rb | 29 +++++++++ .../syntax_parser/payload_parser_spec.rb | 46 ++++++++++++++ spec/models/bridge_spec.rb | 2 +- spec/rails_helper.rb | 3 + spec/requests/bridges_request_spec.rb | 2 +- spec/spec_helper.rb | 3 + spec/support/event_helper.rb | 5 ++ spec/support/factory_bot.rb | 5 ++ .../bridge_api/http/mock_request_handler.rb | 0 .../syntax_parser/mock_headers_parser.rb | 0 .../syntax_parser/mock_payload_parser.rb | 0 spec/support/main_helper.rb | 2 +- 35 files changed, 312 insertions(+), 33 deletions(-) create mode 100644 db/migrate/20201121142638_change_bridge_method.rb create mode 100644 spec/factories/bridge_factory.rb create mode 100644 spec/factories/environment_variable_factory.rb create mode 100644 spec/factories/event_factory.rb create mode 100644 spec/factories/header_factory.rb create mode 100644 spec/factories/user_factory.rb create mode 100644 spec/lib/bridge_api/http/builder_spec.rb create mode 100644 spec/lib/bridge_api/http/deconstructor_spec.rb create mode 100644 spec/lib/bridge_api/http/formatter_spec.rb create mode 100644 spec/lib/bridge_api/http/request_handler_spec.rb create mode 100644 spec/lib/bridge_api/syntax_parser/headers_parser_spec.rb create mode 100644 spec/lib/bridge_api/syntax_parser/payload_parser_spec.rb create mode 100644 spec/support/factory_bot.rb create mode 100644 spec/support/lib/bridge_api/http/mock_request_handler.rb create mode 100644 spec/support/lib/bridge_api/syntax_parser/mock_headers_parser.rb create mode 100644 spec/support/lib/bridge_api/syntax_parser/mock_payload_parser.rb diff --git a/.rubocop.yml b/.rubocop.yml index 87e545f..51269ba 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -8,4 +8,4 @@ AllCops: - "db/*/*" Metrics/BlockLength: Exclude: - - "spec/*/*" + - "spec/**/*" diff --git a/Gemfile b/Gemfile index fd47077..46b3db2 100644 --- a/Gemfile +++ b/Gemfile @@ -48,6 +48,8 @@ end group :test do # Generate test coverage reports gem 'simplecov', require: false + gem 'webmock' + gem 'factory_bot_rails' end # Windows does not include zoneinfo files, so bundle the tzinfo-data gem diff --git a/Gemfile.lock b/Gemfile.lock index ebf9477..f03f04f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -56,6 +56,8 @@ GEM minitest (~> 5.1) tzinfo (~> 1.1) zeitwerk (~> 2.2, >= 2.2.2) + addressable (2.7.0) + public_suffix (>= 2.0.2, < 5.0) ast (2.4.1) bcrypt (3.1.16) bootsnap (1.5.0) @@ -65,13 +67,20 @@ GEM coderay (1.1.3) concurrent-ruby (1.1.7) connection_pool (2.2.3) + crack (0.4.4) crass (1.0.6) diff-lcs (1.4.4) docile (1.3.2) erubi (1.9.0) + factory_bot (6.1.0) + activesupport (>= 5.0.0) + factory_bot_rails (6.1.0) + factory_bot (~> 6.1.0) + railties (>= 5.0.0) ffi (1.13.1) globalid (0.4.2) activesupport (>= 4.2.0) + hashdiff (1.0.1) i18n (1.8.5) concurrent-ruby (~> 1.0) jbuilder (2.10.1) @@ -103,6 +112,7 @@ GEM pry (0.13.1) coderay (~> 1.1) method_source (~> 1.0) + public_suffix (4.0.6) puma (4.3.6) nio4r (~> 2.0) rack (2.2.3) @@ -197,6 +207,10 @@ GEM tzinfo (1.2.8) thread_safe (~> 0.1) unicode-display_width (1.7.0) + webmock (3.10.0) + addressable (>= 2.3.6) + crack (>= 0.3.2) + hashdiff (>= 0.4.0, < 2.0.0) websocket-driver (0.7.3) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) @@ -209,6 +223,7 @@ DEPENDENCIES bcrypt (~> 3.1.7) bootsnap (>= 1.4.2) byebug + factory_bot_rails jbuilder (~> 2.7) jwt listen (~> 3.2) @@ -225,6 +240,7 @@ DEPENDENCIES spring spring-watcher-listen (~> 2.0.0) tzinfo-data + webmock RUBY VERSION ruby 2.6.6p146 diff --git a/app/controllers/bridges_controller.rb b/app/controllers/bridges_controller.rb index e07564d..148b72b 100644 --- a/app/controllers/bridges_controller.rb +++ b/app/controllers/bridges_controller.rb @@ -46,7 +46,7 @@ def destroy def bridge_params params.require(:bridge).permit( :title, - :method, + :http_method, :retries, :delay, :outbound_url, diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index 400f61a..6db99fd 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -28,14 +28,15 @@ def destroy def create event = create_event_object(create_data_object, find_bridge) event.save! - EventWorker.perform_async(event.id) + # EventWorker.perform_async(event.id) + EventWorker.new.perform(event.id) render json: {}, status: 202 # Accepted (asynchronous processing) - rescue ActiveRecord::RecordNotFound - render json: { error: 'a bridge by that id was not found' }, status: 400 - rescue ActiveRecord::RecordInvalid - render json: { error: 'payload, bridge_id, or urls were invalid' }, status: 400 # Bad Request - rescue ActiveRecord::NotNullViolation - render json: { error: 'payload, bridge_id, or urls fields were not submitted' }, status: 400 # Bad Request + # rescue ActiveRecord::RecordNotFound + # render json: { error: 'a bridge by that id was not found' }, status: 400 + # rescue ActiveRecord::RecordInvalid + # render json: { error: 'payload, bridge_id, or urls were invalid' }, status: 400 # Bad Request + # rescue ActiveRecord::NotNullViolation + # render json: { error: 'payload, bridge_id, or urls fields were not submitted' }, status: 400 # Bad Request end private @@ -45,13 +46,15 @@ def event_params end def create_data_object - { 'inbound' => { - 'payload' => JSON.parse(request.body.read), - 'dateTime' => DateTime.now.utc, - 'ip' => request.ip, - 'contentLength' => request.content_length - }, - 'outbound' => [] } + { + 'inbound' => { + 'payload' => JSON.parse(request.body.read), + 'dateTime' => DateTime.now.utc, + 'ip' => request.ip, + 'contentLength' => request.content_length + }, + 'outbound' => [] + } end def create_event_object(data, bridge) diff --git a/app/lib/bridge_api/http/builder.rb b/app/lib/bridge_api/http/builder.rb index 10d492f..147c318 100644 --- a/app/lib/bridge_api/http/builder.rb +++ b/app/lib/bridge_api/http/builder.rb @@ -107,12 +107,12 @@ def scheme # # @return [Net::HTTP::{http_method}] def net_http_request(uri) - "Net::HTTP::#{method}".constantize.new(uri, 'Content-Type' => 'application/json') + "Net::HTTP::#{http_method}".constantize.new(uri, 'Content-Type' => 'application/json') end # @return [String] - def method - bridge.method.capitalize + def http_method + bridge.http_method.capitalize end # @return [Hash(String, JSON)] diff --git a/app/lib/bridge_api/syntax_parser/headers_parser.rb b/app/lib/bridge_api/syntax_parser/headers_parser.rb index 80de767..54295bd 100644 --- a/app/lib/bridge_api/syntax_parser/headers_parser.rb +++ b/app/lib/bridge_api/syntax_parser/headers_parser.rb @@ -33,6 +33,7 @@ def initialize(environment_variables) # in `header.key` & a parsed version of `header.value` as arguments. # # @param [ActiveRecord::Relation(Header)] environment_variables + # @param [Proc] block # # @return [Array(Hash(String, String))] def parse(headers, &block) diff --git a/app/lib/bridge_api/syntax_parser/payload_parser.rb b/app/lib/bridge_api/syntax_parser/payload_parser.rb index 8e10a7a..f2aa261 100644 --- a/app/lib/bridge_api/syntax_parser/payload_parser.rb +++ b/app/lib/bridge_api/syntax_parser/payload_parser.rb @@ -86,7 +86,7 @@ def fetch_payload_data(value) values.each_with_index do |val, idx| next if idx.zero? # skip the $payload - data = data[val] # dig deeper into the request on each iteration + data = data[val.to_s] || data[val.to_sym] # dig deeper into the request on each iteration raise ::InvalidPayloadKey if data.nil? end diff --git a/app/models/bridge.rb b/app/models/bridge.rb index 0747205..9a34e4a 100644 --- a/app/models/bridge.rb +++ b/app/models/bridge.rb @@ -25,13 +25,18 @@ 5 ].freeze +# `data` column: +# { +# "payload" => {} +# "test_payload" => {} +# } class Bridge < ApplicationRecord before_validation :set_inbound_url, on: :create before_validation :set_payloads, on: :create validates :title, presence: true validates :inbound_url, presence: true, uniqueness: true validates :outbound_url, presence: true - validates :method, inclusion: METHODS + validates :http_method, inclusion: METHODS validates :delay, inclusion: DELAYS validates :retries, inclusion: RETRIES validate :validate_payloads diff --git a/app/models/event.rb b/app/models/event.rb index 3480f9b..b3f30e4 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -1,7 +1,60 @@ # frozen_string_literal: true -BOOLEAN = [true, false].freeze - +# `data` column: +# { +# 'inbound' => { +# 'payload' => { +# 'bridge_id' => '1', +# 'top_level_key' => 'present', +# 'nested_key_1' => { +# 'nested_key_2' => 'present' +# } +# }, +# 'dateTime' => '2020-11-21T13:59:47.349Z', +# 'ip' => '0.0.0.0', +# 'contentLength' => 101 +# }, +# 'outbound' => [ +# { +# 'request' => { +# 'payload' => { +# 'first_name' => 'Lee', +# 'last_name' => 'Oswald', +# 'username' => 'GrassyKnoll', +# 'email' => 'kgb63@yandex.ru', +# 'top_level_key' => 'present', +# 'nested_key' => 'present' +# }, +# 'dateTime' => '2020-11-21T13:59:51.076Z', +# 'contentLength' => '141', +# 'uri' => 'https://c41a7126-a18c-4af6-880e-6857771a35c8.mock.pstmn.io/success_event', +# 'headers' => [ +# { +# 'key' => 'content-type', +# 'value' => 'application/json' +# }, +# { +# 'key' => 'should_be_filtered', +# 'value' => 'FILTERED' +# }, +# { +# 'key' => 'not_filtered', +# 'value' => 'bridge api' +# }, +# ] +# }, +# 'response' => { +# 'dateTime' => '2020-11-21T13:59:51.076Z', +# 'statusCode' => '200', +# 'message' => 'OK', +# 'size' => 13, +# 'payload' => { +# 'hello' => 'world' +# } +# } +# } +# ] +# } class Event < ApplicationRecord before_validation :set_urls validates :test, inclusion: [true, false] @@ -38,11 +91,11 @@ def set_urls def data_json_object data = JSON.parse(self.data) %w[inbound outbound].all? { |key| data.include?(key) } && - %w[payload dateTime ip contentLength headers].all? { |key| data['inbound'].include?(key) } || + %w[payload dateTime ip contentLength].all? { |key| data['inbound'].include?(key) } || errors.add( :data, 'must include the keys: "inbound", "outbound", - while "inbound" must include the keys "payload", "dateTime", "ip", "contentLength", "headers"' + while "inbound" must include the keys "payload", "dateTime", "ip", "contentLength"' ) rescue JSON::ParserError, TypeError errors.add(:data, 'object must be a valid json object') diff --git a/app/workers/event_worker.rb b/app/workers/event_worker.rb index 3a754eb..cfbd93c 100644 --- a/app/workers/event_worker.rb +++ b/app/workers/event_worker.rb @@ -10,7 +10,7 @@ class EventWorker def perform(event_id, _retries = 0) event = Event.includes(:bridge).find(event_id) - request_handler = ::BridgeApi::Http::Handler.new(event) + request_handler = ::BridgeApi::Http::RequestHandler.new(event) request_handler.execute event.complete! rescue StandardError => e diff --git a/db/migrate/20201121142638_change_bridge_method.rb b/db/migrate/20201121142638_change_bridge_method.rb new file mode 100644 index 0000000..89828b7 --- /dev/null +++ b/db/migrate/20201121142638_change_bridge_method.rb @@ -0,0 +1,5 @@ +class ChangeBridgeMethod < ActiveRecord::Migration[6.0] + def change + rename_column :bridges, :method, :http_method + end +end diff --git a/db/schema.rb b/db/schema.rb index 540210e..f082f69 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2020_11_13_001336) do +ActiveRecord::Schema.define(version: 2020_11_21_142638) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -19,7 +19,7 @@ t.string "title", null: false t.string "inbound_url", null: false t.string "outbound_url", null: false - t.string "method", null: false + t.string "http_method", null: false t.integer "retries", null: false t.integer "delay", null: false t.jsonb "data", null: false diff --git a/db/seeds.rb b/db/seeds.rb index eb50d3f..eb87189 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -12,7 +12,7 @@ def test_url title: 'My First Bridge', inbound_url: 'bridgeapi.com/249634', outbound_url: 'c41a7126-a18c-4af6-880e-6857771a35c8.mock.pstmn.io/success_event', - method: 'POST', + http_method: 'POST', retries: 5, delay: 15, data: { @@ -38,7 +38,7 @@ def test_url title: 'My Second Bridge', inbound_url: 'bridgeapi.com/746353', outbound_url: test_url, - method: 'PATCH', + http_method: 'PATCH', retries: 3, delay: 0, data: { diff --git a/spec/factories/bridge_factory.rb b/spec/factories/bridge_factory.rb new file mode 100644 index 0000000..9414bfa --- /dev/null +++ b/spec/factories/bridge_factory.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :bridge do + title { 'bridge_1' } + outbound_url { 'myfakeoutbound.com' } + inbound_url { 'myfakeinbound.com' } + http_method { 'POST' } + delay { 0 } + retries { 0 } + data do + { + payload: { + first_name: 'Lee', + last_name: 'Oswald', + username: 'GrassyKnoll', + email: 'kgb63@yandex.ru', + top_level_key: '$payload.top_level_key', + nested_key: { + nested_key_two: '$payload.nested_key_1.nested_key_2' + } + }.to_json, + test_payload: { + "test_key_one": { + "nested": 11 + }, + "test_key_two": 888 + }.to_json + } + end + + association :user + + trait :with_env do + after(:create) do |bridge| + create(:environment_variable, bridge_id: bridge.id) + end + end + + trait :with_header do + after(:create) do |bridge| + create(:header, bridge_id: bridge.id) + end + end + end +end diff --git a/spec/factories/environment_variable_factory.rb b/spec/factories/environment_variable_factory.rb new file mode 100644 index 0000000..8034c62 --- /dev/null +++ b/spec/factories/environment_variable_factory.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :environment_variable do + key { 'API_KEY' } + value { 'hello world' } + end +end diff --git a/spec/factories/event_factory.rb b/spec/factories/event_factory.rb new file mode 100644 index 0000000..a50b1e4 --- /dev/null +++ b/spec/factories/event_factory.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# TODO? +FactoryBot.define do + factory :event do + before(:create) do + create(:bridge) + end + + completed { false } + inbound_url { 'myfakeinbound.com' } + outbound_url { 'myfakeoutbound.com' } + end +end diff --git a/spec/factories/header_factory.rb b/spec/factories/header_factory.rb new file mode 100644 index 0000000..f315ff1 --- /dev/null +++ b/spec/factories/header_factory.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :header do + key { 'hello' } + value { '$env.API_KEY' } + end +end diff --git a/spec/factories/user_factory.rb b/spec/factories/user_factory.rb new file mode 100644 index 0000000..66c3f10 --- /dev/null +++ b/spec/factories/user_factory.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :user do + email { 'demo@demo.com' } + password { 'password' } + end +end diff --git a/spec/lib/bridge_api/http/builder_spec.rb b/spec/lib/bridge_api/http/builder_spec.rb new file mode 100644 index 0000000..f220dd1 --- /dev/null +++ b/spec/lib/bridge_api/http/builder_spec.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe BridgeApi::Http::Builder do + before do + end + + after do + end + + subject do + create(:bridge, :with_env) + end + + it 'is valid when passed valid info' do + expect(subject).to be_valid + end +end diff --git a/spec/lib/bridge_api/http/deconstructor_spec.rb b/spec/lib/bridge_api/http/deconstructor_spec.rb new file mode 100644 index 0000000..e69de29 diff --git a/spec/lib/bridge_api/http/formatter_spec.rb b/spec/lib/bridge_api/http/formatter_spec.rb new file mode 100644 index 0000000..e69de29 diff --git a/spec/lib/bridge_api/http/request_handler_spec.rb b/spec/lib/bridge_api/http/request_handler_spec.rb new file mode 100644 index 0000000..e69de29 diff --git a/spec/lib/bridge_api/syntax_parser/headers_parser_spec.rb b/spec/lib/bridge_api/syntax_parser/headers_parser_spec.rb new file mode 100644 index 0000000..0df055f --- /dev/null +++ b/spec/lib/bridge_api/syntax_parser/headers_parser_spec.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe BridgeApi::SyntaxParser::HeadersParser do + it 'can parse envs' do + bridge = create(:bridge, :with_env, :with_header) + parser = BridgeApi::SyntaxParser::HeadersParser.new(bridge.environment_variables) + data = {} + + parser.parse(bridge.headers) do |k, v| + data[k] = v + end + + expect(data).to eq({ 'hello' => 'hello world' }) + end + + it 'will raise InvalidEnvironmentVariable when no key exists' do + bridge = create(:bridge, :with_header) + parser = BridgeApi::SyntaxParser::HeadersParser.new(bridge.environment_variables) + data = {} + + expect do + parser.parse(bridge.headers) do |k, v| + data[k] = v + end + end.to raise_error InvalidEnvironmentVariable + end +end diff --git a/spec/lib/bridge_api/syntax_parser/payload_parser_spec.rb b/spec/lib/bridge_api/syntax_parser/payload_parser_spec.rb new file mode 100644 index 0000000..e92d6d8 --- /dev/null +++ b/spec/lib/bridge_api/syntax_parser/payload_parser_spec.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe BridgeApi::SyntaxParser::PayloadParser do + subject do + bridge = create(:bridge, :with_env) + BridgeApi::SyntaxParser::PayloadParser.new(bridge.environment_variables) + end + + it 'can parse envs' do + expect( + subject.parse( + Net::HTTP.new(URI('example.com')), + { data: '$env.API_KEY' } + ) + ).to eq({ data: 'hello world' }) + end + + it 'will raise InvalidEnvironmentVariable when no key exists' do + expect do + subject.parse( + Net::HTTP.new(URI('example.com')), + { data: '$env.api_key' } + ) + end.to raise_error InvalidEnvironmentVariable + end + + it 'can access payload data wtih symbols' do + expect( + subject.parse( + { data: 'hello world' }, + { data: '$payload.data' } + ) + ).to eq({ data: 'hello world' }) + end + + it 'can access payload data wtih strings' do + expect( + subject.parse( + { 'data' => 'hello world' }, + { 'data' => '$payload.data' } + ) + ).to eq({ 'data' => 'hello world' }) + end +end diff --git a/spec/models/bridge_spec.rb b/spec/models/bridge_spec.rb index 96768ee..c55c2ba 100644 --- a/spec/models/bridge_spec.rb +++ b/spec/models/bridge_spec.rb @@ -30,7 +30,7 @@ end it 'is invalid without a method' do - subject.method = nil + subject.http_method = nil expect(subject).to_not be_valid end diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index 191f3b9..d508aae 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -4,11 +4,14 @@ require 'spec_helper' ENV['RAILS_ENV'] ||= 'test' require File.expand_path('../config/environment', __dir__) + # Prevent database truncation if the environment is production abort('The Rails environment is running in production mode!') if Rails.env.production? require 'rspec/rails' + # Add additional requires below this line. Rails is not loaded until this point! require './spec/support/main_helper' +require './spec/support/factory_bot' # Requires supporting ruby files with custom matchers and macros, etc, in # spec/support/ and its subdirectories. Files matching `spec/**/*_spec.rb` are # run as spec files by default. This means that files in spec/support that end diff --git a/spec/requests/bridges_request_spec.rb b/spec/requests/bridges_request_spec.rb index 6f9cf86..d10b3ba 100644 --- a/spec/requests/bridges_request_spec.rb +++ b/spec/requests/bridges_request_spec.rb @@ -134,7 +134,7 @@ end it 'doesnt create bridge without method' do invalid_hash = bridge_hash - invalid_hash[:method] = nil + invalid_hash[:http_method] = nil post bridges_path, params: { bridge: invalid_hash }, headers: authenticated_token diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 01f7c97..7e10955 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -93,4 +93,7 @@ # # test failures related to randomization by passing the same `--seed` value # # as the one that triggered the failure. # Kernel.srand config.seed + + # TODO: Maybe + # config.use_transactional_fixtures = true end diff --git a/spec/support/event_helper.rb b/spec/support/event_helper.rb index e69de29..d31fdcf 100644 --- a/spec/support/event_helper.rb +++ b/spec/support/event_helper.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +module EventHelper + +end \ No newline at end of file diff --git a/spec/support/factory_bot.rb b/spec/support/factory_bot.rb new file mode 100644 index 0000000..2e7665c --- /dev/null +++ b/spec/support/factory_bot.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +RSpec.configure do |config| + config.include FactoryBot::Syntax::Methods +end diff --git a/spec/support/lib/bridge_api/http/mock_request_handler.rb b/spec/support/lib/bridge_api/http/mock_request_handler.rb new file mode 100644 index 0000000..e69de29 diff --git a/spec/support/lib/bridge_api/syntax_parser/mock_headers_parser.rb b/spec/support/lib/bridge_api/syntax_parser/mock_headers_parser.rb new file mode 100644 index 0000000..e69de29 diff --git a/spec/support/lib/bridge_api/syntax_parser/mock_payload_parser.rb b/spec/support/lib/bridge_api/syntax_parser/mock_payload_parser.rb new file mode 100644 index 0000000..e69de29 diff --git a/spec/support/main_helper.rb b/spec/support/main_helper.rb index 0b53688..eb8aa93 100644 --- a/spec/support/main_helper.rb +++ b/spec/support/main_helper.rb @@ -30,7 +30,7 @@ def bridge_hash user: @current_user, title: 'bridge', outbound_url: "doggoapi.io/#{(String(rand).split '.')[1]}", - method: 'POST', + http_method: 'POST', retries: 5, delay: 15, data: { payload: '{}', test_payload: '{}' } From f4fe1fc374e3ea8b933ff238db66c2421b805bf5 Mon Sep 17 00:00:00 2001 From: = Date: Sat, 21 Nov 2020 11:12:59 -0500 Subject: [PATCH 46/64] Formatter spec --- spec/factories/event_factory.rb | 25 +++++++- spec/lib/bridge_api/http/formatter_spec.rb | 61 +++++++++++++++++++ spec/lib/bridge_api/http/spec_helper.rb | 8 +++ .../lib/bridge_api/http/mock_deconstructor.rb | 7 +++ 4 files changed, 98 insertions(+), 3 deletions(-) create mode 100644 spec/lib/bridge_api/http/spec_helper.rb create mode 100644 spec/support/lib/bridge_api/http/mock_deconstructor.rb diff --git a/spec/factories/event_factory.rb b/spec/factories/event_factory.rb index a50b1e4..6273d92 100644 --- a/spec/factories/event_factory.rb +++ b/spec/factories/event_factory.rb @@ -3,12 +3,31 @@ # TODO? FactoryBot.define do factory :event do - before(:create) do - create(:bridge) - end + association :bridge completed { false } inbound_url { 'myfakeinbound.com' } outbound_url { 'myfakeoutbound.com' } + data do + { + 'inbound' => { + 'payload' => { + 'bridge_id' => '1', + 'top_level_key' => 'present', + 'nested_key_1' => { + 'nested_key_2' => 'present' + } + }, + 'dateTime' => '2020-11-21T13:59:47.349Z', + 'ip' => '0.0.0.0', + 'contentLength' => 101 + }, + 'outbound' => [] + }.to_json + end + + trait :completed do + # TODO + end end end diff --git a/spec/lib/bridge_api/http/formatter_spec.rb b/spec/lib/bridge_api/http/formatter_spec.rb index e69de29..b928e77 100644 --- a/spec/lib/bridge_api/http/formatter_spec.rb +++ b/spec/lib/bridge_api/http/formatter_spec.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require_relative './spec_helper' + +RSpec.describe BridgeApi::SyntaxParser::HeadersParser do + before do + stub_request(:post, 'http://example.com/some_path?query=string') + .with(headers: { 'Accept' => '*/*', 'User-Agent' => 'Ruby' }) + .to_return(status: 200, body: { data: 'stubbed response' }.to_json, headers: {}) + end + + subject do + BridgeApi::Http::Formatter.new MockDeconstructor.new + end + + it 'can format a successful request' do + event = create(:event) + uri = URI('http://example.com/some_path?query=string') + + Net::HTTP.start(uri.host, uri.port) do |http| + request = Net::HTTP::Post.new uri + request.body = { data: 'hello world' }.to_json + response = http.request request # Net::HTTPResponse object + subject.format! event, request, response + end + + data = JSON.parse(event.data) + + expect(data.keys).to eq %w[inbound outbound] + expect(data['outbound'].first.keys).to eq %w[request response] + expect(data['outbound'].first['request'].keys).to eq %w[payload dateTime contentLength uri headers] + expect(data['outbound'].first['response'].keys).to eq %w[dateTime statusCode message size payload] + expect(data['outbound'].first['request']['payload']).to be_truthy + expect(data['outbound'].first['response']['payload']).to be_truthy + expect(data['outbound'].first['response']['statusCode']).to eq '200' + expect(data['outbound'].first['request']['uri']).to eq 'http://example.com/some_path?query=string' + end + + it 'can format a failed request' do + event = create(:event) + uri = URI('http://example2.com/some_path?query=string') + + Net::HTTP.start(uri.host, uri.port) do |_http| + request = Net::HTTP::Post.new uri + request.body = { data: 'hello world' }.to_json + raise StandardError + rescue StandardError => e + subject.format_error! event, request, e + end + + data = JSON.parse(event.data) + + expect(data.keys).to eq %w[inbound outbound] + expect(data['outbound'].first.keys).to eq %w[request response] + expect(data['outbound'].first['request'].keys).to eq %w[payload dateTime contentLength uri headers] + expect(data['outbound'].first['response'].keys).to eq %w[message] + expect(data['outbound'].first['request']['payload']).to be_truthy + expect(data['outbound'].first['request']['uri']).to eq 'http://example2.com/some_path?query=string' + expect(data['outbound'].first['response']).to eq({ 'message' => 'StandardError' }) + end +end diff --git a/spec/lib/bridge_api/http/spec_helper.rb b/spec/lib/bridge_api/http/spec_helper.rb new file mode 100644 index 0000000..29b83ae --- /dev/null +++ b/spec/lib/bridge_api/http/spec_helper.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +require 'rails_helper' + +require 'webmock/rspec' +WebMock.disable_net_connect!(allow_localhost: true) + +require_relative '../../../support/lib/bridge_api/http/mock_deconstructor' diff --git a/spec/support/lib/bridge_api/http/mock_deconstructor.rb b/spec/support/lib/bridge_api/http/mock_deconstructor.rb new file mode 100644 index 0000000..509c147 --- /dev/null +++ b/spec/support/lib/bridge_api/http/mock_deconstructor.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class MockDeconstructor + def deconstruct(_key, value) + value + end +end From a450457199302f9f680d85a8206b995f0fd44e30 Mon Sep 17 00:00:00 2001 From: = Date: Sat, 21 Nov 2020 11:18:23 -0500 Subject: [PATCH 47/64] Deconstructor spec --- spec/lib/bridge_api/http/deconstructor_spec.rb | 18 ++++++++++++++++++ spec/lib/bridge_api/http/formatter_spec.rb | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/spec/lib/bridge_api/http/deconstructor_spec.rb b/spec/lib/bridge_api/http/deconstructor_spec.rb index e69de29..60c8c12 100644 --- a/spec/lib/bridge_api/http/deconstructor_spec.rb +++ b/spec/lib/bridge_api/http/deconstructor_spec.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require_relative './spec_helper' + +RSpec.describe BridgeApi::Http::Deconstructor do + subject do + bridge = create(:bridge, :with_header) + BridgeApi::Http::Deconstructor.new bridge.headers + end + + it 'can return filtered headers' do + expect(subject.deconstruct('hello', 'world')).to eq 'FILTERED' + end + + it 'can return the value when header doesn\'t exist' do + expect(subject.deconstruct('hi', 'world')).to eq 'world' + end +end diff --git a/spec/lib/bridge_api/http/formatter_spec.rb b/spec/lib/bridge_api/http/formatter_spec.rb index b928e77..fddbe66 100644 --- a/spec/lib/bridge_api/http/formatter_spec.rb +++ b/spec/lib/bridge_api/http/formatter_spec.rb @@ -2,7 +2,7 @@ require_relative './spec_helper' -RSpec.describe BridgeApi::SyntaxParser::HeadersParser do +RSpec.describe BridgeApi::Http::Formatter do before do stub_request(:post, 'http://example.com/some_path?query=string') .with(headers: { 'Accept' => '*/*', 'User-Agent' => 'Ruby' }) From 5ea08a2132a4c482366b446e9757362f93afd86f Mon Sep 17 00:00:00 2001 From: = Date: Sat, 21 Nov 2020 12:32:11 -0500 Subject: [PATCH 48/64] PayloadParser: Add support for deeply nested hashes or arrays --- .../syntax_parser/payload_parser.rb | 35 +++++++++------- .../bridge_api/http/request_handler_spec.rb | 15 +++++++ .../syntax_parser/payload_parser_spec.rb | 40 +++++++++++++++++++ 3 files changed, 76 insertions(+), 14 deletions(-) diff --git a/app/lib/bridge_api/syntax_parser/payload_parser.rb b/app/lib/bridge_api/syntax_parser/payload_parser.rb index f2aa261..ae524e2 100644 --- a/app/lib/bridge_api/syntax_parser/payload_parser.rb +++ b/app/lib/bridge_api/syntax_parser/payload_parser.rb @@ -50,10 +50,8 @@ def initialize(environment_variables) # @return [Hash(String, String)] def parse(incoming_payload, user_data) @incoming_payload = incoming_payload - @user_data = user_data - parse_payload! - - outbound_payload + # @user_data = user_data + parse_payload!(user_data) end private @@ -65,19 +63,28 @@ def parse(incoming_payload, user_data) # Iterates through user defined payload and parse values containing `$env` or `$payload` # Reinitializes & mutates `outbound_payload` - def parse_payload! - @outbound_payload = {} # Reset + def parse_payload!(user_data) + outbound_payload = {} + user_data.each { |key, val| outbound_payload[key] = parse_value(val) } + + outbound_payload + end - user_data.each do |key, val| - outbound_payload[key] = if val.include?('$env') - fetch_environment_variable(val) - elsif val.include?('$payload') - fetch_payload_data(val) - else - val - end + # rubocop:disable Metrics/MethodLength + def parse_value(value) + if value.include?('$env') + fetch_environment_variable(value) + elsif value.include?('$payload') + fetch_payload_data(value) + elsif value.instance_of?(Hash) + parse_payload!(value) + elsif value.instance_of?(Array) + value.map { |i| parse_payload!(i) } + else + value end end + # rubocop:enable Metrics/MethodLength def fetch_payload_data(value) values = value.split('.') diff --git a/spec/lib/bridge_api/http/request_handler_spec.rb b/spec/lib/bridge_api/http/request_handler_spec.rb index e69de29..dfe19b8 100644 --- a/spec/lib/bridge_api/http/request_handler_spec.rb +++ b/spec/lib/bridge_api/http/request_handler_spec.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require_relative './spec_helper' + +RSpec.describe BridgeApi::Http::Deconstructor do + before do + stub_request(:post, 'http://example.com/some_path?query=string') + .with(headers: { 'Accept' => '*/*', 'User-Agent' => 'Ruby' }) + .to_return(status: 200, body: { data: 'stubbed response' }.to_json, headers: {}) + end + subject do + bridge = create(:bridge, :with_header) + BridgeApi::Http::Deconstructor.new bridge.headers + end +end diff --git a/spec/lib/bridge_api/syntax_parser/payload_parser_spec.rb b/spec/lib/bridge_api/syntax_parser/payload_parser_spec.rb index e92d6d8..8267b85 100644 --- a/spec/lib/bridge_api/syntax_parser/payload_parser_spec.rb +++ b/spec/lib/bridge_api/syntax_parser/payload_parser_spec.rb @@ -43,4 +43,44 @@ ) ).to eq({ 'data' => 'hello world' }) end + + it 'can access deeply nested objects' do + expect( + subject.parse( + { + 'data' => 'hello world', + 'nested_data' => { + 'nested_data_1' => [ + { + 'nested_data_2' => 'hello world2' + } + ] + } + }, + { + 'data' => { + 'nested_key' => '$payload.data', + 'nested_array' => [ + 'one' => '$payload.nested_data.nested_data_1' + ] + } + } + ) + ).to eq( + { + 'data' => { + 'nested_array' => [ + { + 'one' => [ + { + 'nested_data_2' => 'hello world2' + } + ] + } + ], + 'nested_key' => 'hello world' + } + } + ) + end end From bb9f38c02b36c6dd45c1b26204cb08399e30cce4 Mon Sep 17 00:00:00 2001 From: = Date: Sat, 21 Nov 2020 12:35:56 -0500 Subject: [PATCH 49/64] Update payload parser spec --- .../syntax_parser/payload_parser_spec.rb | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/spec/lib/bridge_api/syntax_parser/payload_parser_spec.rb b/spec/lib/bridge_api/syntax_parser/payload_parser_spec.rb index 8267b85..34959b1 100644 --- a/spec/lib/bridge_api/syntax_parser/payload_parser_spec.rb +++ b/spec/lib/bridge_api/syntax_parser/payload_parser_spec.rb @@ -61,7 +61,12 @@ 'data' => { 'nested_key' => '$payload.data', 'nested_array' => [ - 'one' => '$payload.nested_data.nested_data_1' + { + 'one' => '$payload.nested_data.nested_data_1' + }, + { + 'two' => '$payload.nested_data.nested_data_1' + } ] } } @@ -76,6 +81,13 @@ 'nested_data_2' => 'hello world2' } ] + }, + { + 'two' => [ + { + 'nested_data_2' => 'hello world2' + } + ] } ], 'nested_key' => 'hello world' From 1c71f80d4792d11513ca8c3d4e57483accd97fea Mon Sep 17 00:00:00 2001 From: = Date: Sat, 21 Nov 2020 12:40:22 -0500 Subject: [PATCH 50/64] Add better docs for payload parser --- .../syntax_parser/payload_parser.rb | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/app/lib/bridge_api/syntax_parser/payload_parser.rb b/app/lib/bridge_api/syntax_parser/payload_parser.rb index ae524e2..a6356ab 100644 --- a/app/lib/bridge_api/syntax_parser/payload_parser.rb +++ b/app/lib/bridge_api/syntax_parser/payload_parser.rb @@ -44,25 +44,25 @@ def initialize(environment_variables) # Parses the user's custom payload replacing any values containing `$env` # or `$payload` with the respective value # - # @param [Hash(String, String)] incoming_payload - # @param [Hash(String, String)] user_data + # @param [Hash(String, String | Hash | Array)] incoming_payload + # @param [Hash(String, String | Hash | Array)] user_data # - # @return [Hash(String, String)] + # @return [Hash(String, String | Hash | Array)] def parse(incoming_payload, user_data) @incoming_payload = incoming_payload - # @user_data = user_data parse_payload!(user_data) end private - attr_reader :user_data, # User defined payload (bridge.data['payload']) - :outbound_payload, # Parsed request returned from `parse_payload` - :incoming_payload, # Inbound payload from service A + attr_reader :incoming_payload, # Inbound payload from service A :environment_variables # Iterates through user defined payload and parse values containing `$env` or `$payload` - # Reinitializes & mutates `outbound_payload` + # + # @param [Hash(String, String | Hash | Array)] user_data + # + # @return [Hash(String, String | Hash | Array)] def parse_payload!(user_data) outbound_payload = {} user_data.each { |key, val| outbound_payload[key] = parse_value(val) } @@ -70,6 +70,13 @@ def parse_payload!(user_data) outbound_payload end + # Handles parsing a value depending on its value or type. + # Will call `parse_payload!` recursively if value is a + # Hash or Array. + # + # @param [String | Hash | Array] value + # + # @return [String | Hash | Array] # rubocop:disable Metrics/MethodLength def parse_value(value) if value.include?('$env') @@ -79,6 +86,7 @@ def parse_value(value) elsif value.instance_of?(Hash) parse_payload!(value) elsif value.instance_of?(Array) + # TODO: Add support for accessing a single element value.map { |i| parse_payload!(i) } else value From 1eaa0f5e1ae3c7b099ad053ce86cb945ac0289aa Mon Sep 17 00:00:00 2001 From: = Date: Sat, 21 Nov 2020 13:06:35 -0500 Subject: [PATCH 51/64] request_handler_spec --- app/lib/bridge_api.rb | 1 + app/lib/bridge_api/http/builder.rb | 4 +-- .../bridge_api/http/request_handler_spec.rb | 29 +++++++++++++++++-- spec/lib/bridge_api/http/spec_helper.rb | 6 ++++ .../lib/bridge_api/http/mock_builders.rb | 23 +++++++++++++++ .../lib/bridge_api/http/mock_formatter.rb | 21 ++++++++++++++ .../syntax_parser/mock_headers_parser.rb | 4 +++ .../syntax_parser/mock_payload_parser.rb | 4 +++ 8 files changed, 87 insertions(+), 5 deletions(-) create mode 100644 spec/support/lib/bridge_api/http/mock_builders.rb create mode 100644 spec/support/lib/bridge_api/http/mock_formatter.rb diff --git a/app/lib/bridge_api.rb b/app/lib/bridge_api.rb index 5a077ab..269fb0e 100644 --- a/app/lib/bridge_api.rb +++ b/app/lib/bridge_api.rb @@ -2,6 +2,7 @@ require_relative './exceptions/invalid_payload_key' require_relative './exceptions/invalid_environment_variable' +require_relative './exceptions/large_status_code' module BridgeApi end diff --git a/app/lib/bridge_api/http/builder.rb b/app/lib/bridge_api/http/builder.rb index 147c318..3a6df11 100644 --- a/app/lib/bridge_api/http/builder.rb +++ b/app/lib/bridge_api/http/builder.rb @@ -54,7 +54,7 @@ def net_http end def http_request - @request = net_http_request(uri) + @request = net_http_request parse_headers! request.body = parsed_payload.to_json @@ -106,7 +106,7 @@ def scheme # HTTP method. # # @return [Net::HTTP::{http_method}] - def net_http_request(uri) + def net_http_request "Net::HTTP::#{http_method}".constantize.new(uri, 'Content-Type' => 'application/json') end diff --git a/spec/lib/bridge_api/http/request_handler_spec.rb b/spec/lib/bridge_api/http/request_handler_spec.rb index dfe19b8..971a590 100644 --- a/spec/lib/bridge_api/http/request_handler_spec.rb +++ b/spec/lib/bridge_api/http/request_handler_spec.rb @@ -4,12 +4,35 @@ RSpec.describe BridgeApi::Http::Deconstructor do before do - stub_request(:post, 'http://example.com/some_path?query=string') + stub_request(:post, 'https://myfakeoutbound.com:80/') .with(headers: { 'Accept' => '*/*', 'User-Agent' => 'Ruby' }) .to_return(status: 200, body: { data: 'stubbed response' }.to_json, headers: {}) + + stub_request(:post, 'https://myfakeoutbound2.com:80/') + .with(headers: { 'Accept' => '*/*', 'User-Agent' => 'Ruby' }) + .to_return(status: 300, body: { data: 'stubbed response' }.to_json, headers: {}) end + subject do - bridge = create(:bridge, :with_header) - BridgeApi::Http::Deconstructor.new bridge.headers + event = create(:event) + handler = BridgeApi::Http::RequestHandler.new event + handler.http_builder = MockSuccessBuilder.new + handler.formatter = MockFormatter.new + handler + end + + it 'can execute' do + subject.execute + end + + it 'can cleanup' do + subject.cleanup(StandardError) + end + + it 'raises an error for non-200 status codes' do + subject.http_builder = MockFailBuilder.new + expect do + subject.execute + end.to raise_error Sidekiq::LargeStatusCode end end diff --git a/spec/lib/bridge_api/http/spec_helper.rb b/spec/lib/bridge_api/http/spec_helper.rb index 29b83ae..49a19bd 100644 --- a/spec/lib/bridge_api/http/spec_helper.rb +++ b/spec/lib/bridge_api/http/spec_helper.rb @@ -6,3 +6,9 @@ WebMock.disable_net_connect!(allow_localhost: true) require_relative '../../../support/lib/bridge_api/http/mock_deconstructor' +require_relative '../../../support/lib/bridge_api/http/mock_builders' +require_relative '../../../support/lib/bridge_api/http/mock_formatter' +require_relative '../../../support/lib/bridge_api/http/mock_request_handler' + +require_relative '../../../support/lib/bridge_api/syntax_parser/mock_headers_parser' +require_relative '../../../support/lib/bridge_api/syntax_parser/mock_payload_parser' diff --git a/spec/support/lib/bridge_api/http/mock_builders.rb b/spec/support/lib/bridge_api/http/mock_builders.rb new file mode 100644 index 0000000..033c7ca --- /dev/null +++ b/spec/support/lib/bridge_api/http/mock_builders.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class MockSuccessBuilder + def generate + http = Net::HTTP.new(URI('myfakeoutbound.com')) + http.use_ssl = true + [ + http, + Net::HTTP::Post.new(URI('https://myfakeoutbound.com'), 'Content-Type' => 'application/json') + ] + end +end + +class MockFailBuilder + def generate + http = Net::HTTP.new(URI('myfakeoutbound2.com')) + http.use_ssl = true + [ + http, + Net::HTTP::Post.new(URI('https://myfakeoutbound2.com'), 'Content-Type' => 'application/json') + ] + end +end \ No newline at end of file diff --git a/spec/support/lib/bridge_api/http/mock_formatter.rb b/spec/support/lib/bridge_api/http/mock_formatter.rb new file mode 100644 index 0000000..2602651 --- /dev/null +++ b/spec/support/lib/bridge_api/http/mock_formatter.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class MockFormatter + def format!(_event, _req, _res) + true + end + + def format_error!(_event, _req, _res) + true + end +end + +class MockFailFormatter + def format!(_event, _req, _res) + raise StandardError + end + + def cleanup(_event, _request, _error) + true + end +end diff --git a/spec/support/lib/bridge_api/syntax_parser/mock_headers_parser.rb b/spec/support/lib/bridge_api/syntax_parser/mock_headers_parser.rb index e69de29..4c42362 100644 --- a/spec/support/lib/bridge_api/syntax_parser/mock_headers_parser.rb +++ b/spec/support/lib/bridge_api/syntax_parser/mock_headers_parser.rb @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +class MockHeadersParser +end diff --git a/spec/support/lib/bridge_api/syntax_parser/mock_payload_parser.rb b/spec/support/lib/bridge_api/syntax_parser/mock_payload_parser.rb index e69de29..bec1f15 100644 --- a/spec/support/lib/bridge_api/syntax_parser/mock_payload_parser.rb +++ b/spec/support/lib/bridge_api/syntax_parser/mock_payload_parser.rb @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +class MockPayloadParser +end From a553ff1dde04f289cf396254c63e109147b0591a Mon Sep 17 00:00:00 2001 From: = Date: Sat, 21 Nov 2020 13:21:44 -0500 Subject: [PATCH 52/64] Couple rubocop errors --- Gemfile | 2 +- spec/lib/bridge_api/http/builder_spec.rb | 6 ------ spec/support/lib/bridge_api/http/mock_builders.rb | 2 +- spec/support/lib/bridge_api/http/mock_request_handler.rb | 4 ++++ 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Gemfile b/Gemfile index 46b3db2..156aafb 100644 --- a/Gemfile +++ b/Gemfile @@ -46,10 +46,10 @@ group :development do end group :test do + gem 'factory_bot_rails' # Generate test coverage reports gem 'simplecov', require: false gem 'webmock' - gem 'factory_bot_rails' end # Windows does not include zoneinfo files, so bundle the tzinfo-data gem diff --git a/spec/lib/bridge_api/http/builder_spec.rb b/spec/lib/bridge_api/http/builder_spec.rb index f220dd1..59a799c 100644 --- a/spec/lib/bridge_api/http/builder_spec.rb +++ b/spec/lib/bridge_api/http/builder_spec.rb @@ -3,12 +3,6 @@ require 'rails_helper' RSpec.describe BridgeApi::Http::Builder do - before do - end - - after do - end - subject do create(:bridge, :with_env) end diff --git a/spec/support/lib/bridge_api/http/mock_builders.rb b/spec/support/lib/bridge_api/http/mock_builders.rb index 033c7ca..5911b16 100644 --- a/spec/support/lib/bridge_api/http/mock_builders.rb +++ b/spec/support/lib/bridge_api/http/mock_builders.rb @@ -20,4 +20,4 @@ def generate Net::HTTP::Post.new(URI('https://myfakeoutbound2.com'), 'Content-Type' => 'application/json') ] end -end \ No newline at end of file +end diff --git a/spec/support/lib/bridge_api/http/mock_request_handler.rb b/spec/support/lib/bridge_api/http/mock_request_handler.rb index e69de29..d840483 100644 --- a/spec/support/lib/bridge_api/http/mock_request_handler.rb +++ b/spec/support/lib/bridge_api/http/mock_request_handler.rb @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +class MockRequestHandler +end From 481d61aa10039b1363999d70f5921e8010790aa5 Mon Sep 17 00:00:00 2001 From: = Date: Sat, 21 Nov 2020 13:42:28 -0500 Subject: [PATCH 53/64] Builder spec --- spec/lib/bridge_api/http/builder_spec.rb | 22 +++++++++++++++---- .../bridge_api/http/mock_request_handler.rb | 6 +++++ .../syntax_parser/mock_headers_parser.rb | 12 ++++++++++ .../syntax_parser/mock_payload_parser.rb | 3 +++ 4 files changed, 39 insertions(+), 4 deletions(-) diff --git a/spec/lib/bridge_api/http/builder_spec.rb b/spec/lib/bridge_api/http/builder_spec.rb index 59a799c..a2b6ae7 100644 --- a/spec/lib/bridge_api/http/builder_spec.rb +++ b/spec/lib/bridge_api/http/builder_spec.rb @@ -1,13 +1,27 @@ # frozen_string_literal: true -require 'rails_helper' +require_relative './spec_helper' RSpec.describe BridgeApi::Http::Builder do subject do - create(:bridge, :with_env) + event = create(:event) + BridgeApi::Http::Builder.new( + MockRequestHandler.new(event), + MockPayloadParser.new, + MockHeadersParser.new + ) end - it 'is valid when passed valid info' do - expect(subject).to be_valid + it 'can generate http objects' do + http, req = subject.generate + + expect(http.use_ssl?).to be true + expect(http.port).to eq 443 + + expect(req['Content-Type']).to eq 'application/json' + expect(req['header_1']).to eq 'value_1' + expect(req['header_2']).to eq 'value_2' + expect(req.uri.to_s).to eq 'https://myfakeoutbound.com' + expect(req.body).to eq '{"data":"hello world"}' end end diff --git a/spec/support/lib/bridge_api/http/mock_request_handler.rb b/spec/support/lib/bridge_api/http/mock_request_handler.rb index d840483..e171fbc 100644 --- a/spec/support/lib/bridge_api/http/mock_request_handler.rb +++ b/spec/support/lib/bridge_api/http/mock_request_handler.rb @@ -1,4 +1,10 @@ # frozen_string_literal: true class MockRequestHandler + attr_reader :event, :bridge + + def initialize(event) + @event = event + @bridge = event.bridge + end end diff --git a/spec/support/lib/bridge_api/syntax_parser/mock_headers_parser.rb b/spec/support/lib/bridge_api/syntax_parser/mock_headers_parser.rb index 4c42362..9c8f9fd 100644 --- a/spec/support/lib/bridge_api/syntax_parser/mock_headers_parser.rb +++ b/spec/support/lib/bridge_api/syntax_parser/mock_headers_parser.rb @@ -1,4 +1,16 @@ # frozen_string_literal: true class MockHeadersParser + def parse(_headers, &block) + [ + { + key: 'header_1', + value: 'value_1' + }, + { + key: 'header_2', + value: 'value_2' + } + ].each { |header| block.call header[:key], header[:value] } + end end diff --git a/spec/support/lib/bridge_api/syntax_parser/mock_payload_parser.rb b/spec/support/lib/bridge_api/syntax_parser/mock_payload_parser.rb index bec1f15..5c86658 100644 --- a/spec/support/lib/bridge_api/syntax_parser/mock_payload_parser.rb +++ b/spec/support/lib/bridge_api/syntax_parser/mock_payload_parser.rb @@ -1,4 +1,7 @@ # frozen_string_literal: true class MockPayloadParser + def parse(_payload, _user_data) + { data: 'hello world' } + end end From 45783c1e35d66ec5584def50f18dabf52ca7865e Mon Sep 17 00:00:00 2001 From: = Date: Sat, 21 Nov 2020 13:46:14 -0500 Subject: [PATCH 54/64] Remove unused event helper --- spec/support/event_helper.rb | 5 ----- spec/support/main_helper.rb | 4 ---- 2 files changed, 9 deletions(-) delete mode 100644 spec/support/event_helper.rb diff --git a/spec/support/event_helper.rb b/spec/support/event_helper.rb deleted file mode 100644 index d31fdcf..0000000 --- a/spec/support/event_helper.rb +++ /dev/null @@ -1,5 +0,0 @@ -# frozen_string_literal: true - -module EventHelper - -end \ No newline at end of file diff --git a/spec/support/main_helper.rb b/spec/support/main_helper.rb index eb8aa93..73d23b4 100644 --- a/spec/support/main_helper.rb +++ b/spec/support/main_helper.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require_relative './event_helper' - def random_email local = [] 8.times { |_| local.push(('a'..'z').to_a.sample) } @@ -10,8 +8,6 @@ def random_email end module MainHelper - include EventHelper - def create_user @current_user = User.create(email: random_email, password: 'password', notifications: false) @token = JsonWebToken.encode(user_id: @current_user.id) From 445154e78df347bb54f979b89b96f4aee258d1ba Mon Sep 17 00:00:00 2001 From: = Date: Sat, 21 Nov 2020 14:55:18 -0500 Subject: [PATCH 55/64] Fix EventsController --- app/controllers/events_controller.rb | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index 6db99fd..3614e7e 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class EventsController < ApplicationController - # before_action :authorize_request + before_action :authorize_request, except: :create before_action :set_events, only: :index before_action :set_event, only: %i[show destroy] @@ -28,15 +28,14 @@ def destroy def create event = create_event_object(create_data_object, find_bridge) event.save! - # EventWorker.perform_async(event.id) - EventWorker.new.perform(event.id) + EventWorker.perform_async(event.id) render json: {}, status: 202 # Accepted (asynchronous processing) - # rescue ActiveRecord::RecordNotFound - # render json: { error: 'a bridge by that id was not found' }, status: 400 - # rescue ActiveRecord::RecordInvalid - # render json: { error: 'payload, bridge_id, or urls were invalid' }, status: 400 # Bad Request - # rescue ActiveRecord::NotNullViolation - # render json: { error: 'payload, bridge_id, or urls fields were not submitted' }, status: 400 # Bad Request + rescue ActiveRecord::RecordNotFound + render json: { error: 'a bridge by that id was not found' }, status: 400 + rescue ActiveRecord::RecordInvalid + render json: { error: 'payload, bridge_id, or urls were invalid' }, status: 400 # Bad Request + rescue ActiveRecord::NotNullViolation + render json: { error: 'payload, bridge_id, or urls fields were not submitted' }, status: 400 # Bad Request end private From 969b40d5ca7ff9e842a260618f68ed38e20f7254 Mon Sep 17 00:00:00 2001 From: = Date: Sun, 22 Nov 2020 12:03:29 -0500 Subject: [PATCH 56/64] Event Specs --- app/controllers/events_controller.rb | 66 +++++++++++---------- spec/requests/events_request.spec.rb | 86 ++++++++++++++++++++++++++-- spec/workers/event_worker_spec.rb | 19 ++++++ spec/workers/spec_helper.rb | 14 +++++ 4 files changed, 146 insertions(+), 39 deletions(-) create mode 100644 spec/workers/event_worker_spec.rb create mode 100644 spec/workers/spec_helper.rb diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index 3614e7e..bdda4a4 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -2,40 +2,38 @@ class EventsController < ApplicationController before_action :authorize_request, except: :create - before_action :set_events, only: :index before_action :set_event, only: %i[show destroy] def index - render json: { events: @events }, status: 200 - rescue ActiveRecord::RecordInvalid - render json: { error: 'neither event_id nor bridge_id were valid' }, status: 400 # Bad Request - rescue ActiveRecord::RecordNotFound - render json: { error: 'events matching that id were not found' }, status: 400 - rescue ActiveRecord::NotNullViolation - render json: { error: 'neither event_id nor bridge_id were not submitted' }, status: 400 # Bad Request + if fetch_events.empty? + render json: { error: 'invalid parameters' }, status: 400 # Bad Request + else + render json: { events: @events }, status: 200 + end end def show - render json: @event, status: 200 - rescue ActiveRecord::RecordNotFound - render json: { error: 'an event by that id was not found' }, status: 400 + if @event + render json: { event: @event }, status: 200 + else + render json: { error: 'an event by that id was not found' }, status: 400 + end end def destroy - @event.destroy + return render json: {}, status: 204 if @event&.destroy + + render json: { error: 'an event by that id was not found' }, status: 400 end def create event = create_event_object(create_data_object, find_bridge) - event.save! - EventWorker.perform_async(event.id) - render json: {}, status: 202 # Accepted (asynchronous processing) - rescue ActiveRecord::RecordNotFound - render json: { error: 'a bridge by that id was not found' }, status: 400 - rescue ActiveRecord::RecordInvalid - render json: { error: 'payload, bridge_id, or urls were invalid' }, status: 400 # Bad Request - rescue ActiveRecord::NotNullViolation - render json: { error: 'payload, bridge_id, or urls fields were not submitted' }, status: 400 # Bad Request + if event.save + EventWorker.perform_async(event.id) + render json: {}, status: 202 # Accepted + else + render json: { error: 'Invalid parameters' }, status: 400 # Bad Request + end end private @@ -44,6 +42,16 @@ def event_params params.permit(:id, :bridge_id, :event_id, :test) end + def fetch_events + @events = if event_params[:bridge_id] + Event.where(bridge_id: event_params[:bridge_id]).order(completed_at: :desc).limit(100) + elsif event_params[:event_id] + Event.where(bridge_id: find_event&.bridge_id).order(completed_at: :desc).limit(100) + else + [] + end + end + def create_data_object { 'inbound' => { @@ -59,30 +67,20 @@ def create_data_object def create_event_object(data, bridge) Event.new( data: data.to_json, - bridge_id: bridge.id, + bridge_id: bridge&.id, test: event_params[:test] || false ) end - def set_events - @events = if event_params[:bridge_id] - Event.where(bridge_id: event_params[:bridge_id]).order(completed_at: :desc).limit(100) - elsif event_params[:event_id] - Event.where(bridge_id: find_event.bridge_id).order(completed_at: :desc).limit(100) - else - raise ActiveRecord::NotNullViolation - end - end - def set_event @event = find_event end def find_event - Event.find(event_params[:event_id]) + Event.find_by(id: event_params[:event_id]) end def find_bridge - Bridge.find(event_params[:bridge_id]) + Bridge.find_by(id: event_params[:bridge_id]) end end diff --git a/spec/requests/events_request.spec.rb b/spec/requests/events_request.spec.rb index 38f5cd6..9d58ad6 100644 --- a/spec/requests/events_request.spec.rb +++ b/spec/requests/events_request.spec.rb @@ -1,11 +1,87 @@ # frozen_string_literal: true -RSpec.describe 'Events', type: :request do - before(:context) do - create_event +require 'rails_helper' + +RSpec.describe 'EventsController', type: :request do + before do + @event = create(:event) + @bridge = @event.bridge + @user = @bridge.user + @token = JsonWebToken.encode(user_id: @user.id) + end + + describe 'GET index' do + it 'returns 200 with bridge_id' do + get '/events', headers: authenticated_token, params: { bridge_id: @bridge.id } + expect(response).to have_http_status(:ok) + end + + it 'returns 200 with event_id' do + get '/events', headers: authenticated_token, params: { event_id: @event.id } + expect(response).to have_http_status(:ok) + end + + it 'returns 400 with no params' do + get '/events', headers: authenticated_token + expect(response).to have_http_status(400) + end + + it 'returns 400 with invalid IDs' do + get '/events', headers: authenticated_token, params: { event_id: '128371283' } + expect(response).to have_http_status(400) + + get '/events', headers: authenticated_token, params: { bridge_id: '128371283' } + expect(response).to have_http_status(400) + end + + it 'requires JWT' do + get '/events', params: { event_id: @event.id } + expect(response).to have_http_status(401) + end + end + + describe 'GET show' do + it 'returns 200 with bridge_id' do + get "/events/#{@event.id}", headers: authenticated_token, params: { event_id: @event.id } + expect(response).to have_http_status(:ok) + end + + it 'returns 400 with invalid IDs' do + get '/events/128371283', headers: authenticated_token, params: { event_id: '128371283' } + expect(response).to have_http_status(400) + + get '/events/128371283', headers: authenticated_token, params: { bridge_id: '128371283' } + expect(response).to have_http_status(400) + end + + it 'requires JWT' do + get "/events/#{@event.id}", params: { event_id: @event.id } + expect(response).to have_http_status(401) + end + end + + describe 'POST destroy' do + it 'returns 204' do + delete "/events/#{@event.id}", headers: authenticated_token, params: { event_id: @event.id } + expect(response).to have_http_status(204) + end + + it 'returns 400 with invalid IDs' do + delete '/events/128371283', headers: authenticated_token, params: { event_id: '128371283' } + expect(response).to have_http_status(400) + + delete '/events/128371283', headers: authenticated_token, params: { bridge_id: '128371283' } + expect(response).to have_http_status(400) + end + + it 'requires JWT' do + delete "/events/#{@event.id}", params: { event_id: @event.id } + expect(response).to have_http_status(401) + end end - after(:context) do - destroy_event + describe 'POST create' do + pending 'creates a job' + pending 'returns 400 with invalid IDs' end end diff --git a/spec/workers/event_worker_spec.rb b/spec/workers/event_worker_spec.rb new file mode 100644 index 0000000..c203b34 --- /dev/null +++ b/spec/workers/event_worker_spec.rb @@ -0,0 +1,19 @@ +require_relative './spec_helper' + +RSpec.describe EventWorker, type: :worker do + let(:time) { (Time.zone.today + 6.hours).to_datetime } + let(:scheduled_job) { described_class.perform_at(time, 'Awesome', true) } + + # it 'Event jobs are enqueued in the scheduled queue' do + # described_class.perform_async + # assert_equal :scheduled, described_class.queue + # end + + # it 'goes into the jobs array for testing environment' do + # expect do + # described_class.perform_async + # end.to change(described_class.jobs, :size).by(1) + # end + + pending 'EventWorker Specs' +end diff --git a/spec/workers/spec_helper.rb b/spec/workers/spec_helper.rb new file mode 100644 index 0000000..4030e1c --- /dev/null +++ b/spec/workers/spec_helper.rb @@ -0,0 +1,14 @@ +require 'rails_helper' +require 'sidekiq/testing' + +Sidekiq::Testing.fake! + +# TODO +# RSpec::Sidekiq.configure do |config| +# # Clears all job queues before each example +# config.clear_all_enqueued_jobs = true # default => true +# # Whether to use terminal colours when outputting messages +# config.enable_terminal_colours = true # default => true +# # Warn when jobs are not enqueued to Redis but to a job array +# config.warn_when_jobs_not_processed_by_sidekiq = true # default => true +# end From a62901250e6573a7615a000866cb9b2791c56538 Mon Sep 17 00:00:00 2001 From: = Date: Mon, 23 Nov 2020 16:19:21 -0500 Subject: [PATCH 57/64] Fast forward fixes --- app/models/environment_variable.rb | 2 +- app/workers/event_worker.rb | 2 +- db/schema.rb | 4 ---- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/app/models/environment_variable.rb b/app/models/environment_variable.rb index 781fb97..4833043 100644 --- a/app/models/environment_variable.rb +++ b/app/models/environment_variable.rb @@ -11,7 +11,7 @@ class EnvironmentVariable < ApplicationRecord delegate :encrypt_and_sign, :decrypt_and_verify, to: :encryptor KEY = ActiveSupport::KeyGenerator.new( - ENV.fetch('SECRET_KEY_BASE') || Rails.application.secrets.secret_key_base + ENV['SECRET_KEY_BASE'] || Rails.application.secrets.secret_key_base ).generate_key( ENV.fetch('ENCRYPTION_KEY_SALT'), ActiveSupport::MessageEncryptor.key_len diff --git a/app/workers/event_worker.rb b/app/workers/event_worker.rb index cfbd93c..ab18898 100644 --- a/app/workers/event_worker.rb +++ b/app/workers/event_worker.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require 'net/http' -require_relative '../lib/exceptions/large_status_code' +require_relative '../lib/sidekiq/large_status_code' class EventWorker include Sidekiq::Worker diff --git a/db/schema.rb b/db/schema.rb index e25a394..bb2bff7 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,11 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -<<<<<<< HEAD ActiveRecord::Schema.define(version: 2020_11_21_142638) do -======= -ActiveRecord::Schema.define(version: 2020_11_20_084233) do ->>>>>>> 37221aa57aa7ae18a799f9d09a8f33a8acfc006e # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" From d6d2d4b118c24bbabe71b0056636504941be952f Mon Sep 17 00:00:00 2001 From: = Date: Mon, 23 Nov 2020 16:48:02 -0500 Subject: [PATCH 58/64] EventController#create spec --- app/controllers/events_controller.rb | 10 ++++++---- app/models/event.rb | 4 ++-- config/routes.rb | 3 ++- .../{events_request.spec.rb => events_request_spec.rb} | 7 ++++++- 4 files changed, 16 insertions(+), 8 deletions(-) rename spec/requests/{events_request.spec.rb => events_request_spec.rb} (91%) diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index bdda4a4..230e39f 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -27,13 +27,15 @@ def destroy end def create - event = create_event_object(create_data_object, find_bridge) + event = create_event(find_bridge) if event.save EventWorker.perform_async(event.id) render json: {}, status: 202 # Accepted else render json: { error: 'Invalid parameters' }, status: 400 # Bad Request end + rescue JSON::ParserError + render json: { error: 'Invalid request. Payload must be in JSON' }, status: 400 # Bad Request end private @@ -48,11 +50,11 @@ def fetch_events elsif event_params[:event_id] Event.where(bridge_id: find_event&.bridge_id).order(completed_at: :desc).limit(100) else - [] + [] # Prevent nil end end - def create_data_object + def data { 'inbound' => { 'payload' => JSON.parse(request.body.read), @@ -64,7 +66,7 @@ def create_data_object } end - def create_event_object(data, bridge) + def create_event(bridge) Event.new( data: data.to_json, bridge_id: bridge&.id, diff --git a/app/models/event.rb b/app/models/event.rb index b3f30e4..b14188e 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -84,8 +84,8 @@ def inbound_payload # TODO: Pass in bridge (or urls) to prevent db hit def set_urls - self.inbound_url = bridge.inbound_url - self.outbound_url = bridge.outbound_url + self.inbound_url = bridge&.inbound_url + self.outbound_url = bridge&.outbound_url end def data_json_object diff --git a/config/routes.rb b/config/routes.rb index cf9e880..abbe1af 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -11,7 +11,8 @@ resources :headers, :environment_variables, only: :destroy post 'login', to: 'sessions#create' - post 'events', to: 'events#create' + + post 'events/:bridge_id', to: 'events#create' get 'events', to: 'events#index' get 'events/:event_id', to: 'events#show' delete 'events/:event_id', to: 'events#destroy' diff --git a/spec/requests/events_request.spec.rb b/spec/requests/events_request_spec.rb similarity index 91% rename from spec/requests/events_request.spec.rb rename to spec/requests/events_request_spec.rb index 9d58ad6..e5a19eb 100644 --- a/spec/requests/events_request.spec.rb +++ b/spec/requests/events_request_spec.rb @@ -82,6 +82,11 @@ describe 'POST create' do pending 'creates a job' - pending 'returns 400 with invalid IDs' + + it 'returns 400 with invalid IDs' do + headers = { 'CONTENT_TYPE' => 'application/json' } + post '/events/128371283', params: '{ "data": { "hello": "world" } }', headers: headers + expect(response).to have_http_status(400) + end end end From 05ce8978ccdaaac824479d477bbdd009606cee63 Mon Sep 17 00:00:00 2001 From: = Date: Mon, 23 Nov 2020 16:49:20 -0500 Subject: [PATCH 59/64] Remove test env in http builder as it wasn't needed --- app/lib/bridge_api/http/builder.rb | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/app/lib/bridge_api/http/builder.rb b/app/lib/bridge_api/http/builder.rb index 3a6df11..f774b21 100644 --- a/app/lib/bridge_api/http/builder.rb +++ b/app/lib/bridge_api/http/builder.rb @@ -90,7 +90,7 @@ def parsed_payload # # @return [JSON] def unparsed_payload - @unparsed_payload ||= data[test_env? ? 'test_payload' : 'payload'] + @unparsed_payload ||= data['payload'] end # Ensures the scheme used is using TSL @@ -129,13 +129,6 @@ def outbound_url def headers @headers ||= bridge.headers end - - # Checks if this request is a test event - # - # @return [Bool] - def test_env? - event.test - end end end end From 2c79927937213508192a3e510defe6d8a6ff2792 Mon Sep 17 00:00:00 2001 From: = Date: Mon, 23 Nov 2020 17:23:31 -0500 Subject: [PATCH 60/64] DI for EventWorker --- app/workers/event_worker.rb | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/app/workers/event_worker.rb b/app/workers/event_worker.rb index ab18898..16c204b 100644 --- a/app/workers/event_worker.rb +++ b/app/workers/event_worker.rb @@ -6,11 +6,12 @@ class EventWorker include Sidekiq::Worker + attr_writer :request_handler + attr_accessor :retry_count def perform(event_id, _retries = 0) - event = Event.includes(:bridge).find(event_id) - request_handler = ::BridgeApi::Http::RequestHandler.new(event) + @event = Event.includes(:bridge).find(event_id) request_handler.execute event.complete! rescue StandardError => e @@ -22,4 +23,12 @@ def perform(event_id, _retries = 0) # should we really tell users "StandardError" if their service replied with 404? raise StandardError end + + private + + attr_reader :event + + def request_handler + @request_handler ||= ::BridgeApi::Http::RequestHandler.new(event) + end end From 9d7dbc1e3723788e04af7de914dd73d81f109727 Mon Sep 17 00:00:00 2001 From: = Date: Mon, 23 Nov 2020 17:25:57 -0500 Subject: [PATCH 61/64] Pending event_worker_specs --- spec/workers/event_worker_spec.rb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/spec/workers/event_worker_spec.rb b/spec/workers/event_worker_spec.rb index c203b34..bd46208 100644 --- a/spec/workers/event_worker_spec.rb +++ b/spec/workers/event_worker_spec.rb @@ -1,9 +1,6 @@ require_relative './spec_helper' RSpec.describe EventWorker, type: :worker do - let(:time) { (Time.zone.today + 6.hours).to_datetime } - let(:scheduled_job) { described_class.perform_at(time, 'Awesome', true) } - # it 'Event jobs are enqueued in the scheduled queue' do # described_class.perform_async # assert_equal :scheduled, described_class.queue @@ -15,5 +12,7 @@ # end.to change(described_class.jobs, :size).by(1) # end - pending 'EventWorker Specs' + pending 'can process an event' + pending 'can retry 3 times' + pending 'can clean up when errors are raised' end From 719abfcf93c747b1c0cebe6fce0eae2a163ce292 Mon Sep 17 00:00:00 2001 From: = Date: Mon, 23 Nov 2020 17:29:41 -0500 Subject: [PATCH 62/64] Remove retry arg for EventWorker --- app/workers/event_worker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/event_worker.rb b/app/workers/event_worker.rb index 16c204b..e72c554 100644 --- a/app/workers/event_worker.rb +++ b/app/workers/event_worker.rb @@ -10,7 +10,7 @@ class EventWorker attr_accessor :retry_count - def perform(event_id, _retries = 0) + def perform(event_id) @event = Event.includes(:bridge).find(event_id) request_handler.execute event.complete! From ca988823839e425c1b827444e0ed1bb05e8c9923 Mon Sep 17 00:00:00 2001 From: = Date: Mon, 23 Nov 2020 17:35:35 -0500 Subject: [PATCH 63/64] Small doc in EventWorker --- app/workers/event_worker.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/workers/event_worker.rb b/app/workers/event_worker.rb index e72c554..bd653f8 100644 --- a/app/workers/event_worker.rb +++ b/app/workers/event_worker.rb @@ -15,6 +15,8 @@ def perform(event_id) request_handler.execute event.complete! rescue StandardError => e + # We can skip clean up on error `Sidekiq::LargeStatusCode` because we did + # recieve a response and it was saved properly. request_handler.cleanup(e) unless e.instance_of? Sidekiq::LargeStatusCode return event.complete! if retry_count&.>= bridge.retries # TODO: Off by one? From 865b58e56120285995743fbc00c61ee6ba2eeb86 Mon Sep 17 00:00:00 2001 From: = Date: Tue, 24 Nov 2020 07:54:34 -0500 Subject: [PATCH 64/64] Rubocop --- spec/support/main_helper.rb | 2 +- spec/workers/event_worker_spec.rb | 2 ++ spec/workers/spec_helper.rb | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/spec/support/main_helper.rb b/spec/support/main_helper.rb index 73d23b4..468404d 100644 --- a/spec/support/main_helper.rb +++ b/spec/support/main_helper.rb @@ -4,7 +4,7 @@ def random_email local = [] 8.times { |_| local.push(('a'..'z').to_a.sample) } domain = ['aol.com', 'nsa.gov', 'jubii.dk', 'kreml.ru', 'hotmail.com', 'netscape.com'][rand(6)] - "#{local.join('')}@#{domain}" + "#{local.join}@#{domain}" end module MainHelper diff --git a/spec/workers/event_worker_spec.rb b/spec/workers/event_worker_spec.rb index bd46208..15c0052 100644 --- a/spec/workers/event_worker_spec.rb +++ b/spec/workers/event_worker_spec.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require_relative './spec_helper' RSpec.describe EventWorker, type: :worker do diff --git a/spec/workers/spec_helper.rb b/spec/workers/spec_helper.rb index 4030e1c..b55e6a5 100644 --- a/spec/workers/spec_helper.rb +++ b/spec/workers/spec_helper.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'rails_helper' require 'sidekiq/testing'