From bb4387d012a1ce867dbc50661776f3f25dcadc6c Mon Sep 17 00:00:00 2001 From: Chen Nakar Date: Tue, 15 Sep 2020 09:24:30 +0300 Subject: [PATCH 1/2] first party --- lib/perimeter_x.rb | 62 +++++++-- lib/perimeterx/configuration.rb | 7 +- .../internal/first_party/px_first_party.rb | 127 ++++++++++++++++++ .../internal/perimeter_x_context.rb | 57 ++++---- lib/perimeterx/utils/px_constants.rb | 4 +- lib/perimeterx/utils/px_http_client.rb | 57 ++++++++ lib/perimeterx/utils/px_template_factory.rb | 4 +- readme.md | 21 +++ 8 files changed, 298 insertions(+), 41 deletions(-) create mode 100644 lib/perimeterx/internal/first_party/px_first_party.rb diff --git a/lib/perimeter_x.rb b/lib/perimeter_x.rb index 0ab5cbb..5c02a2e 100644 --- a/lib/perimeter_x.rb +++ b/lib/perimeter_x.rb @@ -12,13 +12,23 @@ require 'perimeterx/internal/validators/perimeter_x_s2s_validator' require 'perimeterx/internal/validators/perimeter_x_cookie_validator' require 'perimeterx/internal/exceptions/px_config_exception' +require 'perimeterx/internal/first_party/px_first_party' module PxModule # Module expose API def px_verify_request(request_config={}) begin px_instance = PerimeterX.new(request_config) - px_ctx = px_instance.verify(request.env) + req = ActionDispatch::Request.new(request.env) + + # handle first party requests + if px_instance.first_party.is_first_party_request(req) + render_first_party_response(req, px_instance) + return true + end + + # verify request + px_ctx = px_instance.verify(req) px_config = px_instance.px_config msg_title = 'PxModule[px_verify_request]' @@ -46,12 +56,21 @@ def px_verify_request(request_config={}) end is_mobile = px_ctx.context[:cookie_origin] == 'header' ? '1' : '0' - action = px_ctx.context[:block_action][0,1] + action = px_ctx.context[:block_action][0,1] - px_template_object = { + if px_config[:first_party_enabled] + px_template_object = { + js_client_src: "/#{px_config[:app_id][2..-1]}/init.js", + block_script: "/#{px_config[:app_id][2..-1]}/captcha/#{px_config[:app_id]}/captcha.js?a=#{action}&u=#{px_ctx.context[:uuid]}&v=#{px_ctx.context[:vid]}&m=#{is_mobile}", + host_url: "/#{px_config[:app_id][2..-1]}/xhr" + } + else + px_template_object = { + js_client_src: "//#{PxModule::CLIENT_HOST}/#{px_config[:app_id]}/main.min.js", block_script: "//#{PxModule::CAPTCHA_HOST}/#{px_config[:app_id]}/captcha.js?a=#{action}&u=#{px_ctx.context[:uuid]}&v=#{px_ctx.context[:vid]}&m=#{is_mobile}", - js_client_src: "//#{PxModule::CLIENT_HOST}/#{px_config[:app_id]}/main.min.js" - } + host_url: "https://collector-#{px_config[:app_id]}.perimeterx.net" + } + end html = PxTemplateFactory.get_template(px_ctx, px_config, px_template_object) @@ -68,7 +87,7 @@ def px_verify_request(request_config={}) hash_json = { :appId => px_config[:app_id], :jsClientSrc => px_template_object[:js_client_src], - :firstPartyEnabled => false, + :firstPartyEnabled => px_ctx.context[:first_party_enabled], :uuid => px_ctx.context[:uuid], :vid => px_ctx.context[:vid], :hostUrl => "https://collector-#{px_config[:app_id]}.perimeterx.net", @@ -110,6 +129,29 @@ def px_verify_request(request_config={}) end end + def render_first_party_response(req, px_instance) + fp = px_instance.first_party + px_config = px_instance.px_config + + if px_config[:first_party_enabled] + # first party enabled - proxy response + fp_response = fp.get_first_party_response(req) + response.status = fp_response.code + fp_response.to_hash.each do |header_name, header_value_arr| + if header_name!="content-length" + response.headers[header_name] = header_value_arr[0] + end + end + res_type = fp.get_response_content_type(req) + render res_type => fp_response.body + else + # first party disabled - return empty response + response.status = "200" + res_type = fp.get_response_content_type(req) + render res_type => "" + end + end + def self.configure(basic_config) PerimeterX.set_basic_config(basic_config) end @@ -119,6 +161,7 @@ def self.configure(basic_config) class PerimeterX attr_reader :px_config + attr_reader :first_party attr_accessor :px_http_client attr_accessor :px_activity_client @@ -128,7 +171,7 @@ def self.set_basic_config(basic_config) end #Instance Methods - def verify(env) + def verify(req) begin # check module_enabled @@ -137,13 +180,11 @@ def verify(env) @logger.warn('Module is disabled') return nil end - - req = ActionDispatch::Request.new(env) # filter whitelist routes url_path = URI.parse(req.original_url).path if url_path && !url_path.empty? - if check_whitelist_routes(px_config[:whitelist_routes], url_path) + if check_whitelist_routes(px_config[:whitelist_routes], url_path) @logger.debug("PerimeterX[pxVerify]: whitelist route: #{url_path}") return nil end @@ -176,6 +217,7 @@ def initialize(request_config) @px_http_client = PxHttpClient.new(@px_config) @px_activity_client = PerimeterxActivitiesClient.new(@px_config, @px_http_client) + @first_party = FirstPartyManager.new(@px_config, @px_http_client, @logger) @px_cookie_validator = PerimeterxCookieValidator.new(@px_config) @px_s2s_validator = PerimeterxS2SValidator.new(@px_config, @px_http_client) diff --git a/lib/perimeterx/configuration.rb b/lib/perimeterx/configuration.rb index 6ad29a4..861f553 100644 --- a/lib/perimeterx/configuration.rb +++ b/lib/perimeterx/configuration.rb @@ -30,7 +30,8 @@ class Configuration :ip_headers => [], :ip_header_function => nil, :bypass_monitor_header => nil, - :risk_cookie_max_iterations => 5000 + :risk_cookie_max_iterations => 5000, + :first_party_enabled => true } CONFIG_SCHEMA = { @@ -60,7 +61,9 @@ class Configuration :custom_logo => {types: [String], required: false}, :css_ref => {types: [String], required: false}, :js_ref => {types: [String], required: false}, - :custom_uri => {types: [Proc], required: false} + :custom_uri => {types: [Proc], required: false}, + :first_party_enabled => {types: [FalseClass, TrueClass], required: false} + } def self.set_basic_config(basic_config) diff --git a/lib/perimeterx/internal/first_party/px_first_party.rb b/lib/perimeterx/internal/first_party/px_first_party.rb new file mode 100644 index 0000000..02718e9 --- /dev/null +++ b/lib/perimeterx/internal/first_party/px_first_party.rb @@ -0,0 +1,127 @@ +require 'perimeterx/internal/perimeter_x_context' +module PxModule + class FirstPartyManager + def initialize(px_config, px_http_client, logger) + @px_config = px_config + @app_id = px_config[:app_id] + @px_http_client = px_http_client + @logger = logger + @from = [ + "/#{@app_id[2..-1]}/init.js", + "/#{@app_id[2..-1]}/captcha", + "/#{@app_id[2..-1]}/xhr" + ] + end + + def get_first_party_response(req) + uri = URI.parse(req.original_url) + url_path = uri.path + + headers = extract_headers(req) + headers["x-px-first-party"] = "1" + headers["x-px-enforcer-true-ip"] = PerimeterXContext.extract_ip(req, @px_config) + + if url_path.start_with?(@from[0]) + return get_client(req, uri, headers) + elsif url_path.start_with?(@from[1]) + return get_captcha(req, uri, headers) + elsif url_path.start_with?(@from[2]) + return send_xhr(req, uri, headers) + else + return nil + end + end + + def get_client(req, uri, headers) + @logger.debug("FirstPartyManager[get_client]") + + # define host + headers["host"] = PxModule::CLIENT_HOST + + # define request url + url = "#{uri.scheme}://#{PxModule::CLIENT_HOST}/#{@app_id}/main.min.js" + + # send request + return @px_http_client.get(url, headers) + end + + def get_captcha(req, uri, headers) + @logger.debug("FirstPartyManager[get_captcha]") + + # define host + headers["host"] = PxModule::CAPTCHA_HOST + + # define request url + path_and_query = uri.request_uri + uri_suffix = path_and_query.sub "/#{@app_id[2..-1]}/captcha", "" + url = "#{uri.scheme}://#{PxModule::CAPTCHA_HOST}#{uri_suffix}" + + # send request + return @px_http_client.get(url, headers) + end + + def send_xhr(req, uri, headers) + @logger.debug("FirstPartyManager[send_xhr]") + + # handle vid cookies + if !req.cookies.nil? + if req.cookies.key?("_pxvid") + vid = PerimeterXContext.force_utf8(req.cookies["_pxvid"]) + if headers.key?('cookie') + headers['cookie'] += "; pxvid=#{vid}"; + else + headers['cookie'] = "pxvid=#{vid}"; + end + end + end + + # define host + headers["host"] = "collector-#{@app_id.downcase}.perimeterx.net" + + # define request url + path_and_query = uri.request_uri + path_suffix = path_and_query.sub "/#{@app_id[2..-1]}/xhr", "" + url = "#{uri.scheme}://collector-#{@app_id.downcase}.perimeterx.net#{path_suffix}" + + # send request + return @px_http_client.post_xhr(url, req.body.string, headers) + end + + def extract_headers(req) + headers = Hash.new + req.headers.each do |k, v| + if (k.start_with? 'HTTP_') && (!@px_config[:sensitive_headers].include? k) + header = k.to_s.gsub('HTTP_', '') + header = header.gsub('_', '-').downcase + headers[header] = PerimeterXContext.force_utf8(v) + end + end + return headers + end + + # -1 - not first party request + # 0 - /init.js + # 1 - /captcha + # 2 - /xhr + def get_first_party_request_type(req) + url_path = URI.parse(req.original_url).path + @from.each_with_index do |val,index| + if url_path.start_with?(val) + return index + end + end + return -1 + end + + def is_first_party_request(req) + if get_first_party_request_type(req) != -1 + return true + end + return false + end + + def get_response_content_type(req) + return get_first_party_request_type(req) == 2 ? :json : :js + end + end +end \ No newline at end of file diff --git a/lib/perimeterx/internal/perimeter_x_context.rb b/lib/perimeterx/internal/perimeter_x_context.rb index a5fd8d6..994b6d4 100644 --- a/lib/perimeterx/internal/perimeter_x_context.rb +++ b/lib/perimeterx/internal/perimeter_x_context.rb @@ -6,6 +6,28 @@ class PerimeterXContext attr_accessor :context attr_accessor :px_config + + # class methods + + def self.extract_ip(req, px_config) + # Get IP from header/custom function + if px_config[:ip_headers].length() > 0 + px_config[:ip_headers].each do |ip_header| + if req.headers[ip_header] + return PerimeterXContext.force_utf8(req.headers[ip_header]) + end + end + elsif px_config[:ip_header_function] != nil + return px_config[:ip_header_function].call(req) + end + return req.ip + end + + def self.force_utf8(str) + return str.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '') + end + + # instance methods def initialize(px_config, req) @logger = px_config[:logger] @@ -16,33 +38,22 @@ def initialize(px_config, req) @context[:headers] = Hash.new @context[:cookie_origin] = 'cookie' @context[:made_s2s_risk_api_call] = false + @context[:first_party_enabled] = px_config[:first_party_enabled] + cookies = req.cookies - # Get IP from header/custom function - if px_config[:ip_headers].length() > 0 - px_config[:ip_headers].each do |ip_header| - if req.headers[ip_header] - @context[:ip] = force_utf8(req.headers[ip_header]) - end - end - elsif px_config[:ip_header_function] != nil - @context[:ip] = px_config[:ip_header_function].call(req) - end - - if @context[:ip] == nil - @context[:ip] = req.ip - end + @context[:ip] = PerimeterXContext.extract_ip(req, px_config) # Get token from header if req.headers[PxModule::TOKEN_HEADER] @context[:cookie_origin] = 'header' - token = force_utf8(req.headers[PxModule::TOKEN_HEADER]) + token = PerimeterXContext.force_utf8(req.headers[PxModule::TOKEN_HEADER]) if token.match(PxModule::MOBILE_TOKEN_V3_REGEX) @context[:px_cookie][:v3] = token[2..-1] elsif token.match(PxModule::MOBILE_ERROR_REGEX) @context[:mobile_error] = token if req.headers[PxModule::ORIGINAL_TOKEN_HEADER] - token = force_utf8(req.headers[PxModule::ORIGINAL_TOKEN_HEADER]) + token = PerimeterXContext.force_utf8(req.headers[PxModule::ORIGINAL_TOKEN_HEADER]) if token.match(PxModule::MOBILE_TOKEN_V3_REGEX) @context[:px_cookie][:v3] = token[2..-1] end @@ -53,13 +64,13 @@ def initialize(px_config, req) cookies.each do |k, v| case k.to_s when '_px3' - @context[:px_cookie][:v3] = force_utf8(v) + @context[:px_cookie][:v3] = PerimeterXContext.force_utf8(v) when '_px' - @context[:px_cookie][:v1] = force_utf8(v) + @context[:px_cookie][:v1] = PerimeterXContext.force_utf8(v) when '_pxvid' if v.is_a?(String) && v.match(PxModule::VID_REGEX) @context[:vid_source] = "vid_cookie" - @context[:vid] = force_utf8(v) + @context[:vid] = PerimeterXContext.force_utf8(v) end end end #end case @@ -69,10 +80,10 @@ def initialize(px_config, req) if (k.start_with? 'HTTP_') header = k.to_s.gsub('HTTP_', '') header = header.gsub('_', '-').downcase - @context[:headers][header.to_sym] = force_utf8(v) + @context[:headers][header.to_sym] = PerimeterXContext.force_utf8(v) end end #end headers foreach - + @context[:hostname]= req.server_name @context[:user_agent] = req.user_agent ? req.user_agent : '' @context[:uri] = px_config[:custom_uri] ? px_config[:custom_uri].call(req) : req.fullpath @@ -97,10 +108,6 @@ def check_sensitive_route(sensitive_routes, uri) false end - def force_utf8(str) - return str.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '') - end - def set_block_action_type(action) @context[:block_action] = case action when 'c' diff --git a/lib/perimeterx/utils/px_constants.rb b/lib/perimeterx/utils/px_constants.rb index c5a658f..d86d1e7 100644 --- a/lib/perimeterx/utils/px_constants.rb +++ b/lib/perimeterx/utils/px_constants.rb @@ -46,8 +46,8 @@ module PxModule PROP_FIRST_PARTY_ENABLED = :firstPartyEnabled # Hosts - CLIENT_HOST = 'client.px-cloud.net' - CAPTCHA_HOST = 'captcha.px-cloud.net' + CLIENT_HOST = 'client.perimeterx.net' + CAPTCHA_HOST = 'captcha.px-cdn.net' VISIBLE = 'visible' HIDDEN = 'hidden' diff --git a/lib/perimeterx/utils/px_http_client.rb b/lib/perimeterx/utils/px_http_client.rb index 8b46d57..6a80750 100644 --- a/lib/perimeterx/utils/px_http_client.rb +++ b/lib/perimeterx/utils/px_http_client.rb @@ -1,6 +1,7 @@ require 'perimeterx/utils/px_logger' require 'typhoeus' require 'concurrent' +require 'net/http' module PxModule class PxHttpClient @@ -45,5 +46,61 @@ def post(path, body, headers, api_timeout = 1, connection_timeout = 1) return response end + + def post_xhr(url, body, headers) + s = Time.now + begin + @logger.debug("PxHttpClient[post]: sending xhr post request to #{url} with headers {#{headers.to_json()}}") + + #set url + uri = URI(url) + req = Net::HTTP::Post.new(uri) + + # set body + req.body=body + + # set headers + headers.each do |key, value| + req[key] = value + end + + # send request + response = Net::HTTP.start(uri.hostname, uri.port) {|http| + http.request(req) + } + + ensure + e = Time.now + @logger.debug("PxHttpClient[get]: runtime: #{(e-s) * 1000.0}") + end + return response + end + + + def get(url, headers) + s = Time.now + begin + @logger.debug("PxHttpClient[get]: sending get request to #{url} with headers {#{headers.to_json()}}") + + #set url + uri = URI(url) + req = Net::HTTP::Get.new(uri) + + # set headers + headers.each do |key, value| + req[key] = value + end + + # send request + response = Net::HTTP.start(uri.hostname, uri.port) {|http| + http.request(req) + } + + ensure + e = Time.now + @logger.debug("PxHttpClient[get]: runtime: #{(e-s) * 1000.0}") + end + return response + end end end diff --git a/lib/perimeterx/utils/px_template_factory.rb b/lib/perimeterx/utils/px_template_factory.rb index 754b905..acd5c82 100644 --- a/lib/perimeterx/utils/px_template_factory.rb +++ b/lib/perimeterx/utils/px_template_factory.rb @@ -29,11 +29,11 @@ def self.get_template(px_ctx, px_config, px_template_object) view[PxModule::PROP_CUSTOM_LOGO] = px_config[:custom_logo] view[PxModule::PROP_CSS_REF] = px_config[:css_ref] view[PxModule::PROP_JS_REF] = px_config[:js_ref] - view[PxModule::PROP_HOST_URL] = "https://collector-#{px_config[:app_id]}.perimeterx.net" + view[PxModule::PROP_HOST_URL] = px_template_object[:host_url] view[PxModule::PROP_LOGO_VISIBILITY] = px_config[:custom_logo] ? PxModule::VISIBLE : PxModule::HIDDEN view[PxModule::PROP_BLOCK_SCRIPT] = px_template_object[:block_script] view[PxModule::PROP_JS_CLIENT_SRC] = px_template_object[:js_client_src] - view[PxModule::PROP_FIRST_PARTY_ENABLED] = false + view[PxModule::PROP_FIRST_PARTY_ENABLED] = px_ctx.context[:first_party_enabled] return view.render.html_safe end diff --git a/readme.md b/readme.md index 2382fd0..83c5178 100644 --- a/readme.md +++ b/readme.md @@ -31,6 +31,7 @@ Table of Contents * [Debug Mode](#debug-mode) * [Whitelist Routes](#whitelist-routes) * [Update Configuration on Runtime](#update-config) + * [First Party](#first-party) **[Contributing](#contributing)** @@ -347,6 +348,26 @@ class HomeController < ApplicationController end ``` +**First Party** +To enable first party on your enforcer, add the following routes to your `config/routes.rb` file: + +```ruby + get '/:appid_postfix/init.js', to: 'home#index', constraints: { appid_postfix: /XXXXXXXX/ } + get '/:appid_postfix/captcha/:all', to: 'home#index', constraints: { appid_postfix: /XXXXXXXX/, all:/.*/ } + post '/:appid_postfix/xhr/:all', to: 'home#index', constraints: { appid_postfix: /XXXXXXXX/, all:/.*/ } +``` + +Notice that all occurences of `XXXXXXXX` should be replaced with your px_app_id without the PX prefix. For example: 2H4seK9L +In case you are using more than 1 px_app_id, provide all of them with a `|` sign between them. For example: 2H4seK9L|9bMs6K94|Lc5kPMNx + +To disable first_party, you can set `first_party_enabled` to false in your configuration. +The default value of this field is true. + +```ruby +params[:first_party_enabled] = false +``` + + # Contributing # ------------------------------ The following steps are welcome when contributing to our project. From 38630f6d4c6cec82653d464509fc588abcc15431 Mon Sep 17 00:00:00 2001 From: Chen Nakar Date: Tue, 15 Sep 2020 21:47:11 +0300 Subject: [PATCH 2/2] first party changes + version 2.2.0 --- changelog.md | 4 ++++ lib/perimeter_x.rb | 4 ++-- lib/perimeterx/internal/first_party/px_first_party.rb | 7 ++----- lib/perimeterx/version.rb | 2 +- readme.md | 11 ++++++----- 5 files changed, 15 insertions(+), 13 deletions(-) diff --git a/changelog.md b/changelog.md index bb850dd..5ac8590 100644 --- a/changelog.md +++ b/changelog.md @@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [2.2.0] - 2020-09-15 +### Added + - First Party + ## [2.1.0] - 2020-09-01 ### Added - Added option to set a different px configuration on each request diff --git a/lib/perimeter_x.rb b/lib/perimeter_x.rb index 5c02a2e..f45a2db 100644 --- a/lib/perimeter_x.rb +++ b/lib/perimeter_x.rb @@ -135,7 +135,7 @@ def render_first_party_response(req, px_instance) if px_config[:first_party_enabled] # first party enabled - proxy response - fp_response = fp.get_first_party_response(req) + fp_response = fp.send_first_party_request(req) response.status = fp_response.code fp_response.to_hash.each do |header_name, header_value_arr| if header_name!="content-length" @@ -146,7 +146,7 @@ def render_first_party_response(req, px_instance) render res_type => fp_response.body else # first party disabled - return empty response - response.status = "200" + response.status = 200 res_type = fp.get_response_content_type(req) render res_type => "" end diff --git a/lib/perimeterx/internal/first_party/px_first_party.rb b/lib/perimeterx/internal/first_party/px_first_party.rb index 02718e9..2a74c96 100644 --- a/lib/perimeterx/internal/first_party/px_first_party.rb +++ b/lib/perimeterx/internal/first_party/px_first_party.rb @@ -13,7 +13,7 @@ def initialize(px_config, px_http_client, logger) ] end - def get_first_party_response(req) + def send_first_party_request(req) uri = URI.parse(req.original_url) url_path = uri.path @@ -114,10 +114,7 @@ def get_first_party_request_type(req) end def is_first_party_request(req) - if get_first_party_request_type(req) != -1 - return true - end - return false + return get_first_party_request_type(req) != -1 end def get_response_content_type(req) diff --git a/lib/perimeterx/version.rb b/lib/perimeterx/version.rb index 6f71b7c..08a5804 100644 --- a/lib/perimeterx/version.rb +++ b/lib/perimeterx/version.rb @@ -1,3 +1,3 @@ module PxModule - VERSION = '2.1.0' + VERSION = '2.2.0' end diff --git a/readme.md b/readme.md index 83c5178..5875766 100644 --- a/readme.md +++ b/readme.md @@ -357,14 +357,15 @@ To enable first party on your enforcer, add the following routes to your `config post '/:appid_postfix/xhr/:all', to: 'home#index', constraints: { appid_postfix: /XXXXXXXX/, all:/.*/ } ``` -Notice that all occurences of `XXXXXXXX` should be replaced with your px_app_id without the PX prefix. For example: 2H4seK9L -In case you are using more than 1 px_app_id, provide all of them with a `|` sign between them. For example: 2H4seK9L|9bMs6K94|Lc5kPMNx +Notice that all occurences of `XXXXXXXX` should be replaced with your px_app_id without the "PX" prefix. For example, if your px_app_id is `PX2H4seK9L`, reeplace `XXXXXXXX` with `2H4seK9L`. +In case you are using more than one px_app_id, provide all of them with a `|` sign between them. For example: 2H4seK9L|9bMs6K94|Lc5kPMNx -To disable first_party, you can set `first_party_enabled` to false in your configuration. -The default value of this field is true. + +First Party configuration: +Default: true ```ruby -params[:first_party_enabled] = false + params[:first_party_enabled] = false ```