From 8426b71d8ca67f744c83eb7218898cab089bcae8 Mon Sep 17 00:00:00 2001 From: Thomas Bocquet Date: Thu, 17 Nov 2022 18:14:49 +0100 Subject: [PATCH 1/2] Implemented Google Ads Offline Conversion Import action --- src/actions/google/ads/conversion_import.ts | 157 ++++++++++++ src/actions/google/ads/lib/api_client.ts | 225 +++++++++--------- .../conversion_import_data_uploader.ts | 157 ++++++++++++ .../conversion_import_executor.ts | 23 ++ .../conversion_import_form_builder.ts | 180 ++++++++++++++ .../conversion_import_request.ts | 108 +++++++++ src/actions/index.ts | 1 + 7 files changed, 744 insertions(+), 107 deletions(-) create mode 100644 src/actions/google/ads/conversion_import.ts create mode 100644 src/actions/google/ads/lib/conversion_import/conversion_import_data_uploader.ts create mode 100644 src/actions/google/ads/lib/conversion_import/conversion_import_executor.ts create mode 100644 src/actions/google/ads/lib/conversion_import/conversion_import_form_builder.ts create mode 100644 src/actions/google/ads/lib/conversion_import/conversion_import_request.ts diff --git a/src/actions/google/ads/conversion_import.ts b/src/actions/google/ads/conversion_import.ts new file mode 100644 index 000000000..c45d1417b --- /dev/null +++ b/src/actions/google/ads/conversion_import.ts @@ -0,0 +1,157 @@ +import * as winston from "winston" +import * as Hub from "../../../hub" +import { makeBetterErrorMessage, sanitizeError } from "../common/error_utils" +import { MissingAuthError } from "../common/missing_auth_error" +import { GoogleOAuthHelper, UseGoogleOAuthHelper } from "../common/oauth_helper" +import { WrappedResponse } from "../common/wrapped_response" +import { GoogleAdsConversionImportActionRequest } from "./lib/conversion_import/conversion_import_request" + +const LOG_PREFIX = "[G Ads Conversion Import]" + +export class GoogleAdsConversionImport + extends Hub.OAuthAction + implements UseGoogleOAuthHelper { + + /******** Core action properties ********/ + + readonly name = "google_ads_conversion_import" + readonly label = "Google Ads Conversion Import" + readonly iconName = "google/ads/google_ads_icon.svg" + readonly description = "Upload conversions to Google Ads" + readonly supportedActionTypes = [Hub.ActionType.Query] + readonly supportedFormats = [Hub.ActionFormat.JsonLabel] + readonly supportedFormattings = [Hub.ActionFormatting.Unformatted] + readonly supportedVisualizationFormattings = [Hub.ActionVisualizationFormatting.Noapply] + readonly supportedDownloadSettings = [Hub.ActionDownloadSettings.Url] + readonly usesStreaming = true + readonly requiredFields = [] + readonly params = [] + + /******** Other fields + OAuth stuff ********/ + + readonly redirectUri = `${process.env.ACTION_HUB_BASE_URL}/actions/${encodeURIComponent(this.name)}/oauth_redirect` + readonly developerToken: string + readonly oauthClientId: string + readonly oauthClientSecret: string + readonly oauthScopes = [ + "https://www.googleapis.com/auth/adwords", + ] + readonly oauthHelper: GoogleOAuthHelper + + /******** Constructor & Helpers ********/ + + constructor(oauthClientId: string, oauthClientSecret: string, developerToken: string) { + super() + this.developerToken = developerToken + this.oauthClientId = oauthClientId + this.oauthClientSecret = oauthClientSecret + this.oauthHelper = new GoogleOAuthHelper(this, this.makeLogger("oauth")) + } + + makeLogger(webhookId = "") { + return (level: string, ...rest: any[]) => { + return winston.log(level, LOG_PREFIX, `[webhookID=${webhookId}]`, ...rest) + } + } + + makeOAuthClient() { + return this.oauthHelper.makeOAuthClient(this.redirectUri) + } + + /******** OAuth Endpoints ********/ + + async oauthUrl(redirectUri: string, encryptedState: string) { + return this.oauthHelper.oauthUrl(redirectUri, encryptedState) + } + + async oauthFetchInfo(urlParams: { [key: string]: string }, redirectUri: string) { + return this.oauthHelper.oauthFetchInfo(urlParams, redirectUri) + } + + async oauthCheck(_request: Hub.ActionRequest) { + // This part of Hub.OAuthAction is deprecated and unused + return true + } + + /******** Action Endpoints ********/ + + async execute(hubReq: Hub.ActionRequest) { + const wrappedResp = new WrappedResponse(Hub.ActionResponse) + const log = this.makeLogger(hubReq.webhookId) + try { + const adsRequest = await GoogleAdsConversionImportActionRequest.fromHub(hubReq, this, log) + await adsRequest.execute() + log("info", "Execution complete") + return wrappedResp.returnSuccess(adsRequest.userState) + } catch (err) { + sanitizeError(err) + makeBetterErrorMessage(err, hubReq.webhookId) + log("error", "Execution error toString:", err.toString()) + log("error", "Execution error JSON:", JSON.stringify(err)) + return wrappedResp.returnError(err) + } + } + + async form(hubReq: Hub.ActionRequest) { + const wrappedResp = new WrappedResponse(Hub.ActionForm) + const log = this.makeLogger(hubReq.webhookId) + log("debug", "THOMAS - Start form()") + log("debug", "") + try { + const adsWorker = await GoogleAdsConversionImportActionRequest.fromHub(hubReq, this, log) + log("debug", "THOMAS - before makeForm()") + wrappedResp.form = await adsWorker.makeForm() + log("debug", "THOMAS - after makeForm()") + return wrappedResp.returnSuccess(adsWorker.userState) + // Use this code if you need to force a state reset and redo oauth login + // wrappedResp.form = await this.oauthHelper.makeLoginForm(hubReq) + // wrappedResp.resetState() + // return wrappedResp.returnSuccess() + } catch (err) { + sanitizeError(err) + const loginForm = await this.oauthHelper.makeLoginForm(hubReq) + // Token errors that we can detect ahead of time + if (err instanceof MissingAuthError) { + log("debug", "Thomas - Caught MissingAuthError; returning login form. " + err.toString()) + return loginForm + } + log("error", "Form error toString:", err.toString()) + log("error", "Form error JSON:", JSON.stringify(err)) + + // AuthorizationError from API client - this occurs when request contains bad loginCid or targetCid + if (err.code === "403") { + wrappedResp.errorPrefix = `Error loading target account with request: ${err.response.request.responseURL}. ` + + `${err.response.data[0].error.details[0].errors[0].message}` + + ` Please retry loading the form again with the correct login account. ` + return wrappedResp.returnError(err) + } + + // Other errors from the API client - typically an auth problem + if (err.code) { + loginForm.fields[0].label = + `Received error code ${err.code} from the API, so your credentials have been discarded.` + + " Please reauthenticate and try again." + return loginForm + } + // All other errors + wrappedResp.errorPrefix = "Form generation error: " + return wrappedResp.returnError(err) + } + } +} + +/******** Register with Hub if prereqs are satisfied ********/ + +if (process.env.GOOGLE_ADS_CLIENT_ID + && process.env.GOOGLE_ADS_CLIENT_SECRET + && process.env.GOOGLE_ADS_DEVELOPER_TOKEN + ) { + const action = new GoogleAdsConversionImport( + process.env.GOOGLE_ADS_CLIENT_ID, + process.env.GOOGLE_ADS_CLIENT_SECRET, + process.env.GOOGLE_ADS_DEVELOPER_TOKEN, + ) + Hub.addAction(action) +} else { + winston.warn(`${LOG_PREFIX} Action not registered because required environment variables are missing.`) +} diff --git a/src/actions/google/ads/lib/api_client.ts b/src/actions/google/ads/lib/api_client.ts index 9a93b618c..5597a1c1a 100644 --- a/src/actions/google/ads/lib/api_client.ts +++ b/src/actions/google/ads/lib/api_client.ts @@ -5,37 +5,48 @@ import { Logger } from "../../common/logger" export class GoogleAdsApiClient { - constructor(readonly log: Logger, readonly accessToken: string - , readonly developerToken: string, readonly loginCid?: string) {} - - async listAccessibleCustomers() { - const method = "GET" - const path = "customers:listAccessibleCustomers" - return this.apiCall(method, path) + constructor(readonly log: Logger, readonly accessToken: string + , readonly developerToken: string, readonly loginCid?: string) { } + + async uploadClickConversions(clientCid: string, conversions: any[]) { + const method = "POST" + const path = `customers/${clientCid}:uploadClickConversions` + const body = { + conversions, + validate_only: false, + partialFailure: true, } - - async searchOpenUserLists(clientCid: string, uploadKeyType: "MOBILE_ADVERTISING_ID" | "CONTACT_INFO") { - const method = "POST" - const path = `customers/${clientCid}/googleAds:searchStream` - const body = { - query: - "SELECT user_list.id, user_list.name" - + " FROM user_list" - + " WHERE user_list.type = 'CRM_BASED'" - + " AND user_list.read_only = FALSE" - + " AND user_list.account_user_list_status = 'ENABLED'" - + " AND user_list.membership_status = 'OPEN'" - + ` AND user_list.crm_based_user_list.upload_key_type = '${uploadKeyType}'`, - } - return this.apiCall(method, path, body) + return this.apiCall(method, path, body) + } + + async listAccessibleCustomers() { + const method = "GET" + const path = "customers:listAccessibleCustomers" + return this.apiCall(method, path) + } + + async searchOpenUserLists(clientCid: string, uploadKeyType: "MOBILE_ADVERTISING_ID" | "CONTACT_INFO") { + const method = "POST" + const path = `customers/${clientCid}/googleAds:searchStream` + const body = { + query: + "SELECT user_list.id, user_list.name" + + " FROM user_list" + + " WHERE user_list.type = 'CRM_BASED'" + + " AND user_list.read_only = FALSE" + + " AND user_list.account_user_list_status = 'ENABLED'" + + " AND user_list.membership_status = 'OPEN'" + + ` AND user_list.crm_based_user_list.upload_key_type = '${uploadKeyType}'`, } - - async searchClientCustomers(clientCid: string) { - const method = "POST" - const path = `customers/${clientCid}/googleAds:searchStream` - const body = { - query: - `SELECT\ + return this.apiCall(method, path, body) + } + + async searchClientCustomers(clientCid: string) { + const method = "POST" + const path = `customers/${clientCid}/googleAds:searchStream` + const body = { + query: + `SELECT\ customer_client.client_customer\ , customer_client.hidden\ , customer_client.id\ @@ -47,97 +58,97 @@ export class GoogleAdsApiClient { , customer_client.status\ FROM customer_client\ WHERE customer_client.status NOT IN ('CANCELED', 'SUSPENDED')`, - } - return this.apiCall(method, path, body) } - - async createUserList(targetCid: string, newListName: string, newListDescription: string, uploadKeyType: "MOBILE_ADVERTISING_ID" | "CONTACT_INFO", mobileAppId?: string) { - const method = "POST" - const path = `customers/${targetCid}/userLists:mutate` - const body = { - customer_id: targetCid, - operations: [ - { - create: { - name: newListName, - description: newListDescription, - membership_status: "OPEN", - membership_life_span: 10000, - crm_based_user_list: { - upload_key_type: uploadKeyType, - app_id: mobileAppId, - data_source_type: "FIRST_PARTY", - }, + return this.apiCall(method, path, body) + } + + async createUserList(targetCid: string, newListName: string, newListDescription: string, uploadKeyType: "MOBILE_ADVERTISING_ID" | "CONTACT_INFO", mobileAppId?: string) { + const method = "POST" + const path = `customers/${targetCid}/userLists:mutate` + const body = { + customer_id: targetCid, + operations: [ + { + create: { + name: newListName, + description: newListDescription, + membership_status: "OPEN", + membership_life_span: 10000, + crm_based_user_list: { + upload_key_type: uploadKeyType, + app_id: mobileAppId, + data_source_type: "FIRST_PARTY", }, }, - ], - validate_only: false, - } - - return this.apiCall(method, path, body) + }, + ], + validate_only: false, } - async createDataJob(targetCid: string, userListResourceName: string) { - const method = "POST" - const path = `customers/${targetCid}/offlineUserDataJobs:create` - const body = { - customer_id: targetCid, - job: { - external_id: Date.now(), // must be an Int64 so not very useful - type: "CUSTOMER_MATCH_USER_LIST", - customer_match_user_list_metadata: { - user_list: userListResourceName, - }, + return this.apiCall(method, path, body) + } + + async createDataJob(targetCid: string, userListResourceName: string) { + const method = "POST" + const path = `customers/${targetCid}/offlineUserDataJobs:create` + const body = { + customer_id: targetCid, + job: { + external_id: Date.now(), // must be an Int64 so not very useful + type: "CUSTOMER_MATCH_USER_LIST", + customer_match_user_list_metadata: { + user_list: userListResourceName, }, - } - - return this.apiCall(method, path, body) + }, } - async addDataJobOperations(offlineUserDataJobResourceName: string, userIdentifiers: any[]) { - const method = "POST" - const path = `${offlineUserDataJobResourceName}:addOperations` - const body = { - resource_name: offlineUserDataJobResourceName, - enable_partial_failure: true, - operations: userIdentifiers, - } + return this.apiCall(method, path, body) + } - return this.apiCall(method, path, body) + async addDataJobOperations(offlineUserDataJobResourceName: string, userIdentifiers: any[]) { + const method = "POST" + const path = `${offlineUserDataJobResourceName}:addOperations` + const body = { + resource_name: offlineUserDataJobResourceName, + enable_partial_failure: true, + operations: userIdentifiers, } - async runJob(offlineUserDataJobResourceName: string) { - const method = "POST" - const path = `${offlineUserDataJobResourceName}:run` - const body = { - resource_name: offlineUserDataJobResourceName, - } + return this.apiCall(method, path, body) + } - return this.apiCall(method, path, body) + async runJob(offlineUserDataJobResourceName: string) { + const method = "POST" + const path = `${offlineUserDataJobResourceName}:run` + const body = { + resource_name: offlineUserDataJobResourceName, } - async apiCall(method: "GET" | "POST", url: string, data?: any) { - const headers: any = { - "developer-token": this.developerToken, - "Authorization": `Bearer ${this.accessToken}`, - } - if (this.loginCid) { - headers["login-customer-id"] = this.loginCid - } - const response = await gaxios.request({ - method, - url, - data, - headers, - baseURL: "https://googleads.googleapis.com/v11/", - }) - - if (process.env.ACTION_HUB_DEBUG) { - const apiResponse = lodash.cloneDeep(response) - sanitize(apiResponse) - this.log("debug", `Response from ${url}: ${JSON.stringify(apiResponse)}`) - } - - return response.data + return this.apiCall(method, path, body) + } + + async apiCall(method: "GET" | "POST", url: string, data?: any) { + const headers: any = { + "developer-token": this.developerToken, + "Authorization": `Bearer ${this.accessToken}`, } + if (this.loginCid) { + headers["login-customer-id"] = this.loginCid + } + const response = await gaxios.request({ + method, + url, + data, + headers, + baseURL: "https://googleads.googleapis.com/v11/", + }) + + if (process.env.ACTION_HUB_DEBUG) { + const apiResponse = lodash.cloneDeep(response) + sanitize(apiResponse) + this.log("debug", `Response from ${url}: ${JSON.stringify(apiResponse)}`) + } + + return response.data + } } diff --git a/src/actions/google/ads/lib/conversion_import/conversion_import_data_uploader.ts b/src/actions/google/ads/lib/conversion_import/conversion_import_data_uploader.ts new file mode 100644 index 000000000..52aaa8044 --- /dev/null +++ b/src/actions/google/ads/lib/conversion_import/conversion_import_data_uploader.ts @@ -0,0 +1,157 @@ +import * as oboe from "oboe" +import { Readable } from "stream" +import { GoogleAdsConversionImportActionExecutor } from "./conversion_import_executor" + +const BATCH_SIZE = 2000 + +interface OutputCells { + gclid?: string; + conversionAction?: string; + conversionDateTime?: string; + conversionValue?: string; + currencyCode?: string; +} + +export class GoogleAdsConversionImportUploader { + + readonly adsRequest = this.adsExecutor.adsRequest + readonly log = this.adsRequest.log + private batchPromises: Promise[] = [] + private batchQueue: any[] = [] + private currentRequest: Promise | undefined + private isSchemaDetermined = false + private rowQueue: any[] = [] + private schema: { [s: string]: string } = {} + + private regexes = [ + [/gclid|google.*click.*id|click.*id/i, "gclid"], + [/conversion.*name|name|conversion.*action|action/i, "conversionAction"], + [/conversion.*date.*time|conversion.time|time|date.*time|date/i, "conversionDateTime"], + [/conversion.*value|value/i, "conversionValue"], + [/conversion.*currency|currency|currency.*code/i, "currencyCode"], + ] + + constructor(readonly adsExecutor: GoogleAdsConversionImportActionExecutor) { } + + private get batchIsReady() { + return this.rowQueue.length >= BATCH_SIZE + } + + private get numBatches() { + return this.batchPromises.length + } + + async run() { + try { + // The ActionRequest.prototype.stream() method is going to await the callback we pass + // and either resolve the result we return here, or reject with an error from anywhere + await this.adsRequest.streamingDownload(async (downloadStream: Readable) => { + return this.startAsyncParser(downloadStream) + }) + } catch (errorReport) { + // TODO: the oboe fail() handler sends an errorReport object, but that might not be the only thing we catch + this.log("error", "Streaming parse failure toString:", errorReport.toString()) + this.log("error", "Streaming parse failure JSON:", JSON.stringify(errorReport)) + } + await Promise.all(this.batchPromises) + this.log("info", + `Streaming upload complete. Sent ${this.numBatches} batches (batch size = ${BATCH_SIZE})`, + ) + } + + private async startAsyncParser(downloadStream: Readable) { + return new Promise((resolve, reject) => { + oboe(downloadStream) + .node("!.*", (row: any) => { + if (!this.isSchemaDetermined) { + this.determineSchema(row) + } + this.handleRow(row) + this.scheduleBatch() + return oboe.drop + }) + .done(() => { + this.scheduleBatch(true) + resolve() + }) + .fail(reject) + }) + } + + private determineSchema(row: any) { + for (const columnLabel of Object.keys(row)) { + for (const mapping of this.regexes) { + const [regex, outputPath] = mapping + if (columnLabel.match(regex)) { + this.schema[columnLabel] = outputPath as string + } + } + } + this.isSchemaDetermined = true + } + + private handleRow(row: any) { + this.log("info", + `handleRow before transform thomas - ${JSON.stringify(row)} )`, + ) + const output = this.transformRow(row) + this.log("info", + `handleRow thomas - ${JSON.stringify(output)} )`, + ) + this.rowQueue.push(output) + } + + + + private transformRow(row: any) { + const schemaMapping = Object.entries(this.schema) + this.log("info", + `transformRow thomas schema Mapping - ${JSON.stringify(schemaMapping)} )`, + ) + const outputCells: OutputCells = {}; + schemaMapping.forEach(([columnLabel, outputPath]) => { + const outputValue = row[columnLabel]; + if (!outputValue) { + return; + } + outputCells[outputPath as keyof OutputCells] = outputValue; + }) + this.log("info", + `transformRow thomas outputcells - ${JSON.stringify(outputCells)} )`, + ) + + return outputCells; + } + + private scheduleBatch(force = false) { + if (!this.batchIsReady && !force) { + return + } + const batch = this.rowQueue.splice(0, BATCH_SIZE - 1) + this.log("info", + `schedule Batch thomas - ${JSON.stringify(batch)} )`, + ) + this.batchQueue.push(batch) + this.batchPromises.push(this.sendBatch()) + this.log("debug", `Sent batch number: ${this.numBatches}`) + } + + // The Ads API seems to generate a concurrent modification exception if we have multiple + // addDataJobOperations requests in progress at one time. So we use this funky solution + // to run one at a time, without having to refactor the streaming parser and everything too. + private async sendBatch(): Promise { + if (this.currentRequest !== undefined || this.batchQueue.length === 0) { + return + } + await this.adsRequest.checkTokens().then(async () => { + const currentBatch = this.batchQueue.shift() + this.log("info", + `sendBatch thomas - ${JSON.stringify(currentBatch)} )`, + ) + this.currentRequest = this.adsExecutor.importOfflineConversion(currentBatch) + await this.currentRequest + this.currentRequest = undefined + return this.sendBatch() + }) + } +} diff --git a/src/actions/google/ads/lib/conversion_import/conversion_import_executor.ts b/src/actions/google/ads/lib/conversion_import/conversion_import_executor.ts new file mode 100644 index 000000000..c5c760239 --- /dev/null +++ b/src/actions/google/ads/lib/conversion_import/conversion_import_executor.ts @@ -0,0 +1,23 @@ +import { GoogleAdsConversionImportUploader } from "./conversion_import_data_uploader" +import { GoogleAdsConversionImportActionRequest } from "./conversion_import_request" + +export class GoogleAdsConversionImportActionExecutor { + + readonly apiClient = this.adsRequest.apiClient! + readonly log = this.adsRequest.log + readonly targetCid = this.adsRequest.targetCid + + constructor(readonly adsRequest: GoogleAdsConversionImportActionRequest) { + } + + async uploadData() { + this.log("info", "conversion_import_executor uploadData") + const dataUploader = new GoogleAdsConversionImportUploader(this) + return dataUploader.run() + } + + async importOfflineConversion(conversions: any[]) { + this.log("info", "importOfflineConversion before sending to apiclient" + JSON.stringify(conversions)) + return this.apiClient.uploadClickConversions(this.targetCid, conversions) + } +} diff --git a/src/actions/google/ads/lib/conversion_import/conversion_import_form_builder.ts b/src/actions/google/ads/lib/conversion_import/conversion_import_form_builder.ts new file mode 100644 index 000000000..c377b6114 --- /dev/null +++ b/src/actions/google/ads/lib/conversion_import/conversion_import_form_builder.ts @@ -0,0 +1,180 @@ +import * as Hub from "../../../../../hub" +import { GoogleAdsConversionImportActionRequest } from "./conversion_import_request" + +interface SelectFormOption {name: string, label: string} + +interface AdsCustomer { + resourceName: string + manager: boolean + descriptiveName: string + id: string +} + +export class GoogleAdsConversionImportActionFormBuilder { + + readonly apiClient = this.adsRequest.apiClient! + readonly loginCid = this.adsRequest.loginCid + readonly targetCid = this.adsRequest.targetCid + + loginCustomer?: AdsCustomer + targetCustomer?: AdsCustomer + + constructor(readonly adsRequest: GoogleAdsConversionImportActionRequest) {} + + async makeForm() { + const form = new Hub.ActionForm() + + // 0) Fetch objects for fields that have been filled in already + await Promise.all([ + this.maybeSetLoginCustomer(), + this.maybeSetTargetCustomer(), + ]) + + // 1a) User must first pick a login account from the dropdown, which will be the only field to show at first + form.fields.push(await this.loginCidField()) + if (!this.loginCustomer) { return form } + + // 1b) If the chosen login account is a Manager, give the option to pick one of its client accounts as the target. + if (this.loginCustomer.manager) { + form.fields.push(await this.targetCidField()) + if (!this.targetCustomer) { return form } + // If not a manager account, set the targetCustomer to be the same as the loginCustomer + } else { + this.targetCustomer = this.loginCustomer + } + return form + } + + async loginCidField() { + let selectOptions: SelectFormOption[] + let description: string + + if (this.loginCustomer) { + selectOptions = [ + this.selectOptionForCustomer(this.loginCustomer), + ] + description = "To reset this selection, please close and re-open the form." + } else { + selectOptions = await this.getLoginCidOptions() + description = "This is like picking an account to work from using the menu in the Google Ads UI." + + " If you use a manager account to manage clients, choose the relevant manager account here." + + " If you login directly to an Ads account then choose it here." + } + + return { + name: "loginCid", + label: "Step 1) Choose login account", + description, + type: "select" as "select", + options: selectOptions, + default: this.loginCid as string, + interactive: true, + required: true, + } + } + + async targetCidField() { + let selectOptions: SelectFormOption[] + let description: string + + if (this.targetCustomer) { + selectOptions = [ + this.selectOptionForCustomer(this.targetCustomer), + {name: "", label: "Reset..."}, + ] + description = "Select \"Reset\" to go back." + } else { + selectOptions = await this.getTargetCidOptions() + description = "This is the account where you want to send data, i.e. where the audience lists are defined." + } + + return { + name: "targetCid", + label: "Step 1b) Choose target account", + description, + type: "select" as "select", + options: selectOptions, + default: this.targetCid as string, + interactive: true, + required: true, + } + } + + private async maybeSetLoginCustomer() { + if (!this.loginCid) { + return + } + this.loginCustomer = await this.getCustomer(this.loginCid) + } + + private async maybeSetTargetCustomer() { + if (!this.targetCid) { + return + } + this.targetCustomer = await this.getCustomer(this.targetCid) + } + + private async getLoginCidOptions() { + const listCustomersResp = await this.apiClient.listAccessibleCustomers() + const customerResourceNames = listCustomersResp.resourceNames + const customers = await Promise.all(customerResourceNames.map(async (rn: string) => { + const clientCid = rn.replace("customers/", "") + return this.getCustomer(clientCid).catch(() => undefined) // ignore any auth errors from draft accounts + })) + const filteredCustomers = customers.filter(Boolean) as AdsCustomer[] + const sortedCustomers = filteredCustomers.sort(this.sortCustomersCompareFn) + const selectOptions = sortedCustomers.map(this.selectOptionForCustomer) + return selectOptions + } + + private async getCustomer(cId: string) { + return await this.apiClient.searchClientCustomers(cId) + .then((data: any) => { + const cust = data[0].results.filter((c: any) => c.customerClient.id === cId)[0].customerClient + if (!cust.descriptiveName) { cust.descriptiveName = "Untitled" } + return cust as AdsCustomer + }) + } + + private async getTargetCidOptions() { + if (!this.loginCustomer) { + throw new Error("Could not reference the login customer record.") + } + const searchResp = await this.apiClient.searchClientCustomers(this.loginCustomer.id) + const searchResults = searchResp.length ? searchResp[0].results : [] + const clients = searchResults.map((result: any) => { + const client = result.customerClient + if (!client.descriptiveName) { + client.descriptiveName = "Untitled" + } + return client + }) + const sortedClients = clients.sort(this.sortCustomersCompareFn) + const selectOptions = sortedClients.map(this.selectOptionForCustomer) + + return selectOptions + } + + private selectOptionForCustomer(customer: AdsCustomer) { + const name = customer.id + const title = customer.descriptiveName ? customer.descriptiveName : "Untitled" + const prefix = customer.manager ? "[Manager] " : "" + const suffix = `(${customer.id})` + const label = `${prefix}${title} ${suffix}` + + return {name, label} as SelectFormOption + } + + private sortCustomersCompareFn(a: AdsCustomer, b: AdsCustomer) { + if (a.manager && !b.manager) { + return -1 + } + if (!a.manager && b.manager) { + return 1 + } + if (a.descriptiveName < b.descriptiveName) { + return -1 + } + return 0 + } +} diff --git a/src/actions/google/ads/lib/conversion_import/conversion_import_request.ts b/src/actions/google/ads/lib/conversion_import/conversion_import_request.ts new file mode 100644 index 000000000..9775c7408 --- /dev/null +++ b/src/actions/google/ads/lib/conversion_import/conversion_import_request.ts @@ -0,0 +1,108 @@ +import { Credentials } from "google-auth-library" +import * as Hub from "../../../../../hub" +import { Logger } from "../../../common/logger" +import { MissingAuthError } from "../../../common/missing_auth_error" +import { MissingRequiredParamsError } from "../../../common/missing_required_params_error" +import { safeParseJson } from "../../../common/utils" +import { GoogleAdsConversionImport } from "../../conversion_import" +import { GoogleAdsApiClient } from "../api_client" +import { GoogleAdsConversionImportActionExecutor} from "./conversion_import_executor" +import { GoogleAdsConversionImportActionFormBuilder } from "./conversion_import_form_builder" + +interface AdsUserState { + tokens: Credentials + redirect: string +} + +export class GoogleAdsConversionImportActionRequest { + + static async fromHub(hubRequest: Hub.ActionRequest, action: GoogleAdsConversionImport, logger: Logger) { + const adsReq = new GoogleAdsConversionImportActionRequest(hubRequest, action, logger) + await adsReq.checkTokens() + adsReq.setApiClient() + return adsReq + } + + readonly streamingDownload = this.hubRequest.stream.bind(this.hubRequest) + apiClient?: GoogleAdsApiClient + formParams: any + userState: AdsUserState + webhookId?: string + + constructor( + readonly hubRequest: Hub.ActionRequest, + readonly actionInstance: GoogleAdsConversionImport, + readonly log: Logger, + ) { + const state = safeParseJson(hubRequest.params.state_json) + + if (!state || !state.tokens || !state.tokens.access_token || !state.tokens.refresh_token || !state.redirect) { + throw new MissingAuthError("User state was missing or did not contain oauth tokens & redirect") + } + + this.userState = state + this.formParams = hubRequest.formParams + this.webhookId = hubRequest.webhookId + } + + async checkTokens() { + // adding 5 minutes to expiry_date check to handle refresh edge case + if ( this.userState.tokens.expiry_date == null || this.userState.tokens.expiry_date < (Date.now() + 5 * 60000) ) { + this.log("debug", "Tokens appear expired; attempting refresh.") + + const data = await this.actionInstance.oauthHelper.refreshAccessToken(this.userState.tokens) + + if (!data || !data.access_token || !data.expiry_date) { + throw new MissingAuthError("Could not refresh tokens") + } + + this.userState.tokens.access_token = data.access_token + this.userState.tokens.expiry_date = data.expiry_date + this.log("debug", "Set new tokens") + } + } + + setApiClient() { + this.apiClient = new GoogleAdsApiClient(this.log, this.accessToken, this.developerToken, this.loginCid) + } + + get accessToken() { + return this.userState.tokens.access_token! + } + + get developerToken() { + return this.actionInstance.developerToken + } + + get loginCid() { + return this.formParams.loginCid + } + + get targetCid() { + return this.formParams.targetCid + } + + async makeForm() { + const formBuilder = new GoogleAdsConversionImportActionFormBuilder(this) + return formBuilder.makeForm() + } + + async execute() { + + // 0) Do execution specific validations + if (!this.loginCid) { + throw new MissingRequiredParamsError("Login account id is missing") + } + + // 0) If a non-manager account was chosen for login, there will be no targetCid. Fill that in and start the helper. + if (!this.targetCid) { + this.formParams.targetCid = this.loginCid + } + const executor = new GoogleAdsConversionImportActionExecutor(this) + + // 3) Add the data ("user identifiers") to the job + await executor.uploadData() + + return + } +} diff --git a/src/actions/index.ts b/src/actions/index.ts index f2cfdb6a6..dc45b27c3 100644 --- a/src/actions/index.ts +++ b/src/actions/index.ts @@ -10,6 +10,7 @@ import "./digitalocean/digitalocean_object_storage" import "./dropbox/dropbox" import "./facebook/facebook_custom_audiences" import "./firebase/firebase" +import "./google/ads/conversion_import" import "./google/ads/customer_match" import "./google/analytics/data_import" import "./google/drive/google_drive" From 4123e1851a3f967ab72f638f3fa6ef16d63cf755 Mon Sep 17 00:00:00 2001 From: Thomas Bocquet Date: Wed, 14 Dec 2022 15:28:08 +0100 Subject: [PATCH 2/2] Remove debug log lines --- src/actions/google/ads/conversion_import.ts | 5 ----- .../conversion_import_data_uploader.ts | 19 ------------------- 2 files changed, 24 deletions(-) diff --git a/src/actions/google/ads/conversion_import.ts b/src/actions/google/ads/conversion_import.ts index c45d1417b..8dbcc259f 100644 --- a/src/actions/google/ads/conversion_import.ts +++ b/src/actions/google/ads/conversion_import.ts @@ -95,13 +95,9 @@ export class GoogleAdsConversionImport async form(hubReq: Hub.ActionRequest) { const wrappedResp = new WrappedResponse(Hub.ActionForm) const log = this.makeLogger(hubReq.webhookId) - log("debug", "THOMAS - Start form()") - log("debug", "") try { const adsWorker = await GoogleAdsConversionImportActionRequest.fromHub(hubReq, this, log) - log("debug", "THOMAS - before makeForm()") wrappedResp.form = await adsWorker.makeForm() - log("debug", "THOMAS - after makeForm()") return wrappedResp.returnSuccess(adsWorker.userState) // Use this code if you need to force a state reset and redo oauth login // wrappedResp.form = await this.oauthHelper.makeLoginForm(hubReq) @@ -112,7 +108,6 @@ export class GoogleAdsConversionImport const loginForm = await this.oauthHelper.makeLoginForm(hubReq) // Token errors that we can detect ahead of time if (err instanceof MissingAuthError) { - log("debug", "Thomas - Caught MissingAuthError; returning login form. " + err.toString()) return loginForm } log("error", "Form error toString:", err.toString()) diff --git a/src/actions/google/ads/lib/conversion_import/conversion_import_data_uploader.ts b/src/actions/google/ads/lib/conversion_import/conversion_import_data_uploader.ts index 52aaa8044..15cffff05 100644 --- a/src/actions/google/ads/lib/conversion_import/conversion_import_data_uploader.ts +++ b/src/actions/google/ads/lib/conversion_import/conversion_import_data_uploader.ts @@ -91,13 +91,7 @@ export class GoogleAdsConversionImportUploader { } private handleRow(row: any) { - this.log("info", - `handleRow before transform thomas - ${JSON.stringify(row)} )`, - ) const output = this.transformRow(row) - this.log("info", - `handleRow thomas - ${JSON.stringify(output)} )`, - ) this.rowQueue.push(output) } @@ -105,9 +99,6 @@ export class GoogleAdsConversionImportUploader { private transformRow(row: any) { const schemaMapping = Object.entries(this.schema) - this.log("info", - `transformRow thomas schema Mapping - ${JSON.stringify(schemaMapping)} )`, - ) const outputCells: OutputCells = {}; schemaMapping.forEach(([columnLabel, outputPath]) => { const outputValue = row[columnLabel]; @@ -116,10 +107,6 @@ export class GoogleAdsConversionImportUploader { } outputCells[outputPath as keyof OutputCells] = outputValue; }) - this.log("info", - `transformRow thomas outputcells - ${JSON.stringify(outputCells)} )`, - ) - return outputCells; } @@ -128,9 +115,6 @@ export class GoogleAdsConversionImportUploader { return } const batch = this.rowQueue.splice(0, BATCH_SIZE - 1) - this.log("info", - `schedule Batch thomas - ${JSON.stringify(batch)} )`, - ) this.batchQueue.push(batch) this.batchPromises.push(this.sendBatch()) this.log("debug", `Sent batch number: ${this.numBatches}`) @@ -145,9 +129,6 @@ export class GoogleAdsConversionImportUploader { } await this.adsRequest.checkTokens().then(async () => { const currentBatch = this.batchQueue.shift() - this.log("info", - `sendBatch thomas - ${JSON.stringify(currentBatch)} )`, - ) this.currentRequest = this.adsExecutor.importOfflineConversion(currentBatch) await this.currentRequest this.currentRequest = undefined