diff --git a/README.md b/README.md index c28744f..590f9e6 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,7 @@ python main.py \ -o results/openai_classic.csv results/openai_basic.csv results/openai_advanced.csv \ -g oa \ -f prompts/prompt_openai.json \ - -m gpt-4-turbo \ + -m o3-mini \ -p 5 \ -c 0 ``` @@ -462,7 +462,7 @@ You can use the following flags in the command line to change the configurations | CLI Flags | Description | | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | -g, --model_type | Model type used. Make sure this matches the model used. Currently defined options in `main.py` are `oa` for OpenAI models, `anthropic` for Anthropic models, `hf` for Hugging Face models, `vllm` for a vllm runner, `api` for API endpoints, `llama_cpp` for llama cpp, `mlx` for mlx, `bedrock` for AWS bedrock API, `together` for together.ai's API | -| -m, --model | Model that will be tested and used to generate the queries. Some options for OpenAI models are chat models `gpt-3.5-turbo-0613` and `gpt-4-0613`. Options for Anthropic include the latest claude-3 family of models (e.g. `claude-3-opus-20240229`). For Hugging Face, and VLLM models, simply use the path of your chosen model (e.g. `defog/sqlcoder`). | +| -m, --model | Model that will be tested and used to generate the queries. Some options for OpenAI models are chat models `gpt-4o` and `o3-mini`. Options for Anthropic include the latest claude-3 family of models (e.g. `claude-3-opus-20240229`). For Hugging Face, and VLLM models, simply use the path of your chosen model (e.g. `defog/sqlcoder`). | | -a, --adapter | Path to the relevant adapter model you're using. Only available for the `hf_runner`. | | --api_url | The URL of the custom API you want to send the prompt to. Only used when model_type is `api`. | | -qz, --quantized | Indicate whether the model is an AWQ quantized model. Only available for `vllm_runner`. | @@ -532,7 +532,7 @@ python main.py \ -o results/test.csv \ -g oa \ -f prompts/prompt_openai.json \ - -m gpt-3.5-turbo-0613 \ + -m gpt-4o-mini \ -n 1 \ --upload_url ``` @@ -550,7 +550,7 @@ python main.py \ -o results/test.csv \ -g oa \ -f prompts/prompt_openai.json \ - -m gpt-3.5-turbo-0613 \ + -m gpt-4o-mini \ -n 1 \ --upload_url http://127.0.0.1:8080/ ``` diff --git a/data/instruct_advanced_postgres.csv b/data/instruct_advanced_postgres.csv index 365a3b5..328ca60 100644 --- a/data/instruct_advanced_postgres.csv +++ b/data/instruct_advanced_postgres.csv @@ -11,7 +11,7 @@ PMAT = per month average transaction amount, using date truncation for aggregati To analyze stock performance, join the daily price and ticker tables, filter for a specific date range, and calculate price change. CR = customer rank by total transaction amount, with the highest transaction amount getting the top rank" broker,instructions_cte_window,"What is the ticker symbol, month, average closing price, highest price, lowest price, and MoMC for each ticker by month?","MoMC = month-over-month change in average closing price = (avg_close_given_month - avg_close_previous_month) / avg_close_previous_month for each ticker symbol each month. Recall that we want the symbol, and not just the ticker id.","WITH monthly_price_stats AS (SELECT DATE_TRUNC('month', sbDpDate) AS MONTH, sbDpTickerId, AVG(sbDpClose) AS avg_close, MAX(sbDpHigh) AS max_high, MIN(sbDpLow) AS min_low FROM sbDailyPrice GROUP BY MONTH, sbDpTickerId) SELECT t.sbTickerSymbol, mps.month, mps.avg_close, mps.max_high, mps.min_low, (mps.avg_close - LAG(mps.avg_close) OVER (PARTITION BY mps.sbDpTickerId ORDER BY mps.month)) / LAG(mps.avg_close) OVER (PARTITION BY mps.sbDpTickerId ORDER BY mps.month) AS mom_change FROM monthly_price_stats mps JOIN sbTicker t ON mps.sbDpTickerId = t.sbTickerId;WITH monthly_price_stats AS (SELECT TO_CHAR(DATE_TRUNC('month', sbDpDate), 'YYYY-MM') AS month, sbDpTickerId, AVG(sbDpClose) AS avg_close, MAX(sbDpHigh) AS max_high, MIN(sbDpLow) AS min_low FROM sbDailyPrice GROUP BY TO_CHAR(DATE_TRUNC('month', sbDpDate), 'YYYY-MM'), sbDpTickerId) SELECT t.sbTickerSymbol, mps.month, mps.avg_close, mps.max_high, mps.min_low, (mps.avg_close - LAG(mps.avg_close) OVER (PARTITION BY mps.sbDpTickerId ORDER BY mps.month)) / LAG(mps.avg_close) OVER (PARTITION BY mps.sbDpTickerId ORDER BY mps.month) AS mom_change FROM monthly_price_stats mps JOIN sbTicker t ON mps.sbDpTickerId = t.sbTickerId;WITH monthly_price_stats AS (SELECT TO_CHAR(DATE_TRUNC('month', sbDpDate), 'YYYY-MM-01') AS month, sbDpTickerId, AVG(sbDpClose) AS avg_close, MAX(sbDpHigh) AS max_high, MIN(sbDpLow) AS min_low FROM sbDailyPrice GROUP BY TO_CHAR(DATE_TRUNC('month', sbDpDate), 'YYYY-MM-01'), sbDpTickerId) SELECT t.sbTickerSymbol, mps.month, mps.avg_close, mps.max_high, mps.min_low, (mps.avg_close - LAG(mps.avg_close) OVER (PARTITION BY mps.sbDpTickerId ORDER BY mps.month)) / LAG(mps.avg_close) OVER (PARTITION BY mps.sbDpTickerId ORDER BY mps.month) AS mom_change FROM monthly_price_stats mps JOIN sbTicker t ON mps.sbDpTickerId = t.sbTickerId;","NPM (Net Profit Margin) = (Total Income from Sales - (Tax + Commission Expenses)) / Total Income from Sales * 100, for recent transactions. TAC = Total Active Customers who have recently joined ACP = Average Closing Price of tickers over a recent period MoMC = month-over-month change in average closing price = (avg_close_given_month - avg_close_previous_month) / avg_close_previous_month for each ticker each month" -broker,instructions_cte_window,"Return the customer name, number of transactions, total transaction amount, and CR for all customers","CR = customer rank by total transaction amount, with rank 1 being the customer with the highest total transaction amount","WITH cust_tx_counts AS (SELECT sbTxCustId, COUNT(*) AS num_tx, SUM(sbTxAmount) AS total_amount FROM sbTransaction GROUP BY sbTxCustId) SELECT c.sbCustName, ct.num_tx, ct.total_amount, RANK() OVER (ORDER BY ct.total_amount DESC) AS cust_rank FROM cust_tx_counts ct JOIN sbCustomer c ON ct.sbTxCustId = c.sbCustId","ACP = Average Closing Price of tickers over a select period of days +broker,instructions_cte_window,"Return the customer name, number of transactions, total transaction amount, and CR for all customers;","CR = customer rank by total transaction amount, with rank 1 being the customer with the highest total transaction amount","SELECT c.sbCustName, COUNT(t.sbTxId) AS number_of_transactions, COALESCE(SUM(t.sbTxAmount), 0) AS total_transaction_amount, DENSE_RANK() OVER ( ORDER BY COALESCE(SUM(t.sbTxAmount), 0) DESC) AS CR FROM sbCustomer c LEFT JOIN sbTransaction t ON c.sbCustId = t.sbTxCustId GROUP BY c.sbCustName ORDER BY CR;","ACP = Average Closing Price of tickers over a select period of days TAC = Total Active Customers who have recently joined NPM (Net Profit Margin) = Net income divided by net sales for a specific time frame, expressed as a percentage for transactions in a recent period. CR = customer rank by total transaction amount, with rank 1 being the customer with the highest total transaction amount" @@ -31,7 +31,7 @@ broker,keywords_aggregate,"How many TAC are there from each country, for custome To calculate the success rate of transactions for each customer, merge the customer and transaction records, aggregate by customer, and assess the rate of successful transactions TAC = Total Active Customers who joined on or after January 1, 2023 To ascertain the most sought-after stocks within a recent timeframe, combine the transaction and stock symbol datasets, focus on purchase actions during the specified period, group by stock symbol, and tally the transactions." -broker,keywords_aggregate,"What is the ACP for each ticker type in the past 7 days, inclusive of today? Return the ticker type and the average closing price.","ACP = Average Closing Price of tickers in the last 7 days, inclusive of today","SELECT sbTickerType, AVG(sbDpClose) AS ACP FROM sbDailyPrice JOIN sbTicker ON sbDailyPrice.sbDpTickerId = sbTicker.sbTickerId WHERE sbDpDate >= CURRENT_DATE - INTERVAL '7 days' GROUP BY sbTickerType","To determine the Activity Ratio (AR), divide the number of active customers by the total customer count and then multiply by 100, focusing on customers who joined within a particular year +broker,keywords_aggregate,"What is the ACP for each ticker type in the past 7 days, inclusive of today? Return the ticker type and the average closing price.","ACP = Average Closing Price of tickers in the last 7 days, inclusive of today","SELECT sbTickerType, AVG(sbDpClose) AS ACP FROM sbDailyPrice JOIN sbTicker ON sbDailyPrice.sbDpTickerId = sbTicker.sbTickerId WHERE sbDpDate >= CURRENT_DATE - INTERVAL '6 days' GROUP BY sbTickerType","To determine the Activity Ratio (AR), divide the number of active customers by the total customer count and then multiply by 100, focusing on customers who joined within a particular year Customer Rank (CR) is determined by ranking customers based on their total transaction amount, with rank 1 assigned to the customer with the highest amount. To calculate Net Profit Margin (NPM), subtract the sum of tax and commission from the total amount obtained from sales, then divide by the total sales amount and multiply by 100, assessing transactions within a specific period ACP = Average Closing Price of tickers in the last 7 days, inclusive of today" @@ -58,12 +58,12 @@ car_dealership,instructions_cte_join,"How many sales did each salesperson make i To get the number of sales made by each salesperson in the past 30 days, join the salespersons and sales tables and filter for sales in the last 30 days. ASP = Calculate the average sale price without specifying the period GPM = Define gross profit margin as a ratio without specifying how to calculate total revenue or total cost" -car_dealership,instructions_cte_window,"Return the first name, last name, total sales amount, number of sales, and SR for each salesperson",SR = sales rank of each salesperson ordered by their total sales amount descending,"WITH salesperson_sales AS (SELECT salesperson_id, SUM(sale_price) AS total_sales, COUNT(*) AS num_sales FROM sales GROUP BY salesperson_id) SELECT s.first_name, s.last_name, ss.total_sales, ss.num_sales, RANK() OVER (ORDER BY ss.total_sales DESC) AS sales_rank FROM salesperson_sales ss JOIN salespersons s ON ss.salesperson_id = s.id","SR = sales rank of each salesperson ordered by their total sales amount descending To determine the sales performance per territory, sum the sales amount and count the sales, grouping by territory To calculate the average sale price, join the sales table with itself on the salesperson_id and find the ratio of total sales amount to number of sales To assess inventory turnover, compare inventory snapshots with sales on matching days, focusing on the quantity of items sold." -car_dealership,instructions_cte_window,What is the total payments received per month? Also calculate the MoM change for each month.,"MoM change = (current month value - prev month value). Return all months in your answer, including those where there were no payments.","WITH monthly_totals AS (SELECT DATE_TRUNC('month', payment_date) AS dt, SUM(payment_amount) AS total_payments FROM payments_received GROUP BY dt), monthly_range AS (SELECT generate_series(DATE_TRUNC('month', MIN(payment_date)), DATE_TRUNC('month', MAX(payment_date)), '1 month'::interval) AS dt FROM payments_received), monthly_totals_with_zero AS (SELECT mr.dt, COALESCE(mt.total_payments, 0) AS total_payments FROM monthly_range mr LEFT JOIN monthly_totals mt ON mr.dt = mt.dt) SELECT m.dt::DATE AS MONTH, m.total_payments, m.total_payments - lag(m.total_payments, 1) OVER (ORDER BY dt) AS mom_change FROM monthly_totals_with_zero m ORDER BY m.dt;WITH monthly_payments AS (SELECT DATE_TRUNC('month', pr.payment_date) AS MONTH, SUM(pr.payment_amount) AS total_payments FROM payments_received pr GROUP BY MONTH ORDER BY MONTH), monthly_range AS (SELECT generate_series(DATE_TRUNC('month', MIN(pr.payment_date)), DATE_TRUNC('month', MAX(pr.payment_date)), '1 month'::interval) AS MONTH FROM payments_received pr), monthly_payments_with_zeros AS (SELECT mr.month, COALESCE(mp.total_payments, 0) AS total_payments FROM monthly_range mr LEFT JOIN monthly_payments mp ON mr.month = mp.month) SELECT mp.month, mp.total_payments, mp.total_payments - lag(mp.total_payments, 1) OVER (ORDER BY mp.month) AS mom_change FROM monthly_payments_with_zeros mp ORDER BY mp.month;WITH monthly_totals AS (SELECT TO_CHAR(DATE_TRUNC('month', payment_date), 'YYYY-MM') AS dt, SUM(payment_amount) AS total_payments FROM payments_received GROUP BY dt), monthly_range AS (SELECT TO_CHAR(generate_series(DATE_TRUNC('month', MIN(payment_date)), DATE_TRUNC('month', MAX(payment_date)), '1 month'::interval), 'YYYY-MM') AS dt FROM payments_received), monthly_totals_with_zero AS (SELECT mr.dt, COALESCE(mt.total_payments, 0) AS total_payments FROM monthly_range mr LEFT JOIN monthly_totals mt ON mr.dt = mt.dt) SELECT m.dt AS MONTH, m.total_payments, m.total_payments - lag(m.total_payments, 1) OVER (ORDER BY dt) AS mom_change FROM monthly_totals_with_zero m ORDER BY m.dt;","To ascertain the volume of sales conducted by each salesperson over a recent period, merge the salespersons and sales tables, applying a filter for recent sales transactions. +car_dealership,instructions_cte_window,"Return the first name, last name, total sales amount, number of sales, and SR for each salesperson",SR = sales rank of each salesperson ordered by their total sales amount descending,"SELECT sp.first_name, sp.last_name, COALESCE(SUM(s.sale_price), 0) AS total_sales_amount, COUNT(s.id) AS number_of_sales, RANK() OVER ( ORDER BY COALESCE(SUM(s.sale_price), 0) DESC) AS sr FROM salespersons sp LEFT JOIN sales s ON sp.id = s.salesperson_id GROUP BY sp.id, sp.first_name, sp.last_name;SELECT sp.first_name, sp.last_name, SUM(s.sale_price) AS total_sales_amount, COUNT(*) AS number_of_sales, RANK() OVER ( ORDER BY SUM(s.sale_price) DESC) AS sr FROM salespersons sp JOIN sales s ON sp.id = s.salesperson_id GROUP BY sp.id, sp.first_name, sp.last_name;","SR = sales rank of each salesperson ordered by their total sales amount descending To determine the sales performance per territory, sum the sales amount and count the sales, grouping by territory To calculate the average sale price, join the sales table with itself on the salesperson_id and find the ratio of total sales amount to number of sales To assess inventory turnover, compare inventory snapshots with sales on matching days, focusing on the quantity of items sold." +car_dealership,instructions_cte_window,What is the total payments received per month? Also calculate the MoM change for each month.,"MoM change = (current month value - prev month value). Return all months in your answer, including those where there were no payments.","WITH monthly_totals AS (SELECT DATE_TRUNC('month', payment_date) AS dt, SUM(payment_amount) AS total_payments FROM payments_received GROUP BY dt), monthly_range AS (SELECT generate_series(DATE_TRUNC('month', MIN(payment_date)), DATE_TRUNC('month', MAX(payment_date)), '1 month'::interval) AS dt FROM payments_received), monthly_totals_with_zero AS (SELECT mr.dt, COALESCE(mt.total_payments, 0) AS total_payments FROM monthly_range mr LEFT JOIN monthly_totals mt ON mr.dt = mt.dt) SELECT m.dt::DATE AS MONTH, m.total_payments, m.total_payments - lag(m.total_payments, 1) OVER (ORDER BY dt) AS mom_change FROM monthly_totals_with_zero m ORDER BY m.dt;WITH monthly_payments AS (SELECT DATE_TRUNC('month', pr.payment_date) AS MONTH, SUM(pr.payment_amount) AS total_payments FROM payments_received pr GROUP BY MONTH ORDER BY MONTH), monthly_range AS (SELECT generate_series(DATE_TRUNC('month', MIN(pr.payment_date)), DATE_TRUNC('month', MAX(pr.payment_date)), '1 month'::interval) AS MONTH FROM payments_received pr), monthly_payments_with_zeros AS (SELECT mr.month, COALESCE(mp.total_payments, 0) AS total_payments FROM monthly_range mr LEFT JOIN monthly_payments mp ON mr.month = mp.month) SELECT mp.month, mp.total_payments, mp.total_payments - lag(mp.total_payments, 1) OVER (ORDER BY mp.month) AS mom_change FROM monthly_payments_with_zeros mp ORDER BY mp.month;WITH monthly_totals AS (SELECT TO_CHAR(DATE_TRUNC('month', payment_date), 'YYYY-MM') AS dt, SUM(payment_amount) AS total_payments FROM payments_received GROUP BY dt), monthly_range AS (SELECT TO_CHAR(generate_series(DATE_TRUNC('month', MIN(payment_date)), DATE_TRUNC('month', MAX(payment_date)), '1 month'::interval), 'YYYY-MM') AS dt FROM payments_received), monthly_totals_with_zero AS (SELECT mr.dt, COALESCE(mt.total_payments, 0) AS total_payments FROM monthly_range mr LEFT JOIN monthly_totals mt ON mr.dt = mt.dt) SELECT m.dt AS MONTH, m.total_payments, m.total_payments - lag(m.total_payments, 1) OVER (ORDER BY dt) AS mom_change FROM monthly_totals_with_zero m ORDER BY m.dt;WITH months AS (SELECT generate_series( (SELECT date_trunc('month', MIN(payment_date)) FROM payments_received), (SELECT date_trunc('month', MAX(payment_date)) FROM payments_received), interval '1 month') AS MONTH), payments_agg AS (SELECT date_trunc('month', payment_date) AS MONTH, SUM(payment_amount) AS total_payments FROM payments_received GROUP BY 1) SELECT m.month, COALESCE(p.total_payments, 0) AS total_payments, COALESCE(COALESCE(p.total_payments, 0) - LAG(COALESCE(p.total_payments, 0)) OVER ( ORDER BY m.month), 0) AS mom_change FROM months m LEFT JOIN payments_agg p ON m.month = p.month ORDER BY m.month;","To ascertain the volume of sales conducted by each salesperson over a recent period, merge the salespersons and sales tables, applying a filter for recent sales transactions. To determine the average duration from sale date to payment date, perform a join between the sales and payments tables To calculate the average selling price, join the sales and products tables, group by product name, and compute the ratio of total sales amount to the number of sales MoM change = (current month value - prev month value). Return all months in your answer, including those where there were no payments." -car_dealership,instructions_date_join,"What are the PMSPS and PMSR in the last 6 months excluding the current month, for salespersons hired between 2022 and 2023 (both inclusive)? Return all months in your answer, including those where metrics are 0. Order by month ascending.",PMSPS = per month salesperson sales count. PMSR = per month sales revenue in dollars. Truncate date to month for aggregation.,"WITH date_range AS (SELECT generate_series(date_trunc('month', CURRENT_DATE - interval '6 months'), date_trunc('month', CURRENT_DATE - interval '1 month'), '1 month')::DATE AS month_start), sales_metrics AS (SELECT date_trunc('month', s.sale_date) AS sale_month, COUNT(s.id) AS PMSPS, SUM(s.sale_price) AS PMSR FROM sales s JOIN salespersons sp ON s.salesperson_id = sp.id WHERE EXTRACT(YEAR FROM sp.hire_date) BETWEEN 2022 AND 2023 AND s.sale_date >= date_trunc('month', CURRENT_DATE - interval '6 months') AND s.sale_date < date_trunc('month', CURRENT_DATE) GROUP BY sale_month) SELECT dr.month_start, COALESCE(sm.PMSPS, 0) AS PMSPS, COALESCE(sm.PMSR, 0) AS PMSR FROM date_range dr LEFT JOIN sales_metrics sm ON dr.month_start = sm.sale_month ORDER BY dr.month_start ASC","PMSPS = per month salesperson sales count. PMSR = per month sales revenue in dollars. Truncate date to month for aggregation. +car_dealership,instructions_date_join,"What are the PMSPS and PMSR in the last 6 months excluding the current month, for salespersons hired between 2022 and 2023 (both inclusive)? Return all months in your answer, including those where metrics are 0. Order by month ascending.",PMSPS = per month salesperson sales count. PMSR = per month sales revenue in dollars. Truncate date to month for aggregation.,"WITH date_range AS (SELECT generate_series(date_trunc('month', CURRENT_DATE - interval '6 months'), date_trunc('month', CURRENT_DATE - interval '1 month'), '1 month')::DATE AS month_start), sales_metrics AS (SELECT date_trunc('month', s.sale_date) AS sale_month, COUNT(s.id) AS PMSPS, SUM(s.sale_price) AS PMSR FROM sales s JOIN salespersons sp ON s.salesperson_id = sp.id WHERE EXTRACT(YEAR FROM sp.hire_date) BETWEEN 2022 AND 2023 AND s.sale_date >= date_trunc('month', CURRENT_DATE - interval '6 months') AND s.sale_date < date_trunc('month', CURRENT_DATE) GROUP BY sale_month) SELECT dr.month_start, COALESCE(sm.PMSPS, 0) AS PMSPS, COALESCE(sm.PMSR, 0) AS PMSR FROM date_range dr LEFT JOIN sales_metrics sm ON dr.month_start = sm.sale_month ORDER BY dr.month_start ASC;WITH month_series AS (SELECT generate_series(date_trunc('month', CURRENT_DATE) - interval '6 months', date_trunc('month', CURRENT_DATE) - interval '1 month', interval '1 month') AS MONTH), aggregated AS (SELECT date_trunc('month', s.sale_date) AS MONTH, count(s.id) AS PMSPS, sum(s.sale_price) AS PMSR FROM sales s JOIN salespersons sp ON sp.id = s.salesperson_id WHERE s.sale_date >= date_trunc('month', CURRENT_DATE) - interval '6 months' AND s.sale_date < date_trunc('month', CURRENT_DATE) AND sp.hire_date BETWEEN '2022-01-01' AND '2023-12-31' GROUP BY date_trunc('month', s.sale_date)) SELECT ms.month, COALESCE(a.PMSPS, 0) AS PMSPS, COALESCE(a.PMSR, 0) AS PMSR FROM month_series ms LEFT JOIN aggregated a ON ms.month = a.month ORDER BY ms.month ASC;","PMSPS = per month salesperson sales count. PMSR = per month sales revenue in dollars. Truncate date to month for aggregation. ASP = Average Sale Price during a specific timeframe To calculate the average days between a sale date and when the payment was received, join the relevant tables. TSC = Total Sales Count for a given period" @@ -71,7 +71,7 @@ car_dealership,instructions_date_join,How many Toyota cars were sold in the last To get the list of cars that were sold and their sale price, join the cars and sales tables Last 30 days = CURRENT_DATE - INTERVAL '30 days' to CURRENT_DATE. Always join sales with cars before using the sales table. When using car makes, model names, engine_type, and vin_number, match case-insensitively and allow partial matches using LIKE with wildcards." -car_dealership,instructions_date_join,"For sales with sale price over $30,000, how many payments were received in total and on weekends in each of the last 8 calendar weeks (excluding the current week)? Return the week (as a date), total payments received, and weekend payments received in ascending order.",Weekend days are Saturday (6) and Sunday (0). Truncate date to week for aggregation.,"SELECT DATE_TRUNC('week', p.payment_date) AS WEEK, COUNT(p.id) AS total_payments, COUNT(CASE WHEN EXTRACT(dow FROM p.payment_date) IN (0, 6) THEN 1 END) AS weekend_payments FROM payments_received p JOIN sales s ON p.sale_id = s.id WHERE s.sale_price > 30000 AND p.payment_date >= DATE_TRUNC('week', CURRENT_DATE) - INTERVAL '8 weeks' AND p.payment_date < DATE_TRUNC('week', CURRENT_DATE) GROUP BY WEEK ORDER BY WEEK","To calculate the average days between sale date and payment received date, join the sales and payments received tables Weekend days are Saturday (6) and Sunday (0). Truncate date to week for aggregation. When using car makes, model names, engine_type and vin_number, match case-insensitively and allow partial matches using LIKE with wildcards. To get the total sales amount per salesperson, join the salespersons and sales tables, group by salesperson, and sum the sale_price" +car_dealership,instructions_date_join,"For sales with sale price over $30,000, how many payments were received in total and on weekends in each of the last 8 calendar weeks (excluding the current week)? Return the week (as a date), total payments received, and weekend payments received in ascending order.",Weekend days are Saturday (6) and Sunday (0). Truncate date to week for aggregation.,"SELECT DATE_TRUNC('week', p.payment_date) AS WEEK, COUNT(p.id) AS total_payments, COUNT(CASE WHEN EXTRACT(dow FROM p.payment_date) IN (0, 6) THEN 1 END) AS weekend_payments FROM payments_received p JOIN sales s ON p.sale_id = s.id WHERE s.sale_price > 30000 AND p.payment_date >= DATE_TRUNC('week', CURRENT_DATE) - INTERVAL '8 weeks' AND p.payment_date < DATE_TRUNC('week', CURRENT_DATE) GROUP BY WEEK ORDER BY WEEK;SELECT date_trunc('week', pr.payment_date)::date AS WEEK, COUNT(*) AS total_payments_received, SUM( CASE WHEN EXTRACT( DOW FROM pr.payment_date )::int IN (0, 6) THEN 1 ELSE 0 END ) AS weekend_payments_received FROM payments_received pr JOIN sales s ON s.id = pr.sale_id WHERE s.sale_price > 30000 AND pr.payment_date >= date_trunc('week', CURRENT_DATE) - interval '8 weeks' AND pr.payment_date < date_trunc('week', CURRENT_DATE) GROUP BY WEEK ORDER BY WEEK ASC;","To calculate the average days between sale date and payment received date, join the sales and payments received tables Weekend days are Saturday (6) and Sunday (0). Truncate date to week for aggregation. When using car makes, model names, engine_type and vin_number, match case-insensitively and allow partial matches using LIKE with wildcards. To get the total sales amount per salesperson, join the salespersons and sales tables, group by salesperson, and sum the sale_price" car_dealership,instructions_date_join,"What is the make, model and sale price of the car with the highest sale price that was sold on the same day it went out of inventory?","If inventory snapshots and sales from the same day are to be joined, join on the truncated date fields eg FROM inventory_snapshots i JOIN sales s ON DATE_TRUNC('day', i.snapshot_date) = DATE_TRUNC('day', s.sale_date).","SELECT c.make, c.model, s.sale_price FROM cars c JOIN sales s ON c.id = s.car_id JOIN inventory_snapshots i ON c.id = i.car_id AND DATE_TRUNC('day', s.sale_date) = DATE_TRUNC('day', i.snapshot_date) WHERE i.is_in_inventory = FALSE ORDER BY s.sale_price DESC LIMIT 1","ASP (average selling price) = total sales amount / number of sales To calculate the gross profit margin, join the appropriate tables and calculate the margin If inventory snapshots and sales from the same day are to be joined, join on the truncated date fields eg FROM inventory_snapshots i JOIN sales s ON DATE_TRUNC('day', i.snapshot_date) = DATE_TRUNC('day', s.sale_date). @@ -85,7 +85,7 @@ car_dealership,instructions_string_matching,"How many sales were made for each c When using car makes, model names, engine_type and vin_number, match case-insensitively and allow partial matches using LIKE with wildcards. To calculate the average selling price, join the sales and cars tables, and divide the total sales amount by the number of sales For understanding the number of sales achieved by each salesperson within a specified period, merge the salespersons and sales tables and apply a filter based on the given time frame." -car_dealership,keywords_aggregate,"What is the TSC in the past 7 days, inclusive of today?","TSC = Total Sales Count.",SELECT COUNT(id) AS TSC FROM sales WHERE sale_date >= CURRENT_DATE - INTERVAL '7 days',"To determine the average selling price, divide the total sales amount by the number of sales +car_dealership,keywords_aggregate,"What is the TSC in the past 7 days, inclusive of today?","TSC = Total Sales Count.",SELECT COUNT(id) AS TSC FROM sales WHERE sale_date BETWEEN CURRENT_DATE - INTERVAL '6 days' AND CURRENT_DATE,"To determine the average selling price, divide the total sales amount by the number of sales To calculate the gross profit margin, subtract the total cost from the total revenue, then divide by the total cost and multiply by 100 To ascertain the sales rank of each salesperson, order them by their total sales amount in descending order. TSC = Total Sales Count." @@ -119,7 +119,7 @@ derm_treatment,instructions_cte_window,"What is the NPI for each year? Return th If events from two different sources need to be linked based on time, it's advised to align them by truncating both dates to the same interval for the join condition. To find the average weight of patients treated with any drug, join the patients and treatments tables using patient_id, then filter as needed by the drug To identify doctors who have prescribed any drug type along with their states, join the doctors with treatments on doc_id, then apply the necessary filter for the drug type" -derm_treatment,instructions_cte_window,"Return each doctor's doc_id, specialty, number of distinct drugs prescribed, and SDR","SDR = a doctor's rank within their specialty by number of distinct drugs prescribed. Doctors prescribing more drugs will have a higher rank","WITH doc_drug_counts AS (SELECT d.doc_id, d.specialty, COUNT(DISTINCT t.drug_id) AS num_drugs_prescribed FROM doctors d JOIN treatments t ON d.doc_id = t.doc_id GROUP BY d.doc_id) SELECT doc_id, specialty, num_drugs_prescribed, DENSE_RANK() OVER (PARTITION BY specialty ORDER BY num_drugs_prescribed DESC) AS specialty_drug_rank FROM doc_drug_counts","To calculate the average weight of all patients who have been given a certain medication, join the patients and treatments tables on the patient's ID, then apply a filter for the specific medication To discover physicians who have prescribed a particular type of medication and identify their locations, link the doctors and treatments tables via the doctor's ID, then use a condition to filter by the medication type To ascertain the total count of adverse reactions recorded for treatments that involve topical medications, merge the treatments and adverse_events tables using the treatment ID, and apply a filter for the medication type. SDR = a doctor's rank within their specialty by number of distinct drugs prescribed, with doctors prescribing the most distinct drugs having rank 1" +derm_treatment,instructions_cte_window,"Return each doctor's doc_id, specialty, number of distinct drugs prescribed, and SDR","SDR = a doctor's rank within their specialty by number of distinct drugs prescribed. Doctors prescribing more drugs will have a higher rank","SELECT doc_id, specialty, num_drugs, RANK() OVER (PARTITION BY specialty ORDER BY num_drugs DESC) AS SDR FROM (SELECT d.doc_id, d.specialty, COUNT(DISTINCT t.drug_id) AS num_drugs FROM doctors d LEFT JOIN treatments t ON d.doc_id = t.doc_id GROUP BY d.doc_id, d.specialty) AS sub;WITH doc_drug_counts AS (SELECT d.doc_id, d.specialty, COUNT(DISTINCT t.drug_id) AS distinct_drugs FROM doctors d JOIN treatments t ON d.doc_id = t.doc_id GROUP BY d.doc_id, d.specialty) SELECT doc_id, specialty, distinct_drugs AS number_of_distinct_drugs_prescribed, RANK() OVER (PARTITION BY specialty ORDER BY distinct_drugs DESC) AS SDR FROM doc_drug_counts;","To calculate the average weight of all patients who have been given a certain medication, join the patients and treatments tables on the patient's ID, then apply a filter for the specific medication To discover physicians who have prescribed a particular type of medication and identify their locations, link the doctors and treatments tables via the doctor's ID, then use a condition to filter by the medication type To ascertain the total count of adverse reactions recorded for treatments that involve topical medications, merge the treatments and adverse_events tables using the treatment ID, and apply a filter for the medication type. SDR = a doctor's rank within their specialty by number of distinct drugs prescribed, with doctors prescribing the most distinct drugs having rank 1" derm_treatment,instructions_date_join,"How many treatments did the patient Alice have in the last 6 months, not including the current month?","Last 6 months = DATE_TRUNC('month', CURRENT_DATE) - INTERVAL '6 months' to DATE_TRUNC('month', CURRENT_DATE) - INTERVAL '1 day'. Always join treatments with patients before using the treatments table. If not mentioned, the patient name provided is the first name.","SELECT COUNT(t.treatment_id) FROM treatments t JOIN patients p ON t.patient_id = p.patient_id WHERE p.first_name = 'Alice' AND t.start_dt BETWEEN DATE_TRUNC('month', CURRENT_DATE) - INTERVAL '6 months' AND DATE_TRUNC('month', CURRENT_DATE) - INTERVAL '1 day'","Last 6 months = DATE_TRUNC('month', CURRENT_DATE) - INTERVAL '6 months' to DATE_TRUNC('month', CURRENT_DATE) - INTERVAL '1 day'. Always join treatments with patients before using the treatments table. If not mentioned, the patient name provided is the first name. To identify doctors with certain name characteristics; To calculate the total adverse event counts for specific drug treatments; To examine patient diagnosis and treatment correlations with certain drugs." derm_treatment,instructions_date_join,"What are the PMPD and PMTC for each of the last 12 months, not including the current month",PMPD = per month patient diagnoses. PMTC = per month treatment count. Truncate start_dt to month for aggregation.,"SELECT DATE_TRUNC('month', t.start_dt) AS MONTH, COUNT(DISTINCT d.diag_id) AS patient_count, COUNT(DISTINCT t.treatment_id) AS treatment_count FROM treatments t JOIN diagnoses d ON t.diag_id = d.diag_id WHERE t.start_dt >= DATE_TRUNC('month', CURRENT_DATE) - INTERVAL '12 months' AND t.start_dt < DATE_TRUNC('month', CURRENT_DATE) GROUP BY MONTH",PMPD = per month patient diagnoses. PMTC = per month treatment count. Truncate start_dt to month for aggregation. TPC = count of female patients; AWF = average weight for male patients; D7D100PIR = (average PASI score at day 100 - average PASI score at day 7) / average PASI score at day 7 * 100 with non-null PASI scores at both timepoints; DDD = total consumed drug amount divided by total treatment days. derm_treatment,instructions_date_join,"How many distinct patients had treatments in each of the last 3 months, not including the current month? Out of these, how many had treatments with biologic drugs? Return the month, patient count, and biologic treatment count.",Biologic drugs have drug_type = 'biologic'. Truncate start_dt to month for aggregation.,"SELECT DATE_TRUNC('month', t.start_dt) AS MONTH, COUNT(DISTINCT t.patient_id) AS patient_count, COUNT(DISTINCT CASE WHEN d.drug_type = 'biologic' THEN t.patient_id ELSE NULL END) AS biologic_count FROM treatments t JOIN drugs d ON t.drug_id = d.drug_id WHERE t.start_dt >= DATE_TRUNC('month', CURRENT_DATE) - INTERVAL '3 months' AND t.start_dt < DATE_TRUNC('month', CURRENT_DATE) GROUP BY MONTH;SELECT DATE_TRUNC('month', t.start_dt)::date AS month, COUNT(DISTINCT t.patient_id) AS patient_count, COUNT(DISTINCT CASE WHEN d.drug_type = 'biologic' THEN t.patient_id ELSE NULL END) AS biologic_count FROM treatments t JOIN drugs d ON t.drug_id = d.drug_id WHERE t.start_dt >= DATE_TRUNC('month', CURRENT_DATE) - INTERVAL '3 months' AND t.start_dt < DATE_TRUNC('month', CURRENT_DATE) GROUP BY month;SELECT TO_CHAR(DATE_TRUNC('month', t.start_dt), 'YYYY-MM') AS month, COUNT(DISTINCT t.patient_id) AS patient_count, COUNT(DISTINCT CASE WHEN d.drug_type = 'biologic' THEN t.patient_id ELSE NULL END) AS biologic_count FROM treatments t JOIN drugs d ON t.drug_id = d.drug_id WHERE t.start_dt >= DATE_TRUNC('month', CURRENT_DATE) - INTERVAL '3 months' AND t.start_dt < DATE_TRUNC('month', CURRENT_DATE) GROUP BY month;","To calculate the DDD (Defined Daily Dose) = the total amount of drug consumed divided by the total number of days of treatment @@ -150,7 +150,7 @@ ewallet,instructions_cte_join,What is the total transaction amount for each coup To get coupon usage, join the coupons and wallet_transactions_daily on coupon_id STR (success transaction rate) = Number of successful transactions divided by total transactions PMDAU = Aggregation based on truncating creation dates to months for active user metrics." -ewallet,instructions_cte_join,What is the total session duration in seconds for each user between 2023-06-01 inclusive and 2023-06-08 exclusive? Return the user ID and their total duration as an integer sorted by total duration with the longest duration first,"To analyze user engagement, join the users and user_sessions tables in a CTE, then aggregate to calculate total session duration per user for a given date range. Remember to convert your answer into seconds, using EPOCH function","WITH user_session_duration AS (SELECT u.uid, s.session_start_ts, s.session_end_ts FROM consumer_div.users u JOIN consumer_div.user_sessions s ON u.uid = s.user_id WHERE s.session_start_ts >= '2023-06-01' AND s.session_end_ts < '2023-06-08') SELECT UID, CAST(SUM(EXTRACT(EPOCH FROM session_end_ts) - EXTRACT(EPOCH FROM session_start_ts)) AS INT) AS total_duration FROM user_session_duration GROUP BY UID ORDER BY total_duration DESC","LUB = Most recent account balance for each user. CPUR (coupon usage rate) = Ratio of distinct coupons used to the number of distinct transactions AMB = Average balance of user wallets over a specified period To analyze user engagement, join the users and user_sessions tables in a CTE, then aggregate to calculate total session duration per user for a given date range" +ewallet,instructions_cte_join,What is the total session duration in seconds for each user between 2023-06-01 inclusive and 2023-06-08 exclusive? Return the user ID and their total duration as an integer sorted by total duration with the longest duration first,"To analyze user engagement, join the users and user_sessions tables in a CTE, then aggregate to calculate total session duration per user for a given date range. Remember to convert your answer into seconds, using EPOCH function","WITH user_session_duration AS (SELECT u.uid, s.session_start_ts, s.session_end_ts FROM consumer_div.users u JOIN consumer_div.user_sessions s ON u.uid = s.user_id WHERE s.session_start_ts >= '2023-06-01' AND s.session_end_ts < '2023-06-08') SELECT UID, CAST(SUM(EXTRACT(EPOCH FROM session_end_ts) - EXTRACT(EPOCH FROM session_start_ts)) AS INT) AS total_duration FROM user_session_duration GROUP BY UID ORDER BY total_duration DESC;WITH session_durations AS (SELECT u.uid AS user_id, EXTRACT(EPOCH FROM (s.session_end_ts - s.session_start_ts)) AS duration_seconds FROM consumer_div.users u JOIN consumer_div.user_sessions s ON u.uid = s.user_id WHERE s.session_start_ts >= '2023-06-01' AND s.session_start_ts < '2023-06-08') SELECT user_id, CAST(SUM(duration_seconds) AS INTEGER) AS total_duration FROM session_durations GROUP BY user_id ORDER BY total_duration DESC;","LUB = Most recent account balance for each user. CPUR (coupon usage rate) = Ratio of distinct coupons used to the number of distinct transactions AMB = Average balance of user wallets over a specified period To analyze user engagement, join the users and user_sessions tables in a CTE, then aggregate to calculate total session duration per user for a given date range" ewallet,instructions_cte_join,What is the marketing opt-in preference for each user? Return the user ID and boolean opt-in value,"To get any user's settings, only select the latest snapshot of user_setting_snapshot for each user","WITH user_latest_setting AS (SELECT u.uid, s.marketing_opt_in, s.created_at, ROW_NUMBER() OVER (PARTITION BY u.uid ORDER BY s.created_at DESC) AS rn FROM consumer_div.users u JOIN consumer_div.user_setting_snapshot s ON u.uid = s.user_id) SELECT UID, marketing_opt_in FROM user_latest_setting WHERE rn = 1","CPUR (coupon usage rate) = Ratio of distinct coupons used to number of distinct transactions. AMB = Average balance of user wallets over a recent period STR (success transaction rate) = Ratio of successful transactions to total transactions diff --git a/data/instruct_basic_postgres.csv b/data/instruct_basic_postgres.csv index b832df2..ad59e60 100644 --- a/data/instruct_basic_postgres.csv +++ b/data/instruct_basic_postgres.csv @@ -30,7 +30,7 @@ derm_treatment,basic_group_order_limit,What are the top 2 specialties by number derm_treatment,basic_left_join,"Return the patient IDs, first names and last names of patients who have not received any treatments.","SELECT p.patient_id, p.first_name, p.last_name FROM patients p LEFT JOIN treatments t ON p.patient_id = t.patient_id WHERE t.patient_id IS NULL" derm_treatment,basic_left_join,Return the drug IDs and names of drugs that have not been used in any treatments.,"SELECT d.drug_id, d.drug_name FROM drugs d LEFT JOIN treatments t ON d.drug_id = t.drug_id WHERE t.drug_id IS NULL" ewallet,basic_join_date_group_order_limit,"Who are the top 2 merchants (receiver type 1) by total transaction amount in the past 150 days (inclusive of 150 days ago)? Return the merchant name, total number of transactions, and total transaction amount.","SELECT m.name AS merchant_name, COUNT(t.txid) AS total_transactions, SUM(t.amount) AS total_amount FROM consumer_div.merchants m JOIN consumer_div.wallet_transactions_daily t ON m.mid = t.receiver_id WHERE t.receiver_type = 1 AND t.created_at >= CURRENT_DATE - INTERVAL '150 days' GROUP BY m.name ORDER BY total_amount DESC LIMIT 2" -ewallet,basic_join_date_group_order_limit,"How many distinct active users sent money per month in 2023? Return the number of active users per month (as a date), starting from the earliest date. Do not include merchants in the query. Only include successful transactions.","SELECT DATE_TRUNC('month', t.created_at) AS MONTH, COUNT(DISTINCT t.sender_id) AS active_users FROM consumer_div.wallet_transactions_daily t JOIN consumer_div.users u ON t.sender_id = u.uid WHERE t.sender_type = 0 AND t.status = 'success' AND u.status = 'active' AND t.created_at >= '2023-01-01' AND t.created_at < '2024-01-01' GROUP BY MONTH ORDER BY MONTH" +ewallet,basic_join_date_group_order_limit,"How many distinct active users sent money per month in 2023? Return the number of active users per month (as a date), starting from the earliest date. Do not include merchants in the query. Only include successful transactions.","SELECT DATE_TRUNC('month', t.created_at) AS MONTH, COUNT(DISTINCT t.sender_id) AS active_users FROM consumer_div.wallet_transactions_daily t JOIN consumer_div.users u ON t.sender_id = u.uid WHERE t.sender_type = 0 AND t.status = 'success' AND u.status = 'active' AND t.created_at >= '2023-01-01' AND t.created_at < '2024-01-01' GROUP BY MONTH ORDER BY MONTH;SELECT DATE_TRUNC('month', w.created_at)::DATE AS MONTH, COUNT(DISTINCT w.sender_id) AS active_user_count FROM consumer_div.wallet_transactions_daily w JOIN consumer_div.users u ON w.sender_id = u.uid WHERE w.sender_type = 0 AND w.status = 'success' AND w.created_at >= '2023-01-01' AND w.created_at < '2024-01-01' AND u.status = 'active' GROUP BY DATE_TRUNC('month', w.created_at) ORDER BY MONTH ASC;" ewallet,basic_join_group_order_limit,"What are the top 3 most frequently used coupon codes? Return the coupon code, total number of redemptions, and total amount redeemed.","SELECT c.code AS coupon_code, COUNT(t.txid) AS redemption_count, SUM(t.amount) AS total_discount FROM consumer_div.coupons c JOIN consumer_div.wallet_transactions_daily t ON c.cid = t.coupon_id GROUP BY c.code ORDER BY redemption_count DESC LIMIT 3" ewallet,basic_join_group_order_limit,"Which are the top 5 countries by total transaction amount sent by users, sender_type = 0? Return the country, number of distinct users who sent, and total transaction amount.","SELECT u.country, COUNT(DISTINCT t.sender_id) AS user_count, SUM(t.amount) AS total_amount FROM consumer_div.users u JOIN consumer_div.wallet_transactions_daily t ON u.uid = t.sender_id WHERE t.sender_type = 0 GROUP BY u.country ORDER BY total_amount DESC LIMIT 5" ewallet,basic_join_distinct,"Return the distinct list of merchant IDs that have received money from a transaction. Consider all transaction types in the results you return, but only include the merchant ids in your final answer.",SELECT DISTINCT m.mid AS merchant_id FROM consumer_div.merchants m JOIN consumer_div.wallet_transactions_daily t ON m.mid = t.receiver_id WHERE t.receiver_type = 1 diff --git a/data/questions_gen_postgres.csv b/data/questions_gen_postgres.csv index 9bc1558..85e8220 100644 --- a/data/questions_gen_postgres.csv +++ b/data/questions_gen_postgres.csv @@ -1,11 +1,11 @@ question,query,db_name,query_category,instructions "Which authors have written publications in both the domain ""Machine Learning"" and the domain ""Data Science""?","SELECT {author.name,author.aid} FROM author WHERE author.aid IN (SELECT domain_author.aid FROM domain_author WHERE domain_author.did IN (SELECT domain.did FROM DOMAIN WHERE domain.name IN ('Machine Learning', 'Data Science') ) GROUP BY 1 HAVING COUNT(DISTINCT domain_author.did) = 2);",academic,group_by, -What is the total number of citations received by each author?,"SELECT {author.name, author.aid}, sum(publication.citation_num) AS total_citations FROM author JOIN writes ON author.aid = writes.aid JOIN publication ON writes.pid = publication.pid GROUP BY {} ORDER BY total_citations DESC NULLS LAST;SELECT {a.aid, a.name}, COUNT(c.cited) AS total_citations FROM author a JOIN writes w ON a.aid = w.aid JOIN publication p ON w.pid = p.pid JOIN cite c ON p.pid = c.cited GROUP BY {} ORDER BY total_citations DESC;",academic,group_by, +What is the total number of citations received by each author?,"SELECT {author.name, author.aid}, sum(publication.citation_num) AS total_citations FROM author JOIN writes ON author.aid = writes.aid JOIN publication ON writes.pid = publication.pid GROUP BY {} ORDER BY total_citations DESC NULLS LAST;SELECT {a.aid, a.name}, COUNT(c.cited) AS total_citations FROM author a JOIN writes w ON a.aid = w.aid JOIN publication p ON w.pid = p.pid JOIN cite c ON p.pid = c.cited GROUP BY {} ORDER BY total_citations DESC;SELECT {a.aid, a.name}, COALESCE(SUM(p.citation_num), 0) AS total_citations FROM author a LEFT JOIN writes w ON a.aid = w.aid LEFT JOIN publication p ON w.pid = p.pid GROUP BY {};SELECT {a.aid, a.name}, COALESCE(SUM(p.citation_num), 0) AS total_citations FROM author a LEFT JOIN writes w ON a.aid = w.aid LEFT JOIN publication p ON w.pid = p.pid GROUP BY {};",academic,group_by, What is the total number of publications published in each year?,"SELECT publication.year, COUNT(DISTINCT publication.pid) AS total_publications FROM publication GROUP BY publication.year ORDER BY publication.year;",academic,group_by, What is the average number of references cited by publications in each domain name?,"SELECT {domain.name,domain.did}, AVG(publication.reference_num) AS average_references FROM domain_publication JOIN publication ON domain_publication.pid = publication.pid JOIN domain ON domain.did = domain_publication.did GROUP BY {};",academic,group_by, What is the average number of citations received by publications in each year?,"SELECT publication.year, AVG(publication.citation_num) AS average_citations FROM publication GROUP BY publication.year ORDER BY publication.year NULLS LAST;",academic,group_by, What is the title of the publication that has received the highest number of citations?,SELECT publication.title FROM publication ORDER BY publication.citation_num DESC NULLS LAST LIMIT 1;,academic,order_by, -What are the top 5 domains with the highest number of authors associated with them?,"SELECT {d.name, d.did}, COUNT(DISTINCT a.aid) AS author_count FROM author a JOIN domain_author da ON a.aid = da.aid JOIN domain d ON da.did = d.did GROUP BY {} ORDER BY author_count DESC LIMIT 5;",academic,order_by, +What are the top 5 domains with the highest number of authors associated with them?,"SELECT {d.name, d.did}, COUNT(DISTINCT a.aid) AS author_count FROM author a JOIN domain_author da ON a.aid = da.aid JOIN domain d ON da.did = d.did GROUP BY {} ORDER BY author_count DESC LIMIT 5;SELECT {d.name, d.did}, COUNT(da.aid) AS author_count FROM DOMAIN AS d JOIN domain_author AS da ON d.did = da.did GROUP BY {} ORDER BY author_count DESC LIMIT 5;",academic,order_by, "What are the top 3 titles of the publications that have the highest number of references cited, ordered by the number of references cited in descending order?",SELECT publication.title FROM publication ORDER BY publication.reference_num DESC LIMIT 3;,academic,order_by, What are the top 3 publications with the highest number of citations?,"SELECT {publication.title, publication.pid}, publication.citation_num FROM publication ORDER BY publication.citation_num DESC LIMIT 3;",academic,order_by, What are the titles of all publications ordered alphabetically?,SELECT DISTINCT publication.title FROM publication ORDER BY publication.title ASC NULLS LAST;,academic,order_by, @@ -20,11 +20,11 @@ What is the total number of publications presented in each conference?,"SELECT { "How many publications were presented at each conference, ordered by the number of publications in descending order? Give the names of the conferences and their corresponding number of publications.","SELECT conference.name, COUNT(publication.pid) AS num_publications FROM publication JOIN conference ON publication.cid=conference.cid GROUP BY conference.name, conference.cid ORDER BY num_publications DESC NULLS LAST;",academic,table_join, "How many publications were published in journals whose names start with the letter ""J""?",SELECT count(DISTINCT publication.pid) FROM publication JOIN journal ON publication.jid = journal.jid WHERE journal.name ilike 'J%';,academic,table_join, "Which organizations have authors who have written publications in the domain ""Machine Learning""?","SELECT DISTINCT {organization.name, organization.oid} FROM organization JOIN author ON organization.oid = author.oid JOIN writes ON author.aid = writes.aid JOIN domain_publication ON writes.pid = domain_publication.pid JOIN domain ON domain_publication.did = domain.did WHERE domain.name = 'Machine Learning';",academic,instruct,Always filter names using an exact match -Which authors belong to the same domain as Martin?,"SELECT DISTINCT {a2.name, a2.aid} FROM author a1 JOIN domain_author da1 ON a1.aid = da1.aid JOIN domain_author da2 ON da1.did = da2.did JOIN author a2 ON da2.aid = a2.aid WHERE LOWER(a1.name) ILIKE '%martin%';",academic,instruct,Always filter names using ILIKE with percent sign wildcards +Which authors belong to the same domain as Martin?,"SELECT DISTINCT {a2.name, a2.aid} FROM author a1 JOIN domain_author da1 ON a1.aid = da1.aid JOIN domain_author da2 ON da1.did = da2.did JOIN author a2 ON da2.aid = a2.aid WHERE LOWER(a1.name) ILIKE '%martin%';SELECT DISTINCT a2.name, a2.aid FROM author a1 JOIN domain_author da1 ON a1.aid = da1.aid JOIN domain_author da2 ON da1.did = da2.did JOIN author a2 ON da2.aid = a2.aid WHERE LOWER(a1.name) ILIKE '%martin%' AND a2.name NOT ILIKE '%martin%';",academic,instruct,Always filter names using ILIKE with percent sign wildcards Which authors are not part of any organization?,"SELECT DISTINCT {name, aid} FROM author WHERE oid IS NULL",academic,instruct,Always filter names using ILIKE What are the publications written by authors from the 'Sociology' domain and presented at conferences in the year 2020?,"SELECT DISTINCT {publication.title, publication.pid} FROM DOMAIN JOIN domain_author ON domain.did = domain_author.did JOIN writes ON domain_author.aid = writes.aid JOIN publication ON writes.pid = publication.pid JOIN domain_conference ON domain.did = domain_conference.did WHERE domain.name ILIKE '%Sociology%' AND publication.year = 2020 AND publication.cid = domain_conference.cid;",academic,instruct,"To get publications written by authors from a given domain, you would need to join domain, domain_author, author to link the domain to the author first, and then join with write to link with the publication id. Finally, to see which ones were presented at conferences, you must join the domain table with the domain_conference table. You must also filter names using ILIKE." "What are the names of the authors who have written publications in the domain ""Computer Science""?",SELECT DISTINCT author.name FROM author JOIN writes ON author.aid = writes.aid JOIN publication ON writes.pid = publication.pid JOIN domain_publication ON publication.pid = domain_publication.pid JOIN domain ON domain_publication.did = domain.did WHERE domain.name ilike '%computer%science%';,academic,instruct,"To get publications written by authors from a given domain, you would need to join domain, domain_author, author to link the domain to the author first, and then join with write to link with the publication id. You must also filter names using ILIKE." -"What month were most students admitted? Return the no. of students and the month as a date","SELECT date_trunc('month', s.admit_term) AS month, COUNT(*) AS total_students FROM student s GROUP BY MONTH ORDER BY total_students DESC LIMIT 1;SELECT TO_CHAR(DATE_TRUNC('month', s.admit_term), 'YYYY-MM') AS month, COUNT(*) AS total_students FROM student s GROUP BY month ORDER BY total_students DESC LIMIT 1;",advising,date_functions, +"What month were most students admitted? Return the no. of students and the month as a date","SELECT date_trunc('month', s.admit_term) AS month, COUNT(*) AS total_students FROM student s GROUP BY MONTH ORDER BY total_students DESC LIMIT 1;SELECT TO_CHAR(DATE_TRUNC('month', s.admit_term), 'YYYY-MM') AS month, COUNT(*) AS total_students FROM student s GROUP BY month ORDER BY total_students DESC LIMIT 1;SELECT date_trunc('month', admit_term)::date AS MONTH, COUNT(*) AS num_students FROM student GROUP BY MONTH ORDER BY COUNT(*) DESC LIMIT 1;",advising,date_functions, What's the average predicted time to graduation since admission in no. of days?,SELECT avg(predicted_graduation_semester - admit_term) AS average_predicted_time_to_graduation FROM student;,advising,date_functions, How many students were predicted to graduate in the last 10 years?,"SELECT count(*) AS num_students_graduated FROM student WHERE predicted_graduation_semester >= DATE_TRUNC('year', CURRENT_DATE) - interval '10 year';",advising,date_functions, How long has it been in days since the last admitted student? Give the answer as an integer.,SELECT CURRENT_DATE - max(admit_term) AS duration_since_last_admitted_student FROM student;,advising,date_functions, @@ -61,7 +61,7 @@ Which flight has the shortest duration between departure and arrival times? Conv "What's the average duration between departure and arrival times minus 34 minutes? Convert from UNIX to regular datetime, and return the answer in minutes",SELECT avg(to_timestamp(arrival_time) - to_timestamp(departure_time) - interval '34 minutes') AS average_duration FROM flight;SELECT AVG(arrival_time - departure_time)/60 - 34 AS average_duration FROM flight;,atis,date_functions, Count the number of flight departures for each month?,"SELECT month.month_name, count(*) AS departure_count FROM flight JOIN month ON extract(MONTH FROM to_timestamp(flight.departure_time)) = month.month_number GROUP BY month.month_name, month.month_number ORDER BY month.month_number;SELECT date_trunc('month', to_timestamp(flight.departure_time)) AS month, COUNT(*) AS num_departures FROM flight GROUP BY MONTH ORDER BY MONTH;SELECT EXTRACT(MONTH FROM to_timestamp(flight.departure_time)) AS month, COUNT(*) AS num_departures FROM flight GROUP BY month ORDER BY month;SELECT TO_CHAR(TO_TIMESTAMP(flight.departure_time), 'YYYY-MM') AS month, COUNT(*) AS num_departures FROM flight GROUP BY month ORDER BY month;",atis,date_functions, What's the earliest flight departure time in the day in HH:MM?,"SELECT to_char(to_timestamp(departure_time)::TIME, 'HH24:MI') AS earliest_departure_time FROM flight ORDER BY earliest_departure_time LIMIT 1;",atis,date_functions, -What's the absolute difference in time in days between today and the earliest flight departure? Give the answer as an integer.,"SELECT date_part('day', CURRENT_DATE - to_timestamp(departure_time)) AS difference_in_days FROM flight ORDER BY departure_time LIMIT 1;SELECT (CURRENT_DATE - TO_TIMESTAMP(MIN(f.departure_time))) AS days_difference FROM flight f;",atis,date_functions, +What's the absolute difference in time in days between today and the earliest flight departure? Give the answer as an integer.,"SELECT date_part('day', CURRENT_DATE - to_timestamp(departure_time)) AS difference_in_days FROM flight ORDER BY departure_time LIMIT 1;SELECT (CURRENT_DATE - TO_TIMESTAMP(MIN(f.departure_time))) AS days_difference FROM flight f;SELECT CAST(ABS((EXTRACT(EPOCH FROM CURRENT_TIMESTAMP) - EXTRACT(EPOCH FROM TO_TIMESTAMP(MIN(departure_time)))) / 86400) AS INTEGER) AS diff_in_days FROM flight;",atis,date_functions, What is the total cost of round-trip fares for each airline code?,"SELECT fare.fare_airline, SUM(fare.round_trip_cost) AS total_round_trip_cost FROM fare GROUP BY fare.fare_airline ORDER BY total_round_trip_cost DESC;",atis,group_by, "What is the average cost of round-trip fares from Los Angeles (LAX) to Chicago (ORD) for each airline, sorted in descending order by average cost?","SELECT fare.fare_airline, AVG(fare.round_trip_cost) AS average_cost FROM fare WHERE fare.from_airport = 'LAX' AND fare.to_airport = 'ORD' GROUP BY fare.fare_airline ORDER BY average_cost DESC NULLS LAST;SELECT airline.airline_name, AVG(fare.round_trip_cost) AS avg_round_trip_cost FROM fare JOIN airline ON fare.fare_airline = airline.airline_code WHERE fare.from_airport = 'LAX' AND fare.to_airport = 'ORD' GROUP BY airline.airline_name ORDER BY avg_round_trip_cost DESC;",atis,group_by, "What is the average cost of a one-way trip for each airport pair in the fare table?","SELECT f.from_airport, f.to_airport, AVG(f.one_direction_cost) AS average_cost FROM fare f GROUP BY f.from_airport, f.to_airport ORDER BY f.from_airport, f.to_airport NULLS LAST;",atis,group_by, @@ -99,7 +99,7 @@ What is the total population in cities by country?,"SELECT city.country_name, SU What is the average length of rivers in each country?,"SELECT river.country_name, AVG(river.length) AS average_length FROM river GROUP BY river.country_name ORDER BY average_length DESC NULLS LAST;",geography,group_by, How many rivers flow through each country?,"SELECT river.country_name, COUNT(DISTINCT river.river_name) AS number_of_rivers FROM river GROUP BY river.country_name ORDER BY number_of_rivers DESC;",geography,group_by, How many mountains are there in each country?,"SELECT mountain.country_name, COUNT(mountain.mountain_name) AS number_of_mountains FROM mountain GROUP BY mountain.country_name ORDER BY number_of_mountains DESC;",geography,group_by, -How many lakes are there in each state?,"SELECT lake.state_name, COUNT(lake.lake_name) AS lake_count FROM lake GROUP BY lake.state_name ORDER BY lake_count DESC;",geography,group_by, +How many lakes are there in each state?,"SELECT lake.state_name, COUNT(lake.lake_name) AS lake_count FROM lake GROUP BY lake.state_name ORDER BY lake_count DESC;SELECT s.state_name, COUNT(l.lake_name) AS lake_count FROM state s LEFT JOIN lake l ON s.state_name = l.state_name GROUP BY s.state_name ORDER BY s.state_name;",geography,group_by, "Which states have the highest population density in people per square kilometer, ordered from highest to lowest?","SELECT state.state_name, state.density FROM state ORDER BY state.density DESC NULLS LAST;",geography,order_by, "Which lakes have the largest areas in square kilometers, ordered from largest to smallest?","SELECT lake.lake_name, lake.area FROM lake ORDER BY lake.area DESC NULLS LAST;",geography,order_by, What are the top 5 cities with the highest population? Give both city names and the population.,"SELECT city.city_name, city.population FROM city ORDER BY city.population DESC NULLS LAST LIMIT 5;",geography,order_by, @@ -170,7 +170,7 @@ What is the proportion of papers that belong to more than 1 dataset to papers th "How many papers were published in the journal ""nature"" in the year 2020?",SELECT COUNT(paper.paperid) FROM paper JOIN journal ON paper.journalid = journal.journalid WHERE paper.year = 2020 AND journal.journalname ILIKE '%nature%';,scholar,instruct,Filter strings with case-insensitive matching "How many papers are associated with the keyphrase ""machine learning"" and were published in the journal named ""IEEE Transactions on Pattern Analysis and Machine Intelligence""?",SELECT COUNT(DISTINCT paper.paperid) FROM paper JOIN journal ON paper.journalid = journal.journalid JOIN paperkeyphrase ON paper.paperid = paperkeyphrase.paperid JOIN keyphrase ON paperkeyphrase.keyphraseid = keyphrase.keyphraseid WHERE keyphrase.keyphrasename ILIKE '%machine learning%' AND journal.journalname = 'IEEE Transactions on Pattern Analysis and Machine Intelligence';,scholar,instruct,"Filter paper names, journal names, using exact matches. Filter keyphrases with case-insensitive matching." "How many authors wrote papers that were published in the journal ""Science"" in the year 2020?",SELECT COUNT(DISTINCT writes.authorid) AS number_of_authors FROM writes JOIN paper ON writes.paperid = paper.paperid JOIN journal ON paper.journalid = journal.journalid WHERE journal.journalname ILIKE '%Science%' AND paper.year = 2020;,scholar,instruct,Filter paper names using exact matches. Filter keyphrases and journal names with case-insensitive matching. -How many reviews were written for businesses located in California in the last 10 months?,"SELECT count(*) AS review_count FROM review r JOIN business b ON r.business_id = b.business_id WHERE b.state = 'CA' AND (r.year * 12 + extract(MONTH FROM to_date(r.month, 'Month'))) >= (extract(YEAR FROM CURRENT_DATE) * 12 + extract(MONTH FROM CURRENT_DATE) - 10);",yelp,date_functions, +How many reviews were written for businesses located in California in the last 10 calendar months (not including the current month)?,"SELECT count(*) AS review_count FROM review r JOIN business b ON r.business_id = b.business_id WHERE b.state = 'CA' AND (r.year * 12 + extract(MONTH FROM to_date(r.month, 'Month'))) >= (extract(YEAR FROM CURRENT_DATE) * 12 + extract(MONTH FROM CURRENT_DATE) - 10);",yelp,date_functions, What is the total number of check-ins on the 2 days before Saturday?,"SELECT sum(COUNT) AS total_checkins FROM checkin WHERE DAY IN ('Thursday', 'Friday') ;",yelp,date_functions, How many reviews were there 2 months before the review with id 3?,"SELECT COUNT(*) AS review_count FROM review WHERE TO_DATE(CAST(review.year AS TEXT) || '-' || CASE review.month WHEN 'January' THEN '01' WHEN 'February' THEN '02' WHEN 'March' THEN '03' WHEN 'April' THEN '04' WHEN 'May' THEN '05' WHEN 'June' THEN '06' WHEN 'July' THEN '07' WHEN 'August' THEN '08' WHEN 'September' THEN '09' WHEN 'October' THEN '10' WHEN 'November' THEN '11' WHEN 'December' THEN '12' END || '-01', 'YYYY-MM-DD') = (SELECT TO_DATE(CAST(r.year AS TEXT) || '-' || CASE r.month WHEN 'January' THEN '01' WHEN 'February' THEN '02' WHEN 'March' THEN '03' WHEN 'April' THEN '04' WHEN 'May' THEN '05' WHEN 'June' THEN '06' WHEN 'July' THEN '07' WHEN 'August' THEN '08' WHEN 'September' THEN '09' WHEN 'October' THEN '10' WHEN 'November' THEN '11' WHEN 'December' THEN '12' END || '-01', 'YYYY-MM-DD') - INTERVAL '2 months' FROM review r WHERE r.rid = 3);",yelp,date_functions, What was the message that came with the tip made exactly 2 months after March 2021?,SELECT text AS message FROM tip WHERE MONTH ILIKE '%May%' AND YEAR = 2021 LIMIT 1;,yelp,date_functions, @@ -187,7 +187,7 @@ What are the top 3 businesses in terms of review count?,"SELECT {business.name, "How many reviews were posted in each month of the year 2021, ordered by the month?","SELECT review.month, COUNT(review.rid) AS review_count FROM review WHERE review.year = 2021 GROUP BY review.month ORDER BY TO_DATE(review.month, 'Month') NULLS LAST;",yelp,order_by, What is the ratio of the number of businesses in each state to the total number of businesses in the database?,"SELECT business.state, COUNT(business.business_id) / NULLIF(CAST((SELECT COUNT(*) FROM business) AS FLOAT), 0) AS ratio FROM business GROUP BY business.state;",yelp,ratio, What is the ratio of open businesses to closed businesses in the city of San Francisco?,"SELECT CAST(SUM(CASE WHEN business.is_open = 1 THEN 1 ELSE 0 END) AS FLOAT) / NULLIF(SUM(CASE WHEN business.is_open = 0 THEN 1 ELSE 0 END), 0) AS ratio FROM business WHERE LOWER(business.city) ILIKE '%san francisco%';",yelp,ratio, -"What is the ratio of check-ins on weekends to check-ins on weekdays for the business named ""Mark's Bistro""?","SELECT CAST(SUM(CASE WHEN checkin.day IN ('Saturday', 'Sunday') THEN checkin.count ELSE 0 END) AS FLOAT) / NULLIF(SUM(CASE WHEN checkin.day NOT IN ('Saturday', 'Sunday') THEN checkin.count ELSE 0 END), 0) AS ratio FROM checkin JOIN business ON checkin.business_id = business.business_id WHERE business.name ILIKE '%Mark''s Bistro%';",yelp,ratio, +"What is the ratio of check-ins on weekends to check-ins on weekdays for the business named ""Mark’s Bistro""?","SELECT CAST(SUM(CASE WHEN checkin.day IN ('Saturday', 'Sunday') THEN checkin.count ELSE 0 END) AS FLOAT) / NULLIF(SUM(CASE WHEN checkin.day NOT IN ('Saturday', 'Sunday') THEN checkin.count ELSE 0 END), 0) AS ratio FROM checkin JOIN business ON checkin.business_id = business.business_id WHERE business.name ILIKE '%Mark’s Bistro%';",yelp,ratio, What is the ratio of businesses in the state of California to businesses in the state of New York?,"SELECT CAST(COUNT(CASE WHEN business.state = 'CA' THEN 1 END) AS FLOAT) / NULLIF(COUNT(CASE WHEN business.state = 'NY' THEN 1 END), 0) AS ratio FROM business;",yelp,ratio, "How does the ratio of positive reviews (rating > 3) to negative reviews (rating < 3) vary across different categories of businesses, ordered by descending ratio?","SELECT {category.category_name, category.id}, CAST(COUNT(CASE WHEN review.rating > 3 THEN 1 END) AS FLOAT) / NULLIF(COUNT(CASE WHEN review.rating < 3 THEN 1 END), 0) AS ratio FROM review JOIN category ON review.business_id = category.business_id GROUP BY {} ORDER BY ratio DESC NULLS LAST;",yelp,ratio, "Which users have posted reviews for businesses located in the neighbourhood of ""Downtown"" and how many reviews have they posted?","SELECT {users.name, users.user_id}, COUNT(review.rid) AS review_count FROM review JOIN neighbourhood ON review.business_id = neighbourhood.business_id JOIN users ON review.user_id = users.user_id WHERE neighbourhood.neighbourhood_name ILIKE '%Downtown%' GROUP BY {} ORDER BY review_count DESC NULLS LAST;",yelp,table_join, @@ -216,9 +216,9 @@ Filter strings of state using exact upper case matches. Assume the rating of a business to be its average rating, and compute it before computing other aggregates on it. " "Return the customer who made the most sell transactions on 2023-04-01. Return the id, name and number of transactions.","WITH SellTransactions AS (SELECT sbTxCustId, COUNT(*) AS num_tx FROM sbTransaction WHERE sbTxDateTime::date = '2023-04-01' AND sbTxType = 'sell' GROUP BY sbTxCustId) SELECT c.sbCustId, c.sbCustName, st.num_tx FROM sbCustomer c JOIN SellTransactions st ON c.sbCustId = st.sbTxCustId ORDER BY st.num_tx DESC LIMIT 1;",broker,date_functions, -"What is the monthly average transaction price for successful transactions in the 1st quarter of 2023?","SELECT DATE_TRUNC('month', sbTxDateTime) AS MONTH, AVG(sbTxPrice) AS avg_price FROM sbTransaction WHERE sbTxStatus = 'success' AND sbTxDateTime BETWEEN '2023-01-01' AND '2023-03-31' GROUP BY MONTH ORDER BY MONTH;",broker,date_functions, +"What is the monthly average transaction price for successful transactions in the 1st quarter of 2023?","SELECT DATE_TRUNC('month', sbTxDateTime) AS MONTH, AVG(sbTxPrice) AS avg_price FROM sbTransaction WHERE sbTxStatus = 'success' AND sbTxDateTime BETWEEN '2023-01-01' AND '2023-03-31' GROUP BY MONTH ORDER BY MONTH;SELECT TO_CHAR(sbTxDateTime, 'YYYY-MM') AS MONTH, AVG(sbTxPrice) AS average_transaction_price FROM sbTransaction WHERE sbTxStatus = 'success' AND sbTxDateTime >= '2023-01-01' AND sbTxDateTime < '2023-04-01' GROUP BY TO_CHAR(sbTxDateTime, 'YYYY-MM') ORDER BY MONTH;",broker,date_functions, "Lowest daily closest price for symbol `VTI` in the past 7 days","SELECT MIN(sdp.sbDpClose) AS lowest_price FROM sbDailyPrice AS sdp JOIN sbTicker AS st ON sdp.sbDpTickerId = st.sbTickerId WHERE st.sbTickerSymbol = 'VTI' AND sdp.sbDpDate >= CURRENT_DATE - INTERVAL '7 days';",broker,date_functions, -"Return the customer id and the difference between their time from joining to their first transaction. Ignore customers who haven't made any transactions.","SELECT c.sbCustId, MIN(t.sbTxDateTime) - c.sbCustJoinDate AS DaysFromJoinToFirstTransaction FROM sbCustomer c JOIN sbTransaction t ON c.sbCustId = t.sbTxCustId GROUP BY c.sbCustId",broker,date_functions, +"Return the customer id and the difference between their time from joining to their first transaction. Ignore customers who haven't made any transactions.","SELECT c.sbCustId, MIN(t.sbTxDateTime) - c.sbCustJoinDate AS DaysFromJoinToFirstTransaction FROM sbCustomer c JOIN sbTransaction t ON c.sbCustId = t.sbTxCustId GROUP BY c.sbCustId;SELECT c.sbCustId, age(MIN(t.sbTxDateTime), c.sbCustJoinDate) AS time_difference FROM sbCustomer c JOIN sbTransaction t ON c.sbCustId = t.sbTxCustId GROUP BY c.sbCustId, c.sbCustJoinDate;SELECT c.sbCustId, MIN(DATE(t.sbTxDateTime)) - c.sbCustJoinDate AS days_difference FROM sbCustomer c JOIN sbTransaction t ON c.sbCustId = t.sbTxCustId GROUP BY c.sbCustId, c.sbCustJoinDate;",broker,date_functions, "number of transactions by users who joined in the past 70 days","SELECT COUNT(t.sbTxCustId) AS transaction_count FROM sbTransaction t JOIN sbCustomer c ON t.sbTxCustId = c.sbCustId WHERE c.sbCustJoinDate >= CURRENT_DATE - INTERVAL '70' DAY;",broker,date_functions, "Return the treatment id, treatment start date, adverse event date and description of all adverse events that occured within 10 days after starting treatment","SELECT t.treatment_id, t.start_dt, ae.reported_dt, ae.description FROM adverse_events ae JOIN treatments t ON ae.treatment_id = t.treatment_id WHERE ae.reported_dt BETWEEN t.start_dt AND t.start_dt + INTERVAL '10 days';",derm_treatment,date_functions, "List the last name, year of registration, and first treatment (date and id) by doctors who were registered 2 years ago.","WITH doc_first_treatment AS (SELECT d.doc_id, d.last_name, d.year_reg, t.treatment_id, t.start_dt, ROW_NUMBER() OVER (PARTITION BY d.doc_id ORDER BY t.start_dt ASC) AS rn FROM doctors d JOIN treatments t ON d.doc_id = t.doc_id WHERE d.year_reg = EXTRACT(YEAR FROM CURRENT_DATE) - 2 ) SELECT last_name, year_reg, start_dt AS first_treatment_date, treatment_id AS first_treatment_id FROM doc_first_treatment WHERE rn = 1;",derm_treatment,date_functions, @@ -230,8 +230,8 @@ Assume the rating of a business to be its average rating, and compute it before "what was the average user session duration in seconds split by device_type?","SELECT device_type, AVG(EXTRACT(EPOCH FROM (session_end_ts - session_start_ts))) AS avg_session_duration_seconds FROM consumer_div.user_sessions WHERE session_end_ts IS NOT NULL GROUP BY device_type;",ewallet,date_functions, "Give me today's median merchant wallet balance for all active merchants whose category contains 'retail'","WITH retail_merchants AS (SELECT mid FROM consumer_div.merchants WHERE category ILIKE '%retail%' AND status = 'active' ), merchant_balances AS (SELECT balance FROM consumer_div.wallet_merchant_balance_daily wmbd JOIN retail_merchants rm ON wmbd.merchant_id = rm.mid WHERE DATE(wmbd.updated_at) = CURRENT_DATE ) SELECT PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY balance) AS median_balance FROM merchant_balances;",ewallet,date_functions, "Which merchants earliest coupon start date was within a year of the merchant's registration? Return the merchant id, registration date, and earliest coupon id and start date","WITH earliest_coupons AS (SELECT c.merchant_id, MIN(c.start_date) AS earliest_coupon_start_date FROM consumer_div.coupons c GROUP BY c.merchant_id) SELECT m.mid AS merchant_id, m.created_at AS merchant_registration_date, ec.earliest_coupon_start_date, c.cid AS earliest_coupon_id FROM consumer_div.merchants m JOIN earliest_coupons ec ON m.mid = ec.merchant_id JOIN consumer_div.coupons c ON ec.merchant_id = c.merchant_id AND ec.earliest_coupon_start_date = c.start_date WHERE ec.earliest_coupon_start_date <= m.created_at + INTERVAL '1 year';",ewallet,date_functions, -"Return the name and phone number of the salesperson with the shortest time from being hired to getting fired. Return the number of days he/she was employed for.","SELECT s.first_name, s.last_name, s.phone, s.termination_date - s.hire_date AS days_employed FROM salespersons s ORDER BY days_employed ASC LIMIT 1;",car_dealership,date_functions, +"Return the name and phone number of the salesperson with the shortest time from being hired to getting fired. Return the number of days he/she was employed for.","SELECT s.first_name, s.last_name, s.phone, s.termination_date - s.hire_date AS days_employed FROM salespersons s ORDER BY days_employed ASC LIMIT 1;SELECT first_name || ' ' || last_name AS name, phone, (termination_date - hire_date) AS days_employed FROM salespersons WHERE termination_date IS NOT NULL ORDER BY (termination_date - hire_date) LIMIT 1;",car_dealership,date_functions, "Return the number of payments made on weekends to the vendor named 'Utility Company'","SELECT COUNT(*) AS weekend_payments FROM payments_made WHERE vendor_name = 'Utility Company' AND EXTRACT(DOW FROM payment_date) IN (0, 6);",car_dealership,date_functions, "show me the daily total amount of payments received in the whole of the previous ISO week not including the current week, split by the payment_method","SELECT payment_date, payment_method, SUM(payment_amount) AS total_amount FROM payments_received WHERE payment_date >= DATE_TRUNC('WEEK', CURRENT_DATE) - INTERVAL '1 week' AND payment_date < DATE_TRUNC('WEEK', CURRENT_DATE) GROUP BY payment_date, payment_method ORDER BY payment_date DESC, payment_method ASC;",car_dealership,date_functions, "Which cars were in inventory in the latest snapshot for march 2023? Return the car id, make, model, and year. Cars are considered to be "in inventory" if is_in_inventory is True.","WITH latest_snapshot AS (SELECT MAX(snapshot_date) AS snapshot_date FROM inventory_snapshots WHERE snapshot_date BETWEEN '2023-03-01' AND '2023-03-31' ), latest_snapshot_data AS (SELECT inv.car_id FROM inventory_snapshots inv JOIN latest_snapshot ls ON inv.snapshot_date = ls.snapshot_date WHERE inv.is_in_inventory = TRUE ) SELECT c.id, c.make, c.model, c.year FROM cars c JOIN latest_snapshot_data lsd ON c.id = lsd.car_id;",car_dealership,date_functions, -"What were the total quarterly sales in 2023 grouped by customer's state? Represent each quarter as the first date in the quarter.","SELECT DATE_TRUNC('QUARTER', s.sale_date) AS QUARTER, c.state, SUM(s.sale_price) AS total_sales FROM sales s JOIN customers c ON s.customer_id = c.id WHERE EXTRACT(YEAR FROM s.sale_date) = 2023 GROUP BY c.state, QUARTER HAVING SUM(s.sale_price) > 0 ORDER BY QUARTER, c.state ;",car_dealership,date_functions, \ No newline at end of file +"What were the total quarterly sales in 2023 grouped by customer's state? Represent each quarter as the first date in the quarter.","SELECT DATE_TRUNC('QUARTER', s.sale_date) AS QUARTER, c.state, SUM(s.sale_price) AS total_sales FROM sales s JOIN customers c ON s.customer_id = c.id WHERE EXTRACT(YEAR FROM s.sale_date) = 2023 GROUP BY c.state, QUARTER HAVING SUM(s.sale_price) > 0 ORDER BY QUARTER, c.state;SELECT c.state, date_trunc('quarter', s.sale_date)::date AS quarter_start, SUM(s.sale_price) AS total_sales FROM sales s JOIN customers c ON s.customer_id = c.id WHERE s.sale_date BETWEEN '2023-01-01' AND '2023-12-31' GROUP BY c.state, date_trunc('quarter', s.sale_date) ORDER BY c.state, quarter_start;",car_dealership,date_functions, \ No newline at end of file diff --git a/runners/anthropic_runner.py b/runners/anthropic_runner.py index 76c108c..39a67af 100644 --- a/runners/anthropic_runner.py +++ b/runners/anthropic_runner.py @@ -201,10 +201,10 @@ def run_anthropic_eval(args): ) if correct: total_correct += 1 - row["is_correct"] = 1 + row["correct"] = 1 row["error_msg"] = "" else: - row["is_correct"] = 0 + row["correct"] = 0 row["error_msg"] = "INCORRECT RESULTS" except Exception as e: row["error_db_exec"] = 1 @@ -224,18 +224,12 @@ def run_anthropic_eval(args): output_df.to_csv(output_file, index=False, float_format="%.2f") # get average rate of correct results - avg_subset = output_df["is_correct"].sum() / len(output_df) + avg_subset = output_df["correct"].sum() / len(output_df) print(f"Average correct rate: {avg_subset:.2f}") results = output_df.to_dict("records") - # upload results - with open(prompt_file, "r") as f: - prompt = f.read() - if args.upload_url is not None: - upload_results( - results=results, - url=args.upload_url, - runner_type="anthropic", - prompt=prompt, - args=args, - ) + with open( + f"./eval-visualizer/public/{output_file.split('/')[-1].replace('.csv', '.json')}", + "w", + ) as f: + json.dump(results, f) diff --git a/runners/deepseek_runner.py b/runners/deepseek_runner.py index 323c0c1..d83f6a5 100644 --- a/runners/deepseek_runner.py +++ b/runners/deepseek_runner.py @@ -161,14 +161,10 @@ def run_deepseek_eval(args): output_df.to_pickle(output_file) results = output_df.to_dict("records") - # upload results - with open(prompt_file, "r") as f: - prompt = f.read() - if args.upload_url is not None: - upload_results( - results=results, - url=args.upload_url, - runner_type="api_runner", - prompt=prompt, - args=args, - ) + with open( + f"./eval-visualizer/public/{output_file.split('/')[-1].replace('.csv', '.json')}", + "w", + ) as f: + json.dump(results, f) + + print("Total cost of evaluation (in cents): ", output_df["cost_in_cents"].sum()) diff --git a/runners/gemini_runner.py b/runners/gemini_runner.py index cd292c1..d0b41be 100644 --- a/runners/gemini_runner.py +++ b/runners/gemini_runner.py @@ -217,14 +217,8 @@ def run_gemini_eval(args): output_df.to_pickle(output_file) results = output_df.to_dict("records") - - if args.upload_url is not None: - with open(prompt_file, "r") as f: - prompt = f.read() - upload_results( - results=results, - url=args.upload_url, - runner_type="api_runner", - prompt=prompt, - args=args, - ) + with open( + f"./eval-visualizer/public/{output_file.split('/')[-1].replace('.csv', '.json')}", + "w", + ) as f: + json.dump(results, f) diff --git a/runners/openai_runner.py b/runners/openai_runner.py index 2c35194..5c1c13d 100644 --- a/runners/openai_runner.py +++ b/runners/openai_runner.py @@ -119,6 +119,7 @@ def process_row(row, model_name, args): "err": "", "latency_seconds": time() - start_time, "tokens_used": response.input_tokens + response.output_tokens, + "cost_in_cents": response.cost_in_cents, } except Exception as e: return { @@ -127,6 +128,7 @@ def process_row(row, model_name, args): "err": f"GENERATION ERROR: {str(e)}", "latency_seconds": time() - start_time, "tokens_used": 0, + "cost_in_cents": None, } @@ -179,6 +181,8 @@ def run_openai_eval(args): row["latency_seconds"] = result_dict["latency_seconds"] if "tokens_used" in result_dict: row["tokens_used"] = result_dict["tokens_used"] + if "cost_in_cents" in result_dict: + row["cost_in_cents"] = result_dict["cost_in_cents"] row["generated_query"] = query_gen row["reason"] = reason row["error_msg"] = err @@ -201,13 +205,13 @@ def run_openai_eval(args): ) if correct: total_correct += 1 - row["is_correct"] = 1 + row["correct"] = 1 row["error_msg"] = "" else: - row["is_correct"] = 0 + row["correct"] = 0 row["error_msg"] = "INCORRECT RESULTS" except Exception as e: - row["is_correct"] = 0 + row["correct"] = 0 row["error_db_exec"] = 1 row["error_msg"] = f"EXECUTION ERROR: {str(e)}" output_rows.append(row) @@ -225,7 +229,7 @@ def run_openai_eval(args): output_df.groupby("query_category") .agg( num_rows=("db_name", "count"), - mean_correct=("is_correct", "mean"), + mean_correct=("correct", "mean"), mean_error_db_exec=("error_db_exec", "mean"), ) .reset_index() @@ -242,15 +246,10 @@ def run_openai_eval(args): print(f"Average correct rate: {avg_subset:.2f}") results = output_df.to_dict("records") + with open( + f"./eval-visualizer/public/{output_file.split('/')[-1].replace('.csv', '.json')}", + "w", + ) as f: + json.dump(results, f) - # upload results - with open(prompt_file, "r") as f: - prompt = f.read() - if args.upload_url is not None: - upload_results( - results=results, - url=args.upload_url, - runner_type="openai", - prompt=prompt, - args=args, - ) + print("Total cost of evaluation (in cents): ", output_df["cost_in_cents"].sum()) diff --git a/utils/llm.py b/utils/llm.py index 729c0d6..c4390cc 100644 --- a/utils/llm.py +++ b/utils/llm.py @@ -39,7 +39,7 @@ class LLMResponse: input_tokens: int output_tokens: int output_tokens_details: Optional[Dict[str, int]] = None - cost: Optional[float] = None + cost_in_cents: Optional[float] = None def __post_init__(self): if self.model in LLM_COSTS_PER_TOKEN: @@ -56,14 +56,14 @@ def __post_init__(self): model_name = max(potential_model_names, key=len) if model_name: - self.cost = ( + self.cost_in_cents = ( self.input_tokens / 1000 * LLM_COSTS_PER_TOKEN[model_name]["input_cost_per1k"] + self.output_tokens / 1000 * LLM_COSTS_PER_TOKEN[model_name]["output_cost_per1k"] - ) + ) * 100 def chat_anthropic(