GP2_CNVs_no_qc.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# GP2 CNVs Pipeline"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import shutil\n",
    "import numpy as np\n",
    "import glob\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "import statsmodels.api as sm\n",
    "\n",
    "# from QC.utils import shell_do"
   ],
   "outputs": [],
   "metadata": {
    "tags": []
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "source": [
    "# Supress copy warning.\n",
    "pd.options.mode.chained_assignment = None"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "source": [
    "def shell_do(command, log=False, return_log=False):\n",
    "    print(f'Executing: {(\" \").join(command.split())}', file=sys.stderr)\n",
    "\n",
    "    res=subprocess.run(command.split(), stdout=subprocess.PIPE)\n",
    "\n",
    "    if log:\n",
    "        print(res.stdout.decode('utf-8'))\n",
    "    if return_log:\n",
    "        return(res.stdout.decode('utf-8'))"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "source": [
    "wd = '/YOUR/DIRECTORY'"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Begin Pipeline"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "source": [
    "############ SET THIS EVERY RELEASE ############\n",
    "release = '5'\n",
    "\n",
    "key_file = f'/data/GP2/clinical/master_key/GP2_master_key_FINAL_release5_n26728.txt'\n",
    "key = pd.read_csv(f'{key_file}')\n",
    "key.loc[:,'IID'] = key.loc[:,'SentrixBarcode_A'].astype(str) + '_' + key.loc[:,'SentrixPosition_A'].astype(str)\n",
    "key.loc[:,'FID'] = \"0\"\n",
    "\n",
    "swarm_scripts_dir = f'{wd}/swarm'\n",
    "samples_list = key.loc[:,'filename'].unique()\n",
    "barcodes_list = list(set([x.split('_')[0] for x in samples_list]))\n",
    "\n",
    "raw_geno_path = '/data/GP2/raw_genotypes'\n",
    "snp_metrics_path = f'{raw_geno_path}/snp_metrics'\n",
    "cnv_path = f'{wd}/original_CNVs' # personal version\n",
    "os.makedirs(cnv_path, exist_ok=True)\n",
    "\n",
    "# cnv_path = f'/data/GP2/raw_genotypes/cnvs/release{release}' # new release version\n",
    "# covar_path = f'{cnv_path}/key/release{release}_covar.csv'\n",
    "\n",
    "ancestry_labels = [x.split('/')[-1].replace('.bed','').split('_')[-1] for x in glob.glob(f'/data/GP2/quality_control/release{release}/genotype_qc/GP2_round5_APRIL_2023_*.bed') if '_maf_hwe' not in x]\n",
    "\n",
    "## already have these for previous releases - uncomment for new release\n",
    "# key[['FID','GP2sampleID']].to_csv(f'{cnv_path}/key/release{release}.samples', sep='\\t', header=False, index=False)\n",
    "# key[['GP2sampleID','IID']].to_csv(f'{cnv_path}/key/release{release}_sample_id_key.csv')\n",
    "# release_covars = key.loc[:,['FID', 'GP2sampleID','sex_for_qc', 'age', 'age_of_onset']]\n",
    "# release_covars.to_csv(covar_path)\n",
    "\n",
    "cnv_types = ['PERCENT_BAF_INSERTION','PERCENT_L2R_DELETION','PERCENT_L2R_DUPLICATION']"
   ],
   "outputs": [],
   "metadata": {
    "tags": []
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "source": [
    "!ls {snp_metrics_path}/{barcodes_list[0]}/snp_metrics_{barcodes_list[0]}"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "'Sample_ID=204835450068_R01C01'  'Sample_ID=204835450068_R05C01'\n",
      "'Sample_ID=204835450068_R02C01'  'Sample_ID=204835450068_R06C01'\n",
      "'Sample_ID=204835450068_R03C01'  'Sample_ID=204835450068_R07C01'\n",
      "'Sample_ID=204835450068_R04C01'  'Sample_ID=204835450068_R08C01'\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "source": [
    "# Check if all samples in the list have a parquet created with SNP metrics\n",
    "exist_dir = []\n",
    "missing_dir = []\n",
    "\n",
    "for sample in samples_list:\n",
    "    code = sample.split('_')[0]\n",
    "    metrics_file = f'{snp_metrics_path}/{code}/snp_metrics_{code}/Sample_ID={sample}'\n",
    "    \n",
    "    if os.path.isdir(metrics_file):\n",
    "        exist_dir.append(sample)\n",
    "    else:\n",
    "        missing_dir.append(sample)\n",
    "        \n",
    "print(f'metrics exist: {len(exist_dir)}')\n",
    "print(f'missing metrics: {len(missing_dir)}')"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "metrics exist: 26578\n",
      "missing metrics: 150\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "source": [
    "# check ancestries\n",
    "total_sample_n = 0\n",
    "for label in ancestry_labels:\n",
    "\n",
    "    # will need to change for each release\n",
    "    gene_name = f'GP2_round5_APRIL_2023_{label}'\n",
    "    gene_path = f'/data/GP2/quality_control/release{release}/genotype_qc/{gene_name}'\n",
    "    \n",
    "    fam = pd.read_csv(f'{gene_path}.fam', sep='\\s+', header=None, names=['fid','iid','pat','mat','sex','pheno'])\n",
    "    pheno_counts = fam.pheno.value_counts()\n",
    "    ncases = pheno_counts[2]\n",
    "    ncontrols = pheno_counts[1]\n",
    "    ntotal = ncases + ncontrols\n",
    "    total_sample_n += ntotal\n",
    "    nsnps = !cat {out_path}.bim | wc -l\n",
    "    print(f'{label} | total_n: {ntotal} | n_cases: {ncases} | n_controls: {ncontrols} | n_snps: {int(nsnps[0])}')\n",
    "    print()\n",
    "    \n",
    "print(f'total samples across ancestries: {total_sample_n}')"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "AAC | total_n: 1320 | n_cases: 289 | n_controls: 1031 | n_snps: 984555\n",
      "\n",
      "AFR | total_n: 2343 | n_cases: 855 | n_controls: 1488 | n_snps: 984555\n",
      "\n",
      "AJ | total_n: 1129 | n_cases: 742 | n_controls: 387 | n_snps: 984555\n",
      "\n",
      "AMR | total_n: 550 | n_cases: 345 | n_controls: 205 | n_snps: 984555\n",
      "\n",
      "CAS | total_n: 485 | n_cases: 202 | n_controls: 283 | n_snps: 984555\n",
      "\n",
      "EAS | total_n: 3567 | n_cases: 1223 | n_controls: 2344 | n_snps: 984555\n",
      "\n",
      "EUR | total_n: 15359 | n_cases: 9891 | n_controls: 5468 | n_snps: 984555\n",
      "\n",
      "FIN | total_n: 13 | n_cases: 10 | n_controls: 3 | n_snps: 984555\n",
      "\n",
      "MDE | total_n: 139 | n_cases: 113 | n_controls: 26 | n_snps: 984555\n",
      "\n",
      "SAS | total_n: 240 | n_cases: 104 | n_controls: 136 | n_snps: 984555\n",
      "\n",
      "total samples across ancestries: 25145\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "source": [
    "# skip extra QC, only use QC that checked for duplicates and mild variant pruning\n",
    "# if use subset of samples, only include the ancestry_labels applicable to those samples\n",
    "\n",
    "missing_metrics = []\n",
    "included_samples = []\n",
    "\n",
    "with open(f'{swarm_scripts_dir}/cnvs.swarm', 'w') as f:\n",
    "    for label in ancestry_labels:\n",
    "        \n",
    "        # will need to change for each release\n",
    "        geno_name = f'GP2_round5_APRIL_2023_{label}'\n",
    "        geno_path = f'/data/GP2/quality_control/release{release}/genotype_qc/{geno_name}'\n",
    "        \n",
    "        bim_path = f'{geno_path}.bim'\n",
    "        fam = pd.read_csv(f'{geno_path}.fam', sep='\\s+', header=None, names=['FID','IID','pat','mat','sex','pheno'])\n",
    "        fam_key = fam.merge(key, left_on='IID', right_on='GP2sampleID', how='left')\n",
    "        \n",
    "        # may want to change naming convention of directory where CNVs will be output: mine is cnv_calls_no_qc\n",
    "        label_dir = f'{cnv_path}/cnv_calls_no_qc/{label}'\n",
    "        os.makedirs(label_dir, exist_ok=True)\n",
    "\n",
    "        for sample in fam_key.filename.unique():\n",
    "            if pd.notnull(sample): # catch some sample IDs that are NaN\n",
    "                code = sample.split('_')[0]\n",
    "                mfile = f'{snp_metrics_path}/{code}/snp_metrics_{code}/Sample_ID={sample}'\n",
    "                \n",
    "                if os.path.isdir(mfile):\n",
    "                    cnv_out = f'{label_dir}/CNV_{label}_{sample}.parquet'\n",
    "                    intervals = '/data/GP2/utils/ref_dir/glist_hg38_intervals.csv'\n",
    "            \n",
    "                    cmd = f'\\\n",
    "                            python run_cnv_pipeline.py \\\n",
    "                            --metrics {mfile} \\\n",
    "                            --bim {bim_path} \\\n",
    "                            --out_path {cnv_out} \\\n",
    "                            --intervals {intervals} \\\n",
    "                            --min_variants 10 \\\n",
    "                            --kb_window 250 \\\n",
    "                            --min_gentrain 0.2'\n",
    "            \n",
    "                    f.write(f'{cmd}\\n')\n",
    "                    included_samples.append(sample)\n",
    "            \n",
    "                else:\n",
    "                    missing_metrics.append(sample)\n",
    "        \n",
    "f.close()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "source": [
    "# make sure before actually running swarm script\n",
    "print(len(included_samples))"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "24935\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "source": [
    "!swarm -f {swarm_scripts_dir}/cnvs.swarm -g 16 -t 8 --time=00:15:00 --logdir {swarm_scripts_dir}/logs --module python/3.9 --gres=lscratch:20 --partition=norm"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "13171067\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# write dosages per chromosome per ancestry per cnv-type (1 .csv per ancestry per cnv-type)\n",
    "\n",
    "with open(f'{swarm_scripts_dir}/cnv_dosages.swarm', 'w') as f:\n",
    "    for label in ancestry_labels:\n",
    "        label_dir = f'{cnv_path}/dosages/{label}'\n",
    "        os.makedirs(label_dir, exist_ok=True)\n",
    "\n",
    "        # if you want to change naming convention of cnv_calls_no_qc\n",
    "        cnv_files_list = glob.glob(f'{cnv_path}/cnv_calls_no_qc/{label}/CNV_{label}_*.parquet')\n",
    "        cnv_files_df = pd.DataFrame({'filename': cnv_files_list})\n",
    "        cnv_files_df.to_csv(f'{label_dir}/CNV_{label}_files.csv', header=False, index=False)\n",
    "\n",
    "        for cnv_type in cnv_types:\n",
    "\n",
    "            cmd = f'\\\n",
    "                    python run_cnv_dosage_pipeline.py \\\n",
    "                    --files {label_dir}/CNV_{label}_files.csv \\\n",
    "                    --label {label} \\\n",
    "                    --cnv_type {cnv_type} \\\n",
    "                    --out_path {label_dir}/CNV_{label}_{cnv_type}.csv'\n",
    "            f.write(f'{cmd}\\n')\n",
    "f.close()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "!swarm -f {swarm_scripts_dir}/cnv_dosages.swarm -g 8 -t 4 --logdir {swarm_scripts_dir}/logs --module python/3.9 --gres=lscratch:20 --partition=norm"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "### If want to double-check CNVs for 1 sample"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "source": [
    "# check 1 European sample\n",
    "# may need to change directory to your naming conventions\n",
    "test_cnv = pd.read_parquet(f'{wd}/original_CNVs/cnv_calls_qc/EUR/CNV_EUR_{samples_list[0]}.parquet')\n",
    "\n",
    "# not CNVs\n",
    "display(test_cnv[test_cnv.PERCENT_BAF_INSERTION.isnull()])\n",
    "\n",
    "# CNVs\n",
    "display(test_cnv[test_cnv.NUM_VARIANTS>10])"
   ],
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>INTERVAL</th>\n",
       "      <th>NUM_VARIANTS</th>\n",
       "      <th>PERCENT_BAF_INSERTION</th>\n",
       "      <th>PERCENT_L2R_DELETION</th>\n",
       "      <th>PERCENT_L2R_DUPLICATION</th>\n",
       "      <th>START_PLUS_WINDOW</th>\n",
       "      <th>START</th>\n",
       "      <th>STOP</th>\n",
       "      <th>STOP_PLUS_WINDOW</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A1BG</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>58096805</td>\n",
       "      <td>58346805</td>\n",
       "      <td>58353499</td>\n",
       "      <td>58603499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A1BG-AS1</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>58101969</td>\n",
       "      <td>58351969</td>\n",
       "      <td>58355183</td>\n",
       "      <td>58605183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A1CF</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>50549408</td>\n",
       "      <td>50799408</td>\n",
       "      <td>50885675</td>\n",
       "      <td>51135675</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A2M</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8817707</td>\n",
       "      <td>9067707</td>\n",
       "      <td>9115962</td>\n",
       "      <td>9365962</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A2M-AS1</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8815176</td>\n",
       "      <td>9065176</td>\n",
       "      <td>9068055</td>\n",
       "      <td>9318055</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25618</th>\n",
       "      <td>ZYG11A</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>52592510</td>\n",
       "      <td>52842510</td>\n",
       "      <td>52894575</td>\n",
       "      <td>53144575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25619</th>\n",
       "      <td>ZYG11B</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>52476458</td>\n",
       "      <td>52726458</td>\n",
       "      <td>52827341</td>\n",
       "      <td>53077341</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25620</th>\n",
       "      <td>ZYX</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>143131266</td>\n",
       "      <td>143381266</td>\n",
       "      <td>143391111</td>\n",
       "      <td>143641111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25621</th>\n",
       "      <td>ZZEF1</td>\n",
       "      <td>9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3754444</td>\n",
       "      <td>4004444</td>\n",
       "      <td>4142959</td>\n",
       "      <td>4392959</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25622</th>\n",
       "      <td>ZZZ3</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>77314504</td>\n",
       "      <td>77564504</td>\n",
       "      <td>77682658</td>\n",
       "      <td>77932658</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24695 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       INTERVAL  NUM_VARIANTS  PERCENT_BAF_INSERTION  PERCENT_L2R_DELETION  \\\n",
       "0          A1BG             2                    NaN                   NaN   \n",
       "1      A1BG-AS1             2                    NaN                   NaN   \n",
       "2          A1CF             1                    NaN                   NaN   \n",
       "3           A2M             2                    NaN                   NaN   \n",
       "4       A2M-AS1             2                    NaN                   NaN   \n",
       "...         ...           ...                    ...                   ...   \n",
       "25618    ZYG11A             2                    NaN                   NaN   \n",
       "25619    ZYG11B             2                    NaN                   NaN   \n",
       "25620       ZYX             4                    NaN                   NaN   \n",
       "25621     ZZEF1             9                    NaN                   NaN   \n",
       "25622      ZZZ3             0                    NaN                   NaN   \n",
       "\n",
       "       PERCENT_L2R_DUPLICATION  START_PLUS_WINDOW      START       STOP  \\\n",
       "0                          NaN           58096805   58346805   58353499   \n",
       "1                          NaN           58101969   58351969   58355183   \n",
       "2                          NaN           50549408   50799408   50885675   \n",
       "3                          NaN            8817707    9067707    9115962   \n",
       "4                          NaN            8815176    9065176    9068055   \n",
       "...                        ...                ...        ...        ...   \n",
       "25618                      NaN           52592510   52842510   52894575   \n",
       "25619                      NaN           52476458   52726458   52827341   \n",
       "25620                      NaN          143131266  143381266  143391111   \n",
       "25621                      NaN            3754444    4004444    4142959   \n",
       "25622                      NaN           77314504   77564504   77682658   \n",
       "\n",
       "       STOP_PLUS_WINDOW  \n",
       "0              58603499  \n",
       "1              58605183  \n",
       "2              51135675  \n",
       "3               9365962  \n",
       "4               9318055  \n",
       "...                 ...  \n",
       "25618          53144575  \n",
       "25619          53077341  \n",
       "25620         143641111  \n",
       "25621           4392959  \n",
       "25622          77932658  \n",
       "\n",
       "[24695 rows x 9 columns]"
      ]
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>INTERVAL</th>\n",
       "      <th>NUM_VARIANTS</th>\n",
       "      <th>PERCENT_BAF_INSERTION</th>\n",
       "      <th>PERCENT_L2R_DELETION</th>\n",
       "      <th>PERCENT_L2R_DUPLICATION</th>\n",
       "      <th>START_PLUS_WINDOW</th>\n",
       "      <th>START</th>\n",
       "      <th>STOP</th>\n",
       "      <th>STOP_PLUS_WINDOW</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>ABCG1</td>\n",
       "      <td>13</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>41949688</td>\n",
       "      <td>42199688</td>\n",
       "      <td>42297244</td>\n",
       "      <td>42547244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121</th>\n",
       "      <td>ABR</td>\n",
       "      <td>12</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>753517</td>\n",
       "      <td>1003517</td>\n",
       "      <td>1187322</td>\n",
       "      <td>1437322</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>190</th>\n",
       "      <td>ACPT</td>\n",
       "      <td>16</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.062500</td>\n",
       "      <td>0.187500</td>\n",
       "      <td>50540414</td>\n",
       "      <td>50790414</td>\n",
       "      <td>50795224</td>\n",
       "      <td>51045224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>237</th>\n",
       "      <td>ACTR3B</td>\n",
       "      <td>13</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.230769</td>\n",
       "      <td>152509748</td>\n",
       "      <td>152759748</td>\n",
       "      <td>152855379</td>\n",
       "      <td>153105379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>291</th>\n",
       "      <td>ADAMTS2</td>\n",
       "      <td>13</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.076923</td>\n",
       "      <td>178860850</td>\n",
       "      <td>179110850</td>\n",
       "      <td>179345430</td>\n",
       "      <td>179595430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25314</th>\n",
       "      <td>ZNF556</td>\n",
       "      <td>13</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.076923</td>\n",
       "      <td>2617334</td>\n",
       "      <td>2867334</td>\n",
       "      <td>2878503</td>\n",
       "      <td>3128503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25476</th>\n",
       "      <td>ZNF761</td>\n",
       "      <td>17</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.058824</td>\n",
       "      <td>0.117647</td>\n",
       "      <td>53181973</td>\n",
       "      <td>53431973</td>\n",
       "      <td>53458261</td>\n",
       "      <td>53708261</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25479</th>\n",
       "      <td>ZNF765</td>\n",
       "      <td>16</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.062500</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>53145143</td>\n",
       "      <td>53395143</td>\n",
       "      <td>53412009</td>\n",
       "      <td>53662009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25515</th>\n",
       "      <td>ZNF813</td>\n",
       "      <td>17</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.058824</td>\n",
       "      <td>0.117647</td>\n",
       "      <td>53217734</td>\n",
       "      <td>53467734</td>\n",
       "      <td>53494292</td>\n",
       "      <td>53744292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25536</th>\n",
       "      <td>ZNF845</td>\n",
       "      <td>15</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.133333</td>\n",
       "      <td>53083748</td>\n",
       "      <td>53333748</td>\n",
       "      <td>53354869</td>\n",
       "      <td>53604869</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>639 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      INTERVAL  NUM_VARIANTS  PERCENT_BAF_INSERTION  PERCENT_L2R_DELETION  \\\n",
       "85       ABCG1            13                    0.0              0.000000   \n",
       "121        ABR            12                    0.0              0.000000   \n",
       "190       ACPT            16                    0.0              0.062500   \n",
       "237     ACTR3B            13                    0.0              0.000000   \n",
       "291    ADAMTS2            13                    0.0              0.000000   \n",
       "...        ...           ...                    ...                   ...   \n",
       "25314   ZNF556            13                    0.0              0.000000   \n",
       "25476   ZNF761            17                    0.0              0.058824   \n",
       "25479   ZNF765            16                    0.0              0.062500   \n",
       "25515   ZNF813            17                    0.0              0.058824   \n",
       "25536   ZNF845            15                    0.0              0.000000   \n",
       "\n",
       "       PERCENT_L2R_DUPLICATION  START_PLUS_WINDOW      START       STOP  \\\n",
       "85                    0.000000           41949688   42199688   42297244   \n",
       "121                   0.000000             753517    1003517    1187322   \n",
       "190                   0.187500           50540414   50790414   50795224   \n",
       "237                   0.230769          152509748  152759748  152855379   \n",
       "291                   0.076923          178860850  179110850  179345430   \n",
       "...                        ...                ...        ...        ...   \n",
       "25314                 0.076923            2617334    2867334    2878503   \n",
       "25476                 0.117647           53181973   53431973   53458261   \n",
       "25479                 0.125000           53145143   53395143   53412009   \n",
       "25515                 0.117647           53217734   53467734   53494292   \n",
       "25536                 0.133333           53083748   53333748   53354869   \n",
       "\n",
       "       STOP_PLUS_WINDOW  \n",
       "85             42547244  \n",
       "121             1437322  \n",
       "190            51045224  \n",
       "237           153105379  \n",
       "291           179595430  \n",
       "...                 ...  \n",
       "25314           3128503  \n",
       "25476          53708261  \n",
       "25479          53662009  \n",
       "25515          53744292  \n",
       "25536          53604869  \n",
       "\n",
       "[639 rows x 9 columns]"
      ]
     },
     "metadata": {}
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "source": [
    "# double check counts in the ancestry labels\n",
    "\n",
    "for label in ancestry_labels:\n",
    "# for label in ['EUR']: # if want specific label\n",
    "    cnv_dir_ = f'{cnv_path}/cnv_calls/{label}/'\n",
    "    label_cnvs_list = glob.glob(f'{cnv_dir_}/*.parquet')"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "\n"
   ],
   "outputs": [],
   "metadata": {}
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "vscode": {
   "interpreter": {
    "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}