From 59105650bfcad106ff0ae29e9ddfc54e55cb997c Mon Sep 17 00:00:00 2001 From: Takeshi Nakazato Date: Sun, 2 Feb 2025 17:20:26 +0900 Subject: [PATCH 1/4] use __future__.annotations --- src/nro45data/psw/ms2/filler/__init__.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/nro45data/psw/ms2/filler/__init__.py b/src/nro45data/psw/ms2/filler/__init__.py index 3d8e6e9..4a96cff 100644 --- a/src/nro45data/psw/ms2/filler/__init__.py +++ b/src/nro45data/psw/ms2/filler/__init__.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import logging import os from typing import TYPE_CHECKING @@ -24,46 +26,46 @@ LOG = logging.getLogger(__name__) -def fill_antenna(msfile: str, hdu: "BinTableHDU"): +def fill_antenna(msfile: str, hdu: BinTableHDU): columns = _get_antenna_columns(hdu) _fill_antenna_columns(msfile, columns) -def fill_data_description(msfile: str, hdu: "BinTableHDU"): +def fill_data_description(msfile: str, hdu: BinTableHDU): columns = _get_data_description_columns(hdu) _fill_data_description_columns(msfile, columns) -def fill_feed(msfile: str, hdu: "BinTableHDU"): +def fill_feed(msfile: str, hdu: BinTableHDU): columns = _get_feed_columns(hdu) _fill_feed_columns(msfile, columns) -def fill_field(msfile: str, hdu: "BinTableHDU"): +def fill_field(msfile: str, hdu: BinTableHDU): columns = _get_field_columns(hdu) _fill_field_columns(msfile, columns) -def fill_observation(msfile: str, hdu: "BinTableHDU"): +def fill_observation(msfile: str, hdu: BinTableHDU): columns = _get_observation_columns(hdu) _fill_observation_columns(msfile, columns) -def fill_polarization(msfile: str, hdu: "BinTableHDU"): +def fill_polarization(msfile: str, hdu: BinTableHDU): columns = _get_polarization_columns(hdu) _fill_polarization_columns(msfile, columns) -def fill_processor(msfile: str, hdu: "BinTableHDU"): +def fill_processor(msfile: str, hdu: BinTableHDU): columns = _get_processor_columns(hdu) _fill_processor_columns(msfile, columns) -def fill_nothing(msfile: str, hdu: "BinTableHDU"): +def fill_nothing(msfile: str, hdu: BinTableHDU): pass -def fill_ms2(msfile: str, hdu: "BinTableHDU"): +def fill_ms2(msfile: str, hdu: BinTableHDU): if not os.path.exists(msfile): FileNotFoundError("MS must be built before calling fill_ms2") From 1d5b2928fddf90665b4530000e91ee7f4a1f83e8 Mon Sep 17 00:00:00 2001 From: Takeshi Nakazato Date: Sun, 2 Feb 2025 17:31:20 +0900 Subject: [PATCH 2/4] add test for ANTENNA subtable --- tests/ms2_filler/test_forest.py | 15 +++++++++++++++ tests/ms2_filler/test_h40.py | 15 +++++++++++++++ tests/ms2_filler/test_z45.py | 15 +++++++++++++++ 3 files changed, 45 insertions(+) diff --git a/tests/ms2_filler/test_forest.py b/tests/ms2_filler/test_forest.py index 76c7d61..65bedb3 100644 --- a/tests/ms2_filler/test_forest.py +++ b/tests/ms2_filler/test_forest.py @@ -47,6 +47,21 @@ def test_forest_ms2_structure(msfile): num_pols = 2 num_beams = 4 + with open_table(os.path.join(msfile, "ANTENNA")) as tb: + assert tb.nrows() == num_beams + for i in range(num_beams): + assert tb.getcell("NAME", i) == f"NRO45M-BEAM{i}" + assert tb.getcell("DISH_DIAMETER", i) == 45.0 + assert tb.getcell("TYPE", i) == "GROUND-BASED" + assert tb.getcell("MOUNT", i) == "ALT-AZ" + assert tb.getcell("STATION", i) == "NRO45M" + offset = tb.getcell("OFFSET", i) + assert offset.shape == (3,) + assert np.all(offset == 0.0) + position = tb.getcell("POSITION", i) + assert position.shape == (3,) + assert np.allclose(position, np.array([-3871023.46, 3428106.87, 3724039.47])) + with open_table(os.path.join(msfile, "STATE")) as tb: intents_map = dict((i, v) for i, v in enumerate(tb.getcol("OBS_MODE"))) diff --git a/tests/ms2_filler/test_h40.py b/tests/ms2_filler/test_h40.py index 37a6e3b..80b8a0c 100644 --- a/tests/ms2_filler/test_h40.py +++ b/tests/ms2_filler/test_h40.py @@ -49,6 +49,21 @@ def test_h40_ms2_structure(msfile): num_pols = 1 num_beams = 1 + with open_table(os.path.join(msfile, "ANTENNA")) as tb: + assert tb.nrows() == num_beams + for i in range(num_beams): + assert tb.getcell("NAME", i) == f"NRO45M-BEAM{i}" + assert tb.getcell("DISH_DIAMETER", i) == 45.0 + assert tb.getcell("TYPE", i) == "GROUND-BASED" + assert tb.getcell("MOUNT", i) == "ALT-AZ" + assert tb.getcell("STATION", i) == "NRO45M" + offset = tb.getcell("OFFSET", i) + assert offset.shape == (3,) + assert np.all(offset == 0.0) + position = tb.getcell("POSITION", i) + assert position.shape == (3,) + assert np.allclose(position, np.array([-3871023.46, 3428106.87, 3724039.47])) + with open_table(os.path.join(msfile, "STATE")) as tb: intents_map = dict((i, v) for i, v in enumerate(tb.getcol("OBS_MODE"))) diff --git a/tests/ms2_filler/test_z45.py b/tests/ms2_filler/test_z45.py index 0216b90..8c9c939 100644 --- a/tests/ms2_filler/test_z45.py +++ b/tests/ms2_filler/test_z45.py @@ -48,6 +48,21 @@ def test_z45_ms2_structure(msfile): num_pols = 2 num_beams = 1 + with open_table(os.path.join(msfile, "ANTENNA")) as tb: + assert tb.nrows() == num_beams + for i in range(num_beams): + assert tb.getcell("NAME", i) == f"NRO45M-BEAM{i}" + assert tb.getcell("DISH_DIAMETER", i) == 45.0 + assert tb.getcell("TYPE", i) == "GROUND-BASED" + assert tb.getcell("MOUNT", i) == "ALT-AZ" + assert tb.getcell("STATION", i) == "NRO45M" + offset = tb.getcell("OFFSET", i) + assert offset.shape == (3,) + assert np.all(offset == 0.0) + position = tb.getcell("POSITION", i) + assert position.shape == (3,) + assert np.allclose(position, np.array([-3871023.46, 3428106.87, 3724039.47])) + with open_table(os.path.join(msfile, "STATE")) as tb: intents_map = dict((i, v) for i, v in enumerate(tb.getcol("OBS_MODE"))) From 956620d8874beb5b8152fedb0ab17c6aac027116 Mon Sep 17 00:00:00 2001 From: Takeshi Nakazato Date: Mon, 3 Feb 2025 14:15:11 +0900 Subject: [PATCH 3/4] refactor fill_antenna --- src/nro45data/psw/ms2/filler/__init__.py | 8 +- src/nro45data/psw/ms2/filler/antenna.py | 120 ++++++++++------------- src/nro45data/psw/ms2/filler/utils.py | 43 +++++++- 3 files changed, 100 insertions(+), 71 deletions(-) diff --git a/src/nro45data/psw/ms2/filler/__init__.py b/src/nro45data/psw/ms2/filler/__init__.py index 4a96cff..5f76596 100644 --- a/src/nro45data/psw/ms2/filler/__init__.py +++ b/src/nro45data/psw/ms2/filler/__init__.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING import nro45data.psw.ms2._casa as _casa -from .antenna import _fill_antenna_columns, _get_antenna_columns +from .antenna import fill_antenna # _fill_antenna_columns, _get_antenna_columns from .data_description import _fill_data_description_columns, _get_data_description_columns from .feed import _fill_feed_columns, _get_feed_columns from .field import _fill_field_columns, _get_field_columns @@ -26,9 +26,9 @@ LOG = logging.getLogger(__name__) -def fill_antenna(msfile: str, hdu: BinTableHDU): - columns = _get_antenna_columns(hdu) - _fill_antenna_columns(msfile, columns) +# def fill_antenna(msfile: str, hdu: BinTableHDU): +# columns = _get_antenna_columns(hdu) +# _fill_antenna_columns(msfile, columns) def fill_data_description(msfile: str, hdu: BinTableHDU): diff --git a/src/nro45data/psw/ms2/filler/antenna.py b/src/nro45data/psw/ms2/filler/antenna.py index 38b8cef..c726b02 100644 --- a/src/nro45data/psw/ms2/filler/antenna.py +++ b/src/nro45data/psw/ms2/filler/antenna.py @@ -1,10 +1,12 @@ +from __future__ import annotations + import logging -from typing import TYPE_CHECKING +from typing import Generator, TYPE_CHECKING import numpy as np from .._casa import open_table -from .utils import fix_nrow_to, get_array_configuration +from .utils import get_array_configuration, fill_ms_table if TYPE_CHECKING: from astropy.io.fits.hdu.BinTableHDU import BinTableHDU @@ -12,72 +14,58 @@ LOG = logging.getLogger(__name__) -def _get_antenna_columns(hdu: "BinTableHDU") -> dict: +def _get_antenna_row(hdu: BinTableHDU) -> Generator[dict, None, None]: array_conf = get_array_configuration(hdu) beam_list = np.unique(sorted([x[1] for x in array_conf.values()])) num_beam = len(beam_list) - # NAME antenna_name_base = hdu.header["TELESCOP"].strip() - antenna_name = np.array([f"{antenna_name_base}-BEAM{i}" for i in range(num_beam)]) - LOG.debug("antenna_name: %s", antenna_name) - - # POSITION - # in ITRF, from casadata/geodetic/Observatories table - antenna_position = np.array([-3871023.46, 3428106.87, 3724039.47]) - position = np.zeros((3, num_beam), dtype=float) - position[0] = antenna_position[0] - position[1] = antenna_position[1] - position[2] = antenna_position[2] - LOG.debug("position: %s", position) - - # OFFSET - offset = np.zeros(position.shape, dtype=float) - LOG.debug("offset: %s", offset) - - # DIAMETER - diameter = np.array([45.0] * num_beam) - - # TYPE - antenna_type = np.array(["GROUND-BASED"] * num_beam) - LOG.debug("antenna_type: %s", antenna_type) - - # MOUNT - mount = np.array(["ALT-AZ"] * num_beam) - LOG.debug("mount: %s", mount) - - # STATION - station = np.array([antenna_name_base] * num_beam) - LOG.debug("station: %s", station) - - # FLAG_ROW - flag_row = np.zeros(num_beam, dtype=bool) - - columns = { - "NAME": antenna_name, - "POSITION": position, - "OFFSET": offset, - "DISH_DIAMETER": diameter, - "TYPE": antenna_type, - "MOUNT": mount, - "STATION": station, - "FLAG_ROW": flag_row, - } - - return columns - - -def _fill_antenna_columns(msfile: str, columns: dict): - with open_table(msfile + "/ANTENNA", read_only=False) as tb: - nrow = len(columns["NAME"]) - fix_nrow_to(nrow, tb) - - tb.putcol("NAME", columns["NAME"]) - tb.putcol("DISH_DIAMETER", columns["DISH_DIAMETER"]) - tb.putcol("TYPE", columns["TYPE"]) - tb.putcol("MOUNT", columns["MOUNT"]) - tb.putcol("STATION", columns["STATION"]) - tb.putcol("FLAG_ROW", columns["FLAG_ROW"]) - for i in range(nrow): - tb.putcell("POSITION", i, columns["POSITION"][:, i]) - tb.putcell("OFFSET", i, columns["OFFSET"][:, i]) + + for i in range(num_beam): + # NAME + antenna_name = f"{antenna_name_base}-BEAM{i}" + LOG.debug("antenna_name: %s", antenna_name) + + # POSITION + # in ITRF, from casadata/geodetic/Observatories table + position = np.array([-3871023.46, 3428106.87, 3724039.47]) + LOG.debug("position: %s", position) + + # OFFSET + offset = np.zeros(position.shape, dtype=float) + LOG.debug("offset: %s", offset) + + # DIAMETER + diameter = 45.0 + + # TYPE + antenna_type = "GROUND-BASED" + LOG.debug("antenna_type: %s", antenna_type) + + # MOUNT + mount = "ALT-AZ" + LOG.debug("mount: %s", mount) + + # STATION + station = antenna_name_base + LOG.debug("station: %s", station) + + # FLAG_ROW + flag_row = False + + row = { + "NAME": antenna_name, + "POSITION": position, + "OFFSET": offset, + "DISH_DIAMETER": diameter, + "TYPE": antenna_type, + "MOUNT": mount, + "STATION": station, + "FLAG_ROW": flag_row, + } + + yield row + + +def fill_antenna(msfile: str, hdu: BinTableHDU): + fill_ms_table(msfile, hdu, "ANTENNA", _get_antenna_row) diff --git a/src/nro45data/psw/ms2/filler/utils.py b/src/nro45data/psw/ms2/filler/utils.py index 7083cb7..f7c126c 100644 --- a/src/nro45data/psw/ms2/filler/utils.py +++ b/src/nro45data/psw/ms2/filler/utils.py @@ -1,9 +1,13 @@ +from __future__ import annotations + import logging import pprint -from typing import TYPE_CHECKING +from typing import Any, Callable, Optional, TYPE_CHECKING import numpy as np +from .._casa import open_table + if TYPE_CHECKING: import astropy.io.fits.hdu.BinTableHDU as BinTableHDU @@ -271,3 +275,40 @@ def get_processor_map(arry1: str, arry2: str, arry3: str, arry4: str): LOG.debug("arry4: %s", arry4) return processor_sub_type_list, processor_prefix_list + + +def fill_ms_table( + msfile: str, + hdu: BinTableHDU, + table_name: str, + row_generator: Callable, + column_keywords: Optional[dict[str, dict[str, Any]]] = None +): + if table_name.upper() == "MAIN": + table_path = msfile + else: + table_path = f"{msfile}/{table_name.upper()}" + + with open_table(table_path, read_only=False) as tb: + # update table column keywords + if column_keywords is None: + column_keywords = {} + + for colname, colkeywords_new in column_keywords.items(): + colkeywords = tb.getcolkeywords(colname) + for key, value_new in colkeywords_new.items(): + if key in colkeywords and isinstance(value_new, dict): + colkeywords[key].update(value_new) + else: + colkeywords[key] = value_new + tb.putcolkeywords(colname, colkeywords) + + # fill table rows + for row_id, row in enumerate(row_generator(hdu)): + if tb.nrows() <= row_id: + tb.addrows(tb.nrows() - row_id + 1) + + for key, value in row.items(): + LOG.debug("row %d key %s", row_id, key) + tb.putcell(key, row_id, value) + LOG.debug("%s table %d row %s", table_name, row_id, row) From 20fe732dbca9cee26e62638dee0a2826812723b4 Mon Sep 17 00:00:00 2001 From: Takeshi Nakazato Date: Mon, 3 Feb 2025 14:19:56 +0900 Subject: [PATCH 4/4] add docstring for fill_antenna --- src/nro45data/psw/ms2/filler/antenna.py | 14 ++++++++++++++ src/nro45data/psw/ms2/filler/utils.py | 12 ++++++++++++ 2 files changed, 26 insertions(+) diff --git a/src/nro45data/psw/ms2/filler/antenna.py b/src/nro45data/psw/ms2/filler/antenna.py index c726b02..ac37301 100644 --- a/src/nro45data/psw/ms2/filler/antenna.py +++ b/src/nro45data/psw/ms2/filler/antenna.py @@ -15,6 +15,14 @@ def _get_antenna_row(hdu: BinTableHDU) -> Generator[dict, None, None]: + """Provide antenna row information. + + Args: + hdu: NRO45m psw data in the form of BinTableHDU object. + + Yields: + Dictionary containing antenna row information. + """ array_conf = get_array_configuration(hdu) beam_list = np.unique(sorted([x[1] for x in array_conf.values()])) num_beam = len(beam_list) @@ -68,4 +76,10 @@ def _get_antenna_row(hdu: BinTableHDU) -> Generator[dict, None, None]: def fill_antenna(msfile: str, hdu: BinTableHDU): + """Fill MS ANTENNA table. + + Args: + msfile: Name of MS file. + hdu: NRO45m psw data in the form of BinTableHDU object. + """ fill_ms_table(msfile, hdu, "ANTENNA", _get_antenna_row) diff --git a/src/nro45data/psw/ms2/filler/utils.py b/src/nro45data/psw/ms2/filler/utils.py index f7c126c..7660f82 100644 --- a/src/nro45data/psw/ms2/filler/utils.py +++ b/src/nro45data/psw/ms2/filler/utils.py @@ -284,6 +284,18 @@ def fill_ms_table( row_generator: Callable, column_keywords: Optional[dict[str, dict[str, Any]]] = None ): + """Fill MS table. + + Args: + msfile: Name of MS file. + hdu: NRO45m psw data in the form of BinTableHDU object. + table_name: Name of subtable or "MAIN" for MAIN table. + row_generator: Generator to yield table row. + column_keywords: Optional information to update column + keywords. Defaults to None. + Key is column name and value is a dictionary of + column keywords to be updated. + """ if table_name.upper() == "MAIN": table_path = msfile else: