Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft a fid drop monitor plus quick report #10

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Draft a fid drop monitor plus quick report #10

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
"console_scripts": [
"ska_trend_wrong_box_updates=ska_trend.wrong_box_anom.wrong_box_anom:main",
"ska_trend_bad_periscope=ska_trend.bad_periscope_gradient.periscope_update:main",
"ska_trend_fid_drop_mon_update=ska_trend.fid_drop_mon.update:main",
]
}

Expand All @@ -26,12 +27,15 @@
"ska_trend",
"ska_trend.wrong_box_anom",
"ska_trend.bad_periscope_gradient",
"ska_trend.fid_drop_mon",
],
package_data={
"ska_trend": [
"wrong_box_anom/index_template.html",
"wrong_box_anom/task_schedule.cfg",
"bad_periscope_gradient/task_schedule.cfg",
"fid_drop_mon/index_template.html",
"fid_drop_mon/task_schedule.cfg",
]
},
)
Empty file added ska_trend/fid_drop_mon/__init__.py
View file
Open in desktop
Empty file.
100 changes: 100 additions & 0 deletions ska_trend/fid_drop_mon/index_template.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
<HTML>
<HEADER>
<html>
<head>
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta http-equiv="Content-Type" content="text/html; charset=windows-1252">
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.1.3/dist/css/bootstrap.min.css"
rel="stylesheet"
integrity="sha384-1BmE4kWBq78iYhFldvKuhfTAU6auU8tT94WrHftjDbrCEXSU1oBoqyl2QvZ6jIW3"
crossorigin="anonymous">

<style>
h1 {
color: #990000;
}

h2 {
color: #990000;
}
.content-container {
margin-left: 5px; /* Adjust the value as needed */
å}

</style>
<title>Fid Light Drops</title>

</HEADER>
<BODY>

<!--#include virtual="/incl/header.html"-->
<div class="container-md content-container">
<H3>Fid Light Drops</H3>

<div class="accordion" id="accordionExample">
<div class="accordion-item">
<h4 class="accordion-header" id="heading-table">
<button class="accordion-button collapsed bg-primary bg-opacity-25" type="button" data-bs-toggle="collapse"
data-bs-target="#info-table" aria-expanded="true" aria-controls="info-table">
Table Information
</button>
</h4>
<div id="info-table" class="accordion-collapse collapse" aria-labelledby="info-table"
data-bs-parent="#accordionExample">
<div class="accordion-body bg-primary bg-opacity-10">
The table below is of fid drop events.
<UL>
<LI>Interval Start : start of tracking interval - usually the start of Kalman for the obsid dwell,
but if there is a NMAN transition in an interval this can be a later NPNT transition at the same pointing. </LI>
<LI>Interval Stop : end of tracking interval - usually transition to NMAN</LI>
<LI>Obsid : Obsid assigned to the maneuver event to this dwell by kadi events</LI>
<LI>Fid Slot : Fid slot number</LI>
<LI>Trak Fraction : Fraction of the interval that the fid light was tracked (AOACFCTN == "TRAK")</LI>
<LI>Notes : Any notes from the google sheet</LI>
</UL>

The notes are from the google sheet at <a href="{{sheet_url}}">{{sheet_url}}</a>.
</div>
</div>
</div>
</div>

<H4>Fid drops in last year</H4>
<table class="table table-striped table-bordered table-hover">
<TR><TH>Interval Start</TH><TH>Interval Stop</TH></TD><TH>Obsid</TH><TH>Fid Slot</TH><TH>Track Fraction</TH><TH>Notes</TH></TR>
{% for obs in obs_events_last %}
<TR>
<TD>{{obs['start']}}</TD>
<TD>{{obs['stop']}}</TD>
<TD ALIGN="right">{{obs['obsid_telem']}}</TD>
<TD ALIGN="right">{{ obs['slot'] }}</TD>
<TD ALIGN="right">{{ '%.2f' | format(obs['track_fraction'])}}</TD>
<TD ALIGN="right">{{ obs['notes'] }}</TD>
</TR>
{% endfor %}
</TABLE>


<H4>All fid drop events</H4>
<table class="table table-striped table-bordered table-hover">
<TR><TH>Interval Start</TH><TH>Interval Stop</TH><TH>Obsid</TH><TH>Fid Slot</TH><TH>Track Fraction</TH><TH>Notes</TH></TR>
{% for obs in obs_events %}
<TR>
<TD>{{obs['start']}}</TD>
<TD>{{obs['stop']}}</TD>
<TD ALIGN="right">{{obs['obsid_telem']}}</A></TD>
<TD ALIGN="right">{{ obs['slot'] }}</TD>
<TD ALIGN="right">{{ '%.2f' | format(obs['track_fraction'])}}</TD>
<TD ALIGN="right">{{ obs['notes'] }}</TD>
</TR>
{% endfor %}
</TABLE>
</div>

<script
src="https://cdn.jsdelivr.net/npm/bootstrap@5.1.3/dist/js/bootstrap.bundle.min.js"
integrity="sha384-ka7Sk0Gln4gmtz2MlQnikT1wXgYsOg+OMhuP+IlRH9sENBO0LRn5q+8nbTov4+1p"
crossorigin="anonymous">
</script>
</BODY>
</HTML>
265 changes: 265 additions & 0 deletions ska_trend/fid_drop_mon/update.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,265 @@
import argparse
import functools
from pathlib import Path

import astropy.units as u
import jinja2
import numpy as np
from acdc.common import send_mail
from astropy.table import Table, join, vstack
from cheta import fetch
from cheta.utils import logical_intervals
from cxotime import CxoTime, CxoTimeLike
from kadi import events
from kadi.commands.observations import get_starcats
from ska_helpers.logging import basic_logger
from ska_helpers.run_info import log_run_info

from ska_trend import __version__

DOC_ID = "1qHF6-3oz6MSJYd3goQTMdcteXWUjKyHycdWSMCd9S40"
GID = 0
url_start = "https://docs.google.com/spreadsheets/d"
GSHEET_URL = f"{url_start}/{DOC_ID}/export?format=csv&id={DOC_ID}&gid={GID}"
GSHEET_USER_URL = f"{url_start}/{DOC_ID}/edit?usp=sharing"

# Constants and file path definitions
FILE_DIR = Path(__file__).parent


def INDEX_TEMPLATE_PATH():
return FILE_DIR / "index_template.html"


URL = "https://cxc.cfa.harvard.edu/mta/ASPECT/fid_drop_mon/index.html"
LOGGER = basic_logger(__name__, level="INFO")

fetch.data_source.set("cxc", "maude allow_subset=False")


def get_opt():
parser = argparse.ArgumentParser(description="Fid drop monitor")
parser.add_argument("--start", help="Start date")
parser.add_argument("--stop", help="Stop date")
parser.add_argument(
"--out-dir",
default="./out",
help="Output directory",
)
parser.add_argument(
"--email",
action="append",
dest="emails",
default=[],
help="Email address for notificaion",
)
return parser


def get_fid_data(start: CxoTimeLike) -> Table:
"""
Get the fid tracking data for dwells from time start.

Parameters
----------
start : CxoTimeLike
Start time for the data to be fetched.

Returns
-------
Table
Table of fid tracking data.

The returned table has the following columns:
- kalman_start: Start time of the Kalman filter.
- next_nman_start: Start time of the next NMAN.
- starcat_date: Date of the star catalog.
- obsid_starcat: Observation ID from star catalog / kadi starcat
- slot: Slot number.
- track_samples: Number of AOACFCT{n} samples.
- track_ok: Number of AOACFCT samples with "TRAK" status.
- track_fraction: Fraction of samples with "TRAK" status.
"""

manvrs = events.manvrs.filter(kalman_start__gte=start)
starcats = get_starcats(start=CxoTime(start) - 5 * u.day)
dates = [starcat.date for starcat in starcats]

fid_data = []
for i, manvr in enumerate(manvrs):
# get the index of the starcat before the kalman start
idx = np.searchsorted(dates, manvr.kalman_start) - 1
# get the starcat before the kalman start
starcat = starcats[idx]
fid_slots = list(starcat["slot"][(starcat["type"] == "FID")])
if len(fid_slots) == 0:
continue

# If the "last" starcat happened before the last manvr kalman start
# skip, as something isn't right
if i > 0 and starcat.date < manvrs[i - 1].kalman_start:
continue

# Break up the manvr steady interval into NPNT intervals if needed
aopcadmd = fetch.Msid("AOPCADMD", manvr.kalman_start, manvr.next_nman_start)
npnt = logical_intervals(aopcadmd.times, aopcadmd.vals == "NPNT")

for row in npnt:
npnt_start = row["datestart"]
npnt_stop = row["datestop"]

# Check CORADMEN for SCS107 equivalent
radmon = fetch.Msid("CORADMEN", npnt_start, npnt_stop)
# If SCS107, truncate the range to process
if np.any(radmon.vals == "DISA"):
npnt_stop = CxoTime(
radmon.times[np.where(radmon.vals == "DISA")[0][0]]
).date

# If down to less than a minute, skip
if CxoTime(npnt_stop) - CxoTime(npnt_start) < 60 * u.s:
continue

dat = fetch.Msidset(
["AOACASEQ", "AOACFCT0", "AOACFCT1", "AOACFCT2"], npnt_start, npnt_stop
)
dat.interpolate(1.025)

for slot in fid_slots:
track_msid = f"AOACFCT{slot}"
track_telem = dat[track_msid]
ok_kalm = dat["AOACASEQ"].vals == "KALM"
track_samples = np.count_nonzero(ok_kalm)
track_ok = np.count_nonzero(track_telem.vals[ok_kalm] == "TRAK")
track_fraction = track_ok / track_samples
fid_data.append(
{
"start": npnt_start,
"stop": npnt_stop,
"starcat_date": starcat.date,
"obsid_telem": starcat.obsid,
"slot": slot,
"track_samples": track_samples,
"track_ok": track_ok,
"track_fraction": track_fraction,
}
)

return Table(fid_data)


@functools.cache
def get_fid_notes(data_root) -> Table:
"""
Get the high background notes from the Google Sheet.

Parameters
----------
data_root : str
The root directory for the data
Returns
-------
dat : astropy.table.Table
Table of notes
"""
LOGGER.info(f"Reading google sheet {GSHEET_URL}")
dat = None
try:
dat = Table.read(GSHEET_URL, format="ascii.csv")
except Exception as e:
LOGGER.error(f"Failed to read {GSHEET_URL} with error: {e}")

if dat is not None:
dat.write(
Path(data_root) / "notes.csv",
format="ascii.csv",
overwrite=True,
)
else:
dat = Table.read(Path(data_root) / "notes.csv", format="ascii.csv")

return dat


def main(args=None):
"""
Main function to update the fid drop monitor data and web page.
"""
opt = get_opt().parse_args(args)
log_run_info(LOGGER.info, opt, version=__version__)

fid_data_file = Path(opt.out_dir) / "fids_data.dat"
Path(opt.out_dir).mkdir(parents=True, exist_ok=True)

start = None
fid_data_archive = None

if Path(fid_data_file).exists():
fid_data_archive = Table.read(fid_data_file, format="ascii.ecsv")
start = CxoTime(fid_data_archive["start"][-1]) + 10 * u.s
else:
start = "2002:010"

if opt.start is not None:
# Override whatever we have in the archive
start = CxoTime(opt.start)
# And clear the archive after the supplied time
ok = fid_data_archive["start"] < CxoTime(start).date
fid_data_archive = fid_data_archive[ok]

if fid_data_archive is None:
fid_data = get_fid_data(start)
else:
new_fid_data = get_fid_data(start)
fid_data = vstack([fid_data_archive, new_fid_data])
fid_data.sort("start")

fid_data.write(fid_data_file, format="ascii.ecsv", overwrite=True)

# Fid drops just defined as more than 0 and less than 95%
drop_events = fid_data[
(fid_data["track_fraction"] < 0.95) & (fid_data["track_fraction"] > 0)
]

# Get any info from the google sheet
fid_notes = get_fid_notes(opt.out_dir)

# Add the notes to the drop events
if len(drop_events) > 0:
drop_events = join(drop_events, fid_notes, keys="start", join_type="left")

# Last year of events
ok = CxoTime(drop_events["start"]) > CxoTime() - 365 * u.day
drop_events_last = drop_events[ok]

index_template_html = INDEX_TEMPLATE_PATH().read_text()
template = jinja2.Template(index_template_html)
out_html = template.render(
obs_events=drop_events[::-1],
obs_events_last=drop_events_last[::-1],
sheet_url=GSHEET_USER_URL,
)
html_path = Path(opt.out_dir) / "index.html"
LOGGER.info(f"Writing HTML to {html_path}")
html_path.write_text(out_html)

# If any of the drops are new, send emails
ok = drop_events["start"] > CxoTime(start).date
if np.any(ok):
for row in drop_events[ok]:
LOGGER.warning(f"Fid drop in dwell starting at {row['start']}")
if len(opt.emails) > 0:
send_mail(
LOGGER,
opt,
f"Fid drop: Obsid {row['obsid_telem']} {row['start']}",
f"Fid drop in the dwell that starts at {row['start']}\n"
f"Obsid {row['obsid_telem']} Fid slot {row['slot']} tracked "
f"for {row['track_fraction']:.3f} fraction.\n"
f"See {URL}",
__file__,
)


if __name__ == "__main__":
main()