diff --git a/gem2caom2/builder.py b/gem2caom2/builder.py
index caf1f89e..ed72a366 100644
--- a/gem2caom2/builder.py
+++ b/gem2caom2/builder.py
@@ -68,13 +68,15 @@
#
import logging
+import traceback
+from caom2pipe import astro_composable as ac
from caom2pipe import manage_composable as mc
from caom2pipe import name_builder_composable as nbc
from gem2caom2 import gem_name, external_metadata
-__all__ = ['EduQueryBuilder', 'GemObsIDBuilder']
+__all__ = ['EduQueryBuilder', 'GemObsIDBuilder', 'get_instrument']
class EduQueryBuilder(nbc.Builder):
@@ -112,7 +114,8 @@ def build(self, entry):
external_metadata.get_obs_metadata(
gem_name.GemName.remove_extensions(entry))
- storage_name = gem_name.GemName(file_name=entry)
+ instrument = get_instrument()
+ storage_name = gem_name.GemName(file_name=entry, instrument=instrument)
return storage_name
@@ -133,10 +136,49 @@ def build(self, entry):
:return: an instance of StorageName for use in execute_composable.
"""
self._logger.debug(f'Build a StorageName instance for {entry}.')
- if (mc.TaskType.INGEST_OBS in self._config.task_types and
- '.fits' not in entry):
- result = gem_name.GemName(obs_id=entry)
- else:
- result = gem_name.GemName(file_name=entry)
- self._logger.debug('Done build.')
- return result
+ try:
+ if (mc.TaskType.INGEST_OBS in self._config.task_types and
+ '.fits' not in entry):
+ # anything that is NOT ALOPEKE/ZORRO, which are the only
+ # two instruments that change the behaviour of the
+ # GemName constructor - and yeah, that abstraction is leaking
+ # like a sieve.
+ self._logger.debug('INGEST_OBS, hard-coded instrument.')
+ instrument = external_metadata.Inst.CIRPASS
+ result = gem_name.GemName(obs_id=entry, instrument=instrument)
+ elif (mc.TaskType.SCRAPE in self._config.task_types or
+ self._config.use_local_files):
+ self._logger.debug(
+ 'Use a local file to read instrument from the headers.')
+ headers = ac.read_fits_headers(
+ f'{self._config.working_dir}/{entry}')
+ instrument = external_metadata.Inst(headers[0].get('INSTRUME'))
+ result = gem_name.GemName(
+ file_name=entry, instrument=instrument)
+ elif self._config.features.use_file_names:
+ self._logger.debug('Read instrument from archive.gemini.edu.')
+ file_id = gem_name.GemName.remove_extensions(entry)
+ external_metadata.get_obs_metadata(file_id)
+ instrument = get_instrument()
+ result = gem_name.GemName(
+ file_name=entry, instrument=instrument)
+ else:
+ raise mc.CadcException('The need has not been encountered '
+ 'in the real world yet.')
+ self._logger.debug('Done build.')
+ return result
+ except Exception as e:
+ self._logger.error(e)
+ self._logger.debug(traceback.format_exc())
+ raise mc.CadcException(e)
+
+
+def get_instrument():
+ inst = external_metadata.om.get('instrument')
+ if inst == 'ALOPEKE':
+ # because the value in JSON is a different case than the value in
+ # the FITS header
+ inst = 'Alopeke'
+ if inst == 'ZORRO':
+ inst = 'Zorro'
+ return external_metadata.Inst(inst)
diff --git a/gem2caom2/external_metadata.py b/gem2caom2/external_metadata.py
index 25c08255..09ec7301 100644
--- a/gem2caom2/external_metadata.py
+++ b/gem2caom2/external_metadata.py
@@ -159,6 +159,7 @@ class Inst(Enum):
PHOENIX = 'PHOENIX'
TEXES = 'TEXES'
TRECS = 'TReCS'
+ ZORRO = 'Zorro'
def get_obs_metadata(file_id):
@@ -300,6 +301,17 @@ def _repair_filter_name_for_svo(instrument, filter_names):
separated by '+'
:return filter_name the SVO version
"""
+ # Alopeke/ZORRO == FOX in Hawaiian and Spanish
+ FILTER_REPAIR_FOX = {'Red-832': 'EO_832',
+ 'Blue-u': 'u_sdss',
+ 'Blue-466': 'EO_466',
+ 'Blue-g': 'g_sdss',
+ 'Blue-562': 'EO_562',
+ 'Blue-r': 'r_sdss',
+ 'Blue-Halpha': 'Halpha',
+ 'Red-716': 'EO_716',
+ 'Red-i': 'i_sdss',
+ 'Red-z': 'z_sdss'}
FILTER_REPAIR_NICI = {'CH4-H4S': 'ED451',
'CH4-H4L': 'ED449',
'CH4-H1S': 'ED286',
@@ -416,6 +428,8 @@ def _repair_filter_name_for_svo(instrument, filter_names):
elif instrument is Inst.GSAOI:
if temp in FILTER_REPAIR_GSAOI:
temp = FILTER_REPAIR_GSAOI[temp]
+ elif instrument in [Inst.ALOPEKE, Inst.ZORRO]:
+ temp = FILTER_REPAIR_FOX.get(temp)
elif instrument is Inst.F2:
if temp == 'J-lo':
temp = 'Jlow'
@@ -473,7 +487,7 @@ def tap_client(self, value):
self._tap_client = value
def get_obs_id(self, file_id):
- self._logger.error(f'Entering get_obs_id for {file_id}.')
+ self._logger.debug(f'Entering get_obs_id for {file_id}.')
result = super(CachingObsFileRelationship, self).get_obs_id(file_id)
if result is None:
if self._use_local_files:
@@ -498,7 +512,7 @@ def _get_obs_id_from_gemini(self, file_id):
# using the global om structure to look up and store
# metadata will modify the internal index of the class - maintain
# that index here with a save/restore
- self._logger.error(f'Begin _get_obs_id_from_gemini for {file_id}')
+ self._logger.debug(f'Begin _get_obs_id_from_gemini for {file_id}')
global om
current_file_id = om.current
get_obs_metadata(file_id)
diff --git a/gem2caom2/gem_name.py b/gem2caom2/gem_name.py
index f0f4d339..2e1a4ea2 100644
--- a/gem2caom2/gem_name.py
+++ b/gem2caom2/gem_name.py
@@ -89,50 +89,108 @@ class GemName(mc.StorageName):
mixed-case obs id values - the case the inputs are provided in are
assumed to be correct.
- support uncompressed files in storage
+
+ ALOPEKE/ZORRO::
+
+ DB 31-08-20
+ DATALAB can NOT be used for the CAOM2 Observation ID since it appears that
+ the DATALAB value is identical for all files obtained for a single
+ program. e.g. if the program ID is GN-2020A-DD-115 then the DATALAB value
+ is always GN-2020A-DD-115-0-0.
+
+ Instead, use the root of the filename as the observation ID. e.g.
+ N20200819A0003r.fits and N20200819A0003b.fits are two files generated from
+ a single observation (r = red channel, b = blue channel). Use
+ N20200819A0003 as the observation ID with two planes given by the two
+ colours of data.
+
+ DB 01-09-20
+ Gemini has kludged the headers so that every observation for a single
+ program has the same DATALAB in the header. This is what we usually use
+ for the observation ID. Each single ‘observation’ actually produces two
+ files (not a single MEF file) for the red and blue channels so to me it
+ would make the most sense to group these two files as a single observation
+ with two artifacts given by uri’s pointing to the two files. And this is
+ a single plane, correct?
+
+ PD 01-09-20
+ What is the meaning of red and blue channels? different energy bands?
+
+ DB 02-09-20
+ Yes. there’s a dichroic that directs light shortward of 675nm to one
+ detector (through one of several possible filters) and light longward of
+ 675nm to a second detector (through another filter). But instead of
+ generating a single MEF file they generate two files, e.g.
+ N20191219A0004b.fits and N20191219A0004r.fits.
+
+ PD 02-09-20
+ This seems very much like MACHO... if those two files are images in the
+ normal sense then it could make sense to create separate planes with
+ dataProductType = image that end up with the correct (distinct) energy
+ metadata. It is OK for an observation to create two sibling products and
+ two planes probably captures the goal of this instrument/observing mode
+ more directly.
"""
GEM_NAME_PATTERN = '*'
def __init__(self, fname_on_disk=None, file_name=None, obs_id=None,
- file_id=None):
+ file_id=None, instrument=None):
logging.debug('parameters fname_on_disk {} file_name {}'
' obs id {} file id {}'.format(fname_on_disk,
file_name,
obs_id,
file_id))
- # try to set the file name, if that information is available
-
- # file_name is assumed to be the file name in ad
- # because the GEM files are stored uncompressed,
- # while the files available from Gemini are bz2.
- self._file_name = None
- self._file_id = None
- if file_name is not None:
- self._file_id = GemName.get_file_id(file_name)
- self.file_name = file_name
- if fname_on_disk is not None:
- self._file_id = GemName.get_file_id(fname_on_disk)
- self.file_name = fname_on_disk
- if obs_id is not None:
- self._obs_id = obs_id
- super(GemName, self).__init__(
- obs_id=obs_id, collection=ARCHIVE,
- collection_pattern=GemName.GEM_NAME_PATTERN,
- fname_on_disk=self.file_name,
- scheme=SCHEME)
- if self._obs_id is None:
- temp = em.get_gofr().get_obs_id(self._file_id)
- if temp is not None:
- self._obs_id = GemName.remove_extensions(temp)
- if (self._fname_on_disk is None and self._file_name is None and
- self._obs_id is None):
- raise mc.CadcException('Require a name.')
- if (self._file_id is None and self._obs_id is None and
- file_id is not None):
- self._file_id = file_id
- self._obs_id = file_id
- if file_id is not None:
- self._file_id = file_id
+ if instrument in [em.Inst.ALOPEKE, em.Inst.ZORRO]:
+ if file_name is None and fname_on_disk is None:
+ raise mc.CadcException(f'Need a file name of some sort.')
+ if file_name is None:
+ self._file_name = fname_on_disk
+ else:
+ self._file_name = file_name
+ self._file_id = GemName.remove_extensions(self._file_name)
+ self._obs_id = self._file_id[:-1]
+ self._product_id = self._file_id
+ super(GemName, self).__init__(
+ obs_id=self._obs_id, collection=ARCHIVE,
+ collection_pattern=GemName.GEM_NAME_PATTERN,
+ fname_on_disk=self._file_name,
+ scheme=SCHEME)
+ else:
+ # try to set the file name, if that information is available
+
+ # file_name is assumed to be the file name in ad
+ # because the GEM files are stored uncompressed,
+ # while the files available from Gemini are bz2.
+ self._file_name = None
+ self._file_id = None
+ if file_name is not None:
+ self._file_id = GemName.get_file_id(file_name)
+ self.file_name = file_name
+ if fname_on_disk is not None:
+ self._file_id = GemName.get_file_id(fname_on_disk)
+ self.file_name = fname_on_disk
+ if obs_id is not None:
+ self._obs_id = obs_id
+ super(GemName, self).__init__(
+ obs_id=obs_id, collection=ARCHIVE,
+ collection_pattern=GemName.GEM_NAME_PATTERN,
+ fname_on_disk=self.file_name,
+ scheme=SCHEME)
+ if self._obs_id is None:
+ temp = em.get_gofr().get_obs_id(self._file_id)
+ if temp is not None:
+ self._obs_id = GemName.remove_extensions(temp)
+ if (self._fname_on_disk is None and self._file_name is None and
+ self._obs_id is None):
+ raise mc.CadcException('Require a name.')
+ if (self._file_id is None and self._obs_id is None and
+ file_id is not None):
+ self._file_id = file_id
+ self._obs_id = file_id
+ if file_id is not None:
+ self._file_id = file_id
+ self._product_id = self._file_id
self._logger = logging.getLogger(__name__)
self._logger.debug(self)
@@ -175,7 +233,7 @@ def file_id(self, value):
@property
def lineage(self):
- return mc.get_lineage(ARCHIVE, self._file_id, self._file_name,
+ return mc.get_lineage(ARCHIVE, self.product_id, self._file_name,
self.scheme)
@property
@@ -184,7 +242,7 @@ def external_urls(self):
@property
def product_id(self):
- return self._file_id
+ return self._product_id
@property
def thumb_uri(self):
diff --git a/gem2caom2/main_app.py b/gem2caom2/main_app.py
index 0f38f973..607de0df 100644
--- a/gem2caom2/main_app.py
+++ b/gem2caom2/main_app.py
@@ -114,6 +114,8 @@
import gem2caom2.obs_file_relationship as ofr
from gem2caom2.gem_name import GemName, COLLECTION
from gem2caom2.svofps import FilterMetadata
+from gem2caom2.builder import get_instrument
+
__all__ = ['gem_main_app', 'to_caom2', 'update', 'APPLICATION']
@@ -158,7 +160,7 @@ def get_time_delta(header):
def get_calibration_level(uri):
result = CalibrationLevel.RAW_STANDARD
reduction = em.om.get('reduction')
- instrument = _get_instrument()
+ instrument = get_instrument()
if ((reduction is not None and
(('PROCESSED' in reduction) or ('PREPARED' in reduction))) or
(instrument is em.Inst.TEXES and
@@ -197,7 +199,7 @@ def get_art_product_type(header):
else:
result = ProductType.SCIENCE
else:
- instrument = _get_instrument()
+ instrument = get_instrument()
if instrument is not None:
if instrument is em.Inst.PHOENIX:
if _is_phoenix_calibration(header):
@@ -231,7 +233,7 @@ def get_art_product_type(header):
def get_cd11(header):
- instrument = _get_instrument()
+ instrument = get_instrument()
if instrument is em.Inst.HOKUPAA:
result = _get_pix_scale(header)
elif instrument in [em.Inst.OSCIR, em.Inst.TEXES]:
@@ -265,7 +267,7 @@ def get_cd11(header):
def get_cd22(header):
- instrument = _get_instrument()
+ instrument = get_instrument()
if instrument is em.Inst.HOKUPAA:
result = _get_pix_scale(header)
elif instrument in [em.Inst.OSCIR, em.Inst.TEXES]:
@@ -287,7 +289,7 @@ def get_cd22(header):
def get_crpix1(header):
- instrument = _get_instrument()
+ instrument = get_instrument()
if instrument is em.Inst.HOKUPAA:
crpix1 = header.get('CRPIX1')
if crpix1 is None:
@@ -306,7 +308,7 @@ def get_crpix1(header):
def get_crpix2(header):
- instrument = _get_instrument()
+ instrument = get_instrument()
if instrument is em.Inst.HOKUPAA:
crpix2 = header.get('CRPIX2')
if crpix2 is None:
@@ -331,7 +333,7 @@ def get_data_product_type(header):
:param header: The FITS header for the current extension.
:return: The Plane DataProductType, or None if not found.
"""
- instrument = _get_instrument()
+ instrument = get_instrument()
if instrument is em.Inst.FLAMINGOS:
result, ignore = _get_flamingos_mode(header)
elif instrument is em.Inst.GNIRS:
@@ -345,6 +347,20 @@ def get_data_product_type(header):
result = DataProductType.SPECTRUM
elif instrument in [em.Inst.CIRPASS, em.Inst.TEXES]:
result = DataProductType.SPECTRUM
+ elif instrument in [em.Inst.ALOPEKE, em.Inst.ZORRO]:
+ # DB 31-08-20
+ # Both cameras are used for speckle imaging. Datasets consist of
+ # cubes of 256 x 256 x 1000 images. i.e. 1000 short exposures << 1
+ # second long with 256 x 256 pixel images (or smaller images if the CCD
+ # is binned).
+ #
+ # So dataProductType = cube
+ #
+ # PD 02-09-20
+ # if those two files are images in the normal sense then it could make
+ # sense to create separate planes with dataProductType = image that
+ # end up with the correct (distinct) energy metadata.
+ result = DataProductType.IMAGE
else:
mode = em.om.get('mode')
obs_type = _get_obs_type(header)
@@ -397,7 +413,7 @@ def get_dec(header):
:param header: The FITS header for the current extension.
:return: declination, or None if not found.
"""
- instrument = _get_instrument()
+ instrument = get_instrument()
if instrument is em.Inst.HOKUPAA:
ra, dec = _get_sky_coord(header, 'RA', 'DEC')
result = dec
@@ -429,7 +445,7 @@ def get_exposure(header):
:return: The exposure time, or None if not found.
"""
result = em.om.get('exposure_time')
- instrument = _get_instrument()
+ instrument = get_instrument()
if instrument is em.Inst.OSCIR:
# DB - 20-02-19 - json ‘exposure_time’ is in minutes, so multiply
# by 60.0.
@@ -490,7 +506,7 @@ def get_obs_intent(header):
"""
result = ObservationIntentType.CALIBRATION
cal_values = ['GCALflat', 'Bias', 'BIAS', 'Twilight', 'Ar', 'FLAT',
- 'flat', 'ARC', 'Domeflat', 'DARK', 'dark', 'gcal']
+ 'flat', 'ARC', 'Domeflat', 'DARK', 'dark', 'gcal', 'ZERO']
dl = header.get('DATALAB')
lookup = _get_obs_class(header)
logging.debug('observation_class is {} for {}'.format(lookup, dl))
@@ -528,7 +544,7 @@ def get_obs_intent(header):
if type_lookup in cal_values:
result = ObservationIntentType.CALIBRATION
else:
- instrument = _get_instrument()
+ instrument = get_instrument()
if instrument is not None:
if instrument is em.Inst.TRECS:
data_label = header.get('DATALAB')
@@ -564,7 +580,7 @@ def get_obs_type(header):
obs_class = _get_obs_class(header)
if obs_class is not None and (obs_class == 'acq' or obs_class == 'acqCal'):
result = 'ACQUISITION'
- instrument = _get_instrument()
+ instrument = get_instrument()
if instrument is em.Inst.PHOENIX:
result = _get_phoenix_obs_type(header)
elif instrument is em.Inst.HOKUPAA:
@@ -682,7 +698,7 @@ def get_ra(header):
:param header: The FITS header for the current extension.
:return: ra, or None if not found.
"""
- instrument = _get_instrument()
+ instrument = get_instrument()
if instrument is em.Inst.HOKUPAA:
ra, dec = _get_sky_coord(header, 'RA', 'DEC')
result = ra
@@ -730,8 +746,9 @@ def get_target_type(uri):
"""
result = TargetType.FIELD
spectroscopy = em.om.get('spectroscopy')
- instrument = _get_instrument()
- if spectroscopy or instrument is em.Inst.TEXES:
+ instrument = get_instrument()
+ if (spectroscopy or
+ instrument in [em.Inst.ALOPEKE, em.Inst.TEXES, em.Inst.ZORRO]):
result = TargetType.OBJECT
return result
@@ -743,7 +760,7 @@ def get_time_function_val(header):
:param header: The FITS header for the current extension (not used).
:return: The Time WCS value from JSON Summary Metadata.
"""
- instrument = _get_instrument()
+ instrument = get_instrument()
if instrument in [em.Inst.FLAMINGOS, em.Inst.OSCIR]:
# Another FLAMINGOS correction needed: DATE-OBS in header and
# json doesn’t include the time but sets it to 00:00:00. You have
@@ -864,15 +881,6 @@ def _get_data_label():
return em.om.get('data_label')
-def _get_instrument():
- inst = em.om.get('instrument')
- if inst == 'ALOPEKE':
- # because the value in JSON is a different case than the value in
- # the FITS header
- inst = 'Alopeke'
- return em.Inst(inst)
-
-
def _get_sky_coord(header, ra_key, dec_key):
ra_hours = header.get(ra_key)
dec_hours = header.get(dec_key)
@@ -966,7 +974,7 @@ def _is_phoenix_calibration(header):
def _is_gmos_mask(header):
result = False
- instrument = _get_instrument()
+ instrument = get_instrument()
if instrument in [em.Inst.GMOSS, em.Inst.GMOSN, em.Inst.GMOS]:
obs_type = _get_obs_type(header)
if obs_type == 'MASK':
@@ -987,7 +995,7 @@ def accumulate_fits_bp(bp, file_id, uri):
bp.set('Observation.proposal.id', 'get_proposal_id(header)')
bp.clear('Observation.algorithm.name')
- instrument = _get_instrument()
+ instrument = get_instrument()
if instrument in [em.Inst.GMOSN, em.Inst.GMOSS, em.Inst.GMOS]:
bp.set('Observation.instrument.keywords',
'get_provenance_keywords(uri)')
@@ -1012,7 +1020,10 @@ def accumulate_fits_bp(bp, file_id, uri):
# Add IMAGESWV for GRACES
bp.add_fits_attribute('Plane.provenance.producer', 'IMAGESWV')
bp.set_default('Plane.provenance.producer', 'Gemini Observatory')
- if instrument is not em.Inst.TEXES:
+ if instrument in [em.Inst.ALOPEKE, em.Inst.ZORRO]:
+ bp.set('Plane.provenance.reference',
+ f'http://archive.gemini.edu/searchform/filepre={file_id}.fits')
+ elif instrument is not em.Inst.TEXES:
data_label = _get_data_label()
bp.set('Plane.provenance.reference',
'http://archive.gemini.edu/searchform/{}'.format(data_label))
@@ -1054,7 +1065,7 @@ def accumulate_fits_bp(bp, file_id, uri):
# the equinox given at the time specified by the EQUINOX keyword value.
bp.clear('Chunk.position.equinox')
bp.add_fits_attribute('Chunk.position.equinox', 'EQUINOX')
-
+
bp.configure_time_axis(3)
# The Chunk time metadata is calculated using keywords from the
@@ -1068,6 +1079,10 @@ def accumulate_fits_bp(bp, file_id, uri):
bp.set('Chunk.time.axis.error.syser', '1e-07')
bp.set('Chunk.time.axis.error.rnder', '1e-07')
bp.set('Chunk.time.axis.function.naxis', '1')
+ if instrument in [em.Inst.ALOPEKE, em.Inst.ZORRO]:
+ bp.clear('Chunk.time.axis.function.naxis')
+ bp.add_fits_attribute('Chunk.time.axis.function.naxis', 'NAXIS3')
+
bp.set('Chunk.time.axis.function.delta', 'get_time_delta(header)')
bp.set('Chunk.time.axis.function.refCoord.pix', '0.5')
bp.set('Chunk.time.axis.function.refCoord.val',
@@ -1285,6 +1300,12 @@ def update(observation, **kwargs):
_update_chunk_energy_texes(
c, headers[0], plane.data_product_type,
observation.observation_id)
+ elif instrument in [em.Inst.ALOPEKE, em.Inst.ZORRO]:
+ _update_chunk_energy_general(
+ c, instrument, [DataProductType.IMAGE],
+ plane.data_product_type,
+ observation.observation_id,
+ filter_name)
# position WCS
mode = em.om.get('mode')
@@ -1341,6 +1362,9 @@ def update(observation, **kwargs):
_update_chunk_position_trecs(
c, headers, int(part),
observation.observation_id)
+ elif instrument in [em.Inst.ALOPEKE, em.Inst.ZORRO]:
+ _update_chunk_position_fox(
+ c, observation.observation_id)
# time WCS
if instrument is em.Inst.F2:
@@ -1365,6 +1389,10 @@ def update(observation, **kwargs):
c.position_axis_2 = None
c.time_axis = None
c.energy_axis = None
+
+ if instrument in [em.Inst.ALOPEKE, em.Inst.ZORRO]:
+ _update_chunk_time_fox(
+ c, header, observation.observation_id)
if c.naxis is not None and c.naxis <= 2:
if c.position_axis_1 is None:
c.naxis = None
@@ -1449,10 +1477,9 @@ def _build_chunk_energy(chunk, filter_name, fm):
bandpass_name=bandpass_name,
resolving_power=fm.resolving_power)
chunk.energy = energy
- if chunk.naxis > 2:
- chunk.energy_axis = 4
- else:
- chunk.energy_axis = None
+ # no chunk energy is derived from FITS file axis metadata, so no cutouts
+ # to support
+ chunk.energy_axis = None
# values from
@@ -3530,6 +3557,15 @@ def _update_position_from_zeroth_header(artifact, headers, instrument, obs_id):
chunk.position_axis_2 = 2
+def _update_chunk_position_fox(chunk, obs_id):
+ logging.debug(f'Begin _update_chunk_position_fox for {obs_id}')
+ if (chunk is not None and chunk.position is not None and
+ chunk.position.axis is not None and
+ chunk.position.axis.axis1.ctype == 'RA--TAN'):
+ chunk.position.axis.axis1.ctype = 'RA---TAN'
+ logging.debug(f'End _update_chunk_position_fox.')
+
+
def _update_chunk_position_flamingos(chunk, header, obs_id):
# DB - I see nothing in astropy that will do a transformation from crota
# form to CD matrix, but this is it:
@@ -3757,6 +3793,23 @@ def _update_chunk_time_f2(chunk, obs_id):
logging.debug('End _update_chunk_time_f2 {}'.format(obs_id))
+def _update_chunk_time_fox(chunk, header, obs_id):
+ """
+ DB 02-09-20
+ Exposure time using JSON values isn’t correct. I know that for this
+ example Gemini shows the exposure time is 0.02 seconds but there are
+ 1000 x 0.02-second exposures in the cube. The keyword EXPOSURE gives the
+ total exposure time (in seconds), time.exposure, or 20 in this case while
+ the json exposure_time should be the time.resolution.
+ """
+ logging.debug('Begin _update_chunk_time_fox {}'.format(obs_id))
+ mc.check_param(chunk, Chunk)
+ if chunk.time is not None:
+ chunk.time.exposure = header.get('EXPOSURE')
+ # chunk.time.resolution already set by blueprint
+ logging.debug('End _update_chunk_time_fox {}'.format(obs_id))
+
+
def _update_chunk_time_gmos(chunk, obs_id):
""""""
logging.debug('Begin _update_chunk_time_gmos {}'.format(obs_id))
diff --git a/gem2caom2/obs_file_relationship.py b/gem2caom2/obs_file_relationship.py
index 4f8ff58e..4b600bd7 100644
--- a/gem2caom2/obs_file_relationship.py
+++ b/gem2caom2/obs_file_relationship.py
@@ -570,10 +570,12 @@ def is_processed(file_name):
patterns."""
result = True
file_id = gem_name.GemName.remove_extensions(file_name)
- if (file_id.startswith(('S', 'N', 'GN', 'GS', 'c', 'abu')) and
- file_id.endswith(
- ('0', '1', '2', '3', '4', '5', '6', '7', '8', '9'))):
- result = False
+ # ALOPEKE file id ends with 'r' or 'b', so avoid checking that letter
+ if file_id.startswith(('S', 'N', 'GN', 'GS', 'c', 'abu')):
+ if file_id.endswith(('0', '1', '2', '3', '4', '5', '6', '7', '8', '9')):
+ result = False
+ if file_id[:15].endswith(('b', 'r')):
+ result = False
elif file_id.startswith(('2', '02', '01')):
result = False
# TEXES naming patterns
@@ -628,9 +630,6 @@ def repair_data_label(file_name, data_label):
else:
removals = removals + suffix
- if 'fringe' in removals or 'FRINGE' in removals:
- removals = []
-
for ii in removals:
# rreplace
temp = repaired.rsplit(ii, 1)
@@ -674,6 +673,10 @@ def repair_data_label(file_name, data_label):
prefix = ''
suffix = []
+ if (prefix == '' and len(suffix) == 1 and
+ ('FRINGE' in suffix or 'fringe' in suffix)):
+ suffix = []
+
if len(prefix) > 0:
if f'-{prefix.upper()}' not in repaired:
repaired = f'{repaired}-{prefix.upper()}'
diff --git a/gem2caom2/tests/data/F2/S20150101S0261.expected.xml b/gem2caom2/tests/data/F2/S20150101S0261.expected.xml
index 5cb6bfe0..602b7955 100644
--- a/gem2caom2/tests/data/F2/S20150101S0261.expected.xml
+++ b/gem2caom2/tests/data/F2/S20150101S0261.expected.xml
@@ -63,7 +63,6 @@
3
1
2
- 4
3
diff --git a/gem2caom2/tests/data/F2/S20150103S0148.expected.xml b/gem2caom2/tests/data/F2/S20150103S0148.expected.xml
index febc1c79..365ecb92 100644
--- a/gem2caom2/tests/data/F2/S20150103S0148.expected.xml
+++ b/gem2caom2/tests/data/F2/S20150103S0148.expected.xml
@@ -63,7 +63,6 @@
3
1
2
- 4
3
diff --git a/gem2caom2/tests/data/F2/S20150113S0186.expected.xml b/gem2caom2/tests/data/F2/S20150113S0186.expected.xml
index f0f50b8c..61ef8a15 100644
--- a/gem2caom2/tests/data/F2/S20150113S0186.expected.xml
+++ b/gem2caom2/tests/data/F2/S20150113S0186.expected.xml
@@ -62,7 +62,6 @@
3
- 4
3
diff --git a/gem2caom2/tests/data/F2/S20150213S0110.expected.xml b/gem2caom2/tests/data/F2/S20150213S0110.expected.xml
index cacefef5..50dbaff1 100644
--- a/gem2caom2/tests/data/F2/S20150213S0110.expected.xml
+++ b/gem2caom2/tests/data/F2/S20150213S0110.expected.xml
@@ -63,7 +63,6 @@
3
1
2
- 4
3
diff --git a/gem2caom2/tests/data/F2/S20150622S0011.expected.xml b/gem2caom2/tests/data/F2/S20150622S0011.expected.xml
index 4db3a0f6..c7dd0082 100644
--- a/gem2caom2/tests/data/F2/S20150622S0011.expected.xml
+++ b/gem2caom2/tests/data/F2/S20150622S0011.expected.xml
@@ -64,7 +64,6 @@
3
1
2
- 4
3
diff --git a/gem2caom2/tests/data/F2/S20150625S0077.expected.xml b/gem2caom2/tests/data/F2/S20150625S0077.expected.xml
index 78868ade..f8e5a00d 100644
--- a/gem2caom2/tests/data/F2/S20150625S0077.expected.xml
+++ b/gem2caom2/tests/data/F2/S20150625S0077.expected.xml
@@ -64,7 +64,6 @@
3
1
2
- 4
3
diff --git a/gem2caom2/tests/data/F2/S20150625S0230.expected.xml b/gem2caom2/tests/data/F2/S20150625S0230.expected.xml
index b73a543d..4b7be6a3 100644
--- a/gem2caom2/tests/data/F2/S20150625S0230.expected.xml
+++ b/gem2caom2/tests/data/F2/S20150625S0230.expected.xml
@@ -62,7 +62,6 @@
3
- 4
3
diff --git a/gem2caom2/tests/data/F2/S20160220S0125.expected.xml b/gem2caom2/tests/data/F2/S20160220S0125.expected.xml
index 14ac0f5c..6d3d2ff3 100644
--- a/gem2caom2/tests/data/F2/S20160220S0125.expected.xml
+++ b/gem2caom2/tests/data/F2/S20160220S0125.expected.xml
@@ -63,7 +63,6 @@
3
1
2
- 4
3
diff --git a/gem2caom2/tests/data/F2/S20160616S0034.expected.xml b/gem2caom2/tests/data/F2/S20160616S0034.expected.xml
index 46363e65..0c03a327 100644
--- a/gem2caom2/tests/data/F2/S20160616S0034.expected.xml
+++ b/gem2caom2/tests/data/F2/S20160616S0034.expected.xml
@@ -63,7 +63,6 @@
3
1
2
- 4
3
diff --git a/gem2caom2/tests/data/F2/S20170221S0005.expected.xml b/gem2caom2/tests/data/F2/S20170221S0005.expected.xml
index d978c765..caa83d27 100644
--- a/gem2caom2/tests/data/F2/S20170221S0005.expected.xml
+++ b/gem2caom2/tests/data/F2/S20170221S0005.expected.xml
@@ -56,7 +56,6 @@
3
- 4
3
diff --git a/gem2caom2/tests/data/F2/S20170905S0318.expected.xml b/gem2caom2/tests/data/F2/S20170905S0318.expected.xml
index 82abdd76..5b135f4c 100644
--- a/gem2caom2/tests/data/F2/S20170905S0318.expected.xml
+++ b/gem2caom2/tests/data/F2/S20170905S0318.expected.xml
@@ -63,7 +63,6 @@
3
1
2
- 4
3
diff --git a/gem2caom2/tests/data/F2/S20171123S0166.expected.xml b/gem2caom2/tests/data/F2/S20171123S0166.expected.xml
index bdea14da..d1a7bac3 100644
--- a/gem2caom2/tests/data/F2/S20171123S0166.expected.xml
+++ b/gem2caom2/tests/data/F2/S20171123S0166.expected.xml
@@ -63,7 +63,6 @@
3
1
2
- 4
3
diff --git a/gem2caom2/tests/data/F2/S20171123S0216.expected.xml b/gem2caom2/tests/data/F2/S20171123S0216.expected.xml
index 5c4840fb..587c4219 100644
--- a/gem2caom2/tests/data/F2/S20171123S0216.expected.xml
+++ b/gem2caom2/tests/data/F2/S20171123S0216.expected.xml
@@ -63,7 +63,6 @@
3
1
2
- 4
3
diff --git a/gem2caom2/tests/data/F2/S20171226S0358.expected.xml b/gem2caom2/tests/data/F2/S20171226S0358.expected.xml
index 288da739..58192957 100644
--- a/gem2caom2/tests/data/F2/S20171226S0358.expected.xml
+++ b/gem2caom2/tests/data/F2/S20171226S0358.expected.xml
@@ -63,7 +63,6 @@
3
1
2
- 4
3
diff --git a/gem2caom2/tests/data/F2/S20181230S0026.expected.xml b/gem2caom2/tests/data/F2/S20181230S0026.expected.xml
index 5d47454c..fb0a5bd1 100644
--- a/gem2caom2/tests/data/F2/S20181230S0026.expected.xml
+++ b/gem2caom2/tests/data/F2/S20181230S0026.expected.xml
@@ -62,7 +62,6 @@
3
1
2
- 4
3
diff --git a/gem2caom2/tests/data/GS-2019B-Q-222-181-001.expected.xml b/gem2caom2/tests/data/GS-2019B-Q-222-181-001.expected.xml
index 7f931677..3d9ff0fc 100644
--- a/gem2caom2/tests/data/GS-2019B-Q-222-181-001.expected.xml
+++ b/gem2caom2/tests/data/GS-2019B-Q-222-181-001.expected.xml
@@ -59,7 +59,6 @@
2
1
2
- 4
3
@@ -151,7 +150,6 @@
2
1
2
- 4
3
@@ -243,7 +241,6 @@
2
1
2
- 4
3
@@ -335,7 +332,6 @@
2
1
2
- 4
3
diff --git a/gem2caom2/tests/data/TReCS/S20050102S0024.expected.xml b/gem2caom2/tests/data/TReCS/S20050102S0024.expected.xml
index 03af89ee..36548125 100644
--- a/gem2caom2/tests/data/TReCS/S20050102S0024.expected.xml
+++ b/gem2caom2/tests/data/TReCS/S20050102S0024.expected.xml
@@ -61,7 +61,6 @@
4
1
2
- 4
3
@@ -153,7 +152,6 @@
4
1
2
- 4
3
@@ -245,7 +243,6 @@
4
1
2
- 4
3
@@ -337,7 +334,6 @@
4
1
2
- 4
3
@@ -429,7 +425,6 @@
4
1
2
- 4
3
@@ -521,7 +516,6 @@
4
1
2
- 4
3
diff --git a/gem2caom2/tests/data/TReCS/S20050621S0037.expected.xml b/gem2caom2/tests/data/TReCS/S20050621S0037.expected.xml
index 6f3eb20a..1f6fc91c 100644
--- a/gem2caom2/tests/data/TReCS/S20050621S0037.expected.xml
+++ b/gem2caom2/tests/data/TReCS/S20050621S0037.expected.xml
@@ -63,7 +63,6 @@
4
1
2
- 4
3
@@ -155,7 +154,6 @@
4
1
2
- 4
3
@@ -247,7 +245,6 @@
4
1
2
- 4
3
@@ -339,7 +336,6 @@
4
1
2
- 4
3
@@ -431,7 +427,6 @@
4
1
2
- 4
3
@@ -523,7 +518,6 @@
4
1
2
- 4
3
@@ -615,7 +609,6 @@
4
1
2
- 4
3
@@ -707,7 +700,6 @@
4
1
2
- 4
3
@@ -799,7 +791,6 @@
4
1
2
- 4
3
@@ -891,7 +882,6 @@
4
1
2
- 4
3
diff --git a/gem2caom2/tests/data/TReCS/S20050918S0058.expected.xml b/gem2caom2/tests/data/TReCS/S20050918S0058.expected.xml
index d44b421c..8680b436 100644
--- a/gem2caom2/tests/data/TReCS/S20050918S0058.expected.xml
+++ b/gem2caom2/tests/data/TReCS/S20050918S0058.expected.xml
@@ -63,7 +63,6 @@
4
1
2
- 4
3
diff --git a/gem2caom2/tests/data/TReCS/S20120922S0372.expected.xml b/gem2caom2/tests/data/TReCS/S20120922S0372.expected.xml
index c7ae252a..33e77c07 100644
--- a/gem2caom2/tests/data/TReCS/S20120922S0372.expected.xml
+++ b/gem2caom2/tests/data/TReCS/S20120922S0372.expected.xml
@@ -63,7 +63,6 @@
4
1
2
- 4
3
@@ -155,7 +154,6 @@
4
1
2
- 4
3
@@ -247,7 +245,6 @@
4
1
2
- 4
3
@@ -339,7 +336,6 @@
4
1
2
- 4
3
@@ -431,7 +427,6 @@
4
1
2
- 4
3
@@ -523,7 +518,6 @@
4
1
2
- 4
3
@@ -615,7 +609,6 @@
4
1
2
- 4
3
@@ -707,7 +700,6 @@
4
1
2
- 4
3
@@ -799,7 +791,6 @@
4
1
2
- 4
3
@@ -891,7 +882,6 @@
4
1
2
- 4
3
@@ -983,7 +973,6 @@
4
1
2
- 4
3
@@ -1075,7 +1064,6 @@
4
1
2
- 4
3
@@ -1167,7 +1155,6 @@
4
1
2
- 4
3
@@ -1259,7 +1246,6 @@
4
1
2
- 4
3
@@ -1351,7 +1337,6 @@
4
1
2
- 4
3
@@ -1443,7 +1428,6 @@
4
1
2
- 4
3
@@ -1535,7 +1519,6 @@
4
1
2
- 4
3
diff --git a/gem2caom2/tests/data/config.yml b/gem2caom2/tests/data/config.yml
index c4efe798..8b41babd 100644
--- a/gem2caom2/tests/data/config.yml
+++ b/gem2caom2/tests/data/config.yml
@@ -8,7 +8,7 @@ todo_file_name: todo.txt
# values True False
use_local_files: False
# values DEBUG INFO WARN ERROR
-logging_level: DEBUG
+logging_level: INFO
# values True False
log_to_file: True
# fully qualified name for a directory to write log files
diff --git a/gem2caom2/tests/data/fix_sc2.sh b/gem2caom2/tests/data/fix_sc2.sh
deleted file mode 100755
index 753f256e..00000000
--- a/gem2caom2/tests/data/fix_sc2.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/bash
-
-for ii in GMOS/GN-CAL20150925-2-007.actual.xml GMOS/GN-CAL20150217-2-003.actual.xml GMOS/GN-2015A-Q-91-5-002.actual.xml GMOS/GN-2015A-C-4-24-086.actual.xml GMOS/GN-2015A-C-2-96-002.actual.xml GMOS/GN-2013B-Q-28-150-002.actual.xml
-do
- obs_id=$(echo $ii | awk -F'/' '{print $2}' | awk -F '.' '{print $1}')
- caom2-repo delete --cert $HOME/.ssl/cadcproxy.pem --resource-id ivo://cadc.nrc.ca/sc2repo GEMINI ${obs_id}
- caom2-repo create --cert $HOME/.ssl/cadcproxy.pem --resource-id ivo://cadc.nrc.ca/sc2repo ${ii}
-done
diff --git a/gem2caom2/tests/data/get_check_stuff.sh b/gem2caom2/tests/data/get_check_stuff.sh
deleted file mode 100755
index 7ed17e15..00000000
--- a/gem2caom2/tests/data/get_check_stuff.sh
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/bin/bash
-
-#instrument="GNIRS"
-#for ii in GN-2011B-Q-7-193-010 GN-2011B-Q-63-101-006 GN-2011B-Q-68-116-013 GN-2011B-Q-63-126-005 GN-2011B-Q-7-193-044 GN-2011B-Q-63-126-005
-#instrument="GRACES"
-#for ii in GN-2015B-Q-1-12-1003 GN-CAL20150604-1000-1072 GN-CAL20150604-1000-1081 GN-CAL20150807-1000-1035
-#instrument="NIFS"
-#for ii in GN-2012B-Q-51-108-001 GN-2012B-Q-73-172-002 GN-2016B-Q-27-31-007 GN-2012A-Q-57-151-005 GN-2011B-Q-59-70-016 GN-2012A-Q-2-22-007
-#instrument="GSAOI"
-#for ii in GS-2013B-Q-61-8-008 GS-2012B-SV-499-21-002 GS-2013B-DD-1-13-002 GS-CAL20181023-5-001 GS-CAL20140109-3-009 GS-CAL20130201-3-017 GS-2013B-Q-26-19-004
-#instrument="F2"
-#for ii in GS-2014B-Q-60-8-071 GS-2015A-Q-60-126-001 GS-2014B-Q-17-69-002 GS-2014B-Q-17-53-030 GS-2014B-Q-60-11-012 GS-2017A-Q-28-280-012 GS-2016A-FT-18-54-002 GS-2016A-C-1-86-003 GS-2017B-Q-18-96-006 GS-2017B-Q-45-156-079 GS-2015A-Q-63-328-014 GS-2018B-SV-301-144-020 GS-2015A-Q-63-365-016 GS-2015A-Q-63-406-001
-#instrument=GPI
-#for ii in GS-2018A-FT-101-5-043 GS-CAL20150410-1-007 GS-CAL20140323-6-014 GS-CAL20160224-12-017 GS-2014A-SV-408-6-003 GS-CAL20140320-7-011 GS-CAL20140315-4-004 GS-CAL20140316-5-012
-#instrument=Michelle
-#for ii in GN-2008A-Q-43-6-002 GN-2009B-C-1-62-001 GN-CAL20060413-7-001 GN-CAL20080308-8-016 GN-2006A-Q-58-9-007 GN-2007A-C-11-234-001 GN-2010B-C-3-21-002 GN-2006A-C-14-49-002
-#instrument=TReCS
-#for ii in GS-CAL20050102-1-001 GS-2005A-Q-15-1-001 GS-2005B-Q-10-22-003 GS-2012A-Q-7-31-001 GS-2008A-C-5-35-002
-#instrument=bHROS
-#for ii in GS-2006B-Q-47-76-003 GS-2006B-Q-7-32-008 GS-2005B-SV-302-20-001 GS-2005B-SV-301-16-005
-#instrument=hrwfs
-#for ii in GS-2003A-Q-6-1-002 GS-CAL20030303-7-0006 GS-CAL20030730-10-006 GS-CAL20031218-1-034 GS-CAL20030105-2-0072
-# instrument=Phoenix
-# for ii in GS-2006B-C-8-2-0052 GS-2006A-DD-1-1-0073 GS-2003B-Q-51-27-0073 GS-2006A-C-10-1-0258
-# instrument = OSCIR
-#for ii in GS-2001B-Q-31-9-007
-# instrument = Flamingos
-# for ii in GS-2002B-DD-2-6-0230
-# instrument = HOKUPAA
-#for ii in GN-2002A-DD-1-319-591 GN-CAL20020424-1-007
-# instrument = GMOS
-for ii in GS-2003B-Q-5-29-003 GS-2005B-Q-27-4-001 GS-CAL20060125-1-002 GS-2005B-Q-27-33-001 GS-2008A-Q-41-26-013 GN2009AQ021-04 GS-2005B-Q-22-17-006 GS-2005B-Q-22-29-004 GS-2005B-Q-54-2-004 GS-2006A-Q-48-15-002 GS-2009A-Q-30-6-007 GN-2009B-Q-121-15-003 GN-2010A-Q-35-10-002 GN-2009B-Q-121-15-001 GN-2007B-Q-112-14-018
-do
- echo $ii
- # caom2-repo read --cert $HOME/.ssl/cadcproxy.pem --resource-id ivo://cadc.nrc.ca/ams GEMINI $ii > "${ii}.xml"
- curl "http://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/caom2ops/meta?ID=caom:GEMINI/${ii}" > "${ii}.xml"
-done
diff --git a/gem2caom2/tests/data/get_programs.sh b/gem2caom2/tests/data/get_programs.sh
deleted file mode 100755
index b50e3748..00000000
--- a/gem2caom2/tests/data/get_programs.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/bin/bash
-for ii in $(ls -d *)
-do
- if [[ -d ${ii} && ${ii} != 'votable' ]]
- then
- echo " # $ii"
-# mkdir -p ${ii}/program
-# for jj in $(grep program_id ${ii}/json/* | awk -F'"' '{print $4}')
-# do
-# echo $jj
-# curl "https://archive.gemini.edu/programinfo/${jj}" > ${ii}/program/${jj}.xml
-# done
- for jj in $(ls ${ii}/*.in.xml)
- do
- obs_id=$( grep observationID ${jj} | awk -F'>' '{print $2}' | awk -F'<' '{print $1}' )
- file_id=$( grep uri ${jj} | grep fits | awk -F'>' '{print $2}' | awk -F'<' '{print $1}' | awk -F'/' '{print $2}' | awk -F'.' '{print $1"."$2}' )
- program_id=$(grep program_id ${ii}/json/${obs_id}.json | awk -F'"' '{print $4}')
- echo " '${file_id}': ['${obs_id}', '${ii}', '${program_id}'], "
- done
- fi
-done
-date
-exit 0
diff --git a/gem2caom2/tests/data/hrwfs/S20030218S0042.expected.xml b/gem2caom2/tests/data/hrwfs/S20030218S0042.expected.xml
index 8bf252ff..5c56769e 100644
--- a/gem2caom2/tests/data/hrwfs/S20030218S0042.expected.xml
+++ b/gem2caom2/tests/data/hrwfs/S20030218S0042.expected.xml
@@ -12,6 +12,9 @@
GS-2003A-Q-6
Jane Gregorio-Hetem
The nature of unidentified X-ray sources in CMa star formation region- Part II
+
+ Moleculear clouds, Neutron stars, T Tauri stars, Young star clusters
+
src17
diff --git a/gem2caom2/tests/data/json/N20190313A0002b.json b/gem2caom2/tests/data/json/N20190313A0002b.json
new file mode 100644
index 00000000..30591a51
--- /dev/null
+++ b/gem2caom2/tests/data/json/N20190313A0002b.json
@@ -0,0 +1,68 @@
+[
+ {
+ "name": "N20190313A0002b.fits",
+ "filename": "N20190313A0002b.fits.bz2",
+ "path": "",
+ "compressed": true,
+ "file_size": 37670946,
+ "data_size": 104872320,
+ "file_md5": "3a6f57da8634eb170faa83bfd924d44a",
+ "data_md5": "eb57edfe159d841ee02bafcf0ebfa221",
+ "lastmod": "2020-06-26 00:35:12.713745+00:00",
+ "mdready": true,
+ "entrytime": "2020-06-26 00:35:12.795311+00:00",
+ "size": 37670946,
+ "md5": "3a6f57da8634eb170faa83bfd924d44a",
+ "program_id": "GN-2019A-Q-214",
+ "engineering": false,
+ "science_verification": false,
+ "calibration_program": false,
+ "observation_id": "GN-2019A-Q-214-0",
+ "data_label": "GN-2019A-Q-214-0-0",
+ "telescope": "Gemini-North",
+ "instrument": "ALOPEKE",
+ "ut_datetime": "2019-03-13 22:28:05.001600",
+ "local_time": null,
+ "observation_type": "DARK",
+ "observation_class": null,
+ "object": "Zenith",
+ "ra": null,
+ "dec": null,
+ "azimuth": null,
+ "elevation": null,
+ "cass_rotator_pa": null,
+ "airmass": 1.0,
+ "filter_name": null,
+ "exposure_time": 15.0,
+ "disperser": null,
+ "camera": null,
+ "central_wavelength": null,
+ "wavelength_band": null,
+ "focal_plane_mask": null,
+ "detector_binning": "1x1",
+ "detector_gain_setting": null,
+ "detector_roi_setting": "Fixed",
+ "detector_readspeed_setting": null,
+ "detector_welldepth_setting": null,
+ "detector_readmode_setting": "Image",
+ "spectroscopy": false,
+ "mode": "imaging",
+ "adaptive_optics": false,
+ "laser_guide_star": false,
+ "wavefront_sensor": null,
+ "gcal_lamp": null,
+ "raw_iq": null,
+ "raw_cc": null,
+ "raw_wv": null,
+ "raw_bg": null,
+ "requested_iq": null,
+ "requested_cc": null,
+ "requested_wv": null,
+ "requested_bg": null,
+ "qa_state": "Undefined",
+ "release": "2020-03-12",
+ "reduction": "RAW",
+ "types": "{'UNPREPARED', 'GEMINI', 'NORTH', 'RAW'}",
+ "phot_standard": null
+ }
+]
\ No newline at end of file
diff --git a/gem2caom2/tests/data/json/N20190313A0002r.json b/gem2caom2/tests/data/json/N20190313A0002r.json
new file mode 100644
index 00000000..23e059c7
--- /dev/null
+++ b/gem2caom2/tests/data/json/N20190313A0002r.json
@@ -0,0 +1,68 @@
+[
+ {
+ "name": "N20190313A0002r.fits",
+ "filename": "N20190313A0002r.fits.bz2",
+ "path": "",
+ "compressed": true,
+ "file_size": 37464408,
+ "data_size": 104872320,
+ "file_md5": "c76c67acdfc5a58db03092095cd76d18",
+ "data_md5": "d849865e4451c58d293c288a2c372f70",
+ "lastmod": "2020-06-26 00:32:00.624440+00:00",
+ "mdready": true,
+ "entrytime": "2020-06-26 00:32:00.712622+00:00",
+ "size": 37464408,
+ "md5": "c76c67acdfc5a58db03092095cd76d18",
+ "program_id": "GN-2019A-Q-214",
+ "engineering": false,
+ "science_verification": false,
+ "calibration_program": false,
+ "observation_id": "GN-2019A-Q-214-0",
+ "data_label": "GN-2019A-Q-214-0-0",
+ "telescope": "Gemini-North",
+ "instrument": "ALOPEKE",
+ "ut_datetime": "2019-03-13 22:28:05.001600",
+ "local_time": null,
+ "observation_type": "DARK",
+ "observation_class": null,
+ "object": "Zenith",
+ "ra": null,
+ "dec": null,
+ "azimuth": null,
+ "elevation": null,
+ "cass_rotator_pa": null,
+ "airmass": 1.0,
+ "filter_name": null,
+ "exposure_time": 15.0,
+ "disperser": null,
+ "camera": null,
+ "central_wavelength": null,
+ "wavelength_band": null,
+ "focal_plane_mask": null,
+ "detector_binning": "1x1",
+ "detector_gain_setting": null,
+ "detector_roi_setting": "Fixed",
+ "detector_readspeed_setting": null,
+ "detector_welldepth_setting": null,
+ "detector_readmode_setting": "Image",
+ "spectroscopy": false,
+ "mode": "imaging",
+ "adaptive_optics": false,
+ "laser_guide_star": false,
+ "wavefront_sensor": null,
+ "gcal_lamp": null,
+ "raw_iq": null,
+ "raw_cc": null,
+ "raw_wv": null,
+ "raw_bg": null,
+ "requested_iq": null,
+ "requested_cc": null,
+ "requested_wv": null,
+ "requested_bg": null,
+ "qa_state": "Undefined",
+ "release": "2020-03-12",
+ "reduction": "RAW",
+ "types": "{'GEMINI', 'RAW', 'NORTH', 'UNPREPARED'}",
+ "phot_standard": null
+ }
+]
\ No newline at end of file
diff --git a/gem2caom2/tests/data/json/N20191219A0001b.json b/gem2caom2/tests/data/json/N20191219A0001b.json
new file mode 100644
index 00000000..d4077311
--- /dev/null
+++ b/gem2caom2/tests/data/json/N20191219A0001b.json
@@ -0,0 +1,68 @@
+[
+ {
+ "name": "N20191219A0001b.fits",
+ "filename": "N20191219A0001b.fits.bz2",
+ "path": "",
+ "compressed": true,
+ "file_size": 76394294,
+ "data_size": 131094720,
+ "file_md5": "95041b08ff4cfcff17b4ff93c648f076",
+ "data_md5": "ab7a9ad50229f615ddbf58dcbdea96ae",
+ "lastmod": "2020-06-07 18:21:08.112331+00:00",
+ "mdready": true,
+ "entrytime": "2020-06-07 18:21:08.265400+00:00",
+ "size": 76394294,
+ "md5": "95041b08ff4cfcff17b4ff93c648f076",
+ "program_id": "GN-2020A-Q-132",
+ "engineering": false,
+ "science_verification": false,
+ "calibration_program": false,
+ "observation_id": "GN-2020A-Q-132-0",
+ "data_label": "GN-2020A-Q-132-0-0",
+ "telescope": "Gemini-North",
+ "instrument": "ALOPEKE",
+ "ut_datetime": "2019-12-19 16:02:30.019200",
+ "local_time": null,
+ "observation_type": "OBJECT",
+ "observation_class": null,
+ "object": "HD118203",
+ "ra": null,
+ "dec": null,
+ "azimuth": null,
+ "elevation": null,
+ "cass_rotator_pa": null,
+ "airmass": 1.325,
+ "filter_name": null,
+ "exposure_time": 0.06,
+ "disperser": null,
+ "camera": null,
+ "central_wavelength": null,
+ "wavelength_band": null,
+ "focal_plane_mask": null,
+ "detector_binning": "1x1",
+ "detector_gain_setting": null,
+ "detector_roi_setting": "Fixed",
+ "detector_readspeed_setting": null,
+ "detector_welldepth_setting": null,
+ "detector_readmode_setting": "Image",
+ "spectroscopy": false,
+ "mode": "imaging",
+ "adaptive_optics": false,
+ "laser_guide_star": false,
+ "wavefront_sensor": null,
+ "gcal_lamp": null,
+ "raw_iq": null,
+ "raw_cc": null,
+ "raw_wv": null,
+ "raw_bg": null,
+ "requested_iq": null,
+ "requested_cc": null,
+ "requested_wv": null,
+ "requested_bg": null,
+ "qa_state": "Undefined",
+ "release": "2020-12-18",
+ "reduction": "RAW",
+ "types": "{'GEMINI', 'RAW', 'NORTH', 'UNPREPARED'}",
+ "phot_standard": null
+ }
+]
\ No newline at end of file
diff --git a/gem2caom2/tests/data/json/N20191219A0001r.json b/gem2caom2/tests/data/json/N20191219A0001r.json
new file mode 100644
index 00000000..0a6e8e28
--- /dev/null
+++ b/gem2caom2/tests/data/json/N20191219A0001r.json
@@ -0,0 +1,68 @@
+[
+ {
+ "name": "N20191219A0001r.fits",
+ "filename": "N20191219A0001r.fits.bz2",
+ "path": "",
+ "compressed": true,
+ "file_size": 70542890,
+ "data_size": 131094720,
+ "file_md5": "2f8f51d2b388461c55f20d2da9a0ca9c",
+ "data_md5": "124c4edd88ad4f235b3544c565384c02",
+ "lastmod": "2020-06-07 18:20:59.934289+00:00",
+ "mdready": true,
+ "entrytime": "2020-06-07 18:21:00.078952+00:00",
+ "size": 70542890,
+ "md5": "2f8f51d2b388461c55f20d2da9a0ca9c",
+ "program_id": "GN-2020A-Q-132",
+ "engineering": false,
+ "science_verification": false,
+ "calibration_program": false,
+ "observation_id": "GN-2020A-Q-132-0",
+ "data_label": "GN-2020A-Q-132-0-0",
+ "telescope": "Gemini-North",
+ "instrument": "ALOPEKE",
+ "ut_datetime": "2019-12-19 16:02:30.019200",
+ "local_time": null,
+ "observation_type": "OBJECT",
+ "observation_class": null,
+ "object": "HD118203",
+ "ra": null,
+ "dec": null,
+ "azimuth": null,
+ "elevation": null,
+ "cass_rotator_pa": null,
+ "airmass": 1.325,
+ "filter_name": null,
+ "exposure_time": 0.06,
+ "disperser": null,
+ "camera": null,
+ "central_wavelength": null,
+ "wavelength_band": null,
+ "focal_plane_mask": null,
+ "detector_binning": "1x1",
+ "detector_gain_setting": null,
+ "detector_roi_setting": "Fixed",
+ "detector_readspeed_setting": null,
+ "detector_welldepth_setting": null,
+ "detector_readmode_setting": "Image",
+ "spectroscopy": false,
+ "mode": "imaging",
+ "adaptive_optics": false,
+ "laser_guide_star": false,
+ "wavefront_sensor": null,
+ "gcal_lamp": null,
+ "raw_iq": null,
+ "raw_cc": null,
+ "raw_wv": null,
+ "raw_bg": null,
+ "requested_iq": null,
+ "requested_cc": null,
+ "requested_wv": null,
+ "requested_bg": null,
+ "qa_state": "Undefined",
+ "release": "2020-12-18",
+ "reduction": "RAW",
+ "types": "{'GEMINI', 'RAW', 'NORTH', 'UNPREPARED'}",
+ "phot_standard": null
+ }
+]
\ No newline at end of file
diff --git a/gem2caom2/tests/data/json/N20200215A0004b.json b/gem2caom2/tests/data/json/N20200215A0004b.json
new file mode 100644
index 00000000..ea2df11c
--- /dev/null
+++ b/gem2caom2/tests/data/json/N20200215A0004b.json
@@ -0,0 +1,68 @@
+[
+ {
+ "name": "N20200215A0004b.fits",
+ "filename": "N20200215A0004b.fits.bz2",
+ "path": "",
+ "compressed": true,
+ "file_size": 49874165,
+ "data_size": 104872320,
+ "file_md5": "a3166604367bf0fb2837b4d1d68ddd94",
+ "data_md5": "00a9f610b2d6d4abc97d7e9e4d8be33a",
+ "lastmod": "2020-06-06 08:31:00.373788+00:00",
+ "mdready": true,
+ "entrytime": "2020-06-06 08:31:00.480437+00:00",
+ "size": 49874165,
+ "md5": "a3166604367bf0fb2837b4d1d68ddd94",
+ "program_id": "GN-2019B-Q-415",
+ "engineering": false,
+ "science_verification": false,
+ "calibration_program": false,
+ "observation_id": "GN-2019B-Q-415-0",
+ "data_label": "GN-2019B-Q-415-0-0",
+ "telescope": "Gemini-North",
+ "instrument": "ALOPEKE",
+ "ut_datetime": "2020-02-15 04:48:55.036800",
+ "local_time": null,
+ "observation_type": "FLAT",
+ "observation_class": null,
+ "object": "ALT_HD224688",
+ "ra": null,
+ "dec": null,
+ "azimuth": null,
+ "elevation": null,
+ "cass_rotator_pa": 0.015,
+ "airmass": 1.797,
+ "filter_name": null,
+ "exposure_time": 0.6,
+ "disperser": null,
+ "camera": null,
+ "central_wavelength": null,
+ "wavelength_band": null,
+ "focal_plane_mask": null,
+ "detector_binning": "1x1",
+ "detector_gain_setting": null,
+ "detector_roi_setting": "Fixed",
+ "detector_readspeed_setting": null,
+ "detector_welldepth_setting": null,
+ "detector_readmode_setting": "Image",
+ "spectroscopy": false,
+ "mode": "imaging",
+ "adaptive_optics": false,
+ "laser_guide_star": false,
+ "wavefront_sensor": null,
+ "gcal_lamp": null,
+ "raw_iq": null,
+ "raw_cc": null,
+ "raw_wv": null,
+ "raw_bg": null,
+ "requested_iq": null,
+ "requested_cc": null,
+ "requested_wv": null,
+ "requested_bg": null,
+ "qa_state": "Undefined",
+ "release": "2021-02-14",
+ "reduction": "RAW",
+ "types": "{'UNPREPARED', 'GEMINI', 'NORTH', 'RAW'}",
+ "phot_standard": null
+ }
+]
\ No newline at end of file
diff --git a/gem2caom2/tests/data/json/N20200215A0004r.json b/gem2caom2/tests/data/json/N20200215A0004r.json
new file mode 100644
index 00000000..301a57d8
--- /dev/null
+++ b/gem2caom2/tests/data/json/N20200215A0004r.json
@@ -0,0 +1,68 @@
+[
+ {
+ "name": "N20200215A0004r.fits",
+ "filename": "N20200215A0004r.fits.bz2",
+ "path": "",
+ "compressed": true,
+ "file_size": 52708997,
+ "data_size": 104872320,
+ "file_md5": "a6943f932aaa0eeaef719025f5904bb1",
+ "data_md5": "fb2e134f26045510129b625db2971c0e",
+ "lastmod": "2020-06-06 04:13:33.206849+00:00",
+ "mdready": true,
+ "entrytime": "2020-06-06 04:13:33.320485+00:00",
+ "size": 52708997,
+ "md5": "a6943f932aaa0eeaef719025f5904bb1",
+ "program_id": "GN-2019B-Q-415",
+ "engineering": false,
+ "science_verification": false,
+ "calibration_program": false,
+ "observation_id": "GN-2019B-Q-415-0",
+ "data_label": "GN-2019B-Q-415-0-0",
+ "telescope": "Gemini-North",
+ "instrument": "ALOPEKE",
+ "ut_datetime": "2020-02-15 04:48:55.036800",
+ "local_time": null,
+ "observation_type": "FLAT",
+ "observation_class": null,
+ "object": "ALT_HD224688",
+ "ra": null,
+ "dec": null,
+ "azimuth": null,
+ "elevation": null,
+ "cass_rotator_pa": 0.015,
+ "airmass": 1.797,
+ "filter_name": null,
+ "exposure_time": 0.6,
+ "disperser": null,
+ "camera": null,
+ "central_wavelength": null,
+ "wavelength_band": null,
+ "focal_plane_mask": null,
+ "detector_binning": "1x1",
+ "detector_gain_setting": null,
+ "detector_roi_setting": "Fixed",
+ "detector_readspeed_setting": null,
+ "detector_welldepth_setting": null,
+ "detector_readmode_setting": "Image",
+ "spectroscopy": false,
+ "mode": "imaging",
+ "adaptive_optics": false,
+ "laser_guide_star": false,
+ "wavefront_sensor": null,
+ "gcal_lamp": null,
+ "raw_iq": null,
+ "raw_cc": null,
+ "raw_wv": null,
+ "raw_bg": null,
+ "requested_iq": null,
+ "requested_cc": null,
+ "requested_wv": null,
+ "requested_bg": null,
+ "qa_state": "Undefined",
+ "release": "2021-02-14",
+ "reduction": "RAW",
+ "types": "{'GEMINI', 'RAW', 'NORTH', 'UNPREPARED'}",
+ "phot_standard": null
+ }
+]
\ No newline at end of file
diff --git a/gem2caom2/tests/data/json/S20190716Z0925b.json b/gem2caom2/tests/data/json/S20190716Z0925b.json
new file mode 100644
index 00000000..2d78ea20
--- /dev/null
+++ b/gem2caom2/tests/data/json/S20190716Z0925b.json
@@ -0,0 +1,68 @@
+[
+ {
+ "name": "S20190716Z0925b.fits",
+ "filename": "S20190716Z0925b.fits.bz2",
+ "path": "",
+ "compressed": true,
+ "file_size": 88013053,
+ "data_size": 131094720,
+ "file_md5": "8cf0414c3c1e8211b22a6fbac321e3e1",
+ "data_md5": "4f0a662fe78a4e1c844371fcac90dfef",
+ "lastmod": "2020-05-27 13:45:56.235934+00:00",
+ "mdready": true,
+ "entrytime": "2020-05-27 13:45:56.411699+00:00",
+ "size": 88013053,
+ "md5": "8cf0414c3c1e8211b22a6fbac321e3e1",
+ "program_id": "GS-2019A-DD-109",
+ "engineering": false,
+ "science_verification": false,
+ "calibration_program": false,
+ "observation_id": "GS-2019A-DD-109-0",
+ "data_label": "GS-2019A-DD-109-0-0",
+ "telescope": "Gemini-South",
+ "instrument": "ZORRO",
+ "ut_datetime": "2019-07-16 10:28:29.020800",
+ "local_time": null,
+ "observation_type": "OBJECT",
+ "observation_class": null,
+ "object": "HIP 28909",
+ "ra": null,
+ "dec": null,
+ "azimuth": null,
+ "elevation": null,
+ "cass_rotator_pa": null,
+ "airmass": 1.736,
+ "filter_name": null,
+ "exposure_time": 0.06,
+ "disperser": null,
+ "camera": null,
+ "central_wavelength": null,
+ "wavelength_band": null,
+ "focal_plane_mask": null,
+ "detector_binning": "1x1",
+ "detector_gain_setting": null,
+ "detector_roi_setting": "Fixed",
+ "detector_readspeed_setting": null,
+ "detector_welldepth_setting": null,
+ "detector_readmode_setting": "Image",
+ "spectroscopy": false,
+ "mode": "imaging",
+ "adaptive_optics": false,
+ "laser_guide_star": false,
+ "wavefront_sensor": null,
+ "gcal_lamp": null,
+ "raw_iq": null,
+ "raw_cc": null,
+ "raw_wv": null,
+ "raw_bg": null,
+ "requested_iq": null,
+ "requested_cc": null,
+ "requested_wv": null,
+ "requested_bg": null,
+ "qa_state": "Undefined",
+ "release": "2020-07-15",
+ "reduction": "RAW",
+ "types": "{'UNPREPARED', 'GEMINI', 'SOUTH', 'RAW'}",
+ "phot_standard": null
+ }
+]
\ No newline at end of file
diff --git a/gem2caom2/tests/data/json/S20190716Z0925r.json b/gem2caom2/tests/data/json/S20190716Z0925r.json
new file mode 100644
index 00000000..3fd15fd8
--- /dev/null
+++ b/gem2caom2/tests/data/json/S20190716Z0925r.json
@@ -0,0 +1,68 @@
+[
+ {
+ "name": "S20190716Z0925r.fits",
+ "filename": "S20190716Z0925r.fits.bz2",
+ "path": "",
+ "compressed": true,
+ "file_size": 78514633,
+ "data_size": 131094720,
+ "file_md5": "361da32466d46f4a5e99bb6051b48fc2",
+ "data_md5": "601c29a81410ef985e8f8a0e90f57052",
+ "lastmod": "2020-05-27 13:45:21.830752+00:00",
+ "mdready": true,
+ "entrytime": "2020-05-27 13:45:21.988960+00:00",
+ "size": 78514633,
+ "md5": "361da32466d46f4a5e99bb6051b48fc2",
+ "program_id": "GS-2019A-DD-109",
+ "engineering": false,
+ "science_verification": false,
+ "calibration_program": false,
+ "observation_id": "GS-2019A-DD-109-0",
+ "data_label": "GS-2019A-DD-109-0-0",
+ "telescope": "Gemini-South",
+ "instrument": "ZORRO",
+ "ut_datetime": "2019-07-16 10:28:29.020800",
+ "local_time": null,
+ "observation_type": "OBJECT",
+ "observation_class": null,
+ "object": "HIP 28909",
+ "ra": null,
+ "dec": null,
+ "azimuth": null,
+ "elevation": null,
+ "cass_rotator_pa": null,
+ "airmass": 1.736,
+ "filter_name": null,
+ "exposure_time": 0.06,
+ "disperser": null,
+ "camera": null,
+ "central_wavelength": null,
+ "wavelength_band": null,
+ "focal_plane_mask": null,
+ "detector_binning": "1x1",
+ "detector_gain_setting": null,
+ "detector_roi_setting": "Fixed",
+ "detector_readspeed_setting": null,
+ "detector_welldepth_setting": null,
+ "detector_readmode_setting": "Image",
+ "spectroscopy": false,
+ "mode": "imaging",
+ "adaptive_optics": false,
+ "laser_guide_star": false,
+ "wavefront_sensor": null,
+ "gcal_lamp": null,
+ "raw_iq": null,
+ "raw_cc": null,
+ "raw_wv": null,
+ "raw_bg": null,
+ "requested_iq": null,
+ "requested_cc": null,
+ "requested_wv": null,
+ "requested_bg": null,
+ "qa_state": "Undefined",
+ "release": "2020-07-15",
+ "reduction": "RAW",
+ "types": "{'UNPREPARED', 'GEMINI', 'SOUTH', 'RAW'}",
+ "phot_standard": null
+ }
+]
\ No newline at end of file
diff --git a/gem2caom2/tests/data/json/S20190912Z0264b.json b/gem2caom2/tests/data/json/S20190912Z0264b.json
new file mode 100644
index 00000000..b3739124
--- /dev/null
+++ b/gem2caom2/tests/data/json/S20190912Z0264b.json
@@ -0,0 +1,68 @@
+[
+ {
+ "name": "S20190912Z0264b.fits",
+ "filename": "S20190912Z0264b.fits.bz2",
+ "path": "",
+ "compressed": true,
+ "file_size": 51242703,
+ "data_size": 131094720,
+ "file_md5": "1271aa501633136daa409c7733148e33",
+ "data_md5": "56685678eee2b3acda8986adbe0f88e8",
+ "lastmod": "2020-04-30 04:45:52.204956+00:00",
+ "mdready": true,
+ "entrytime": "2020-04-30 04:45:52.313235+00:00",
+ "size": 51242703,
+ "md5": "1271aa501633136daa409c7733148e33",
+ "program_id": "GS-2019B-FT-103",
+ "engineering": false,
+ "science_verification": false,
+ "calibration_program": false,
+ "observation_id": "GS-2019B-FT-103-0",
+ "data_label": "GS-2019B-FT-103-0-0",
+ "telescope": "Gemini-South",
+ "instrument": "ZORRO",
+ "ut_datetime": "2019-09-12 22:46:10.012800",
+ "local_time": null,
+ "observation_type": null,
+ "observation_class": null,
+ "object": "Zenith",
+ "ra": null,
+ "dec": null,
+ "azimuth": null,
+ "elevation": null,
+ "cass_rotator_pa": null,
+ "airmass": 1.0,
+ "filter_name": null,
+ "exposure_time": 0.06,
+ "disperser": null,
+ "camera": null,
+ "central_wavelength": null,
+ "wavelength_band": null,
+ "focal_plane_mask": null,
+ "detector_binning": "1x1",
+ "detector_gain_setting": null,
+ "detector_roi_setting": "Fixed",
+ "detector_readspeed_setting": null,
+ "detector_welldepth_setting": null,
+ "detector_readmode_setting": "Image",
+ "spectroscopy": false,
+ "mode": "imaging",
+ "adaptive_optics": false,
+ "laser_guide_star": false,
+ "wavefront_sensor": null,
+ "gcal_lamp": null,
+ "raw_iq": null,
+ "raw_cc": null,
+ "raw_wv": null,
+ "raw_bg": null,
+ "requested_iq": null,
+ "requested_cc": null,
+ "requested_wv": null,
+ "requested_bg": null,
+ "qa_state": "Undefined",
+ "release": "2020-03-13",
+ "reduction": "RAW",
+ "types": "{'GEMINI', 'SOUTH', 'RAW', 'UNPREPARED'}",
+ "phot_standard": null
+ }
+]
\ No newline at end of file
diff --git a/gem2caom2/tests/data/json/S20190912Z0264r.json b/gem2caom2/tests/data/json/S20190912Z0264r.json
new file mode 100644
index 00000000..d66c2cf6
--- /dev/null
+++ b/gem2caom2/tests/data/json/S20190912Z0264r.json
@@ -0,0 +1,68 @@
+[
+ {
+ "name": "S20190912Z0264r.fits",
+ "filename": "S20190912Z0264r.fits.bz2",
+ "path": "",
+ "compressed": true,
+ "file_size": 52541648,
+ "data_size": 131094720,
+ "file_md5": "e0da9eeb10021f91af6a9b3d3900cdb8",
+ "data_md5": "f7249563691e31dec16bd3c76f2acc31",
+ "lastmod": "2020-04-30 13:56:16.168534+00:00",
+ "mdready": true,
+ "entrytime": "2020-04-30 13:56:16.281391+00:00",
+ "size": 52541648,
+ "md5": "e0da9eeb10021f91af6a9b3d3900cdb8",
+ "program_id": "GS-2019B-FT-103",
+ "engineering": false,
+ "science_verification": false,
+ "calibration_program": false,
+ "observation_id": "GS-2019B-FT-103-0",
+ "data_label": "GS-2019B-FT-103-0-0",
+ "telescope": "Gemini-South",
+ "instrument": "ZORRO",
+ "ut_datetime": "2019-09-12 22:46:10.012800",
+ "local_time": null,
+ "observation_type": null,
+ "observation_class": null,
+ "object": "Zenith",
+ "ra": null,
+ "dec": null,
+ "azimuth": null,
+ "elevation": null,
+ "cass_rotator_pa": null,
+ "airmass": 1.0,
+ "filter_name": null,
+ "exposure_time": 0.06,
+ "disperser": null,
+ "camera": null,
+ "central_wavelength": null,
+ "wavelength_band": null,
+ "focal_plane_mask": null,
+ "detector_binning": "1x1",
+ "detector_gain_setting": null,
+ "detector_roi_setting": "Fixed",
+ "detector_readspeed_setting": null,
+ "detector_welldepth_setting": null,
+ "detector_readmode_setting": "Image",
+ "spectroscopy": false,
+ "mode": "imaging",
+ "adaptive_optics": false,
+ "laser_guide_star": false,
+ "wavefront_sensor": null,
+ "gcal_lamp": null,
+ "raw_iq": null,
+ "raw_cc": null,
+ "raw_wv": null,
+ "raw_bg": null,
+ "requested_iq": null,
+ "requested_cc": null,
+ "requested_wv": null,
+ "requested_bg": null,
+ "qa_state": "Undefined",
+ "release": "2020-03-13",
+ "reduction": "RAW",
+ "types": "{'GEMINI', 'SOUTH', 'RAW', 'UNPREPARED'}",
+ "phot_standard": null
+ }
+]
\ No newline at end of file
diff --git a/gem2caom2/tests/data/json/S20200316Z0308b.json b/gem2caom2/tests/data/json/S20200316Z0308b.json
new file mode 100644
index 00000000..950b04b3
--- /dev/null
+++ b/gem2caom2/tests/data/json/S20200316Z0308b.json
@@ -0,0 +1,68 @@
+[
+ {
+ "name": "S20200316Z0308b.fits",
+ "filename": "S20200316Z0308b.fits.bz2",
+ "path": "",
+ "compressed": true,
+ "file_size": 70709904,
+ "data_size": 131094720,
+ "file_md5": "4807b068cf8577b1121c846e50b9450d",
+ "data_md5": "c9ad5c9ec511639c989c67385db6f95b",
+ "lastmod": "2020-04-15 00:02:52.630774+00:00",
+ "mdready": true,
+ "entrytime": "2020-04-15 00:02:52.774146+00:00",
+ "size": 70709904,
+ "md5": "4807b068cf8577b1121c846e50b9450d",
+ "program_id": "GS-2020A-Q-321",
+ "engineering": false,
+ "science_verification": false,
+ "calibration_program": false,
+ "observation_id": "GS-2020A-Q-321-0",
+ "data_label": "GS-2020A-Q-321-0-0",
+ "telescope": "Gemini-South",
+ "instrument": "ZORRO",
+ "ut_datetime": "2020-03-16 10:05:59.971200",
+ "local_time": null,
+ "observation_type": "OBJECT",
+ "observation_class": null,
+ "object": "HD146596",
+ "ra": null,
+ "dec": null,
+ "azimuth": null,
+ "elevation": null,
+ "cass_rotator_pa": null,
+ "airmass": 1.091,
+ "filter_name": null,
+ "exposure_time": 0.06,
+ "disperser": null,
+ "camera": null,
+ "central_wavelength": null,
+ "wavelength_band": null,
+ "focal_plane_mask": null,
+ "detector_binning": "1x1",
+ "detector_gain_setting": null,
+ "detector_roi_setting": "Fixed",
+ "detector_readspeed_setting": null,
+ "detector_welldepth_setting": null,
+ "detector_readmode_setting": "Image",
+ "spectroscopy": false,
+ "mode": "imaging",
+ "adaptive_optics": false,
+ "laser_guide_star": false,
+ "wavefront_sensor": null,
+ "gcal_lamp": null,
+ "raw_iq": null,
+ "raw_cc": null,
+ "raw_wv": null,
+ "raw_bg": null,
+ "requested_iq": null,
+ "requested_cc": null,
+ "requested_wv": null,
+ "requested_bg": null,
+ "qa_state": "Undefined",
+ "release": "2021-03-16",
+ "reduction": "RAW",
+ "types": "{'GEMINI', 'SOUTH', 'RAW', 'UNPREPARED'}",
+ "phot_standard": null
+ }
+]
\ No newline at end of file
diff --git a/gem2caom2/tests/data/json/S20200316Z0308r.json b/gem2caom2/tests/data/json/S20200316Z0308r.json
new file mode 100644
index 00000000..4062785c
--- /dev/null
+++ b/gem2caom2/tests/data/json/S20200316Z0308r.json
@@ -0,0 +1,68 @@
+[
+ {
+ "name": "S20200316Z0308r.fits",
+ "filename": "S20200316Z0308r.fits.bz2",
+ "path": "",
+ "compressed": true,
+ "file_size": 63678182,
+ "data_size": 131094720,
+ "file_md5": "3e5e9397c3db2e7b9d03113ba1e8109c",
+ "data_md5": "38f8e744e5f442ca23fed3749557e632",
+ "lastmod": "2020-04-15 00:01:31.063349+00:00",
+ "mdready": true,
+ "entrytime": "2020-04-15 00:01:31.196206+00:00",
+ "size": 63678182,
+ "md5": "3e5e9397c3db2e7b9d03113ba1e8109c",
+ "program_id": "GS-2020A-Q-321",
+ "engineering": false,
+ "science_verification": false,
+ "calibration_program": false,
+ "observation_id": "GS-2020A-Q-321-0",
+ "data_label": "GS-2020A-Q-321-0-0",
+ "telescope": "Gemini-South",
+ "instrument": "ZORRO",
+ "ut_datetime": "2020-03-16 10:05:59.971200",
+ "local_time": null,
+ "observation_type": "OBJECT",
+ "observation_class": null,
+ "object": "HD146596",
+ "ra": null,
+ "dec": null,
+ "azimuth": null,
+ "elevation": null,
+ "cass_rotator_pa": null,
+ "airmass": 1.091,
+ "filter_name": null,
+ "exposure_time": 0.06,
+ "disperser": null,
+ "camera": null,
+ "central_wavelength": null,
+ "wavelength_band": null,
+ "focal_plane_mask": null,
+ "detector_binning": "1x1",
+ "detector_gain_setting": null,
+ "detector_roi_setting": "Fixed",
+ "detector_readspeed_setting": null,
+ "detector_welldepth_setting": null,
+ "detector_readmode_setting": "Image",
+ "spectroscopy": false,
+ "mode": "imaging",
+ "adaptive_optics": false,
+ "laser_guide_star": false,
+ "wavefront_sensor": null,
+ "gcal_lamp": null,
+ "raw_iq": null,
+ "raw_cc": null,
+ "raw_wv": null,
+ "raw_bg": null,
+ "requested_iq": null,
+ "requested_cc": null,
+ "requested_wv": null,
+ "requested_bg": null,
+ "qa_state": "Undefined",
+ "release": "2021-03-16",
+ "reduction": "RAW",
+ "types": "{'GEMINI', 'SOUTH', 'RAW', 'UNPREPARED'}",
+ "phot_standard": null
+ }
+]
\ No newline at end of file
diff --git a/gem2caom2/tests/data/michelle/N20060413S0129.expected.xml b/gem2caom2/tests/data/michelle/N20060413S0129.expected.xml
index 8303e5bf..f24d5230 100644
--- a/gem2caom2/tests/data/michelle/N20060413S0129.expected.xml
+++ b/gem2caom2/tests/data/michelle/N20060413S0129.expected.xml
@@ -61,7 +61,6 @@
4
1
2
- 4
3
diff --git a/gem2caom2/tests/data/michelle/N20060418S0123.expected.xml b/gem2caom2/tests/data/michelle/N20060418S0123.expected.xml
index 4ec78410..41ad95a6 100644
--- a/gem2caom2/tests/data/michelle/N20060418S0123.expected.xml
+++ b/gem2caom2/tests/data/michelle/N20060418S0123.expected.xml
@@ -63,7 +63,6 @@
4
1
2
- 4
3
@@ -155,7 +154,6 @@
4
1
2
- 4
3
@@ -247,7 +245,6 @@
4
1
2
- 4
3
@@ -339,7 +336,6 @@
4
1
2
- 4
3
@@ -431,7 +427,6 @@
4
1
2
- 4
3
@@ -523,7 +518,6 @@
4
1
2
- 4
3
@@ -615,7 +609,6 @@
4
1
2
- 4
3
@@ -707,7 +700,6 @@
4
1
2
- 4
3
diff --git a/gem2caom2/tests/data/michelle/N20070310S0156.expected.xml b/gem2caom2/tests/data/michelle/N20070310S0156.expected.xml
index d22086c0..57bf69f1 100644
--- a/gem2caom2/tests/data/michelle/N20070310S0156.expected.xml
+++ b/gem2caom2/tests/data/michelle/N20070310S0156.expected.xml
@@ -63,7 +63,6 @@
4
1
2
- 4
3
@@ -155,7 +154,6 @@
4
1
2
- 4
3
@@ -247,7 +245,6 @@
4
1
2
- 4
3
@@ -339,7 +336,6 @@
4
1
2
- 4
3
diff --git a/gem2caom2/tests/data/michelle/N20080612S0038.expected.xml b/gem2caom2/tests/data/michelle/N20080612S0038.expected.xml
index 57e901d3..2de2a8ce 100644
--- a/gem2caom2/tests/data/michelle/N20080612S0038.expected.xml
+++ b/gem2caom2/tests/data/michelle/N20080612S0038.expected.xml
@@ -63,7 +63,6 @@
4
1
2
- 4
3
@@ -155,7 +154,6 @@
4
1
2
- 4
3
@@ -247,7 +245,6 @@
4
1
2
- 4
3
@@ -339,7 +336,6 @@
4
1
2
- 4
3
@@ -431,7 +427,6 @@
4
1
2
- 4
3
@@ -523,7 +518,6 @@
4
1
2
- 4
3
@@ -615,7 +609,6 @@
4
1
2
- 4
3
@@ -707,7 +700,6 @@
4
1
2
- 4
3
@@ -799,7 +791,6 @@
4
1
2
- 4
3
@@ -891,7 +882,6 @@
4
1
2
- 4
3
@@ -983,7 +973,6 @@
4
1
2
- 4
3
@@ -1075,7 +1064,6 @@
4
1
2
- 4
3
@@ -1167,7 +1155,6 @@
4
1
2
- 4
3
@@ -1259,7 +1246,6 @@
4
1
2
- 4
3
@@ -1351,7 +1337,6 @@
4
1
2
- 4
3
@@ -1443,7 +1428,6 @@
4
1
2
- 4
3
@@ -1535,7 +1519,6 @@
4
1
2
- 4
3
@@ -1627,7 +1610,6 @@
4
1
2
- 4
3
@@ -1719,7 +1701,6 @@
4
1
2
- 4
3
@@ -1811,7 +1792,6 @@
4
1
2
- 4
3
@@ -1903,7 +1883,6 @@
4
1
2
- 4
3
@@ -1995,7 +1974,6 @@
4
1
2
- 4
3
@@ -2087,7 +2065,6 @@
4
1
2
- 4
3
@@ -2179,7 +2156,6 @@
4
1
2
- 4
3
@@ -2271,7 +2247,6 @@
4
1
2
- 4
3
@@ -2363,7 +2338,6 @@
4
1
2
- 4
3
@@ -2455,7 +2429,6 @@
4
1
2
- 4
3
@@ -2547,7 +2520,6 @@
4
1
2
- 4
3
@@ -2639,7 +2611,6 @@
4
1
2
- 4
3
@@ -2731,7 +2702,6 @@
4
1
2
- 4
3
@@ -2823,7 +2793,6 @@
4
1
2
- 4
3
@@ -2915,7 +2884,6 @@
4
1
2
- 4
3
@@ -3007,7 +2975,6 @@
4
1
2
- 4
3
@@ -3099,7 +3066,6 @@
4
1
2
- 4
3
@@ -3191,7 +3157,6 @@
4
1
2
- 4
3
@@ -3283,7 +3248,6 @@
4
1
2
- 4
3
@@ -3375,7 +3339,6 @@
4
1
2
- 4
3
@@ -3467,7 +3430,6 @@
4
1
2
- 4
3
@@ -3559,7 +3521,6 @@
4
1
2
- 4
3
@@ -3651,7 +3612,6 @@
4
1
2
- 4
3
diff --git a/gem2caom2/tests/data/michelle/N20100131S0131.expected.xml b/gem2caom2/tests/data/michelle/N20100131S0131.expected.xml
index 7379b508..21d60903 100644
--- a/gem2caom2/tests/data/michelle/N20100131S0131.expected.xml
+++ b/gem2caom2/tests/data/michelle/N20100131S0131.expected.xml
@@ -63,7 +63,6 @@
4
1
2
- 4
3
diff --git a/gem2caom2/tests/data/michelle/rN20050228S0074.expected.xml b/gem2caom2/tests/data/michelle/rN20050228S0074.expected.xml
index e097e5c8..4c13cef8 100644
--- a/gem2caom2/tests/data/michelle/rN20050228S0074.expected.xml
+++ b/gem2caom2/tests/data/michelle/rN20050228S0074.expected.xml
@@ -234,7 +234,6 @@
4
1
2
- 4
3
@@ -326,7 +325,6 @@
4
1
2
- 4
3
@@ -418,7 +416,6 @@
4
1
2
- 4
3
@@ -510,7 +507,6 @@
4
1
2
- 4
3
@@ -732,7 +728,6 @@
2
1
2
- 4
3
diff --git a/gem2caom2/tests/data/multi_plane/GS-2003B-Q-23-17-001.expected.xml b/gem2caom2/tests/data/multi_plane/GS-2003B-Q-23-17-001.expected.xml
index dfb76b74..adbecb53 100644
--- a/gem2caom2/tests/data/multi_plane/GS-2003B-Q-23-17-001.expected.xml
+++ b/gem2caom2/tests/data/multi_plane/GS-2003B-Q-23-17-001.expected.xml
@@ -58,7 +58,6 @@
4
1
2
- 4
3
@@ -150,7 +149,6 @@
4
1
2
- 4
3
@@ -242,7 +240,6 @@
4
1
2
- 4
3
@@ -334,7 +331,6 @@
4
1
2
- 4
3
@@ -426,7 +422,6 @@
4
1
2
- 4
3
@@ -518,7 +513,6 @@
4
1
2
- 4
3
@@ -610,7 +604,6 @@
4
1
2
- 4
3
@@ -702,7 +695,6 @@
4
1
2
- 4
3
@@ -794,7 +786,6 @@
4
1
2
- 4
3
@@ -886,7 +877,6 @@
4
1
2
- 4
3
@@ -978,7 +968,6 @@
4
1
2
- 4
3
@@ -1070,7 +1059,6 @@
4
1
2
- 4
3