diff --git a/py/fiberassign/assign.py b/py/fiberassign/assign.py index 548d00b0..35509307 100644 --- a/py/fiberassign/assign.py +++ b/py/fiberassign/assign.py @@ -818,6 +818,58 @@ def merge_results_tile_initialize(tgbufs, tgdtypes, tgshapes, skybufs, merge_results_tile_skyshapes = skyshapes return +# minimal set of columns to read from the target file +minimal_target_columns = OrderedDict([ + ('BRICKNAME', 'S8'), + ('BRICKID', '>i4') , + ('BRICK_OBJID', '>i4'), + ('MORPHTYPE', 'S4'), + ('RA', '>f8'), + ('DEC', '>f8'), + ('EBV', '>f4'), + ('FLUX_G', '>f4'), + ('FLUX_R', '>f4'), + ('FLUX_Z', '>f4'), + ('FLUX_IVAR_G', '>f4'), + ('FLUX_IVAR_R', '>f4'), + ('FLUX_IVAR_Z', '>f4'), + ('FLUX_W1', '>f4'), + ('FLUX_W2', '>f4'), + ('FIBERFLUX_G', '>f4'), + ('FIBERFLUX_R', '>f4'), + ('FIBERFLUX_Z', '>f4'), + ('FIBERTOTFLUX_G', '>f4'), + ('FIBERTOTFLUX_R', '>f4'), + ('FIBERTOTFLUX_Z', '>f4'), + ('REF_EPOCH', '>f4'), + ('MASKBITS', '>i2'), + ('FRACDEV', '>f4'), + ('SHAPEDEV_R', '>f4'), + ('SHAPEDEV_E1', '>f4'), + ('SHAPEDEV_E2', '>f4'), + ('SHAPEEXP_R', '>f4'), + ('SHAPEEXP_E1', '>f4'), + ('SHAPEEXP_E2', '>f4'), + ('REF_ID', '>i8'), + ('REF_CAT', 'S2'), + ('GAIA_PHOT_G_MEAN_MAG', '>f4'), + ('GAIA_PHOT_BP_MEAN_MAG', '>f4'), + ('GAIA_PHOT_RP_MEAN_MAG', '>f4'), + ('PARALLAX', '>f4'), + ('PMRA', '>f4'), + ('PMDEC', '>f4'), + ('PHOTSYS', 'S1'), + ('TARGETID', '>i8'), + ('DESI_TARGET', '>i8'), + ('BGS_TARGET', '>i8'), + ('MWS_TARGET', '>i8'), + ('CMX_TARGET', '>i8'), + ('SV0_TARGET', '>i8'), + ('SUBPRIORITY', '>f8'), + ('OBSCONDITIONS', '>i8'), + ('PRIORITY_INIT', '>i8'), + ('NUMOBS_INIT', '>i8') +]) merged_fiberassign_swap = { "RA": "TARGET_RA", @@ -1271,30 +1323,10 @@ def merge_results(targetfiles, skyfiles, tiles, result_dir=".", survey = None - some_dt = np.dtype([('BRICKNAME', 'S8'), ('BRICKID', '>i4') , ('BRICK_OBJID', '>i4'), - ('MORPHTYPE', 'S4'), - ('RA', '>f8'), ('DEC', '>f8'), - ('EBV', '>f4'), - ('FLUX_G', '>f4'), ('FLUX_R', '>f4'), ('FLUX_Z', '>f4'), - ('FLUX_IVAR_G', '>f4'), ('FLUX_IVAR_R', '>f4'), ('FLUX_IVAR_Z', '>f4'), - ('FLUX_W1', '>f4'), ('FLUX_W2', '>f4'), - ('FIBERFLUX_G', '>f4'), ('FIBERFLUX_R', '>f4'), ('FIBERFLUX_Z', '>f4'), - ('FIBERTOTFLUX_G', '>f4'), ('FIBERTOTFLUX_R', '>f4'), ('FIBERTOTFLUX_Z', '>f4'), - ('REF_EPOCH', '>f4'), ('MASKBITS', '>i2'), - ('FRACDEV', '>f4'), - ('SHAPEDEV_R', '>f4'), ('SHAPEDEV_E1', '>f4'), - ('SHAPEDEV_E2', '>f4'), - ('SHAPEEXP_R', '>f4'), ('SHAPEEXP_E1', '>f4'), - ('SHAPEEXP_E2', '>f4'), - ('REF_ID', '>i8'), ('REF_CAT', 'S2'), - ('GAIA_PHOT_G_MEAN_MAG', '>f4'), ('GAIA_PHOT_BP_MEAN_MAG', '>f4'), - ('GAIA_PHOT_RP_MEAN_MAG', '>f4'), - ('PARALLAX', '>f4'), ('PMRA', '>f4'), ('PMDEC', '>f4'), - ('PHOTSYS', 'S1'), ('TARGETID', '>i8'), - ('DESI_TARGET', '>i8'), ('BGS_TARGET', '>i8'), ('MWS_TARGET', '>i8'), - ('SUBPRIORITY', '>f8'), ('OBSCONDITIONS', '>i8'), ('PRIORITY_INIT', '>i8'), - ('NUMOBS_INIT', '>i8')]) - some_columns = list(some_dt.fields.keys()) +# minimal_target_columns to read + minimal_dcolnames = [x for x in minimal_target_columns.keys()] + minimal_dcols = [(x, y) for x, y in minimal_target_columns.items()] + for tf in targetfiles: tm = Timer() tm.start() @@ -1304,6 +1336,18 @@ def merge_results(targetfiles, skyfiles, tiles, result_dir=".", tglen = fd[1].get_nrows() tgshape[tf] = (tglen,) #tgdtype[tf], tempoff, tempisvararray = fd[1].get_rec_dtype() + + #select what subset of the 'minimal_dcolnames' are present in the data. + file_tgdtype, tempoff, tempisvararray = fd[1].get_rec_dtype() + file_dcolnames = [x for x in file_tgdtype.names] + dcols_to_read = [] + for i in range(len(minimal_dcolnames)): + if minimal_dcolnames[i] in file_dcolnames: + dcols_to_read.append(minimal_dcols[i]) + some_dt = np.dtype(dcols_to_read) + some_columns = list(some_dt.fields.keys()) + + #print(file_tgdtype) tgdtype[tf] = some_dt tgbytes = tglen * tgdtype[tf].itemsize tgdata[tf] = RawArray("B", tgbytes)