From 58c14e4c776721adb0059c612b1f969401566907 Mon Sep 17 00:00:00 2001 From: Heike Konow Date: Fri, 22 Jan 2021 12:00:31 +0100 Subject: [PATCH 01/13] Add table with radiometer time offsets for EUREC4A --- functions/radiometerTimeOffset.m | 62 ++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 functions/radiometerTimeOffset.m diff --git a/functions/radiometerTimeOffset.m b/functions/radiometerTimeOffset.m new file mode 100644 index 0000000..b57e05a --- /dev/null +++ b/functions/radiometerTimeOffset.m @@ -0,0 +1,62 @@ +function [timeOffsetRange, timeOffsetValue] = radiometerTimeOffset(flightdate, frequency) + +timeOffsetsAll = {... + '20200119', 'HAMP-WF', ':', -141; + '20200122', 'none ', ':', 0; + '20200124', 'HAMP-KV', ':', -1; + '20200124', 'HAMP-WF', ':', -2; + '20200124', 'HAMP-G ', '1:21001', -7; + '20200124', 'HAMP-G ', '21001:29401', -3; + '20200126', 'HAMP-KV', '1:23351', -2; + '20200126', 'HAMP-WF', '1:23351', 1; + '20200126', 'HAMP-G ', '23701:', -3; + '20200128', 'none ', ':', 0; + '20200130', 'HAMP-WF', ':', -1; + '20200130', 'HAMP-G ', '21201:', -3; + '20200202', 'HAMP-KV', '4001:6211', -2; + '20200202', 'HAMP-WF', '1:2001', -3; + '20200202', 'HAMP-WF', '2001:4001', -1; + '20200202', 'HAMP-WF', '4001:6255', -3; + '20200202', 'HAMP-WF', '6255:6721', -2; + '20200202', 'HAMP-WF', '6721:', -1; + '20200202', 'HAMP-G ', ':', -2; + '20200205', 'HAMP-KV', ':', -3; + '20200205', 'HAMP-WF', ':', -4; + '20200207', 'HAMP-KV', '21301:23261', -2; + '20200207', 'HAMP-WF', '1:22401', 2; + '20200209', 'HAMP-G ', ':', -2; + '20200211', 'HAMP-KV', '1:22901', -2; + '20200211', 'HAMP-G ', '21001:23501', -5; + '20200213', 'none ', ':', 0; + '20200215', 'HAMP-KV', '20484:', -1; + '20200215', 'HAMP-WF', '2034:28757', 2; + '20200218', 'HAMP-WF', ':', -1; + '20200218', 'HAMP-G ', ':', -4; + }; + +offsetDates = timeOffsetsAll(:,1); +offsetModules = timeOffsetsAll(:,2); + +indexDate = strcmp(flightdate, offsetDates); + +% Explanation for different radiometer modules +% 1: 183, f>180 (G band) +% 2: 11990, f>=90 & f<180 (WF band) +% 3: KV, f<90 (KV band) + +if frequency >= 180 + freqString = 'HAMP-G '; +elseif frequency>=90 && frequency<180 + freqString = 'HAMP-WF'; +elseif frequency < 90 + freqString = 'HAMP-KV'; +else + error('Frequency not found') +end + +indexFrequency = strcmp(freqString, offsetModules); + +indexEntry = indexFrequency & indexDate; + +timeOffsetRange = timeOffsetsAll(indexEntry, 3); +timeOffsetValue = cell2mat(timeOffsetsAll(indexEntry, 4)); \ No newline at end of file From b6e4f69cb1dcf6451dfdbea8be0abc7434dc92d6 Mon Sep 17 00:00:00 2001 From: Heike Konow Date: Tue, 26 Jan 2021 09:38:20 +0100 Subject: [PATCH 02/13] include time correction in file --- functions/radiometerTimeOffset.m | 44 ++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/functions/radiometerTimeOffset.m b/functions/radiometerTimeOffset.m index b57e05a..6fa8307 100644 --- a/functions/radiometerTimeOffset.m +++ b/functions/radiometerTimeOffset.m @@ -1,4 +1,4 @@ -function [timeOffsetRange, timeOffsetValue] = radiometerTimeOffset(flightdate, frequency) +function [time, timeOffsetRange, timeOffsetValue] = radiometerTimeOffset(flightdate, frequency, time) timeOffsetsAll = {... '20200119', 'HAMP-WF', ':', -141; @@ -34,9 +34,11 @@ '20200218', 'HAMP-G ', ':', -4; }; +% Copy data to variables offsetDates = timeOffsetsAll(:,1); offsetModules = timeOffsetsAll(:,2); +% Find date index indexDate = strcmp(flightdate, offsetDates); % Explanation for different radiometer modules @@ -44,6 +46,7 @@ % 2: 11990, f>=90 & f<180 (WF band) % 3: KV, f<90 (KV band) +% Translate frequency to string from table above if frequency >= 180 freqString = 'HAMP-G '; elseif frequency>=90 && frequency<180 @@ -54,9 +57,46 @@ error('Frequency not found') end +% Find frequency index indexFrequency = strcmp(freqString, offsetModules); +% Find row where frequency and data match given date and frequency indexEntry = indexFrequency & indexDate; +% Extract entries from table timeOffsetRange = timeOffsetsAll(indexEntry, 3); -timeOffsetValue = cell2mat(timeOffsetsAll(indexEntry, 4)); \ No newline at end of file +timeOffsetValue = cell2mat(timeOffsetsAll(indexEntry, 4)); + +%%%%%%%%%%% +% Apply time offset values +for i=1:length(timeOffsetRange) + + % Get colon position from string + colPos = regexp(timeOffsetRange{i}, ':'); + + % Analyse time offset index + if strcmp(timeOffsetRange{i}, ':') % ':' + ind(1) = 1; + ind(2) = length(time); + + elseif strncmp(timeOffsetRange{i}, ':', 1) % ':yyyy' + ind(1) = 1; + a = timeOffsetRange{i}(2:end); + ind(2) = str2double(a); + + elseif colPos==length(timeOffsetRange{i}) + a = timeOffsetRange{i}(1:colPos-1); % 'xxxx:' + + ind(1) = str2double(a); + ind(2) = length(time); + + else % 'xxxx:yyyy' + ind{1} = timeOffsetRange{i}(1:colPos-1); + ind{2} = timeOffsetRange{i}(colPos+1:end); + ind = cellfun(@str2double, ind); + end + + % Apply offset to time array + time(ind(1):ind(2)) = time(ind(1):ind(2)) + timeOffsetValue(i) ./24./60./60; + clear ind +end \ No newline at end of file From 1e5b8001d1bcdc002907d43bc431a089743eb543 Mon Sep 17 00:00:00 2001 From: Heike Konow Date: Tue, 26 Jan 2021 09:38:58 +0100 Subject: [PATCH 03/13] implement radiometer time correction --- functions/run_unifyGrid.m | 5 +++-- functions/unifyGrid_radiometer.m | 10 +++++++++- start_hamp_processing.m | 9 +++++---- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/functions/run_unifyGrid.m b/functions/run_unifyGrid.m index 98b21ed..a57bcb2 100644 --- a/functions/run_unifyGrid.m +++ b/functions/run_unifyGrid.m @@ -1,5 +1,6 @@ function run_unifyGrid(version, subversion, flightdates_use, comment, contact, altitudeThreshold, ... - rollThreshold, radarmask, radarClutter, missingvalue, fillvalue, filenameprefix) + rollThreshold, radarmask, radarClutter, correctRadiometerTime, ... + missingvalue, fillvalue, filenameprefix) tic %% Switches @@ -71,7 +72,7 @@ function run_unifyGrid(version, subversion, flightdates_use, comment, contact, a % Radiometer unifyGrid_radiometer(pathtofolder,flightdates_use{i},uniTime,radiometerVars,... - altitudeThreshold, rollThreshold, missingvalue, fillvalue) + altitudeThreshold, rollThreshold, missingvalue, fillvalue, correctRadiometerTime) % Radar unifyGrid_radar(pathtofolder,flightdates_use{i},uniHeight,uniTime,radarVars) diff --git a/functions/unifyGrid_radiometer.m b/functions/unifyGrid_radiometer.m index d5d0ce8..df7e6cf 100644 --- a/functions/unifyGrid_radiometer.m +++ b/functions/unifyGrid_radiometer.m @@ -1,5 +1,6 @@ function unifyGrid_radiometer(pathtofolder, flightdate, uniTime, radiometerVars, ... - altitudeThreshold, rollThreshold, missingvalue, fillvalue) + altitudeThreshold, rollThreshold, missingvalue, fillvalue, ... + correctRadiometerTime) interpolate = 1; @@ -86,6 +87,13 @@ function unifyGrid_radiometer(pathtofolder, flightdate, uniTime, radiometerVars, % Round time to seconds to avoid numerical deviations radiometerTime = dateround(radiometerTime', 'second'); + if correctRadiometerTime + % freq(1) + % flightdate + [radiometerTime, ~, ~] = radiometerTimeOffset(flightdate, freq(1), radiometerTime); + + end + % Remove times in the future and past ind_off = find(radiometerTime > datenum(flightdate,'yyyymmdd')+2 | ... radiometerTime < datenum(flightdate,'yyyymmdd')-2); diff --git a/start_hamp_processing.m b/start_hamp_processing.m index 13cbae3..d0c92bc 100644 --- a/start_hamp_processing.m +++ b/start_hamp_processing.m @@ -15,9 +15,9 @@ %% Specify time frame for data conversion % Start date -t1 = '20200119'; +t1 = '20200124'; % End date -t2 = '20200228'; +t2 = '20200124'; % ! Add flight information to file flight_dates.m if they aren't already in % there @@ -28,10 +28,11 @@ quicklooks = true; removeClutter = true; removeRadiometerErrors = true; % Only possible if errors have been identified using run_assessData.m +correctRadiometerTime = true; %% Set version information version = 0; -subversion = 8; +subversion = 9; %% Missing value % Set value for missing value (pixels with no measured signal). This should @@ -76,7 +77,7 @@ if unifyGrid % Unify data from bahamas, dropsondes, radar, radiometer onto common grid run_unifyGrid(version, subversion, flightdates_use, comment, contact, altitudeThreshold, ... - rollThreshold, addRadarMask, removeClutter, missingvalue, fillvalue, filenameprefix) + rollThreshold, addRadarMask, removeClutter, correctRadiometerTime, missingvalue, fillvalue, filenameprefix) end if removeRadiometerErrors From 2b21579ae1c75f1a8606ff3bf5ac13dc3c9ec9e9 Mon Sep 17 00:00:00 2001 From: Heike Konow Date: Tue, 9 Feb 2021 13:32:56 +0100 Subject: [PATCH 04/13] Implement comments for time correction --- functions/radiometerTimeOffset.m | 39 ++++++++++++++++++++------------ functions/unifyGrid_radiometer.m | 20 ++++++++++++---- start_hamp_processing.m | 10 ++++---- 3 files changed, 45 insertions(+), 24 deletions(-) diff --git a/functions/radiometerTimeOffset.m b/functions/radiometerTimeOffset.m index 6fa8307..715f545 100644 --- a/functions/radiometerTimeOffset.m +++ b/functions/radiometerTimeOffset.m @@ -1,37 +1,39 @@ -function [time, timeOffsetRange, timeOffsetValue] = radiometerTimeOffset(flightdate, frequency, time) +function [time, timeOffsetRange, timeOffsetValue, commentstr] = ... + radiometerTimeOffset(flightdate, frequency, time) timeOffsetsAll = {... '20200119', 'HAMP-WF', ':', -141; - '20200122', 'none ', ':', 0; + '20200122', 'none', ':', 0; '20200124', 'HAMP-KV', ':', -1; - '20200124', 'HAMP-WF', ':', -2; - '20200124', 'HAMP-G ', '1:21001', -7; - '20200124', 'HAMP-G ', '21001:29401', -3; + '20200124', 'HAMP-WF', '1:20501', -2; + '20200124', 'HAMP-G', '1:21001', -7; + '20200124', 'HAMP-G', '21001:29401', -3; '20200126', 'HAMP-KV', '1:23351', -2; '20200126', 'HAMP-WF', '1:23351', 1; - '20200126', 'HAMP-G ', '23701:', -3; - '20200128', 'none ', ':', 0; - '20200130', 'HAMP-WF', ':', -1; - '20200130', 'HAMP-G ', '21201:', -3; + '20200126', 'HAMP-G', '23701:', -3; + '20200128', 'none', ':', 0; + '20200130', 'none', ':', 0; + '20200131', 'HAMP-WF', ':', -1; + '20200131', 'HAMP-G', '21201:', -3; '20200202', 'HAMP-KV', '4001:6211', -2; '20200202', 'HAMP-WF', '1:2001', -3; '20200202', 'HAMP-WF', '2001:4001', -1; '20200202', 'HAMP-WF', '4001:6255', -3; '20200202', 'HAMP-WF', '6255:6721', -2; '20200202', 'HAMP-WF', '6721:', -1; - '20200202', 'HAMP-G ', ':', -2; + '20200202', 'HAMP-G', ':', -2; '20200205', 'HAMP-KV', ':', -3; '20200205', 'HAMP-WF', ':', -4; '20200207', 'HAMP-KV', '21301:23261', -2; '20200207', 'HAMP-WF', '1:22401', 2; - '20200209', 'HAMP-G ', ':', -2; + '20200209', 'HAMP-G', ':', -2; '20200211', 'HAMP-KV', '1:22901', -2; - '20200211', 'HAMP-G ', '21001:23501', -5; - '20200213', 'none ', ':', 0; + '20200211', 'HAMP-G', '21001:23501', -5; + '20200213', 'none', ':', 0; '20200215', 'HAMP-KV', '20484:', -1; '20200215', 'HAMP-WF', '2034:28757', 2; '20200218', 'HAMP-WF', ':', -1; - '20200218', 'HAMP-G ', ':', -4; + '20200218', 'HAMP-G', ':', -4; }; % Copy data to variables @@ -48,7 +50,7 @@ % Translate frequency to string from table above if frequency >= 180 - freqString = 'HAMP-G '; + freqString = 'HAMP-G'; elseif frequency>=90 && frequency<180 freqString = 'HAMP-WF'; elseif frequency < 90 @@ -63,12 +65,17 @@ % Find row where frequency and data match given date and frequency indexEntry = indexFrequency & indexDate; +% List Modules in variable +modules = offsetModules(indexEntry); + % Extract entries from table timeOffsetRange = timeOffsetsAll(indexEntry, 3); timeOffsetValue = cell2mat(timeOffsetsAll(indexEntry, 4)); %%%%%%%%%%% % Apply time offset values + +commentstr = cell(1, length(timeOffsetRange)); for i=1:length(timeOffsetRange) % Get colon position from string @@ -99,4 +106,6 @@ % Apply offset to time array time(ind(1):ind(2)) = time(ind(1):ind(2)) + timeOffsetValue(i) ./24./60./60; clear ind + + commentstr{i} = [modules{i} ': (' timeOffsetRange{i} ') ' num2str(timeOffsetValue(i)) ' s']; end \ No newline at end of file diff --git a/functions/unifyGrid_radiometer.m b/functions/unifyGrid_radiometer.m index df7e6cf..a654a36 100644 --- a/functions/unifyGrid_radiometer.m +++ b/functions/unifyGrid_radiometer.m @@ -20,6 +20,7 @@ function unifyGrid_radiometer(pathtofolder, flightdate, uniTime, radiometerVars, % radiometerVars = {'183','11990','KV'}; interpolate_flag = cell(1,3); +corrComment = cell(1, length(radiometerVars)); for i=1:length(radiometerVars) @@ -87,10 +88,11 @@ function unifyGrid_radiometer(pathtofolder, flightdate, uniTime, radiometerVars, % Round time to seconds to avoid numerical deviations radiometerTime = dateround(radiometerTime', 'second'); + % If time offsets for radiometer were found, apply offsets to time + % arrays if correctRadiometerTime - % freq(1) - % flightdate - [radiometerTime, ~, ~] = radiometerTimeOffset(flightdate, freq(1), radiometerTime); + + [radiometerTime, ~, ~, corrComment{i}] = radiometerTimeOffset(flightdate, freq(1), radiometerTime); end @@ -257,7 +259,17 @@ function unifyGrid_radiometer(pathtofolder, flightdate, uniTime, radiometerVars, clear indTimeUni indTimeRadiometer uniDataRadiometer data freq radiometerTime end -% Combine measurements from all modules into one variable +%% Create comment string with time offset information +% Create cell with delimiters for comments +c = cell(4, 1); +c(1:3) = {', '}; +c(4) = {' '}; + +corrComment = join([[corrComment{:}]' c]); +corrCommentString = join(['time corrections: ', corrComment']); + + +%% Combine measurements from all modules into one variable uniRadiometer = [uniRadiometerKV;... uniRadiometer11990;... uniRadiometer183]; diff --git a/start_hamp_processing.m b/start_hamp_processing.m index d0c92bc..4c18d2f 100644 --- a/start_hamp_processing.m +++ b/start_hamp_processing.m @@ -22,12 +22,12 @@ % there %% Processing steps -correctAttitude = true; -addRadarMask = true; +correctAttitude = false; +addRadarMask = false; unifyGrid = true; -quicklooks = true; -removeClutter = true; -removeRadiometerErrors = true; % Only possible if errors have been identified using run_assessData.m +quicklooks = false; +removeClutter = false; +removeRadiometerErrors = false; % Only possible if errors have been identified using run_assessData.m correctRadiometerTime = true; %% Set version information From 9c67bcaa4b623fea05ad1cbf742793721325c061 Mon Sep 17 00:00:00 2001 From: Heike Konow Date: Wed, 10 Feb 2021 17:07:02 +0100 Subject: [PATCH 05/13] Add function to flatten cells --- functions/cellflat.m | 59 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 functions/cellflat.m diff --git a/functions/cellflat.m b/functions/cellflat.m new file mode 100644 index 0000000..2039658 --- /dev/null +++ b/functions/cellflat.m @@ -0,0 +1,59 @@ +function out = cellflat(celllist) +% CELLFLAT is a helper function to flatten nested cell arrays. +% +% CELLFLAT(celllist) searches every cell element in cellist and put them on +% the top most level. Therefore, CELLFLAT linearizes a cell array tree +% structure. +% +% Example: cellflat({[1 2 3], [4 5 6],{[7 8 9 10],[11 12 13 14 15]},{'abc',{'defg','hijk'},'lmnop'}}) +% +% Output: +%Columns 1 through 7 +% [1x3 double] [1x3 double] [1x4 double] [1x5 double] 'abc' 'defg' 'hijk' +% Column 8 +% 'lmnop' +% +% cellflat(({{1 {2 3}} 'z' {'y' 'x' 'w'} {4 @iscell 5} 6}) ) +% Output: +% [1] [2] [3] 'z' 'y' 'x' 'w' [4] @iscell [5] [6] +% +% Version: 1.0 +% Author: Yung-Yeh Chang, Ph.D. (yungyeh@hotmail.com) +% Date: 12/31/2014 +% Copyright 2015, Yung-Yeh Chang, Ph.D. +% See Also: cell +% +% Copyright (c) 2017, Yung-Yeh +% All rights reserved. +% +% Redistribution and use in source and binary forms, with or without +% modification, are permitted provided that the following conditions are met: +% +% * Redistributions of source code must retain the above copyright notice, this +% list of conditions and the following disclaimer. +% +% * Redistributions in binary form must reproduce the above copyright notice, +% this list of conditions and the following disclaimer in the documentation +% and/or other materials provided with the distribution +% THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +% AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +% IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +% DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +% FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +% DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +% SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +% CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +% OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +% OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +if ~iscell(celllist) + error('CELLFLAT:ImproperInputAugument','Input argument must be a cell array'); +end +out = {}; +for idx_c = 1:numel(celllist) + if iscell(celllist{idx_c}) + out = [out cellflat(celllist{idx_c})]; + else + out = [out celllist(idx_c)]; + end +end \ No newline at end of file From 10a7840050a39896c0b3297dbdb870968889b40d Mon Sep 17 00:00:00 2001 From: Heike Konow Date: Wed, 10 Feb 2021 17:07:54 +0100 Subject: [PATCH 06/13] Add time correction information as attribute to output NetCDF files --- functions/prepareMat2NetCDF.m | 4 ++ functions/radiometerTimeOffset.m | 91 +++++++++++++++++--------------- functions/run_unifyGrid.m | 13 ++++- functions/unifyGrid_radiometer.m | 34 +++++++++--- start_hamp_processing.m | 10 ++-- 5 files changed, 98 insertions(+), 54 deletions(-) diff --git a/functions/prepareMat2NetCDF.m b/functions/prepareMat2NetCDF.m index eef4c17..028d8ef 100644 --- a/functions/prepareMat2NetCDF.m +++ b/functions/prepareMat2NetCDF.m @@ -45,6 +45,10 @@ sizes(strcmp(varnames,'extra_info')) = []; % Delete variable name of variable 'extra_info' varnames(strcmp(varnames,'extra_info')) = []; +% Delete size information of variable 'corrCommentString' +sizes(strcmp(varnames,'corrCommentString')) = []; +% Delete variable name of variable 'corrCommentString' +varnames(strcmp(varnames,'corrCommentString')) = []; % Concatenate dimension sizes and variable size % Here, just the order is changed so that the dimensions will be written diff --git a/functions/radiometerTimeOffset.m b/functions/radiometerTimeOffset.m index 715f545..5f5c62e 100644 --- a/functions/radiometerTimeOffset.m +++ b/functions/radiometerTimeOffset.m @@ -65,47 +65,54 @@ % Find row where frequency and data match given date and frequency indexEntry = indexFrequency & indexDate; -% List Modules in variable -modules = offsetModules(indexEntry); - -% Extract entries from table -timeOffsetRange = timeOffsetsAll(indexEntry, 3); -timeOffsetValue = cell2mat(timeOffsetsAll(indexEntry, 4)); - -%%%%%%%%%%% -% Apply time offset values - -commentstr = cell(1, length(timeOffsetRange)); -for i=1:length(timeOffsetRange) - - % Get colon position from string - colPos = regexp(timeOffsetRange{i}, ':'); - - % Analyse time offset index - if strcmp(timeOffsetRange{i}, ':') % ':' - ind(1) = 1; - ind(2) = length(time); - - elseif strncmp(timeOffsetRange{i}, ':', 1) % ':yyyy' - ind(1) = 1; - a = timeOffsetRange{i}(2:end); - ind(2) = str2double(a); - - elseif colPos==length(timeOffsetRange{i}) - a = timeOffsetRange{i}(1:colPos-1); % 'xxxx:' - - ind(1) = str2double(a); - ind(2) = length(time); - - else % 'xxxx:yyyy' - ind{1} = timeOffsetRange{i}(1:colPos-1); - ind{2} = timeOffsetRange{i}(colPos+1:end); - ind = cellfun(@str2double, ind); +if any(indexEntry) + + % List Modules in variable + modules = offsetModules(indexEntry); + + % Extract entries from table + timeOffsetRange = timeOffsetsAll(indexEntry, 3); + timeOffsetValue = cell2mat(timeOffsetsAll(indexEntry, 4)); + + %%%%%%%%%%% + % Apply time offset values + + commentstr = cell(1, length(timeOffsetRange)); + for i=1:length(timeOffsetRange) + + % Get colon position from string + colPos = regexp(timeOffsetRange{i}, ':'); + + % Analyse time offset index + if strcmp(timeOffsetRange{i}, ':') % ':' + ind(1) = 1; + ind(2) = length(time); + + elseif strncmp(timeOffsetRange{i}, ':', 1) % ':yyyy' + ind(1) = 1; + a = timeOffsetRange{i}(2:end); + ind(2) = str2double(a); + + elseif colPos==length(timeOffsetRange{i}) + a = timeOffsetRange{i}(1:colPos-1); % 'xxxx:' + + ind(1) = str2double(a); + ind(2) = length(time); + + else % 'xxxx:yyyy' + ind{1} = timeOffsetRange{i}(1:colPos-1); + ind{2} = timeOffsetRange{i}(colPos+1:end); + ind = cellfun(@str2double, ind); + end + + % Apply offset to time array + time(ind(1):ind(2)) = time(ind(1):ind(2)) + timeOffsetValue(i) ./24./60./60; + clear ind + + commentstr{i} = [modules{i} ': (' timeOffsetRange{i} ') ' num2str(timeOffsetValue(i)) ' s']; end - - % Apply offset to time array - time(ind(1):ind(2)) = time(ind(1):ind(2)) + timeOffsetValue(i) ./24./60./60; - clear ind - - commentstr{i} = [modules{i} ': (' timeOffsetRange{i} ') ' num2str(timeOffsetValue(i)) ' s']; +else + commentstr = {}; + timeOffsetRange = {}; + timeOffsetValue = {}; end \ No newline at end of file diff --git a/functions/run_unifyGrid.m b/functions/run_unifyGrid.m index a57bcb2..1cac91e 100644 --- a/functions/run_unifyGrid.m +++ b/functions/run_unifyGrid.m @@ -95,7 +95,7 @@ function run_unifyGrid(version, subversion, flightdates_use, comment, contact, a %% Export to netcdf -instr = {'radar','bahamas','radiometer','dropsondes'}; +instr = {'radiometer','radar','bahamas','dropsondes'}; % instr = {'bahamas','radar','radiometer'}; % instr = {'radar'}; % instr = {'bahamas'}; @@ -220,6 +220,11 @@ function run_unifyGrid(version, subversion, flightdates_use, comment, contact, a removeClutter(outfile, missingvalue, fillvalue) end + %% Add comment about radiometer time correction + if correctRadiometerTime && strcmp(instr{j}, 'radiometer') + addRadiometerTimeComment(outfile, infile) + end + else disp(['No ' instr{j} ' data found']) end @@ -430,3 +435,9 @@ function removeSideLobes(outfile, rollThreshold, fillvalue, radarmask) ncwriteatt(outfile, 'data_flag', 'long_name', longNameAtt) end end + +function addRadiometerTimeComment(outfile, infile) + string = load(infile, 'corrCommentString'); + + ncwriteatt(outfile, 'tb', 'comment', string.corrCommentString) +end diff --git a/functions/unifyGrid_radiometer.m b/functions/unifyGrid_radiometer.m index a654a36..ac3b762 100644 --- a/functions/unifyGrid_radiometer.m +++ b/functions/unifyGrid_radiometer.m @@ -261,12 +261,34 @@ function unifyGrid_radiometer(pathtofolder, flightdate, uniTime, radiometerVars, %% Create comment string with time offset information % Create cell with delimiters for comments -c = cell(4, 1); -c(1:3) = {', '}; -c(4) = {' '}; -corrComment = join([[corrComment{:}]' c]); -corrCommentString = join(['time corrections: ', corrComment']); +if correctRadiometerTime + + % Reshape cells to include multiple comments per radiometer module + corrComment = [corrComment{:}]; + + c = cell(length(corrComment), 1); + c(1:length(corrComment)-1) = {', '}; + if ~isempty(corrComment) + c(length(corrComment)) = {' '}; + else + corrComment{1} = 'none'; + end + + % Delete empty comment string entries (where no correction was applied) + c(cellfun(@isempty, corrComment)) = []; + corrComment(cellfun(@isempty, corrComment)) = []; + + if length(corrComment)==1 +% corrComment = corrComment{1}{1}; + corrComment = cellflat(corrComment); + corrCommentString = ['time corrections: ', corrComment{1}]; + else +% corrComment = join([[corrComment{:}]' c]); + corrComment = join([corrComment' c]); + corrCommentString = cell2mat(join(['time corrections: ', corrComment'])); + end +end %% Combine measurements from all modules into one variable @@ -307,7 +329,7 @@ function unifyGrid_radiometer(pathtofolder, flightdate, uniTime, radiometerVars, clear uniData unitsTemp % Save data to file -save(outfile,'uni*','flightdate','extra_info','interpolate_flag') +save(outfile,'uni*','flightdate','extra_info','interpolate_flag', 'corrCommentString') end diff --git a/start_hamp_processing.m b/start_hamp_processing.m index 4c18d2f..da93355 100644 --- a/start_hamp_processing.m +++ b/start_hamp_processing.m @@ -15,9 +15,9 @@ %% Specify time frame for data conversion % Start date -t1 = '20200124'; +t1 = '20200119'; % End date -t2 = '20200124'; +t2 = '20200218'; % ! Add flight information to file flight_dates.m if they aren't already in % there @@ -25,9 +25,9 @@ correctAttitude = false; addRadarMask = false; unifyGrid = true; -quicklooks = false; -removeClutter = false; -removeRadiometerErrors = false; % Only possible if errors have been identified using run_assessData.m +quicklooks = true; +removeClutter = true; +removeRadiometerErrors = true; % Only possible if errors have been identified using run_assessData.m correctRadiometerTime = true; %% Set version information From 8a65e2f8f7b768bafd7f6f78f703e73db6e6f72c Mon Sep 17 00:00:00 2001 From: Heike Konow Date: Fri, 12 Feb 2021 07:53:26 +0100 Subject: [PATCH 07/13] Add error indices for one flight --- functions/radiometerErrorsSingleChannelLookup.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/radiometerErrorsSingleChannelLookup.m b/functions/radiometerErrorsSingleChannelLookup.m index ef18e1d..8449549 100644 --- a/functions/radiometerErrorsSingleChannelLookup.m +++ b/functions/radiometerErrorsSingleChannelLookup.m @@ -15,7 +15,7 @@ '20200122', 23.84, {[115703 115760], [119740 119780]};... '20200207', 90, {[94442 94449]};... '20200130', 190.81, {[44438 63007]};... - '', 190.81, {[]};... + '20200211', 23.84, {[22209 22240]};... '', 190.81, {[]};... '', 190.81, {[]};... }; \ No newline at end of file From a3dee1f83b9553c18be227e23bf815f32a69e8d4 Mon Sep 17 00:00:00 2001 From: Heike Konow Date: Fri, 12 Feb 2021 08:15:45 +0100 Subject: [PATCH 08/13] Add function to get indices in raw measurements from given time interval --- findRadiometerErrorindex.m | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 findRadiometerErrorindex.m diff --git a/findRadiometerErrorindex.m b/findRadiometerErrorindex.m new file mode 100644 index 0000000..97feb9b --- /dev/null +++ b/findRadiometerErrorindex.m @@ -0,0 +1,29 @@ +function [idx] = findRadiometerErrorindex(time, frequency, varargin) + +% Convert time format +if isdatetime(time) + time = datenum(time); +elseif nargin>2 + time = datenum(time, varargin{1}); +end + +% Check time format +if ~(all(time>700000) && all(time<800000)) + error('Time must either be given as serial date number, datetime, or string with string format as third input.') +end + +% Get string for frequency +freqString = getHAMPfrequencyString(frequency); + +% Get date as string +day = datestr(time(1), 'yyyymmdd'); + +% Get path to measurement file for current date +filepath = listFiles([getPathPrefix getCampaignFolder(day) 'radiometer/' freqString '/*' day(3:end) '*BRT*NC'], 'full', 'mat'); + +% Read time and convert to serial date number +t = time2001_2sdn(ncread(filepath, 'time')); + +% Find indices of given time +idx(1) = find(t>=time(1), 1, 'first'); +idx(2) = find(t<=time(2), 1, 'last'); \ No newline at end of file From 45cc173dc6d60a074a392cbb31b71785306e8aae Mon Sep 17 00:00:00 2001 From: Heike Konow Date: Fri, 12 Feb 2021 08:19:22 +0100 Subject: [PATCH 09/13] Add function to get module name string for a given frequency --- functions/getHAMPfrequencyString.m | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 functions/getHAMPfrequencyString.m diff --git a/functions/getHAMPfrequencyString.m b/functions/getHAMPfrequencyString.m new file mode 100644 index 0000000..b21a313 --- /dev/null +++ b/functions/getHAMPfrequencyString.m @@ -0,0 +1,22 @@ +%% freqString = getHAMPfrequencyString(frequency) +% +% Use this function to get the module name string for a given frequency +% +% Explanation for different radiometer modules +% 1: 183, f>180 (G band) +% 2: 11990, f>=90 & f<180 (WF band) +% 3: KV, f<90 (KV band) + +function freqString = getHAMPfrequencyString(frequency) + + +% Translate frequency to string from table above +if frequency >= 180 + freqString = '183'; +elseif frequency>=90 && frequency<180 + freqString = '11990'; +elseif frequency < 90 + freqString = 'KV'; +else + error('Frequency not found') +end \ No newline at end of file From 76985a568b2594ab7eae179017fed4deed3a70bc Mon Sep 17 00:00:00 2001 From: Heike Konow Date: Fri, 12 Feb 2021 08:19:43 +0100 Subject: [PATCH 10/13] Add description --- functions/radiometerErrorsSingleChannelLookup.m | 3 +++ 1 file changed, 3 insertions(+) diff --git a/functions/radiometerErrorsSingleChannelLookup.m b/functions/radiometerErrorsSingleChannelLookup.m index 8449549..e60446b 100644 --- a/functions/radiometerErrorsSingleChannelLookup.m +++ b/functions/radiometerErrorsSingleChannelLookup.m @@ -2,6 +2,9 @@ % Numbers represent indices in raw measurement data and are not yet % converted to times % +% Indices can be found by using findRadiometerErrorindex for given time +% intervals. +% % Note individual indices or intervals in the form [ii jj] function errors = radiometerErrorsSingleChannelLookup From 25b46c9e14079f722ca09e00177329ab49af8ce0 Mon Sep 17 00:00:00 2001 From: Heike Konow Date: Fri, 12 Feb 2021 13:32:09 +0100 Subject: [PATCH 11/13] Correct minor processing hickups --- functions/convertRadarErrorTimes.m | 7 ++++++- functions/convertRadiometerErrorTimesSingleChannel.m | 3 ++- functions/listFiles.m | 3 ++- functions/removeTBErrors.m | 2 +- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/functions/convertRadarErrorTimes.m b/functions/convertRadarErrorTimes.m index 2379998..373a4be 100644 --- a/functions/convertRadarErrorTimes.m +++ b/functions/convertRadarErrorTimes.m @@ -70,6 +70,11 @@ function convertRadarErrorTimes(campaign) % Copy errors to variables errorsDay = errors{indDay,2}; + + % If variable is not cell, convert to cell + if ~iscell(errorsDay) + errorsDay = {errorsDay}; + end % Read time from unified data timeUni = ncread(filepathnc,'time'); @@ -80,7 +85,7 @@ function convertRadarErrorTimes(campaign) timeErrorFlag = zeros(size(timeUni)); % If errors array is not empty - if sum(cellfun(@isempty,errorsDay))==0 + if sum(cellfun(@isempty,errorsDay))==0 && ~isempty(timeRaw) % Loop all errors for current date for k=1:length(errorsDay) diff --git a/functions/convertRadiometerErrorTimesSingleChannel.m b/functions/convertRadiometerErrorTimesSingleChannel.m index e14241f..eac4671 100644 --- a/functions/convertRadiometerErrorTimesSingleChannel.m +++ b/functions/convertRadiometerErrorTimesSingleChannel.m @@ -141,4 +141,5 @@ function convertRadiometerErrorTimesSingleChannel(campaign) checkandcreate(basefolder, 'aux') % Save flags to file -save([basefolder 'aux/errorFlagRadiometer.mat'],'errorFlagSingleChannel','frequencySingleChannel','dateSingleChannel','instrSingleChannel','-append') \ No newline at end of file +% save([basefolder 'aux/errorFlagRadiometer.mat'],'errorFlagSingleChannel','frequencySingleChannel','dateSingleChannel','instrSingleChannel','-append') +save([basefolder 'aux/errorFlagRadiometer.mat'],'errorFlagSingleChannel','frequencySingleChannel','dateSingleChannel','-append') \ No newline at end of file diff --git a/functions/listFiles.m b/functions/listFiles.m index cd86329..8972a10 100644 --- a/functions/listFiles.m +++ b/functions/listFiles.m @@ -111,7 +111,8 @@ end if any(strcmp(varargin, 'mat')) - Files = [Files{:}]; +% Files = [Files{:}]; + Files = cell2mat(Files); end end diff --git a/functions/removeTBErrors.m b/functions/removeTBErrors.m index 1ee5dc9..72173f1 100644 --- a/functions/removeTBErrors.m +++ b/functions/removeTBErrors.m @@ -10,7 +10,7 @@ if any(strcmp(extractfield(vars, 'name'), 'errorFlagSingleChannel')) - load(datafile, 'dateSingleChannel', 'errorFlagSingleChannel', 'frequencySingleChannel', 'instrSingleChannel') + load(datafile, 'dateSingleChannel', 'errorFlagSingleChannel', 'frequencySingleChannel') singleChannelError = true; end From d819b0d9b5ad1fa90041deadf90161bd65e23675 Mon Sep 17 00:00:00 2001 From: Heike Konow Date: Fri, 12 Feb 2021 13:32:37 +0100 Subject: [PATCH 12/13] Remove errors in one channel on 20200211 --- functions/radiometerErrorsSingleChannelLookup.m | 2 +- run_assessData.m | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/functions/radiometerErrorsSingleChannelLookup.m b/functions/radiometerErrorsSingleChannelLookup.m index e60446b..e15b879 100644 --- a/functions/radiometerErrorsSingleChannelLookup.m +++ b/functions/radiometerErrorsSingleChannelLookup.m @@ -18,7 +18,7 @@ '20200122', 23.84, {[115703 115760], [119740 119780]};... '20200207', 90, {[94442 94449]};... '20200130', 190.81, {[44438 63007]};... - '20200211', 23.84, {[22209 22240]};... + '20200211', 23.84, {[22207 22240]};... '', 190.81, {[]};... '', 190.81, {[]};... }; \ No newline at end of file diff --git a/run_assessData.m b/run_assessData.m index b8983ce..fbe408b 100644 --- a/run_assessData.m +++ b/run_assessData.m @@ -53,7 +53,7 @@ % Set if error time steps should be calculated from indices calc = false; % Set if overview figure should be produced -overview = true; +overview = false; % Set if you want to check the results of error removal check = false; % Set campaign to analyse From 3800bffc42e37b124028d91b07a6d66ccdb5deb5 Mon Sep 17 00:00:00 2001 From: Heike Konow Date: Fri, 12 Feb 2021 13:33:13 +0100 Subject: [PATCH 13/13] Process time corrected and error corrected radiometer data v0.9 --- start_hamp_processing.m | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/start_hamp_processing.m b/start_hamp_processing.m index da93355..1b113fd 100644 --- a/start_hamp_processing.m +++ b/start_hamp_processing.m @@ -15,15 +15,15 @@ %% Specify time frame for data conversion % Start date -t1 = '20200119'; +t1 = '20200118'; % End date -t2 = '20200218'; +t2 = '20200219'; % ! Add flight information to file flight_dates.m if they aren't already in % there %% Processing steps correctAttitude = false; -addRadarMask = false; +addRadarMask = true; unifyGrid = true; quicklooks = true; removeClutter = true;