From 2ca0f97ed3fb97a9a711914f7d62fb0339de165d Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 16 Apr 2024 08:37:11 -0700 Subject: [PATCH 01/16] Improve validation on requestor --- .../queries/labpurchasing/purchases.js | 25 +++++++++++++++++++ .../LabPurchasingTriggerHelper.java | 12 +++++++++ 2 files changed, 37 insertions(+) diff --git a/LabPurchasing/resources/queries/labpurchasing/purchases.js b/LabPurchasing/resources/queries/labpurchasing/purchases.js index 602ce37ff..cbcfa7d50 100644 --- a/LabPurchasing/resources/queries/labpurchasing/purchases.js +++ b/LabPurchasing/resources/queries/labpurchasing/purchases.js @@ -4,6 +4,31 @@ var console = require("console"); var triggerHelper = new org.labkey.labpurchasing.LabPurchasingTriggerHelper(LABKEY.Security.currentUser.id, LABKEY.Security.currentContainer.id); function beforeInsert(row, errors){ + beforeUpsert(row, errors) +} + +function beforeUpdate(row, oldRow, errors){ + beforeUpsert(row, errors) +} + +function beforeUpsert(row, errors){ + // Validate requestor: + if (row.requestor) { + if (!isNaN(row.requestor) && !triggerHelper.isValidUserId(row.requestor)) { + errors.requestor = 'Unknown userId for requestor: ' + row.requestor; + } + // Try to resolve strings: + else if (isNaN(row.requestor)) { + var id = triggerHelper.resolveUserId(String(row.requestor)); + if (!id) { + errors.requestor = 'Unknown userId for requestor: ' + row.requestor; + } + else { + row.requestor = id; + } + } + } + // The purpose of this is to allow the user to provide a string value for // vendorId or vendorName, and attempt to resolve this against known vendors: if (!row.vendorId || isNaN(row.vendorId)) { diff --git a/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingTriggerHelper.java b/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingTriggerHelper.java index f79c3688e..2625a0dca 100644 --- a/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingTriggerHelper.java +++ b/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingTriggerHelper.java @@ -112,4 +112,16 @@ public void sendNotifications(List rowIds) { _log.error("Unable to send purchasing email", e); } } + + public boolean isValidUserId(int userId) + { + return UserManager.getUser(userId) != null; + } + + public Integer resolveUserId(String userNameOrEmail) + { + User u = UserManager.getUserByDisplayName(userNameOrEmail); + + return u == null ? null : u.getUserId(); + } } From 3081251660d4e60837bd7e0f493520a23647d648 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 17 Apr 2024 08:26:03 -0700 Subject: [PATCH 02/16] Data validation for SNPRC ETL --- mcc/resources/etls/snprc-datasets.xml | 4 ++++ mcc/resources/queries/study/demographics.js | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/mcc/resources/etls/snprc-datasets.xml b/mcc/resources/etls/snprc-datasets.xml index 981a4855b..401ebba93 100644 --- a/mcc/resources/etls/snprc-datasets.xml +++ b/mcc/resources/etls/snprc-datasets.xml @@ -136,6 +136,10 @@ weight objectid + + + + diff --git a/mcc/resources/queries/study/demographics.js b/mcc/resources/queries/study/demographics.js index c1dc87412..1b0016228 100644 --- a/mcc/resources/queries/study/demographics.js +++ b/mcc/resources/queries/study/demographics.js @@ -110,6 +110,10 @@ function onUpsert(helper, scriptErrors, row, oldRow){ idToMccAlias[row.sire] = row.sireMccAlias; } } + + if (!row.date) { + row.date = new Date(); + } } function onComplete(event, errors, helper){ From ded68ba5b53fb1c0b35788f3a0681146c0b39f1f Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 17 Apr 2024 13:25:39 -0700 Subject: [PATCH 03/16] Updates to MCC import --- mcc/resources/web/mcc/panel/MccImportPanel.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mcc/resources/web/mcc/panel/MccImportPanel.js b/mcc/resources/web/mcc/panel/MccImportPanel.js index 43c0bc4d1..ed5f9637f 100644 --- a/mcc/resources/web/mcc/panel/MccImportPanel.js +++ b/mcc/resources/web/mcc/panel/MccImportPanel.js @@ -251,6 +251,10 @@ Ext4.define('MCC.panel.MccImportPanel', { sex: function(val, panel, row) { val = panel.stripLeadingNumbers(val); + if (val && val.toLowerCase() === 'tbd') { + val = 'unknown'; + } + val = panel.enforceAllowableValues(val, ['male', 'female', 'unknown'], row); return(val); @@ -1215,7 +1219,7 @@ Ext4.define('MCC.panel.MccImportPanel', { commands.push({ command: 'insert', schemaName: 'study', - queryName: 'departures', + queryName: 'departure', rows: departureInserts }); } From 4d597eac3489870332e1fbdc5f2bbdbaa27d3bb4 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 22 Apr 2024 09:55:09 -0700 Subject: [PATCH 04/16] Add field aliases to MCC import --- mcc/resources/web/mcc/panel/MccImportPanel.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mcc/resources/web/mcc/panel/MccImportPanel.js b/mcc/resources/web/mcc/panel/MccImportPanel.js index ed5f9637f..743d479c0 100644 --- a/mcc/resources/web/mcc/panel/MccImportPanel.js +++ b/mcc/resources/web/mcc/panel/MccImportPanel.js @@ -93,7 +93,7 @@ Ext4.define('MCC.panel.MccImportPanel', { transform: 'damOrSire' },{ name: 'weight', - labels: ['Weight (g)', 'Weight (grams)'], + labels: ['Weight (g)', 'Weight (grams)', 'current weight (g)', 'current weight (grams)'], allowRowSpan: false, allowBlank: true, transform: 'weight' From 76240229f27175ea5ac5c5736c4d98aad7ab40c1 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 23 Apr 2024 08:39:25 -0500 Subject: [PATCH 05/16] Expand import validation in MCC (#157) * Expand import validation in MCC --- .../mcc/exampleData/MCC_Data_Template.xlsx | Bin 12783 -> 14358 bytes mcc/resources/web/mcc/panel/MccImportPanel.js | 54 +++++++++++++----- .../org/labkey/test/tests/mcc/MccTest.java | 2 +- 3 files changed, 41 insertions(+), 15 deletions(-) diff --git a/mcc/resources/web/mcc/exampleData/MCC_Data_Template.xlsx b/mcc/resources/web/mcc/exampleData/MCC_Data_Template.xlsx index ae2622e210c7a332db7324cb86af29679073dd28..916878789c50f9193e630fefc4ac6a63807cae7c 100644 GIT binary patch delta 8022 zcma)hWl&v9x9!G*yA#}P;}YDRK(OHMF2P};!F2u5E^1F5e+Ob z>eR=9)v0qRCf%hS6)dHt1&?7E&xUgarz}iM5asBDhyrXz>1t}RN>X0qu+;9eaZep# z>ZoM1Qsuhx5w2Iz;M9cA4W+f-LOY7zQcJwkE??0d86;6dL%Q*L8eM^-H*n-6#7_d! zaAHtM!ZR*Se605*3nGoBm^zvF%HV$y6EbJ=e zjrjKU$_yA!?KK$9^4x5%DU#^qZm&e-6{xV%IpLAZHu z;-8CJL#gUS+C$rL`eJ=EzsS5AD<5`T;R~pUOT{V5Oi$gk#+?cGH1_wFZ4&_{eO&z% z-m!1lzi=PVImTK%oLLL634_7&UmqA=4|u({@Hg6we&s$SFx!UMaeYg3Ah@T9Fk}9s zyQHHvX!D!3M$-=-X;EDQxDKI`w-GyAR6V^a9+)%S&q12l4m4yifK}2Ai2#wt4 zf_Vg=KJNkC8kg1G77_^b@&Z?13daM7+z#`^DFO!q89*qJiGWq7RROF3@C<-;TP2@D zM@K+S*ku-46qB3jXpDx(BO_Pg^EEl;B=?~ERIMqiFi-&;q7qGZel{^dd~v^S6m2pd zyQ`P_%6Nd$B>FJAJOby*ulrTCn#pT*d~Cg191*UVjkt}j)(zWwUE5A%koYpr!W6Fz zo~|CZ!g07$IsiMaN2)-EHYsI6+AnWuKa_>1jx+2-i>z}Hq?W#9a&QgR;G1hmRc?ig zGz}5Rb++I5My@iu)T3a==ZO1`jWp=JYwx-_QZ%eWOeWG7bX{Z@zl3@d=dDlu6vH#L zPpT$wRu$ds8W!xmU}nAsUuC=q#B&5qIJNFo)Zxs!2zm&c-l-z#qn#`4>Kg{`L?wk-1jU6Q|v5GKNrKy>Ki7+1;Jmc z^m>oo`gWr;^y_@5$;HU_X}>51Emr284aQfd1%5nob!t!loc@&%@*~LESCpvZShn;6g31<4kGmeI}!>u?5~V6DEIqaZXKE zw5^b?S}c$?G3(m$e&WKxKNVwE_zq(#y(1UeP$fDxDD{aE!7(dCG%>)r*Y)egVboCb z?QK}sOWse%&CdR3lgEMODN*Ec%+M$6D&|fW^+;To!}DF2C1SD5v4z|T)2u#*h}Lc* zpd=XJoT_@H&5L(l7SZjE^`bf#R=Nt=JnnIZ!3;C6_3j1x%GL-a2HPwlm9bD)RdBN% z!xl4Vy*!Wn4yF+Qi5JM=uJJ+Z(jQ&j00&z$H^*8jXwz0<);4c50Y@Tl#ea#8ck-eT z&eG4ze*J*N7wZjDw+l%Ec-xd@=u(29NT*+jHf)XlJ5M&<<*}P+=3A~DD*zWC_#yo3 zu=1cK;t07mcRbz+URYvLCL6r46mc45rpy2d;Mv=#g3B2k>b2fXPJ3DBi>D$ zwU+7iL*)UVxf|h&i<_jvX0DJq#?u5L9!Ann@!$jfa2FoAmI>9`;g?uu+v*;xsOD41 z+U3mbCM#3kkL(Nmg23_FC2#+HNj}h`<-hi?*3X*>Gb^i}IO})Y)(Cndrl-3Nfd}7i|G>zQy0$kl4_e2M zewmfGE!*32olIqq>N!-Nbl)`~p|TNEuzKu1?awI880%u5SJP;QK*hTEWJr`|_W zUX#j?GA}EML-U3JEfHB19Hw(T`MN!tU*%CV21Irq1q5mZWEy%(HxOxXNB?$F<^HZ2 z^U7Fk2Yb7_CVhIXyY=NyLj1oMt{MLorH=zm2B6!4%aW2NGL-ne6S@WVV7HH1=bDY| zKGkgcu0QqN$Zb;mM`>OP*V>(+T^yZWy&l$~Js|u^OT)}mrFO6~BdfR^mtq(#>b|vY zGaHNczGiJ~ZiyUQPkYxiPrQoeZU!dLdJ$od;c!KopW&Y0wvkfn*e*rsCbMC1KLDI2 zhaqjZ`F8}uf&M3{YWa44=Zy#gu@HhlM1L&29l&1BZuagrmX?rWY;+(iWi@`~FAQS2 zeoeiJJNz9>JWQ(*|4>nD!PBpWFCHdFpr}c(@vz(S?BlX$cq73d^RRk z2EHBmy3xqvWi;}>cQJ79NNz5;BSwbs=f zEi3PWLO2uqI@4z2t`eykx4S|-q;TTe;!Tgg(jf=dR@N#mnr01soGE|BVNBO!gbG9L zgFf0%vXwJ{YtmH)liy%Nqy?XDFG+?G5gK-Dr=1!E2=A?Z%W&Cs*u}O%1CO#XoAE!t zwLINjki~Y{wu|2~Q;LUgQc_`PHh-@@x-MYwpu7^>)mxJ7$!1K44OfNZQjj02C&GG& za@F6$Q^MwWP|kHyk4PG>knG{tH3^PRvt5t{fPY@G zE=M^uK)MuH^XteNlQBEgAQ9?0dt5B~g9P)Xnn5hfnh) z)qNGZ@w2(idFh8|cUogO`#sfmf(?s^hS75e4cXH z4b^@_@E--$%w9zm(1ZncsJ@p@R0q?~BULLMYUyB?vFM9Ao~M83iF8=bb^QDxc2a1_ zjh|j{deGgd9WSbn;Pv8od~GTzx00X5i2Hid>_-LUlV6KeaNT$_5~oD)hCr@JXl{Wh z$1CQw0;xx{!~^oOV$>>xPEhwS&xr)FAUYz<5M2zR>|&)n$%bbxoqCE!$!=mZ4_5QqzPv@+CmWry(oMb0!RDPBu+lPd0);P5Kzdz1TM%Vv8!};u}|Ezh8$)@HGXHpc) zl86BPx$JaqE;jUCMY;v~*ve9OyYhkddqQXSpoLaa$`x~n2DX@3YwN;w;7nG&XX$e+ zxZIs8-xJ#fR#*Dlv+eEQ#lqjnGoYoC@CQkrSZH$c53%U(LS=}?v0DRI~Zn#zOzo>18CaxLaq zPop3n#Af5zhUwBs_!gAr<|e1dcY+K`9~*lSuxAy{kV>iXV+3@i`_K1BuVi~r>Efm| z&h5WdiFR+F5&e!(_wLZ|zDvNDKN=dbVgEQ2(a2MVgnf#F|J#gD0Av-afdjmx&KqW} zeV`pY?z{h#B7-?AXsDMODtAkeU@D`yyOQQFhj`*NeaBVO0VTR|b=y>rz0D^gEsZlE zM(}UlQqa?3wBxa}qw=npM^3o$_>01Ta1`S1cLYF;QAt2G#z(yaWF z+Xdo-U>(&d*=uz4@*VQ=nA9|xy}kjPbabn3(21mac6)Nxi^IdSz~qRi4IYI0Ur`k$ zTeT;KZA~)X%v@tjwNJJg0_Lk_czeWFA&_qy$zy@s4|l~S@APGV@r5y>QhMXBl71Ou zlGJLW0uZX$WfTNDM#EWaLP`w^%2MF1Q=dyrM!eRs@YCJeWzq?Lx{IT*iYW|x!Cas8wDA>jNHh6j^dF_Uem}H^&>cM1LRS(bW2NGqa6Wo-3q@m_lT=NT_qn2 zKXs@8KBJWW?Zb%|=?P{0RjBz>j4?aA0VAH8?0QH&BjIDsFZ_yfg!+4gxan?9O$hMYryRx4_;AYn|myh&O*DpX6-yRzcM| z-UjYBu`YQ7b_H{fMlY3T>WAqX&~IBA4M$ws1-wD~RhKu6>z?jq2ET0``!Dj?wWaCz zj%Ql$M0+nJ>l>fIjz_%2Eicj#kw`sQ{0ET&ON@9vz3HZ(=gQCaX`;m-KZR#kU?Ong z5p(4T@e_l?=eH+oFEXW=%fnB3l{Yi|)l1xODblLz9z>3a0F&iJxXXEBL4hVX44BDa zaa}*tE z8k^rP1TlO|?NdQ;8N3!0#LfWPrwFxY2)q3-)t%y|5W3%n4&K#!SvLdj#X1`j=opPg!m)J>S%UeGbWA&ZhoFMNZFE!&0Uq2D>pm&Ob+s{>v`z%T#f*edxg z<&e8RWLf^pLF$Z_F275i)87vv?bWRn{d*^IMfjnlb;nsJvE4!=X6^j8ll=bp6TenP z&!ukOoEb2*x`|lh{ix-}sD@=g76!26_nIYet{gC@I%wKQE>~8Y_(ap0_0s7D_<-e4 zlF26bn5wyk^fTYZsOr06$p4m~{zYTWR6a{h{~;30C=f>)GN3npl@F^EX+#XoV+D3< ztX;Rk)>Ke@@l{QPqn{0iVEmUPnG|aLhm@MU~31pqsPv z`>FHC)s8fXfY6r@yh#3ShKHmK2Dgf|^KUv0*#T91ZTh^mCVGoeuorTpY=G@oLAZiH`#h#uZ#F3UUTBuln-% z=B07b4O8xhei(K|PeE5Kcj-tpbit~ldo6uZc6Pt80BoDLEf?m9z3@~u+->(S|5gT( zYSkA-ObXOjtBh<2Ry`GRoe^4FX=IM;H{vnN&4v$gj&;Jiv`oVXI_rNLzlG*Y(#7Lg zQg0Uj^e8tdp{^?~bS<$;bO*^1ANJh6fg6p* z`mKrj04yrF{%r9c^Xa7OLIj4G#wWgsA9t=0Gf~6PsEY}}Jc!}!%ay}*OqMX<>IDT`5}+k-vYfjq|rmU;DMp6np+z} zbjY09ST3qT;x>BetZdd8m32V&j~8Dls%7s11CkL|7CsVxVnEVCx6i>$bKBwT-aFO< zzDvi?F*law)^HnE_!DEYU#^bFDv_rS5!NlqaYCkEGPr2TG8Ht!Z`VrcdsK!4M9Kn1 zdqRf&Ff*}5kRoQ~LQGI#b!aeC&n3b=tL{rB9fcYLG$@M?lDjE7C-RkkuxI~$ z0AWSR_W0G23eqWpn0DZX*RQ&Bn=_hQC39={EPYgYY2Tuo42{pJ&z4ul(+F24^OE#; zA)um7s?-Y|^H>V=zeWz~L#468*n4>%4EOtxt#qK=H+H-{+~g}`zyvvJn!CJY!QoF) z*2+J38K|vWG~_b#T@`e1N5N>o=WLNm0wkx$=4`KO*um)SpEu=%(|9+QTuB|`x9&$F z(~}KWXmPH0(J>UI*Ctz&2puFF3%i_x?T%lO);={Kw=+~)MGh*R#r&!me(-a<_@ae@ z&K3W&tqQ4ac;LM%8vK{9`=A@%r&80-8qRP_)N9%I$-O_#v?Y;_o___B5a9J5GphFb zTt8}pDClXB*3Ml-KJI?pnCWO7%&vtKb0;9`FZ6OB44lw0Kd&CBb#HRqJ2a%cGq5>z zssF|2H}{>YW#k8PxY0(+a!wK6i|^43OuZi#HVm?mBbQR|A0{aYNynlDxY)fN9oXG0 z9o)kbdZYU|@MLb_o~69~AdBH>QX2f!En53}fMfa8!qI@`HNAF@tXDZ6<950NK8o;WJr>r{xI-zew-swIc#|KadI~8w!`1er-3dHw!uAAkX0Y`5U>uoQVh)khV3}yN3mV4k$q& zhX1O;UEFM)JS^QbEImAIovhu#|Jrl1o7p;DSidrRW#VE{W5EI22SP+ZL`THrIO$Iu zQJY8{QybVqpj8UuqDMm%FX>5~NSscD8=&kN>6`G32$t);1K@;c;Ar|;s{G>tJj%&w ztlk0mLYnmS;4=Rn`jj9QIQxP~ldwa?ugc=As1}b%upUx|!cUT^M5-=jXB}8wxfNrf z)3cO#3bKgoHOrK988woJcd%wSBlMa*Z(H#T=o9f^YdmCPi*U2h(0aS5j3^sn(S(9J z)Hz6C3#k6b1pL|KNp6L6z=c}d1Vy0KRKMfgKbf5Koe1V~ptz7Sa1-`00N1Lzx6NY* zn8lo=#SHUgLG|9Uu($e3U+)$!#-Cx5`56?AzkW3?$n3ve@bqJAPzx6>>-Q$$HZ<#b zuROh+wtAwN4x2+>swm*CyZc?=G#FFq16%WY>rRa;U{rjD5Th`aKwKoNxs$aqs{ql&UYK+s}>IsQnx-AXa~?fXtYs zW727iEXTh6!uYubC)vN}z_mm=FWMw?uFU?e@sLvE4gGYiO18JlQqnqqJlSY7;oUeH z;WPmku<7f$9{o-3G)wcyc@pFK>F#u{Eqtj;sHG+B3VO4fm0nQwD16k+pQxYn`?+TN zVe~Y=OCXHRo^T}%%Chid=Vo+g_PHC^z(8^C`H8<&yJn^9;=;ImcTKGNyxHJhi0!aB zQ1oRz9coVIMA{nPEu!~qk?{FkKipDj_Nnt4IO2Jiqx}?nycXCjyEM2E!Uh=ldeYrs<2MoBsdH0)< z@6Pw^GW__9*1~Tu*2zgxu<-4#pX(zJ5Ub_q6M?SX?TDltd_5}nC9QzM1`2+GHhXwS_tCyYkIg4po6@^6ToOt zoGwijo6nnLa>`*FR6kzxD}J->)J%H((J?kLXKQKU$y}dTyJ$HTl=9^rhiUM`n3dNW zOa1QGdS<@!e6F-{yqhblPjp^^_={cIl>6Ed4j6l(_Lljrv!^B+kE=LFHmI+2V2Hyb zkKZ3ubf6J7-{DyD%B~L63OQycC;Lwq@K3e>N9_J@V*?T51g3(ZfJxxx zQ9vNb2_gxrni32w8|a_D?thvdSpOwK{^K5k2I=BpCi}NW`Hul2HUx@I0#WB+hG~HK zbI_Chr&7cQfzbc4`?KPvfRuwN$o?UUpg%|W@56`&;z~mRA>qV=jDbmDjTs=>Sac9n z4lJ_&WS&3GXPW=+0y1FyV?>JbPcnk&ax%jhLBcp0^!|>WlZ2U%69lqwHdk|Vc5w%ry14w68r75#{;0ZtT?Z!UFFhKD I=da!W0SaK*Pyhe` delta 6525 zcmZ8mbyyVMwjE%Q#zBVeW+=%a2Bf>BJ0v8BZiE4(1r-L66p*12>29POkxuDGKsuiH z-uJ%W`|keheBW8;{BzFUd#$xo+>WiuU#emN!GI?KYybej0NBQ0Uv@_W08%k4$(hg* zLoQvspcaEYNx3%tm{3_gJs@EawFzqSDevz{L*}|oGxob6;jP>f6=`ggc~xvHOu#(Q zPQ+Z;yDBWCch2;@`m<68sRk8ntY`aNCTj~CDQK;XHnSNRg*`SGp!u{r7JGeec)9#GzzK$4@8&S}-w=GR1k zQ0#_7#;nJtjK4*O3=wBCXDOl*1&fU1v?kQ=U8flWRUvbit}`qRE=RxOD9G|_g)ZxM z^KUezSnzPY9_6lDr>EgCGfGFV;Y=a$rI)3Xawr276<3p2D^N7g1cgho=EXVWB5WBB zuNhC$#GX8&-d-(*#;pvQ5WUi%rgqT5Fu%QF+JS7TV2;y){bauJtg>C;(4bx~qEU3$ zW?KLyddJQLa7LM6${}a2uUtvs-g^|-=oBKU@-yQ!OKq>3}n9`VYLk|7dqk#*?uuG1^_^V1pqw#`#65i&wX7zoUB}3op}7<&SiRLt{J@G zpzNAQ%j>K#oN-{XuDVRE%D%3*!|k(Tj}9FDj@NY#+IsiL&8!T&eX28rEYPiA>z;4M zgxCr*O5~oZIq=Ab4J1Sn5vL6|8nRx!4mdjJn~aRjOwq0YN+S5 z03fl>2tMzFCNWM!`9BF|X(9bGC9#y`CHbXJJ@(~zy=r<*ZQJrL@{JuP;f6s#Ssqex zYvME^65`i*UksHA#%D0Uc}MBn(uI~584J}S2TGnqi#~7kcb4WWsk*X)puU_vvvF05 zhT;Yj-Nj(VuOPkxbA46y#C5cXB1F;ceN}I=f`2uJx6Fr)+)7-LYp-%#bifHO%XH6` z^(SaK#X^XVhwOo6KLc1IM{b#o8@9LB*9O8q9r!qz$?e~eeRVESiBl<2PxtIT1=4lD zET3CLE7De%aP-5`TG{_|J4b^vW87 zf8|!VU;5dpvM&M{mFIL(xE3q@DzsgSvqtdUuBnBVPL4t{4sBmG6~U>Ur};=)x_YWP zh0z#K-})S$=j3Jb$pDj4-_T7M{Qn%|3Ksj`3yE(lz z#Eh*qFac4Vx+qD|GJc#bUrZd4H-!^`I>y(2?(zPu*BjdFH9vk)#1lS_SOH}!or6<& zF7aYDVGvalDuYL+rD5$lv=>?5Ouf+V&@@|3;;dO`GCF;;iVu^^fbn=VYE;O!fb#lg zy1l7d)*B&G8}xwgoWp9mxaWA5YS__T(XS&VD-e*ZZeq^PdU4{r*cF?hV``p0=(nrJ z=h$^@93Xfg^E}6@4AsvL4mYkYG?y{c=HC)0!OjNSWTKT~7Nfag1~qb`Lgzi;)%xsT z_2xU%L-uLXjkD$Ijv$3#yg}$zA+(!#r?Z&aE1?4hyLb}1p5Ho-bAz$6izTr_fkiAy z!1v*YvKTHr{jiYvfey>jgxCEtzKRR4WLS5?8PlT9sg7KWpv`%=se{;gsnVy}sJk8c z2qHnmGee08e$Z7yz>F z>jSWVn*o#Pv>Ri=$(wZ&06CRXzgnEr)fHcB^3pcKp=n-o8E{x`8KCd&4&4{jYD}QG zipj%=uO44Yn$|>!eEdMT`8>F};esQc-sVTV_Agcz<+!?pRSc=4GOvpnKWq|7%#=G|>)zMdeWR^Ec<$_fTD*J!YeSb5IjNHk(tEnS{yJx?LCZZd+Kzw173R&u*npB(Hv-iv3N$tdUi#drmxS?;#v< z2^{_K0hbI<+r-G)EUR7VQ9v-n)A+~K4p6#Q`FEbT$WhWogJ=p*a|P7J-{LomR!l2Q z&ABbUIxJUW{^3)TcPu_)7*Wtq>(3n;cb~FcMW(p^73m!ymiU-fP{?e_)-U;CUQ+O( zb+l=D7bj}llkM_S>CcO8zuEfpHmij*-_3RYNUGXRMXX?4u|K~2vzMaEnqPn)h7UZi zX`5XG`-=M_tgfbh)VH&UNI8hL?1l&(UVUH++3r0G^vvelp))#*pKF>Bipdr@WF zMuSyAWgtcK?Qa+&aUxchrA8YuiB-%kcW~6_s%`=@OlDwOP8mma{wM~=FOB)08LTff zGoLFwyxAZqSK5fryZ%Z>;IdU;KQR0BJ_8|^b&hvqzi`zp55i|cJa)W$v+wwQL!!xNPr7;;#%2ukYsSyyH>%^8RQd?Y{0qSkCTZ z{ON?8j?25%m-q6%Bw@fG-^fGj5~OY)ONJjX6N$h`INLq0U#CBHA0_}mO@gE)qD6c+ zgnQ15fF6PcF6jgxGA(WBneoRRGEnOC%;y1zL8J;HtW}#x8|>D`$Kd7^I4@LicUV33 zdkWoG+QV|6KhQU4VCAY`^VlRG#OECY!~1g6WBm9|+wT2?0*;8Rjl`|Bd){oo8-6dx zAs*ok`Zp6?qV+&>D}mFLX;zFCVZ>UZ;&6>_mvvDZCk-E7cEDPOoF{aMQhxI3{GouP z72^eUx0Ga*?8r9b#wQLnCR!84FO4>&=fbK9sbSo>*|KDO_o=Lfxv|f3QgI;%tW1BD zz=STOm6OgKokwi*whaE0CuALn!X4IbCkc7RsllQ~I9m0wS9u0v&EkrS3z4>WA^9%S zCuqa~9m`FXF`kGn? zcAlQx*>!)QDwai9$5~%DS4bOP8x&@1HFwVWNxX(yESL5=SNB}VM#Luwd++p&7DHPM zWbIcn#gw=tRSorW;(vCKf{4&By9(ryzCl}j^)Lz!Wn^FbWPFf<7hhgolx@$y?wO); zYd+BaeV9jDV1DmUlil|HX9L}b+-bT-LoV*Ai;pnfXVI(5jG@XnmHnR;Xhqz!kY2_$ zK@-x1*24CTMPvoNB}F@|+wS;FnP|&c6gfxO80Sg5f`J=giS}fcui^V}IPXf+z5e)H z)YHl}6KB!W9God~DTDR1-1e|LLeCgqZ=Ct!*cMmNuD(1^7Pg`B#gA_)razVE^p~ z25b2Ef1rrJSe(iQxfr+Psz^0z?ZWS(#W4cir6_OvL|K9w@(zM&>#5^i<0w|qDXBpE zJNIHIe0$UW^k?+l8T=tGQD0#>_g30%!)_;v^(j`J(^E^N7W^&6c(X4^#q_AA;@p~H z^<~4B?$@KJ5aG{!N=1|8TTiDdjO7nlL%*hidl3QQ%Ei;i%dA*@3JjX7ye{ss{SF4Btm4dR#jhQy+Tnk=hX#7aUyu9A_oF=mX- zzWLROUkh>d!-?_6a;p zy={>7oqQ%FnbwcJr&ki8YV@`1CCbhiL{t%*LNCg14#KgxhJZ;5H*(@*#lLGMg<}q= zv|m*GI{w}HzK<(E+b{rP4{>74+>%3Y7qLsWZzb6cTqFr^H!X=uA|Yw;8$DGEp3w;U zrevU5nCO0*+1+9>5rKV$GRg=410> zj#2-vkPe=`=~}j|%-JXzbo5uYSBlrDZo}1HA`SGRUF+-&;EUo(XsoGvygR)4*oj+e z1>KXt#nOZH{c7*5r-1( z9wxWuNrgZ{(FcU4#Gq+B$Vc{Evc5ZL8gd^Ih!w#6$@IwRfg9E@;7kV|-mbR!jZH|k zmVIzRHKJjMqkRao6%`39HL!8)lm3!m5Y+xt`^79lBwB&wUlAQyR`H)YJ(kNOo4U{Dt%a$u z?MlidIXrpQ-#o>~6mN5}n-FV}Iha(io7pXbu$KlsDr3{rjI4irqKLNFEoUjNkT)Fq zWU@XlCPCIy5$icT>s~_gwLWPjM}o#(aJXR_2A}Tod%rcnTNSqMO4)g`-Uedh=pl#b z!9{5~!d?WSS1-kF*KGK_l`c{0Ge!8Z_UNMAK@WhnA(?~Aoz%|z576!>sLmSHYvTD*!erZL1y#NVK^-_M>jN=2 zld~JOBRJf)&kC!2MVGq5rFIEM+XSJFPM#&Np}7c3=}(kyVzustUibyb=0MfhxT)T7 zA3hwNIoO)lsjFQrv-qssePo1MPc0vt57SglLhw%tu7Eu2%^l0S3Xfgm*@`-&_46Z|e9r<|bQE3X%CB4+}YiAHnhGbPVuhAyr z9dH{3P$BsCf$K&fv+P`l{bo$9%O7>`6Uq5QJi_;y9WcpcU3HKC{92nPqaDPfgJ*{`ReIB$NsEUct?#ZDRzS8Ctty zRPPS%H7UIHgZ4LqO&gjArb~0COB)DJTL#wyk_*S&cZUwmnsg7kyrQxV9i~mV!8Gz? z(5YXATDIG*vk0a5eRRs!m#TB=Iv;T-u7iy+Z1W~LF374l-^-L2v=*5Wue3D=ycrhG zxZ9thD@ca62Z_(UAnwqom_Dyad;Pu^XAU;TI?REA_1~>MjJqQ znbt{Xx~-aK{!5;$*-RqYr~Xog5}4tG2rt{KG3G1;_6ew1KU zmlb-OiRG4*kCutBqfaC13XOByY0>@kUbQfD8up#)#4f#xtL)u(hN7%w4kBKnKY3&T z+>|AlQjx{D`tPid{!188+3V1?m{rg$XLnQylez+q8pf88G(_1!NUqdQ0I#w%po+Zr zXQn~+xHckbn*!jFH41m`sG~S+b>ghA|MABHR-L5^eWkkUi9T8T^^7Q(2uqN7h;!a2 z0rYE+@-^~{@o`CclB0CJ{LY}ZOc5*yx@tDjQD&+=*k3+5x07W)Zbn4UuyZ#iDr>be znqI=3WL`_;B6hg16@P|CD!vprtG46s#=b-h)Ikt6r27>)M|1dWm(X#N3l|+qVtT}t zU*Ap)F*Y(E@%NL3EQ3ka?Mvp0yyVpwbBAqpHDwks*2vDwvCDO_Yew66vULasJn>S0 zpW-LsJC^>y{!qG6)?4A)iLSa`AWH3%$^SVc6ezWpOk_v6)&AHOq3E5ZvLn|u_`+QY z9*>A^(5{*i7(PwM7o*_d6#lG=!H(8qL2|Z^C!bNyU0#uO zqq{)sp*s3v8vm%%Uj_V$VhBM5=5*4bJzA>V1W?s2Nk`Eg1K(RyPLbJ8LK~4Yw*El`H;fh+jMFyR!AZ9M>m^ zaP;W`Y(ngC1f-yi<$F4dLMm+Ulk z9l?l`SNhRZSne6yhLw~!+9~vM%ry~#_Ai>{3<087%khn%Iph;IGg4|AK}M`AwJ!c?YFLt3;9svmqyiDbZ^=kw?Omgam&LNu$3# KB!uT*2mS{cgAf@2 diff --git a/mcc/resources/web/mcc/panel/MccImportPanel.js b/mcc/resources/web/mcc/panel/MccImportPanel.js index 743d479c0..a27d981d6 100644 --- a/mcc/resources/web/mcc/panel/MccImportPanel.js +++ b/mcc/resources/web/mcc/panel/MccImportPanel.js @@ -33,7 +33,8 @@ Ext4.define('MCC.panel.MccImportPanel', { allowRowSpan: false, alwaysShow: true, transform: 'animalId', - allowBlank: false + allowBlank: false, + expectInImport: true },{ name: 'alternateIds', labels: ['Alternate Ids', 'previous Ids'], @@ -46,7 +47,8 @@ Ext4.define('MCC.panel.MccImportPanel', { labels: ['Source Colony', 'Source'], allowRowSpan: false, allowBlank: true, - alwaysShow: true + alwaysShow: true, + expectInImport: true },{ name: 'shippingDestination', labels: ['Shipping Destination'], @@ -65,13 +67,15 @@ Ext4.define('MCC.panel.MccImportPanel', { labels: ['Birth', 'DOB', 'DOB (mm/dd/yyyy)'], allowRowSpan: false, allowBlank: true, - transform: 'genericDate' + transform: 'genericDate', + expectInImport: true },{ name: 'gender', labels: ['Sex'], allowRowSpan: false, allowBlank: true, - transform: 'sex' + transform: 'sex', + expectInImport: true },{ name: 'status', labels: ['Status'], @@ -84,26 +88,32 @@ Ext4.define('MCC.panel.MccImportPanel', { labels: ['Dam', 'maternal ID'], allowRowSpan: false, allowBlank: true, - transform: 'damOrSire' + alwaysShow: true, + transform: 'damOrSire', + expectInImport: true },{ name: 'sire', labels: ['Sire', 'paternal ID'], allowRowSpan: false, allowBlank: true, - transform: 'damOrSire' + alwaysShow: true, + transform: 'damOrSire', + expectInImport: true },{ name: 'weight', labels: ['Weight (g)', 'Weight (grams)', 'current weight (g)', 'current weight (grams)'], allowRowSpan: false, allowBlank: true, - transform: 'weight' + transform: 'weight', + expectInImport: true },{ name: 'weightDate', labels: ['Date of Weight', 'date of weight', 'date of weight (mm/dd/yyyy)'], alwaysShow: true, allowRowSpan: false, transform: 'genericDate', - allowBlank: true + allowBlank: true, + expectInImport: true },{ name: 'date', labels: ['Observation Date', 'date'], @@ -117,13 +127,15 @@ Ext4.define('MCC.panel.MccImportPanel', { alwaysShow: false, allowRowSpan: false, allowBlank: false, - transform: 'u24' + transform: 'u24', + expectInImport: true },{ name: 'availability', labels: ['Available to Transfer', 'available to transfer'], allowRowSpan: false, allowBlank: true, - transform: 'available' + transform: 'available', + expectInImport: true },{ name: 'breedingPartnerId', labels: ['Breeding Partner Id'], @@ -134,25 +146,29 @@ Ext4.define('MCC.panel.MccImportPanel', { labels: ['Current Housing Status'], allowRowSpan: false, allowBlank: true, - transform: 'housingStatus' + transform: 'housingStatus', + expectInImport: true },{ name: 'infantHistory', labels: ['Infant History'], allowRowSpan: false, alwaysShow: true, - transform: 'infantHistory' + transform: 'infantHistory', + expectInImport: true },{ name: 'fertilityStatus', labels: ['Fertility Status'], allowRowSpan: false, alwaysShow: true, - transform: 'fertilityStatus' + transform: 'fertilityStatus', + expectInImport: true },{ name: 'medicalHistory', labels: ['Medical History'], allowRowSpan: false, allowBlank: true, - transform: 'medicalHistory' + transform: 'medicalHistory', + expectInImport: true },{ name: 'currentUsage', labels: ['Usage (Current)', 'usage (current)'], @@ -437,6 +453,16 @@ Ext4.define('MCC.panel.MccImportPanel', { var rows = LDK.Utils.CSVToArray(text, '\t'); var colArray = this.parseHeader(rows.shift()); + var foundColsNames = colArray.map(x => x.name) + var missingExpectedCols = this.COLUMNS.filter(x => x.expectInImport).map(x => x.name).filter(x => foundColsNames.indexOf(x) === -1) + if (missingExpectedCols.length) { + // now convert from name to label: + var colNameToLabel = {} + this.COLUMNS.forEach(x => colNameToLabel[x.name] = x.labels[0]) + Ext4.Msg.alert('Error', 'The following columns were expected but not found:
' + missingExpectedCols.map(x => colNameToLabel[x]).join('
')) + return null; + } + var errorsMsgs = []; var parsedRows = this.parseRows(colArray, rows, errorsMsgs); diff --git a/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java b/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java index b1da3960d..89d94101c 100644 --- a/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java +++ b/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java @@ -68,7 +68,7 @@ public void testMccModule() throws Exception testAnimalImportAndTransfer(); } - private static final String ANIMAL_DATA_HEADER = "animal ID\tprevious IDs\tsource\t\"DOB\n(MM/DD/YYYY)\"\tsex\tmaternal ID\tpaternal ID\t\"weight(grams)\"\t\"date of weight\n(MM/DD/YY)\"\tU24 status\tavailable to transfer\tcurrent housing status\tinfant history\tfertility status\tmedical history\n"; + private static final String ANIMAL_DATA_HEADER = "animal ID\tprevious IDs\tsource\t\"DOB\n(MM/DD/YYYY)\"\tsex\tmaternal ID\tpaternal ID\t\"weight (grams)\"\t\"date of weight\n(MM/DD/YY)\"\tU24 status\tavailable to transfer\tcurrent housing status\tinfant history\tfertility status\tmedical history\n"; private static final String ANIMAL_DATA1 = "12345\t\t\t7/10/2011\t0 - male\t23456\t23453\t382.8\t5/19/2021\t0 - not assigned to U24 breeding colony\t0 - not available for transfer\t1 - natal family unit\t3 - successful rearing of offspring\t2 - successful offspring produced\t0 - naive animal\n"; From 3dfbbe331addbe54d16bdbd3982d247e2ba54a2f Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 24 Apr 2024 11:47:38 -0700 Subject: [PATCH 06/16] Improve MCC validation and remove some elements from snapshot panel --- .../web/mcc/panel/MccClinicalSnapshotPanel.js | 77 +++++++++++++++++++ mcc/resources/web/mcc/panel/MccImportPanel.js | 21 ++++- mcc/src/org/labkey/mcc/MccModule.java | 5 ++ .../mcc/ehr/NoOpClinicalHistorySource.java | 44 +++++++++++ 4 files changed, 145 insertions(+), 2 deletions(-) create mode 100644 mcc/resources/web/mcc/panel/MccClinicalSnapshotPanel.js create mode 100644 mcc/src/org/labkey/mcc/ehr/NoOpClinicalHistorySource.java diff --git a/mcc/resources/web/mcc/panel/MccClinicalSnapshotPanel.js b/mcc/resources/web/mcc/panel/MccClinicalSnapshotPanel.js new file mode 100644 index 000000000..f50b1f896 --- /dev/null +++ b/mcc/resources/web/mcc/panel/MccClinicalSnapshotPanel.js @@ -0,0 +1,77 @@ +EHR.reports.clinicalHistoryPanelXtype = 'mcc-snapshotpanel'; + +Ext4.define('MCC.panel.SnapshotPanel', { + extend: 'EHR.panel.SnapshotPanel', + alias: 'widget.mcc-snapshotpanel', + + showLocationDuration: false, + + minWidth: 800, + + initComponent: function () { + + this.defaultLabelWidth = 120; + this.callParent(); + }, + + getBaseItems: function(){ + return [{ + xtype: 'container', + border: false, + defaults: { + border: false + }, + items: [{ + xtype: 'container', + html: 'Summary:
' + },{ + bodyStyle: 'padding-bottom: 20px;', + layout: 'column', + defaults: { + border: false + }, + items: [{ + xtype: 'container', + columnWidth: 0.25, + defaults: { + labelWidth: this.defaultLabelWidth, + style: 'margin-right: 20px;' + }, + items: [{ + xtype: 'displayfield', + fieldLabel: 'Status', + name: 'calculated_status' + },{ + xtype: 'displayfield', + fieldLabel: 'Gender', + name: 'gender' + },{ + xtype: 'displayfield', + fieldLabel: 'Species', + name: 'species' + }] + },{ + xtype: 'container', + columnWidth: 0.25, + defaults: { + labelWidth: this.defaultLabelWidth, + style: 'margin-right: 20px;' + }, + items: [{ + xtype: 'displayfield', + fieldLabel: 'Age', + name: 'age' + }, { + xtype: 'displayfield', + fieldLabel: 'Source', + name: 'source' + },{ + xtype: 'displayfield', + fieldLabel: 'Weights', + name: 'weights' + }] + }] + }] + }]; + } +}); \ No newline at end of file diff --git a/mcc/resources/web/mcc/panel/MccImportPanel.js b/mcc/resources/web/mcc/panel/MccImportPanel.js index a27d981d6..705e85744 100644 --- a/mcc/resources/web/mcc/panel/MccImportPanel.js +++ b/mcc/resources/web/mcc/panel/MccImportPanel.js @@ -131,7 +131,8 @@ Ext4.define('MCC.panel.MccImportPanel', { expectInImport: true },{ name: 'availability', - labels: ['Available to Transfer', 'available to transfer'], + // NOTE: availalble was a typo in one generation of the input templates: + labels: ['Available to Transfer', 'available to transfer', 'availalble to transfer'], allowRowSpan: false, allowBlank: true, transform: 'available', @@ -348,6 +349,14 @@ Ext4.define('MCC.panel.MccImportPanel', { row.errors.push('Suspicious weight value'); } + return val; + }, + + alternateIds: function(val) { + if (val) { + val = val.split(/[ ]*[;,]+[ ]*/g).join(',') + } + return val; } }, @@ -520,7 +529,7 @@ Ext4.define('MCC.panel.MccImportPanel', { else { row.objectId = existingRecord.objectid; - var fields = ['birth', 'dam', 'sire', 'source', 'alternateIds']; + var fields = ['birth', 'dam', 'sire', 'source']; for (var idx in fields) { var fn = fields[idx]; @@ -528,10 +537,18 @@ Ext4.define('MCC.panel.MccImportPanel', { if (fn === 'birth' && existingRecord[fn]) { existingRecord[fn] = Ext4.Date.format(LDK.ConvertUtils.parseDate(existingRecord[fn]), 'Y-m-d'); } + if (row[fn] && existingRecord[fn] && row[fn] !== existingRecord[fn]) { row.errors.push('Does not match existing row for ' + fn + ': ' + existingRecord[fn]); } } + + // The goal of this is to take the union of the existing/new aliases: + if (row.alternateIds && existingRecord.alternateIds) { + row.alternateIds = row.alternateIds.split(/[ ]*[;,]+[ ]*/g) + existingRecord.alternateIds = existingRecord.alternateIds.split(/[ ]*[;,]+[ ]*/g) + row.alternateIds = Ext4.unique(row.alternateIds.concat(existingRecord.alternateIds)).sort().join(',') + } } if (existingRecord['Id/death/date']) { diff --git a/mcc/src/org/labkey/mcc/MccModule.java b/mcc/src/org/labkey/mcc/MccModule.java index c7c4fe169..a970d6a08 100644 --- a/mcc/src/org/labkey/mcc/MccModule.java +++ b/mcc/src/org/labkey/mcc/MccModule.java @@ -31,7 +31,9 @@ import org.labkey.api.security.roles.RoleManager; import org.labkey.api.util.SystemMaintenance; import org.labkey.api.view.WebPartFactory; +import org.labkey.api.view.template.ClientDependency; import org.labkey.api.writer.ContainerUser; +import org.labkey.mcc.ehr.NoOpClinicalHistorySource; import org.labkey.mcc.query.MarkShippedButton; import org.labkey.mcc.query.MccEhrCustomizer; import org.labkey.mcc.query.RenameIdButton; @@ -125,6 +127,9 @@ private void registerEHRResources() EHRService.get().registerMoreActionsButton(new MarkShippedButton(), "study", "demographics"); EHRService.get().registerMoreActionsButton(new RenameIdButton(), "study", "demographics"); LDKService.get().registerQueryButton(new ShowEditUIButton(this, MccSchema.NAME, MccSchema.TABLE_CENSUS), MccSchema.NAME, MccSchema.TABLE_CENSUS); + + EHRService.get().registerHistoryDataSource(new NoOpClinicalHistorySource("Case Opened")); + EHRService.get().registerClientDependency(ClientDependency.supplierFromPath("mcc/panel/MccClinicalSnapshotPanel.js"), this); } @Override diff --git a/mcc/src/org/labkey/mcc/ehr/NoOpClinicalHistorySource.java b/mcc/src/org/labkey/mcc/ehr/NoOpClinicalHistorySource.java new file mode 100644 index 000000000..13629ad94 --- /dev/null +++ b/mcc/src/org/labkey/mcc/ehr/NoOpClinicalHistorySource.java @@ -0,0 +1,44 @@ +package org.labkey.mcc.ehr; + +import org.jetbrains.annotations.NotNull; +import org.labkey.api.data.Container; +import org.labkey.api.data.Results; +import org.labkey.api.data.TableInfo; +import org.labkey.api.ehr.history.AbstractDataSource; +import org.labkey.api.ehr.history.HistoryRow; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.security.User; +import org.labkey.mcc.MccModule; + +import java.sql.SQLException; +import java.util.Collections; +import java.util.Date; +import java.util.List; + +public class NoOpClinicalHistorySource extends AbstractDataSource +{ + public NoOpClinicalHistorySource(String categoryText) + { + super(null, null, categoryText, ModuleLoader.getInstance().getModule(MccModule.class)); + + } + + @Override + protected TableInfo getTableInfo(Container c, User u) + { + throw new UnsupportedOperationException("This should never be called"); + } + + @Override + @NotNull + public List getRows(Container c, User u, final String subjectId, Date minDate, Date maxDate, boolean redacted) + { + return Collections.emptyList(); + } + + @Override + protected String getHtml(Container c, Results rs, boolean redacted) throws SQLException + { + return null; + } +} From 5f01338233e370afde75a9558b1ad3cc1127f3cb Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 24 Apr 2024 13:32:06 -0700 Subject: [PATCH 07/16] Add code to automatically update luceneIndex of release record --- .../pipeline/IndexVariantsForMgapStep.java | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/mGAP/src/org/labkey/mgap/pipeline/IndexVariantsForMgapStep.java b/mGAP/src/org/labkey/mgap/pipeline/IndexVariantsForMgapStep.java index 35ca79d17..90e6dfec0 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/IndexVariantsForMgapStep.java +++ b/mGAP/src/org/labkey/mgap/pipeline/IndexVariantsForMgapStep.java @@ -5,12 +5,16 @@ import org.json.JSONObject; import org.labkey.api.data.Container; import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; import org.labkey.api.jbrowse.JBrowseService; import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.query.BatchValidationException; import org.labkey.api.query.FieldKey; +import org.labkey.api.query.InvalidKeyException; import org.labkey.api.query.QueryService; +import org.labkey.api.query.QueryUpdateServiceException; import org.labkey.api.security.User; import org.labkey.api.sequenceanalysis.SequenceOutputFile; import org.labkey.api.sequenceanalysis.pipeline.AbstractVariantProcessingStepProvider; @@ -28,8 +32,11 @@ import javax.annotation.Nullable; import java.io.File; +import java.sql.SQLException; import java.util.Arrays; +import java.util.Collections; import java.util.List; +import java.util.Map; public class IndexVariantsForMgapStep extends AbstractCommandPipelineStep implements VariantProcessingStep { @@ -97,4 +104,41 @@ public Output processVariants(File inputVCF, File outputDirectory, ReferenceGeno return output; } + + @Override + public void complete(PipelineJob job, List inputs, List outputsCreated, SequenceAnalysisJobSupport support) throws PipelineJobException + { + String releaseVersion = getProvider().getParameterByName("releaseVersion").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), String.class); + + List of = outputsCreated.stream().filter(x -> CATEGORY.equals(x.getCategory())).toList(); + if (of.size() != 1) + { + throw new PipelineJobException("Expected a single output, found: " + of.size()); + } + + Container target = job.getContainer().isWorkbook() ? job.getContainer().getParent() : job.getContainer(); + TableInfo ti = QueryService.get().getUserSchema(job.getUser(), target, mGAPSchema.NAME).getTable(mGAPSchema.TABLE_VARIANT_CATALOG_RELEASES); + TableSelector ts = new TableSelector(ti, PageFlowUtil.set("rowId", "container"), new SimpleFilter(FieldKey.fromString("version"), releaseVersion), null); + if (ts.exists()) + { + job.getLogger().info("Updating release record"); + Map row = ts.getValueMap(); + row.put("luceneIndex", of.get(0).getRowid()); + + try + { + BatchValidationException bve = new BatchValidationException(); + Map oldKeys = Map.of("rowId", row.get("rowId")); + ti.getUpdateService().updateRows(job.getUser(), target, Collections.singletonList(row), Collections.singletonList(oldKeys), bve, null, null); + } + catch (BatchValidationException | InvalidKeyException | QueryUpdateServiceException | SQLException e) + { + throw new PipelineJobException(e); + } + } + else + { + job.getLogger().info("No release record found, will not update"); + } + } } From 3bb50b67973ddbcc9a0d72dafb30104ff95789b7 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 24 Apr 2024 15:50:48 -0700 Subject: [PATCH 08/16] Update mGAP ETL to perform file copy and rsync outside of DB transaction --- mGAP/resources/etls/prime-seq.xml | 15 +++ .../AbstractVariantTransform.java | 33 +---- .../columnTransforms/ExpDataTransform.java | 50 ------- .../LuceneIndexTransform.java | 8 +- .../mgap/etl/ClearEtlWorkQueueTask.java | 43 ++++++ .../org/labkey/mgap/etl/EtlQueueManager.java | 122 ++++++++++++++++++ .../mgap/etl/PerformQueuedEtlWorkTask.java | 42 ++++++ 7 files changed, 230 insertions(+), 83 deletions(-) delete mode 100644 mGAP/src/org/labkey/mgap/columnTransforms/ExpDataTransform.java create mode 100644 mGAP/src/org/labkey/mgap/etl/ClearEtlWorkQueueTask.java create mode 100644 mGAP/src/org/labkey/mgap/etl/EtlQueueManager.java create mode 100644 mGAP/src/org/labkey/mgap/etl/PerformQueuedEtlWorkTask.java diff --git a/mGAP/resources/etls/prime-seq.xml b/mGAP/resources/etls/prime-seq.xml index 25a9ac50a..3f798eb5a 100644 --- a/mGAP/resources/etls/prime-seq.xml +++ b/mGAP/resources/etls/prime-seq.xml @@ -3,6 +3,13 @@ PRIMe-Seq ETLs Syncs Anonymized Data PRIMe-Seq to mGAP + + + + + + + Copy to local table @@ -190,6 +197,14 @@ + + + + + + + + diff --git a/mGAP/src/org/labkey/mgap/columnTransforms/AbstractVariantTransform.java b/mGAP/src/org/labkey/mgap/columnTransforms/AbstractVariantTransform.java index 9490c39e6..5345e4fc3 100644 --- a/mGAP/src/org/labkey/mgap/columnTransforms/AbstractVariantTransform.java +++ b/mGAP/src/org/labkey/mgap/columnTransforms/AbstractVariantTransform.java @@ -1,6 +1,5 @@ package org.labkey.mgap.columnTransforms; -import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.data.Results; @@ -21,13 +20,12 @@ import org.labkey.api.query.QueryService; import org.labkey.api.util.FileUtil; import org.labkey.api.util.PageFlowUtil; +import org.labkey.mgap.etl.EtlQueueManager; import org.labkey.mgap.mGAPManager; import java.io.File; -import java.io.IOException; import java.net.URI; import java.sql.SQLException; -import java.util.Arrays; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -107,7 +105,7 @@ protected Integer getOrCreateOutputFile(Object dataFileUrl, Object folderName, S { if (dataFileUrl == null) { - throw new IllegalArgumentException("DataFileUrl was null."); + throw new IllegalArgumentException("DataFileUrl was null"); } URI uri = new URI(String.valueOf(dataFileUrl)); @@ -215,20 +213,8 @@ protected File doFileCopy(File f, File subdir, String name) throws PipelineJobEx if (doCopy) { - getStatusLogger().info("copying file locally: " + localCopy.getPath()); - if (localCopy.exists()) - { - localCopy.delete(); - } - - try - { - FileUtils.copyFile(f, localCopy); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } + getStatusLogger().info("queueing file copy: " + localCopy.getPath()); + EtlQueueManager.get().queueFileCopy(getContainerUser().getContainer(), f, localCopy); } File index = new File(f.getPath() + ".tbi"); @@ -243,15 +229,8 @@ protected File doFileCopy(File f, File subdir, String name) throws PipelineJobEx if (!indexLocal.exists()) { - getStatusLogger().info("copying index locally: " + indexLocal.getPath()); - try - { - FileUtils.copyFile(index, indexLocal); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } + getStatusLogger().info("queueing copy of index: " + indexLocal.getPath()); + EtlQueueManager.get().queueFileCopy(getContainerUser().getContainer(), index, indexLocal); } } diff --git a/mGAP/src/org/labkey/mgap/columnTransforms/ExpDataTransform.java b/mGAP/src/org/labkey/mgap/columnTransforms/ExpDataTransform.java deleted file mode 100644 index 0c6a66ba0..000000000 --- a/mGAP/src/org/labkey/mgap/columnTransforms/ExpDataTransform.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.labkey.mgap.columnTransforms; - -import org.labkey.api.di.columnTransform.ColumnTransform; -import org.labkey.api.exp.api.DataType; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExperimentService; - -import java.io.File; -import java.net.URI; -import java.net.URISyntaxException; - -/** - * Created by bimber on 5/1/2017. - */ -public class ExpDataTransform extends ColumnTransform -{ - @Override - protected Object doTransform(Object inputValue) - { - if (null == inputValue) - return null; - - try - { - URI uri = new URI(String.valueOf(inputValue)); - File f = new File(uri); - if (!f.exists()) - { - getStatusLogger().error("File not found: " + uri); - } - - ExpData d = ExperimentService.get().getExpDataByURL(String.valueOf(inputValue), getContainerUser().getContainer()); - if (d == null) - { - d = ExperimentService.get().createData(getContainerUser().getContainer(), new DataType("Variant Catalog")); - d.setDataFileURI(uri); - d.setName(f.getName()); - d.save(getContainerUser().getUser()); - } - - return d.getRowId(); - } - catch (URISyntaxException e) - { - getStatusLogger().error("Error syncing file: " + inputValue, e); - } - - return null; - } -} diff --git a/mGAP/src/org/labkey/mgap/columnTransforms/LuceneIndexTransform.java b/mGAP/src/org/labkey/mgap/columnTransforms/LuceneIndexTransform.java index 14d4f8c03..35e9787db 100644 --- a/mGAP/src/org/labkey/mgap/columnTransforms/LuceneIndexTransform.java +++ b/mGAP/src/org/labkey/mgap/columnTransforms/LuceneIndexTransform.java @@ -4,6 +4,7 @@ import org.jetbrains.annotations.Nullable; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper; +import org.labkey.mgap.etl.EtlQueueManager; import java.io.File; import java.io.IOException; @@ -26,12 +27,7 @@ protected File doFileCopy(File f, File subdir, @Nullable String name) throws Pip // NOTE: lucene is a special case since the DB tracks one file, but we need this whole folder: File sourceDir = f.getParentFile(); File targetDir = new File(subdir, "LuceneIndex"); - - // NOTE: rsync should no-op if there are no source changes - getStatusLogger().info("Copying lucene index dir to: " + targetDir.getPath()); - new SimpleScriptWrapper(getStatusLogger()).execute(Arrays.asList( - "rsync", "-r", "-a", "--delete", "--no-owner", "--no-group", "--chmod=D2770,F660", sourceDir.getPath(), targetDir.getPath() - )); + EtlQueueManager.get().queueRsyncCopy(getContainerUser().getContainer(), sourceDir, targetDir); return new File(targetDir, sourceDir.getName() + "/" + f.getName()); } diff --git a/mGAP/src/org/labkey/mgap/etl/ClearEtlWorkQueueTask.java b/mGAP/src/org/labkey/mgap/etl/ClearEtlWorkQueueTask.java new file mode 100644 index 000000000..4ee9f0e1d --- /dev/null +++ b/mGAP/src/org/labkey/mgap/etl/ClearEtlWorkQueueTask.java @@ -0,0 +1,43 @@ +package org.labkey.mgap.etl; + +import org.apache.xmlbeans.XmlException; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.di.TaskRefTask; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.pipeline.RecordedActionSet; +import org.labkey.api.writer.ContainerUser; + +import java.util.List; +import java.util.Map; + +public class ClearEtlWorkQueueTask implements TaskRefTask +{ + private ContainerUser _containerUser = null; + + @Override + public RecordedActionSet run(@NotNull PipelineJob job) throws PipelineJobException + { + EtlQueueManager.get().clearQueue(_containerUser.getContainer(), job.getLogger()); + + return new RecordedActionSet(); + } + + @Override + public List getRequiredSettings() + { + return null; + } + + @Override + public void setSettings(Map settings) throws XmlException + { + + } + + @Override + public void setContainerUser(ContainerUser containerUser) + { + _containerUser = containerUser; + } +} diff --git a/mGAP/src/org/labkey/mgap/etl/EtlQueueManager.java b/mGAP/src/org/labkey/mgap/etl/EtlQueueManager.java new file mode 100644 index 000000000..950454383 --- /dev/null +++ b/mGAP/src/org/labkey/mgap/etl/EtlQueueManager.java @@ -0,0 +1,122 @@ +package org.labkey.mgap.etl; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.logging.log4j.Logger; +import org.labkey.api.data.Container; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class EtlQueueManager +{ + private static EtlQueueManager _instance = new EtlQueueManager(); + + private Map>> _pendingFileCopy = new HashMap<>(); + + private Map>> _pendingRsyncCopy = new HashMap<>(); + + private EtlQueueManager() + { + + } + + public static EtlQueueManager get() + { + return _instance; + } + + public void clearQueue(Container container, Logger log) + { + if (_pendingFileCopy.containsKey(container) && !_pendingFileCopy.get(container).isEmpty()) + { + log.error("The file copy queue was not empty!"); + } + + if (_pendingRsyncCopy.containsKey(container) && !_pendingRsyncCopy.get(container).isEmpty()) + { + log.error("The rsync copy queue was not empty!"); + } + + _pendingFileCopy.clear(); + _pendingRsyncCopy.clear(); + } + + public void performQueuedWork(Container container, Logger log) + { + List> queue = _pendingFileCopy.get(container); + if (queue != null && !queue.isEmpty()) + { + queue.forEach(x -> copyFile(x.getLeft(), x.getRight(), log)); + } + _pendingFileCopy.clear(); + + List> rsyncQueue = _pendingRsyncCopy.get(container); + if (rsyncQueue != null && !rsyncQueue.isEmpty()) + { + rsyncQueue.forEach(x -> doRsyncCopy(x.getLeft(), x.getRight(), log)); + } + _pendingRsyncCopy.clear(); + } + + public void queueFileCopy(Container c, File source, File destination) + { + if (!_pendingFileCopy.containsKey(c)) + { + _pendingFileCopy.put(c, new ArrayList<>()); + } + + _pendingFileCopy.get(c).add(Pair.of(source, destination)); + } + + public void queueRsyncCopy(Container c, File source, File destination) + { + if (!_pendingRsyncCopy.containsKey(c)) + { + _pendingRsyncCopy.put(c, new ArrayList<>()); + } + + _pendingRsyncCopy.get(c).add(Pair.of(source, destination)); + } + + private void doRsyncCopy(File sourceDir, File destination, Logger log) + { + // NOTE: rsync should no-op if there are no source changes + log.info("Performing rsync from: " + sourceDir.getPath() + " to " + destination.getPath()); + try + { + new SimpleScriptWrapper(log).execute(Arrays.asList( + "rsync", "-r", "-a", "--delete", "--no-owner", "--no-group", "--chmod=D2770,F660", sourceDir.getPath(), destination.getPath() + )); + } + catch (PipelineJobException e) + { + log.error("Error running rsync", e); + } + } + + private void copyFile(File source, File destination, Logger log) + { + if (destination.exists()) + { + destination.delete(); + } + + try + { + log.info("Copying file: " + source.getPath()); + FileUtils.copyFile(source, destination); + } + catch (IOException e) + { + log.error("Error copying file", e); + } + } +} diff --git a/mGAP/src/org/labkey/mgap/etl/PerformQueuedEtlWorkTask.java b/mGAP/src/org/labkey/mgap/etl/PerformQueuedEtlWorkTask.java new file mode 100644 index 000000000..712b53970 --- /dev/null +++ b/mGAP/src/org/labkey/mgap/etl/PerformQueuedEtlWorkTask.java @@ -0,0 +1,42 @@ +package org.labkey.mgap.etl; + +import org.apache.xmlbeans.XmlException; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.di.TaskRefTask; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.pipeline.RecordedActionSet; +import org.labkey.api.writer.ContainerUser; + +import java.util.List; +import java.util.Map; + +public class PerformQueuedEtlWorkTask implements TaskRefTask +{ + private ContainerUser _containerUser = null; + + @Override + public RecordedActionSet run(@NotNull PipelineJob job) throws PipelineJobException + { + EtlQueueManager.get().performQueuedWork(_containerUser.getContainer(), job.getLogger()); + return new RecordedActionSet(); + } + + @Override + public List getRequiredSettings() + { + return null; + } + + @Override + public void setSettings(Map settings) throws XmlException + { + + } + + @Override + public void setContainerUser(ContainerUser containerUser) + { + _containerUser = containerUser; + } +} From f3dc78d155b96839a4f2d3a6178fe545fb957314 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 29 Apr 2024 09:29:08 -0700 Subject: [PATCH 09/16] Make ETL step IDs unique --- mGAP/resources/etls/prime-seq.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mGAP/resources/etls/prime-seq.xml b/mGAP/resources/etls/prime-seq.xml index 3f798eb5a..8b2bcecc9 100644 --- a/mGAP/resources/etls/prime-seq.xml +++ b/mGAP/resources/etls/prime-seq.xml @@ -198,7 +198,7 @@ - + From 6873487fe2f6e023b6f05f15cb7b38d91b6796b7 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 6 May 2024 09:12:36 -0700 Subject: [PATCH 10/16] Update MCC landing text --- mcc/resources/views/begin.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mcc/resources/views/begin.html b/mcc/resources/views/begin.html index bc85b030f..14d820175 100644 --- a/mcc/resources/views/begin.html +++ b/mcc/resources/views/begin.html @@ -7,7 +7,7 @@ return; Ext4.get(webpart.wrapperDivId).update( - 'This is the future home of the BRAIN Initiative Marmoset Coordinating Center. This site is currently under active development - please come back soon!' + + 'This is the home of the BRAIN Initiative Marmoset Coordinating Center. Please use the links below to access the main website features:' + '

' + '' + '
' + From 9923ae3e79d9467c50862cbf1c7e8af6c2fdad7e Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 6 May 2024 13:06:34 -0700 Subject: [PATCH 11/16] Add code to cache the first page of each mGAP JBrowse session --- .../mgap/etl/PerformQueuedEtlWorkTask.java | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/mGAP/src/org/labkey/mgap/etl/PerformQueuedEtlWorkTask.java b/mGAP/src/org/labkey/mgap/etl/PerformQueuedEtlWorkTask.java index 712b53970..d1534bd33 100644 --- a/mGAP/src/org/labkey/mgap/etl/PerformQueuedEtlWorkTask.java +++ b/mGAP/src/org/labkey/mgap/etl/PerformQueuedEtlWorkTask.java @@ -2,11 +2,20 @@ import org.apache.xmlbeans.XmlException; import org.jetbrains.annotations.NotNull; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; import org.labkey.api.di.TaskRefTask; +import org.labkey.api.jbrowse.JBrowseService; import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.pipeline.RecordedActionSet; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.UserSchema; +import org.labkey.api.util.PageFlowUtil; import org.labkey.api.writer.ContainerUser; +import org.labkey.mgap.mGAPSchema; import java.util.List; import java.util.Map; @@ -19,6 +28,26 @@ public class PerformQueuedEtlWorkTask implements TaskRefTask public RecordedActionSet run(@NotNull PipelineJob job) throws PipelineJobException { EtlQueueManager.get().performQueuedWork(_containerUser.getContainer(), job.getLogger()); + + // clear/warm caches: + final TableInfo dbm = QueryService.get().getUserSchema(_containerUser.getUser(), _containerUser.getContainer(), "jbrowse").getTable("database_members"); + new TableSelector(QueryService.get().getUserSchema(_containerUser.getUser(), _containerUser.getContainer(), mGAPSchema.NAME).getTable(mGAPSchema.TABLE_VARIANT_CATALOG_RELEASES), PageFlowUtil.set("objectId", "jbrowseId")).forEachResults(rs -> { + String jbrowseId = rs.getString(FieldKey.fromString("jbrowseId")); + List trackIds = new TableSelector(dbm, PageFlowUtil.set("objectid"), new SimpleFilter(FieldKey.fromString("database"), jbrowseId).addCondition(FieldKey.fromString("jsonfile/outputfile/name"), "mGAP Release"), null).getArrayList(String.class); + if (trackIds.isEmpty()) + { + job.getLogger().error("No mGAP Release track found for session: " + jbrowseId); + } + else if (trackIds.size() > 1) + { + job.getLogger().error("More than one mGAP Release track found for session: " + jbrowseId); + } + else + { + JBrowseService.get().cacheDefaultQuery(_containerUser.getUser(), rs.getString(FieldKey.fromString("objectId")), trackIds.get(0)); + } + }); + return new RecordedActionSet(); } From 28dcb33d060a85144d7d8dd34eb39ff65475243c Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 7 May 2024 06:51:45 -0700 Subject: [PATCH 12/16] Revert "Add code to cache the first page of each mGAP JBrowse session" This reverts commit 9923ae3e79d9467c50862cbf1c7e8af6c2fdad7e. --- .../mgap/etl/PerformQueuedEtlWorkTask.java | 29 ------------------- 1 file changed, 29 deletions(-) diff --git a/mGAP/src/org/labkey/mgap/etl/PerformQueuedEtlWorkTask.java b/mGAP/src/org/labkey/mgap/etl/PerformQueuedEtlWorkTask.java index d1534bd33..712b53970 100644 --- a/mGAP/src/org/labkey/mgap/etl/PerformQueuedEtlWorkTask.java +++ b/mGAP/src/org/labkey/mgap/etl/PerformQueuedEtlWorkTask.java @@ -2,20 +2,11 @@ import org.apache.xmlbeans.XmlException; import org.jetbrains.annotations.NotNull; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; import org.labkey.api.di.TaskRefTask; -import org.labkey.api.jbrowse.JBrowseService; import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.pipeline.RecordedActionSet; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.UserSchema; -import org.labkey.api.util.PageFlowUtil; import org.labkey.api.writer.ContainerUser; -import org.labkey.mgap.mGAPSchema; import java.util.List; import java.util.Map; @@ -28,26 +19,6 @@ public class PerformQueuedEtlWorkTask implements TaskRefTask public RecordedActionSet run(@NotNull PipelineJob job) throws PipelineJobException { EtlQueueManager.get().performQueuedWork(_containerUser.getContainer(), job.getLogger()); - - // clear/warm caches: - final TableInfo dbm = QueryService.get().getUserSchema(_containerUser.getUser(), _containerUser.getContainer(), "jbrowse").getTable("database_members"); - new TableSelector(QueryService.get().getUserSchema(_containerUser.getUser(), _containerUser.getContainer(), mGAPSchema.NAME).getTable(mGAPSchema.TABLE_VARIANT_CATALOG_RELEASES), PageFlowUtil.set("objectId", "jbrowseId")).forEachResults(rs -> { - String jbrowseId = rs.getString(FieldKey.fromString("jbrowseId")); - List trackIds = new TableSelector(dbm, PageFlowUtil.set("objectid"), new SimpleFilter(FieldKey.fromString("database"), jbrowseId).addCondition(FieldKey.fromString("jsonfile/outputfile/name"), "mGAP Release"), null).getArrayList(String.class); - if (trackIds.isEmpty()) - { - job.getLogger().error("No mGAP Release track found for session: " + jbrowseId); - } - else if (trackIds.size() > 1) - { - job.getLogger().error("More than one mGAP Release track found for session: " + jbrowseId); - } - else - { - JBrowseService.get().cacheDefaultQuery(_containerUser.getUser(), rs.getString(FieldKey.fromString("objectId")), trackIds.get(0)); - } - }); - return new RecordedActionSet(); } From f2f3b8fbcb20f6f9350ee2c4e1a8490429d25293 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 7 May 2024 12:54:47 -0700 Subject: [PATCH 13/16] Add ETL to sync data from MCC --- primeseq/resources/etls/mcc.xml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 primeseq/resources/etls/mcc.xml diff --git a/primeseq/resources/etls/mcc.xml b/primeseq/resources/etls/mcc.xml new file mode 100644 index 000000000..2277dbbc5 --- /dev/null +++ b/primeseq/resources/etls/mcc.xml @@ -0,0 +1,32 @@ + + + MCC_Data + MCC Demographics and Pedigree Data + + + Copy to target + + + Id + dam + sire + gender + + + + + + + + + + + + + + + + + + + From 349f77f282d170e988aeccc7a0844968880b256b Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 9 May 2024 11:44:39 -0700 Subject: [PATCH 14/16] Debug IndexVariantsForMgapStep --- .../pipeline/IndexVariantsForMgapStep.java | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/mGAP/src/org/labkey/mgap/pipeline/IndexVariantsForMgapStep.java b/mGAP/src/org/labkey/mgap/pipeline/IndexVariantsForMgapStep.java index 90e6dfec0..6f8b3d83c 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/IndexVariantsForMgapStep.java +++ b/mGAP/src/org/labkey/mgap/pipeline/IndexVariantsForMgapStep.java @@ -118,17 +118,33 @@ public void complete(PipelineJob job, List inputs, List 1) + { + throw new IllegalStateException("More than one row found matching: " + releaseVersion); + } + job.getLogger().info("Updating release record"); - Map row = ts.getValueMap(); + Map row = ts.getMap(); + if (!row.containsKey("rowid") || row.get("rowid") == null) + { + job.getLogger().error("Missing rowId, found: "); + for (String key : row.keySet()) + { + job.getLogger().debug(key + ": " + row.get(key)); + } + + throw new IllegalStateException("Missing rowId from release record"); + } + row.put("luceneIndex", of.get(0).getRowid()); try { BatchValidationException bve = new BatchValidationException(); - Map oldKeys = Map.of("rowId", row.get("rowId")); + Map oldKeys = Map.of("rowId", row.get("rowid")); ti.getUpdateService().updateRows(job.getUser(), target, Collections.singletonList(row), Collections.singletonList(oldKeys), bve, null, null); } catch (BatchValidationException | InvalidKeyException | QueryUpdateServiceException | SQLException e) From e870f7d308f7edc4e4840c03075c495dda90dce8 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 9 May 2024 15:16:14 -0700 Subject: [PATCH 15/16] Use CaseInsensitiveHashMap for IndexVariantsForMgapStep --- .../org/labkey/mgap/pipeline/IndexVariantsForMgapStep.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mGAP/src/org/labkey/mgap/pipeline/IndexVariantsForMgapStep.java b/mGAP/src/org/labkey/mgap/pipeline/IndexVariantsForMgapStep.java index 6f8b3d83c..204bfbf79 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/IndexVariantsForMgapStep.java +++ b/mGAP/src/org/labkey/mgap/pipeline/IndexVariantsForMgapStep.java @@ -3,6 +3,7 @@ import htsjdk.samtools.util.Interval; import org.apache.commons.lang3.StringUtils; import org.json.JSONObject; +import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.data.Container; import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.TableInfo; @@ -127,7 +128,7 @@ public void complete(PipelineJob job, List inputs, List row = ts.getMap(); + Map row = new CaseInsensitiveHashMap<>(ts.getMap()); if (!row.containsKey("rowid") || row.get("rowid") == null) { job.getLogger().error("Missing rowId, found: "); @@ -144,7 +145,7 @@ public void complete(PipelineJob job, List inputs, List oldKeys = Map.of("rowId", row.get("rowid")); + Map oldKeys = new CaseInsensitiveHashMap<>(Map.of("rowid", row.get("rowid"))); ti.getUpdateService().updateRows(job.getUser(), target, Collections.singletonList(row), Collections.singletonList(oldKeys), bve, null, null); } catch (BatchValidationException | InvalidKeyException | QueryUpdateServiceException | SQLException e) From 097c254a4bb386b3e76f61c2a29ea1255ecbaed8 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 10 May 2024 06:25:34 -0700 Subject: [PATCH 16/16] Reduce length of description field --- mGAP/src/org/labkey/mgap/pipeline/IndexVariantsForMgapStep.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mGAP/src/org/labkey/mgap/pipeline/IndexVariantsForMgapStep.java b/mGAP/src/org/labkey/mgap/pipeline/IndexVariantsForMgapStep.java index 204bfbf79..0644a0e3f 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/IndexVariantsForMgapStep.java +++ b/mGAP/src/org/labkey/mgap/pipeline/IndexVariantsForMgapStep.java @@ -101,7 +101,7 @@ public Output processVariants(File inputVCF, File outputDirectory, ReferenceGeno throw new PipelineJobException("Unable to find file: " + idx.getPath()); } - output.addSequenceOutput(idx, "mGAP Lucene Index: " + releaseVersion, CATEGORY, null, null, genome.getGenomeId(), "Fields indexed: " + infoFieldsRaw); + output.addSequenceOutput(idx, "mGAP Lucene Index: " + releaseVersion, CATEGORY, null, null, genome.getGenomeId(), "Fields indexed: " + infoFields.size()); return output; }