From b6b2358cbe62a103f947c63d111369ab7d83abe9 Mon Sep 17 00:00:00 2001 From: Juan Calero Date: Wed, 17 Jan 2024 13:46:45 +0100 Subject: [PATCH 001/174] Initial commit Ultrabasic connector that does nothing, but compile and launch --- .gitignore | 11 + README.md | 15 ++ base/build.gradle.kts | 34 ++++ build.gradle.kts | 60 ++++++ gradle.properties | 1 + gradle/libs.versions.toml | 26 +++ gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 63721 bytes gradle/wrapper/gradle-wrapper.properties | 7 + gradlew | 249 +++++++++++++++++++++++ gradlew.bat | 92 +++++++++ settings.gradle.kts | 31 +++ 11 files changed, 526 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 base/build.gradle.kts create mode 100644 build.gradle.kts create mode 100644 gradle.properties create mode 100644 gradle/libs.versions.toml create mode 100644 gradle/wrapper/gradle-wrapper.jar create mode 100644 gradle/wrapper/gradle-wrapper.properties create mode 100755 gradlew create mode 100644 gradlew.bat create mode 100644 settings.gradle.kts diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1c93825 --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +# Ignore Gradle project-specific cache directory +.gradle + +# Ignore Gradle build output directory +build + +.idea +.vs +.vscode + +.env diff --git a/README.md b/README.md new file mode 100644 index 0000000..00efc1e --- /dev/null +++ b/README.md @@ -0,0 +1,15 @@ +# InesData Connector + +Dataspaces connector for the InesData project, based on the EDC dataspaces framework + +## Launching the connector + +- Compile: + ``` + gradle base:build + ``` + +- Launch: + ``` + java -jar base/build/libs/connector-app.jar + ``` diff --git a/base/build.gradle.kts b/base/build.gradle.kts new file mode 100644 index 0000000..d7d5628 --- /dev/null +++ b/base/build.gradle.kts @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2020, 2021 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial API and implementation + * Fraunhofer Institute for Software and Systems Engineering - added dependencies + * + */ + +plugins { + `java-library` + id("application") + alias(libs.plugins.shadow) +} + +dependencies { + implementation(libs.edc.boot) + implementation(libs.edc.connector.core) +} + +application { + mainClass.set("org.eclipse.edc.boot.system.runtime.BaseRuntime") +} + +tasks.withType { + mergeServiceFiles() + archiveFileName.set("connector-app.jar") +} diff --git a/build.gradle.kts b/build.gradle.kts new file mode 100644 index 0000000..83279cf --- /dev/null +++ b/build.gradle.kts @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2022 Fraunhofer Institute for Software and Systems Engineering + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation + * + */ + + +plugins { + `java-library` +} + +repositories { + mavenCentral() +} + +buildscript { + dependencies { + classpath(libs.edc.build.plugin) + } +} + +val edcVersion = libs.versions.edc + +allprojects { + apply(plugin = "$group.edc-build") + + // configure which version of the annotation processor to use. defaults to the same version as the plugin + configure { + processorVersion.set(edcVersion) + outputDirectory.set(project.buildDir) + } + + configure { + versions { + // override default dependency versions here + metaModel.set(edcVersion) + } + publish.set(false) + } + + configure { + configFile = rootProject.file("resources/edc-checkstyle-config.xml") + configDirectory.set(rootProject.file("resources")) + } + + // EdcRuntimeExtension uses this to determine the runtime classpath of the module to run. + tasks.register("printClasspath") { + doLast { + println(sourceSets["main"].runtimeClasspath.asPath) + } + } +} \ No newline at end of file diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 0000000..858fd3f --- /dev/null +++ b/gradle.properties @@ -0,0 +1 @@ +group=org.eclipse.edc diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml new file mode 100644 index 0000000..60b609f --- /dev/null +++ b/gradle/libs.versions.toml @@ -0,0 +1,26 @@ +[metadata] +format.version = "1.1" + +[versions] +assertj = "3.25.1" +edc = "0.4.1" +junit-pioneer = "2.2.0" +jupiter = "5.10.1" +rsApi = "3.1.0" + +[libraries] +assertj = { module = "org.assertj:assertj-core", version.ref = "assertj" } +edc-api-core = { module = "org.eclipse.edc:api-core", version.ref = "edc" } +edc-boot = { module = "org.eclipse.edc:boot", version.ref = "edc" } +edc-build-plugin = { module = "org.eclipse.edc.edc-build:org.eclipse.edc.edc-build.gradle.plugin", version.ref = "edc" } +edc-configuration-filesystem = { module = "org.eclipse.edc:configuration-filesystem", version.ref = "edc" } +edc-connector-core = { module = "org.eclipse.edc:connector-core", version.ref = "edc" } +edc-http = { module = "org.eclipse.edc:http", version.ref = "edc" } +jakarta-rsApi = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version.ref = "rsApi" } +junit-jupiter-api = { module = "org.junit.jupiter:junit-jupiter-api", version.ref = "jupiter" } +junit-jupiter-engine = { module = "org.junit.jupiter:junit-jupiter-engine", version.ref = "jupiter" } +junit-jupiter-params = { module = "org.junit.jupiter:junit-jupiter-params", version.ref = "jupiter" } +junit-pioneer = { module = "org.junit-pioneer:junit-pioneer", version.ref = "junit-pioneer" } + +[plugins] +shadow = { id = "com.github.johnrengelman.shadow", version = "8.1.1" } diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..7f93135c49b765f8051ef9d0a6055ff8e46073d8 GIT binary patch literal 63721 zcmb5Wb9gP!wgnp7wrv|bwr$&XvSZt}Z6`anZSUAlc9NHKf9JdJ;NJVr`=eI(_pMp0 zy1VAAG3FfAOI`{X1O)&90s;U4K;XLp008~hCjbEC_fbYfS%6kTR+JtXK>nW$ZR+`W ze|#J8f4A@M|F5BpfUJb5h>|j$jOe}0oE!`Zf6fM>CR?!y@zU(cL8NsKk`a z6tx5mAkdjD;J=LcJ;;Aw8p!v#ouk>mUDZF@ zK>yvw%+bKu+T{Nk@LZ;zkYy0HBKw06_IWcMHo*0HKpTsEFZhn5qCHH9j z)|XpN&{`!0a>Vl+PmdQc)Yg4A(AG-z!+@Q#eHr&g<9D?7E)_aEB?s_rx>UE9TUq|? z;(ggJt>9l?C|zoO@5)tu?EV0x_7T17q4fF-q3{yZ^ipUbKcRZ4Qftd!xO(#UGhb2y>?*@{xq%`(-`2T^vc=#< zx!+@4pRdk&*1ht2OWk^Z5IAQ0YTAXLkL{(D*$gENaD)7A%^XXrCchN&z2x+*>o2FwPFjWpeaL=!tzv#JOW#( z$B)Nel<+$bkH1KZv3&-}=SiG~w2sbDbAWarg%5>YbC|}*d9hBjBkR(@tyM0T)FO$# zPtRXukGPnOd)~z=?avu+4Co@wF}1T)-uh5jI<1$HLtyDrVak{gw`mcH@Q-@wg{v^c zRzu}hMKFHV<8w}o*yg6p@Sq%=gkd~;`_VGTS?L@yVu`xuGy+dH6YOwcP6ZE`_0rK% zAx5!FjDuss`FQ3eF|mhrWkjux(Pny^k$u_)dyCSEbAsecHsq#8B3n3kDU(zW5yE|( zgc>sFQywFj5}U*qtF9Y(bi*;>B7WJykcAXF86@)z|0-Vm@jt!EPoLA6>r)?@DIobIZ5Sx zsc@OC{b|3%vaMbyeM|O^UxEYlEMHK4r)V-{r)_yz`w1*xV0|lh-LQOP`OP`Pk1aW( z8DSlGN>Ts|n*xj+%If~+E_BxK)~5T#w6Q1WEKt{!Xtbd`J;`2a>8boRo;7u2M&iOop4qcy<)z023=oghSFV zST;?S;ye+dRQe>ygiJ6HCv4;~3DHtJ({fWeE~$H@mKn@Oh6Z(_sO>01JwH5oA4nvK zr5Sr^g+LC zLt(i&ecdmqsIJGNOSUyUpglvhhrY8lGkzO=0USEKNL%8zHshS>Qziu|`eyWP^5xL4 zRP122_dCJl>hZc~?58w~>`P_s18VoU|7(|Eit0-lZRgLTZKNq5{k zE?V=`7=R&ro(X%LTS*f+#H-mGo_j3dm@F_krAYegDLk6UV{`UKE;{YSsn$ z(yz{v1@p|p!0>g04!eRSrSVb>MQYPr8_MA|MpoGzqyd*$@4j|)cD_%^Hrd>SorF>@ zBX+V<@vEB5PRLGR(uP9&U&5=(HVc?6B58NJT_igiAH*q~Wb`dDZpJSKfy5#Aag4IX zj~uv74EQ_Q_1qaXWI!7Vf@ZrdUhZFE;L&P_Xr8l@GMkhc#=plV0+g(ki>+7fO%?Jb zl+bTy7q{w^pTb{>(Xf2q1BVdq?#f=!geqssXp z4pMu*q;iiHmA*IjOj4`4S&|8@gSw*^{|PT}Aw~}ZXU`6=vZB=GGeMm}V6W46|pU&58~P+?LUs%n@J}CSrICkeng6YJ^M? zS(W?K4nOtoBe4tvBXs@@`i?4G$S2W&;$z8VBSM;Mn9 zxcaEiQ9=vS|bIJ>*tf9AH~m&U%2+Dim<)E=}KORp+cZ^!@wI`h1NVBXu{@%hB2Cq(dXx_aQ9x3mr*fwL5!ZryQqi|KFJuzvP zK1)nrKZ7U+B{1ZmJub?4)Ln^J6k!i0t~VO#=q1{?T)%OV?MN}k5M{}vjyZu#M0_*u z8jwZKJ#Df~1jcLXZL7bnCEhB6IzQZ-GcoQJ!16I*39iazoVGugcKA{lhiHg4Ta2fD zk1Utyc5%QzZ$s3;p0N+N8VX{sd!~l*Ta3|t>lhI&G`sr6L~G5Lul`>m z{!^INm?J|&7X=;{XveF!(b*=?9NAp4y&r&N3(GKcW4rS(Ejk|Lzs1PrxPI_owB-`H zg3(Rruh^&)`TKA6+_!n>RdI6pw>Vt1_j&+bKIaMTYLiqhZ#y_=J8`TK{Jd<7l9&sY z^^`hmi7^14s16B6)1O;vJWOF$=$B5ONW;;2&|pUvJlmeUS&F;DbSHCrEb0QBDR|my zIs+pE0Y^`qJTyH-_mP=)Y+u^LHcuZhsM3+P||?+W#V!_6E-8boP#R-*na4!o-Q1 zVthtYhK{mDhF(&7Okzo9dTi03X(AE{8cH$JIg%MEQca`S zy@8{Fjft~~BdzWC(di#X{ny;!yYGK9b@=b|zcKZ{vv4D8i+`ilOPl;PJl{!&5-0!w z^fOl#|}vVg%=n)@_e1BrP)`A zKPgs`O0EO}Y2KWLuo`iGaKu1k#YR6BMySxQf2V++Wo{6EHmK>A~Q5o73yM z-RbxC7Qdh0Cz!nG+7BRZE>~FLI-?&W_rJUl-8FDIaXoNBL)@1hwKa^wOr1($*5h~T zF;%f^%<$p8Y_yu(JEg=c_O!aZ#)Gjh$n(hfJAp$C2he555W5zdrBqjFmo|VY+el;o z=*D_w|GXG|p0**hQ7~9-n|y5k%B}TAF0iarDM!q-jYbR^us(>&y;n^2l0C%@2B}KM zyeRT9)oMt97Agvc4sEKUEy%MpXr2vz*lb zh*L}}iG>-pqDRw7ud{=FvTD?}xjD)w{`KzjNom-$jS^;iw0+7nXSnt1R@G|VqoRhE%12nm+PH?9`(4rM0kfrZzIK9JU=^$YNyLvAIoxl#Q)xxDz!^0@zZ zSCs$nfcxK_vRYM34O<1}QHZ|hp4`ioX3x8(UV(FU$J@o%tw3t4k1QPmlEpZa2IujG&(roX_q*%e`Hq|);0;@k z0z=fZiFckp#JzW0p+2A+D$PC~IsakhJJkG(c;CqAgFfU0Z`u$PzG~-9I1oPHrCw&)@s^Dc~^)#HPW0Ra}J^=|h7Fs*<8|b13ZzG6MP*Q1dkoZ6&A^!}|hbjM{2HpqlSXv_UUg1U4gn z3Q)2VjU^ti1myodv+tjhSZp%D978m~p& z43uZUrraHs80Mq&vcetqfQpQP?m!CFj)44t8Z}k`E798wxg&~aCm+DBoI+nKq}&j^ zlPY3W$)K;KtEajks1`G?-@me7C>{PiiBu+41#yU_c(dITaqE?IQ(DBu+c^Ux!>pCj zLC|HJGU*v+!it1(;3e`6igkH(VA)-S+k(*yqxMgUah3$@C zz`7hEM47xr>j8^g`%*f=6S5n>z%Bt_Fg{Tvmr+MIsCx=0gsu_sF`q2hlkEmisz#Fy zj_0;zUWr;Gz}$BS%Y`meb(=$d%@Crs(OoJ|}m#<7=-A~PQbyN$x%2iXP2@e*nO0b7AwfH8cCUa*Wfu@b)D_>I*%uE4O3 z(lfnB`-Xf*LfC)E}e?%X2kK7DItK6Tf<+M^mX0Ijf_!IP>7c8IZX%8_#0060P{QMuV^B9i<^E`_Qf0pv9(P%_s8D`qvDE9LK9u-jB}J2S`(mCO&XHTS04Z5Ez*vl^T%!^$~EH8M-UdwhegL>3IQ*)(MtuH2Xt1p!fS4o~*rR?WLxlA!sjc2(O znjJn~wQ!Fp9s2e^IWP1C<4%sFF}T4omr}7+4asciyo3DntTgWIzhQpQirM$9{EbQd z3jz9vS@{aOqTQHI|l#aUV@2Q^Wko4T0T04Me4!2nsdrA8QY1%fnAYb~d2GDz@lAtfcHq(P7 zaMBAGo}+NcE-K*@9y;Vt3*(aCaMKXBB*BJcD_Qnxpt75r?GeAQ}*|>pYJE=uZb73 zC>sv)18)q#EGrTG6io*}JLuB_jP3AU1Uiu$D7r|2_zlIGb9 zjhst#ni)Y`$)!fc#reM*$~iaYoz~_Cy7J3ZTiPm)E?%`fbk`3Tu-F#`{i!l5pNEn5 zO-Tw-=TojYhzT{J=?SZj=Z8#|eoF>434b-DXiUsignxXNaR3 zm_}4iWU$gt2Mw5NvZ5(VpF`?X*f2UZDs1TEa1oZCif?Jdgr{>O~7}-$|BZ7I(IKW`{f;@|IZFX*R8&iT= zoWstN8&R;}@2Ka%d3vrLtR|O??ben;k8QbS-WB0VgiCz;<$pBmIZdN!aalyCSEm)crpS9dcD^Y@XT1a3+zpi-`D}e#HV<} z$Y(G&o~PvL-xSVD5D?JqF3?B9rxGWeb=oEGJ3vRp5xfBPlngh1O$yI95EL+T8{GC@ z98i1H9KhZGFl|;`)_=QpM6H?eDPpw~^(aFQWwyXZ8_EEE4#@QeT_URray*mEOGsGc z6|sdXtq!hVZo=d#+9^@lm&L5|q&-GDCyUx#YQiccq;spOBe3V+VKdjJA=IL=Zn%P} zNk=_8u}VhzFf{UYZV0`lUwcD&)9AFx0@Fc6LD9A6Rd1=ga>Mi0)_QxM2ddCVRmZ0d z+J=uXc(?5JLX3=)e)Jm$HS2yF`44IKhwRnm2*669_J=2LlwuF5$1tAo@ROSU@-y+;Foy2IEl2^V1N;fk~YR z?&EP8#t&m0B=?aJeuz~lHjAzRBX>&x=A;gIvb>MD{XEV zV%l-+9N-)i;YH%nKP?>f`=?#`>B(`*t`aiPLoQM(a6(qs4p5KFjDBN?8JGrf3z8>= zi7sD)c)Nm~x{e<^jy4nTx${P~cwz_*a>%0_;ULou3kHCAD7EYkw@l$8TN#LO9jC( z1BeFW`k+bu5e8Ns^a8dPcjEVHM;r6UX+cN=Uy7HU)j-myRU0wHd$A1fNI~`4;I~`zC)3ul#8#^rXVSO*m}Ag>c%_;nj=Nv$rCZ z*~L@C@OZg%Q^m)lc-kcX&a*a5`y&DaRxh6O*dfhLfF+fU5wKs(1v*!TkZidw*)YBP za@r`3+^IHRFeO%!ai%rxy;R;;V^Fr=OJlpBX;(b*3+SIw}7= zIq$*Thr(Zft-RlY)D3e8V;BmD&HOfX+E$H#Y@B3?UL5L~_fA-@*IB-!gItK7PIgG9 zgWuGZK_nuZjHVT_Fv(XxtU%)58;W39vzTI2n&)&4Dmq7&JX6G>XFaAR{7_3QB6zsT z?$L8c*WdN~nZGiscY%5KljQARN;`w$gho=p006z;n(qIQ*Zu<``TMO3n0{ARL@gYh zoRwS*|Niw~cR!?hE{m*y@F`1)vx-JRfqET=dJ5_(076st(=lFfjtKHoYg`k3oNmo_ zNbQEw8&sO5jAYmkD|Zaz_yUb0rC})U!rCHOl}JhbYIDLzLvrZVw0~JO`d*6f;X&?V=#T@ND*cv^I;`sFeq4 z##H5;gpZTb^0Hz@3C*~u0AqqNZ-r%rN3KD~%Gw`0XsIq$(^MEb<~H(2*5G^<2(*aI z%7}WB+TRlMIrEK#s0 z93xn*Ohb=kWFc)BNHG4I(~RPn-R8#0lqyBBz5OM6o5|>x9LK@%HaM}}Y5goCQRt2C z{j*2TtT4ne!Z}vh89mjwiSXG=%DURar~=kGNNaO_+Nkb+tRi~Rkf!7a$*QlavziD( z83s4GmQ^Wf*0Bd04f#0HX@ua_d8 z23~z*53ePD6@xwZ(vdl0DLc=>cPIOPOdca&MyR^jhhKrdQO?_jJh`xV3GKz&2lvP8 zEOwW6L*ufvK;TN{=S&R@pzV^U=QNk^Ec}5H z+2~JvEVA{`uMAr)?Kf|aW>33`)UL@bnfIUQc~L;TsTQ6>r-<^rB8uoNOJ>HWgqMI8 zSW}pZmp_;z_2O5_RD|fGyTxaxk53Hg_3Khc<8AUzV|ZeK{fp|Ne933=1&_^Dbv5^u zB9n=*)k*tjHDRJ@$bp9mrh}qFn*s}npMl5BMDC%Hs0M0g-hW~P*3CNG06G!MOPEQ_ zi}Qs-6M8aMt;sL$vlmVBR^+Ry<64jrm1EI1%#j?c?4b*7>)a{aDw#TfTYKq+SjEFA z(aJ&z_0?0JB83D-i3Vh+o|XV4UP+YJ$9Boid2^M2en@APw&wx7vU~t$r2V`F|7Qfo z>WKgI@eNBZ-+Og<{u2ZiG%>YvH2L3fNpV9J;WLJoBZda)01Rn;o@){01{7E#ke(7U zHK>S#qZ(N=aoae*4X!0A{)nu0R_sKpi1{)u>GVjC+b5Jyl6#AoQ-1_3UDovNSo`T> z?c-@7XX*2GMy?k?{g)7?Sv;SJkmxYPJPs!&QqB12ejq`Lee^-cDveVWL^CTUldb(G zjDGe(O4P=S{4fF=#~oAu>LG>wrU^z_?3yt24FOx>}{^lCGh8?vtvY$^hbZ)9I0E3r3NOlb9I?F-Yc=r$*~l`4N^xzlV~N zl~#oc>U)Yjl0BxV>O*Kr@lKT{Z09OXt2GlvE38nfs+DD7exl|&vT;)>VFXJVZp9Np zDK}aO;R3~ag$X*|hRVY3OPax|PG`@_ESc8E!mHRByJbZQRS38V2F__7MW~sgh!a>98Q2%lUNFO=^xU52|?D=IK#QjwBky-C>zOWlsiiM&1n z;!&1((Xn1$9K}xabq~222gYvx3hnZPg}VMF_GV~5ocE=-v>V=T&RsLBo&`)DOyIj* zLV{h)JU_y*7SdRtDajP_Y+rBkNN*1_TXiKwHH2&p51d(#zv~s#HwbNy?<+(=9WBvo zw2hkk2Dj%kTFhY+$T+W-b7@qD!bkfN#Z2ng@Pd=i3-i?xYfs5Z*1hO?kd7Sp^9`;Y zM2jeGg<-nJD1er@Pc_cSY7wo5dzQX44=%6rn}P_SRbpzsA{6B+!$3B0#;}qwO37G^ zL(V_5JK`XT?OHVk|{_$vQ|oNEpab*BO4F zUTNQ7RUhnRsU`TK#~`)$icsvKh~(pl=3p6m98@k3P#~upd=k*u20SNcb{l^1rUa)>qO997)pYRWMncC8A&&MHlbW?7i^7M`+B$hH~Y|J zd>FYOGQ;j>Zc2e7R{KK7)0>>nn_jYJy&o@sK!4G>-rLKM8Hv)f;hi1D2fAc$+six2 zyVZ@wZ6x|fJ!4KrpCJY=!Mq0;)X)OoS~{Lkh6u8J`eK%u0WtKh6B>GW_)PVc zl}-k`p09qwGtZ@VbYJC!>29V?Dr>>vk?)o(x?!z*9DJ||9qG-&G~#kXxbw{KKYy}J zQKa-dPt~M~E}V?PhW0R26xdA%1T*%ra6SguGu50YHngOTIv)@N|YttEXo#OZfgtP7;H?EeZZxo<}3YlYxtBq znJ!WFR^tmGf0Py}N?kZ(#=VtpC@%xJkDmfcCoBTxq zr_|5gP?u1@vJZbxPZ|G0AW4=tpb84gM2DpJU||(b8kMOV1S3|(yuwZJ&rIiFW(U;5 zUtAW`O6F6Zy+eZ1EDuP~AAHlSY-+A_eI5Gx)%*uro5tljy}kCZU*_d7)oJ>oQSZ3* zneTn`{gnNC&uJd)0aMBzAg021?YJ~b(fmkwZAd696a=0NzBAqBN54KuNDwa*no(^O z6p05bioXUR^uXjpTol*ppHp%1v9e)vkoUAUJyBx3lw0UO39b0?^{}yb!$yca(@DUn zCquRF?t=Zb9`Ed3AI6|L{eX~ijVH`VzSMheKoP7LSSf4g>md>`yi!TkoG5P>Ofp+n z(v~rW+(5L96L{vBb^g51B=(o)?%%xhvT*A5btOpw(TKh^g^4c zw>0%X!_0`{iN%RbVk+A^f{w-4-SSf*fu@FhruNL##F~sF24O~u zyYF<3el2b$$wZ_|uW#@Ak+VAGk#e|kS8nL1g>2B-SNMjMp^8;-FfeofY2fphFHO!{ z*!o4oTb{4e;S<|JEs<1_hPsmAlVNk?_5-Fp5KKU&d#FiNW~Y+pVFk@Cua1I{T+1|+ zHx6rFMor)7L)krbilqsWwy@T+g3DiH5MyVf8Wy}XbEaoFIDr~y;@r&I>FMW{ z?Q+(IgyebZ)-i4jNoXQhq4Muy9Fv+OxU;9_Jmn+<`mEC#%2Q_2bpcgzcinygNI!&^ z=V$)o2&Yz04~+&pPWWn`rrWxJ&}8khR)6B(--!9Q zubo}h+1T)>a@c)H^i``@<^j?|r4*{;tQf78(xn0g39IoZw0(CwY1f<%F>kEaJ zp9u|IeMY5mRdAlw*+gSN^5$Q)ShM<~E=(c8QM+T-Qk)FyKz#Sw0EJ*edYcuOtO#~Cx^(M7w5 z3)rl#L)rF|(Vun2LkFr!rg8Q@=r>9p>(t3Gf_auiJ2Xx9HmxYTa|=MH_SUlYL`mz9 zTTS$`%;D-|Jt}AP1&k7PcnfFNTH0A-*FmxstjBDiZX?}%u%Yq94$fUT&z6od+(Uk> zuqsld#G(b$G8tus=M!N#oPd|PVFX)?M?tCD0tS%2IGTfh}3YA3f&UM)W$_GNV8 zQo+a(ml2Km4o6O%gKTCSDNq+#zCTIQ1*`TIJh~k6Gp;htHBFnne))rlFdGqwC6dx2+La1&Mnko*352k0y z+tQcwndQlX`nc6nb$A9?<-o|r*%aWXV#=6PQic0Ok_D;q>wbv&j7cKc!w4~KF#-{6 z(S%6Za)WpGIWf7jZ3svNG5OLs0>vCL9{V7cgO%zevIVMH{WgP*^D9ws&OqA{yr|m| zKD4*07dGXshJHd#e%x%J+qmS^lS|0Bp?{drv;{@{l9ArPO&?Q5=?OO9=}h$oVe#3b z3Yofj&Cb}WC$PxmRRS)H%&$1-)z7jELS}!u!zQ?A^Y{Tv4QVt*vd@uj-^t2fYRzQj zfxGR>-q|o$3sGn^#VzZ!QQx?h9`njeJry}@x?|k0-GTTA4y3t2E`3DZ!A~D?GiJup z)8%PK2^9OVRlP(24P^4_<|D=H^7}WlWu#LgsdHzB%cPy|f8dD3|A^mh4WXxhLTVu_ z@abE{6Saz|Y{rXYPd4$tfPYo}ef(oQWZ=4Bct-=_9`#Qgp4ma$n$`tOwq#&E18$B; z@Bp)bn3&rEi0>fWWZ@7k5WazfoX`SCO4jQWwVuo+$PmSZn^Hz?O(-tW@*DGxuf)V1 zO_xm&;NVCaHD4dqt(-MlszI3F-p?0!-e$fbiCeuaw66h^TTDLWuaV<@C-`=Xe5WL) zwooG7h>4&*)p3pKMS3O!4>-4jQUN}iAMQ)2*70?hP~)TzzR?-f@?Aqy$$1Iy8VGG$ zMM?8;j!pUX7QQD$gRc_#+=raAS577ga-w?jd`vCiN5lu)dEUkkUPl9!?{$IJNxQys z*E4e$eF&n&+AMRQR2gcaFEjAy*r)G!s(P6D&TfoApMFC_*Ftx0|D0@E-=B7tezU@d zZ{hGiN;YLIoSeRS;9o%dEua4b%4R3;$SugDjP$x;Z!M!@QibuSBb)HY!3zJ7M;^jw zlx6AD50FD&p3JyP*>o+t9YWW8(7P2t!VQQ21pHJOcG_SXQD;(5aX#M6x##5H_Re>6lPyDCjxr*R(+HE%c&QN+b^tbT zXBJk?p)zhJj#I?&Y2n&~XiytG9!1ox;bw5Rbj~)7c(MFBb4>IiRATdhg zmiEFlj@S_hwYYI(ki{}&<;_7(Z0Qkfq>am z&LtL=2qc7rWguk3BtE4zL41@#S;NN*-jWw|7Kx7H7~_%7fPt;TIX}Ubo>;Rmj94V> zNB1=;-9AR7s`Pxn}t_6^3ahlq53e&!Lh85uG zec0vJY_6e`tg7LgfrJ3k!DjR)Bi#L@DHIrZ`sK=<5O0Ip!fxGf*OgGSpP@Hbbe&$9 z;ZI}8lEoC2_7;%L2=w?tb%1oL0V+=Z`7b=P&lNGY;yVBazXRYu;+cQDKvm*7NCxu&i;zub zAJh#11%?w>E2rf2e~C4+rAb-&$^vsdACs7 z@|Ra!OfVM(ke{vyiqh7puf&Yp6cd6{DptUteYfIRWG3pI+5< zBVBI_xkBAc<(pcb$!Y%dTW(b;B;2pOI-(QCsLv@U-D1XJ z(Gk8Q3l7Ws46Aktuj>|s{$6zA&xCPuXL-kB`CgYMs}4IeyG*P51IDwW?8UNQd+$i~ zlxOPtSi5L|gJcF@DwmJA5Ju8HEJ>o{{upwIpb!f{2(vLNBw`7xMbvcw<^{Fj@E~1( z?w`iIMieunS#>nXlmUcSMU+D3rX28f?s7z;X=se6bo8;5vM|O^(D6{A9*ChnGH!RG zP##3>LDC3jZPE4PH32AxrqPk|yIIrq~`aL-=}`okhNu9aT%q z1b)7iJ)CN=V#Ly84N_r7U^SH2FGdE5FpTO2 z630TF$P>GNMu8`rOytb(lB2};`;P4YNwW1<5d3Q~AX#P0aX}R2b2)`rgkp#zTxcGj zAV^cvFbhP|JgWrq_e`~exr~sIR$6p5V?o4Wym3kQ3HA+;Pr$bQ0(PmADVO%MKL!^q z?zAM8j1l4jrq|5X+V!8S*2Wl@=7*pPgciTVK6kS1Ge zMsd_u6DFK$jTnvVtE;qa+8(1sGBu~n&F%dh(&c(Zs4Fc#A=gG^^%^AyH}1^?|8quj zl@Z47h$){PlELJgYZCIHHL= z{U8O>Tw4x3<1{?$8>k-P<}1y9DmAZP_;(3Y*{Sk^H^A=_iSJ@+s5ktgwTXz_2$~W9>VVZsfwCm@s0sQ zeB50_yu@uS+e7QoPvdCwDz{prjo(AFwR%C?z`EL{1`|coJHQTk^nX=tvs1<0arUOJ z!^`*x&&BvTYmemyZ)2p~{%eYX=JVR?DYr(rNgqRMA5E1PR1Iw=prk=L2ldy3r3Vg@27IZx43+ywyzr-X*p*d@tZV+!U#~$-q=8c zgdSuh#r?b4GhEGNai)ayHQpk>5(%j5c@C1K3(W1pb~HeHpaqijJZa-e6vq_8t-^M^ zBJxq|MqZc?pjXPIH}70a5vt!IUh;l}<>VX<-Qcv^u@5(@@M2CHSe_hD$VG-eiV^V( zj7*9T0?di?P$FaD6oo?)<)QT>Npf6Og!GO^GmPV(Km0!=+dE&bk#SNI+C9RGQ|{~O*VC+tXK3!n`5 zHfl6>lwf_aEVV3`0T!aHNZLsj$paS$=LL(?b!Czaa5bbSuZ6#$_@LK<(7yrrl+80| z{tOFd=|ta2Z`^ssozD9BINn45NxUeCQis?-BKmU*Kt=FY-NJ+)8S1ecuFtN-M?&42 zl2$G>u!iNhAk*HoJ^4v^9#ORYp5t^wDj6|lx~5w45#E5wVqI1JQ~9l?nPp1YINf++ zMAdSif~_ETv@Er(EFBI^@L4BULFW>)NI+ejHFP*T}UhWNN`I)RRS8za? z*@`1>9ZB}An%aT5K=_2iQmfE;GcBVHLF!$`I99o5GO`O%O_zLr9AG18>&^HkG(;=V z%}c!OBQ~?MX(9h~tajX{=x)+!cbM7$YzTlmsPOdp2L-?GoW`@{lY9U3f;OUo*BwRB z8A+nv(br0-SH#VxGy#ZrgnGD(=@;HME;yd46EgWJ`EL%oXc&lFpc@Y}^>G(W>h_v_ zlN!`idhX+OjL+~T?19sroAFVGfa5tX-D49w$1g2g_-T|EpHL6}K_aX4$K=LTvwtlF zL*z}j{f+Uoe7{-px3_5iKPA<_7W=>Izkk)!l9ez2w%vi(?Y;i8AxRNLSOGDzNoqoI zP!1uAl}r=_871(G?y`i&)-7{u=%nxk7CZ_Qh#!|ITec zwQn`33GTUM`;D2POWnkqngqJhJRlM>CTONzTG}>^Q0wUunQyn|TAiHzyX2_%ATx%P z%7gW)%4rA9^)M<_%k@`Y?RbC<29sWU&5;@|9thf2#zf8z12$hRcZ!CSb>kUp=4N#y zl3hE#y6>kkA8VY2`W`g5Ip?2qC_BY$>R`iGQLhz2-S>x(RuWv)SPaGdl^)gGw7tjR zH@;jwk!jIaCgSg_*9iF|a);sRUTq30(8I(obh^|}S~}P4U^BIGYqcz;MPpC~Y@k_m zaw4WG1_vz2GdCAX!$_a%GHK**@IrHSkGoN>)e}>yzUTm52on`hYot7cB=oA-h1u|R ztH$11t?54Qg2L+i33FPFKKRm1aOjKST{l1*(nps`>sv%VqeVMWjl5+Gh+9);hIP8? zA@$?}Sc z3qIRpba+y5yf{R6G(u8Z^vkg0Fu&D-7?1s=QZU`Ub{-!Y`I?AGf1VNuc^L3v>)>i# z{DV9W$)>34wnzAXUiV^ZpYKw>UElrN_5Xj6{r_3| z$X5PK`e5$7>~9Dj7gK5ash(dvs`vwfk}&RD`>04;j62zoXESkFBklYaKm5seyiX(P zqQ-;XxlV*yg?Dhlx%xt!b0N3GHp@(p$A;8|%# zZ5m2KL|{on4nr>2_s9Yh=r5ScQ0;aMF)G$-9-Ca6%wA`Pa)i?NGFA|#Yi?{X-4ZO_ z^}%7%vkzvUHa$-^Y#aA+aiR5sa%S|Ebyn`EV<3Pc?ax_f>@sBZF1S;7y$CXd5t5=WGsTKBk8$OfH4v|0?0I=Yp}7c=WBSCg!{0n)XmiU;lfx)**zZaYqmDJelxk$)nZyx5`x$6R|fz(;u zEje5Dtm|a%zK!!tk3{i9$I2b{vXNFy%Bf{50X!x{98+BsDr_u9i>G5%*sqEX|06J0 z^IY{UcEbj6LDwuMh7cH`H@9sVt1l1#8kEQ(LyT@&+K}(ReE`ux8gb0r6L_#bDUo^P z3Ka2lRo52Hdtl_%+pwVs14=q`{d^L58PsU@AMf(hENumaxM{7iAT5sYmWh@hQCO^ zK&}ijo=`VqZ#a3vE?`7QW0ZREL17ZvDfdqKGD?0D4fg{7v%|Yj&_jcKJAB)>=*RS* zto8p6@k%;&^ZF>hvXm&$PCuEp{uqw3VPG$9VMdW5$w-fy2CNNT>E;>ejBgy-m_6`& z97L1p{%srn@O_JQgFpa_#f(_)eb#YS>o>q3(*uB;uZb605(iqM$=NK{nHY=+X2*G) zO3-_Xh%aG}fHWe*==58zBwp%&`mge<8uq8;xIxOd=P%9EK!34^E9sk|(Zq1QSz-JVeP12Fp)-`F|KY$LPwUE?rku zY@OJ)Z9A!ojfzfeyJ9;zv2EM7ZQB)AR5xGa-tMn^bl)FmoIiVyJ@!~@%{}qXXD&Ns zPnfe5U+&ohKefILu_1mPfLGuapX@btta5C#gPB2cjk5m4T}Nfi+Vfka!Yd(L?-c~5 z#ZK4VeQEXNPc4r$K00Fg>g#_W!YZ)cJ?JTS<&68_$#cZT-ME`}tcwqg3#``3M3UPvn+pi}(VNNx6y zFIMVb6OwYU(2`at$gHba*qrMVUl8xk5z-z~fb@Q3Y_+aXuEKH}L+>eW__!IAd@V}L zkw#s%H0v2k5-=vh$^vPCuAi22Luu3uKTf6fPo?*nvj$9(u)4$6tvF-%IM+3pt*cgs z_?wW}J7VAA{_~!?))?s6{M=KPpVhg4fNuU*|3THp@_(q!b*hdl{fjRVFWtu^1dV(f z6iOux9hi&+UK=|%M*~|aqFK{Urfl!TA}UWY#`w(0P!KMe1Si{8|o))Gy6d7;!JQYhgMYmXl?3FfOM2nQGN@~Ap6(G z3+d_5y@=nkpKAhRqf{qQ~k7Z$v&l&@m7Ppt#FSNzKPZM z8LhihcE6i=<(#87E|Wr~HKvVWhkll4iSK$^mUHaxgy8*K$_Zj;zJ`L$naPj+^3zTi z-3NTaaKnD5FPY-~?Tq6QHnmDDRxu0mh0D|zD~Y=vv_qig5r-cIbCpxlju&8Sya)@{ zsmv6XUSi)@(?PvItkiZEeN*)AE~I_?#+Ja-r8$(XiXei2d@Hi7Rx8+rZZb?ZLa{;@*EHeRQ-YDadz~M*YCM4&F-r;E#M+@CSJMJ0oU|PQ^ z=E!HBJDMQ2TN*Y(Ag(ynAL8%^v;=~q?s4plA_hig&5Z0x_^Oab!T)@6kRN$)qEJ6E zNuQjg|G7iwU(N8pI@_6==0CL;lRh1dQF#wePhmu@hADFd3B5KIH#dx(2A zp~K&;Xw}F_N6CU~0)QpQk7s$a+LcTOj1%=WXI(U=Dv!6 z{#<#-)2+gCyyv=Jw?Ab#PVkxPDeH|sAxyG`|Ys}A$PW4TdBv%zDz z^?lwrxWR<%Vzc8Sgt|?FL6ej_*e&rhqJZ3Y>k=X(^dytycR;XDU16}Pc9Vn0>_@H+ zQ;a`GSMEG64=JRAOg%~L)x*w{2re6DVprNp+FcNra4VdNjiaF0M^*>CdPkt(m150rCue?FVdL0nFL$V%5y6N z%eLr5%YN7D06k5ji5*p4v$UMM)G??Q%RB27IvH7vYr_^3>1D-M66#MN8tWGw>WED} z5AhlsanO=STFYFs)Il_0i)l)f<8qn|$DW7ZXhf5xI;m+7M5-%P63XFQrG9>DMqHc} zsgNU9nR`b}E^mL5=@7<1_R~j@q_2U^3h|+`7YH-?C=vme1C3m`Fe0HC>pjt6f_XMh zy~-i-8R46QNYneL4t@)<0VU7({aUO?aH`z4V2+kxgH5pYD5)wCh75JqQY)jIPN=U6 z+qi8cGiOtXG2tXm;_CfpH9ESCz#i5B(42}rBJJF$jh<1sbpj^8&L;gzGHb8M{of+} zzF^8VgML2O9nxBW7AvdEt90vp+#kZxWf@A)o9f9}vKJy9NDBjBW zSt=Hcs=YWCwnfY1UYx*+msp{g!w0HC<_SM!VL1(I2PE?CS}r(eh?{I)mQixmo5^p# zV?2R!R@3GV6hwTCrfHiK#3Orj>I!GS2kYhk1S;aFBD_}u2v;0HYFq}Iz1Z(I4oca4 zxquja8$+8JW_EagDHf$a1OTk5S97umGSDaj)gH=fLs9>_=XvVj^Xj9a#gLdk=&3tl zfmK9MNnIX9v{?%xdw7568 zNrZ|roYs(vC4pHB5RJ8>)^*OuyNC>x7ad)tB_}3SgQ96+-JT^Qi<`xi=)_=$Skwv~ zdqeT9Pa`LYvCAn&rMa2aCDV(TMI#PA5g#RtV|CWpgDYRA^|55LLN^uNh*gOU>Z=a06qJ;$C9z8;n-Pq=qZnc1zUwJ@t)L;&NN+E5m zRkQ(SeM8=l-aoAKGKD>!@?mWTW&~)uF2PYUJ;tB^my`r9n|Ly~0c%diYzqs9W#FTjy?h&X3TnH zXqA{QI82sdjPO->f=^K^f>N`+B`q9&rN0bOXO79S&a9XX8zund(kW7O76f4dcWhIu zER`XSMSFbSL>b;Rp#`CuGJ&p$s~G|76){d?xSA5wVg##_O0DrmyEYppyBr%fyWbbv zp`K84JwRNP$d-pJ!Qk|(RMr?*!wi1if-9G#0p>>1QXKXWFy)eB3ai)l3601q8!9JC zvU#ZWWDNKq9g6fYs?JQ)Q4C_cgTy3FhgKb8s&m)DdmL5zhNK#8wWg!J*7G7Qhe9VU zha?^AQTDpYcuN!B+#1dE*X{<#!M%zfUQbj=zLE{dW0XeQ7-oIsGY6RbkP2re@Q{}r_$iiH0xU%iN*ST`A)-EH6eaZB$GA#v)cLi z*MpA(3bYk$oBDKAzu^kJoSUsDd|856DApz={3u8sbQV@JnRkp2nC|)m;#T=DvIL-O zI4vh;g7824l}*`_p@MT4+d`JZ2%6NQh=N9bmgJ#q!hK@_<`HQq3}Z8Ij>3%~<*= zcv=!oT#5xmeGI92lqm9sGVE%#X$ls;St|F#u!?5Y7syhx6q#MVRa&lBmmn%$C0QzU z);*ldgwwCmzM3uglr}!Z2G+?& zf%Dpo&mD%2ZcNFiN-Z0f;c_Q;A%f@>26f?{d1kxIJD}LxsQkB47SAdwinfMILZdN3 zfj^HmTzS3Ku5BxY>ANutS8WPQ-G>v4^_Qndy==P3pDm+Xc?>rUHl-4+^%Sp5atOja z2oP}ftw-rqnb}+khR3CrRg^ibi6?QYk1*i^;kQGirQ=uB9Sd1NTfT-Rbv;hqnY4neE5H1YUrjS2m+2&@uXiAo- zrKUX|Ohg7(6F(AoP~tj;NZlV#xsfo-5reuQHB$&EIAhyZk;bL;k9ouDmJNBAun;H& zn;Of1z_Qj`x&M;5X;{s~iGzBQTY^kv-k{ksbE*Dl%Qf%N@hQCfY~iUw!=F-*$cpf2 z3wix|aLBV0b;W@z^%7S{>9Z^T^fLOI68_;l@+Qzaxo`nAI8emTV@rRhEKZ z?*z_{oGdI~R*#<2{bkz$G~^Qef}$*4OYTgtL$e9q!FY7EqxJ2`zk6SQc}M(k(_MaV zSLJnTXw&@djco1~a(vhBl^&w=$fa9{Sru>7g8SHahv$&Bl(D@(Zwxo_3r=;VH|uc5 zi1Ny)J!<(KN-EcQ(xlw%PNwK8U>4$9nVOhj(y0l9X^vP1TA>r_7WtSExIOsz`nDOP zs}d>Vxb2Vo2e5x8p(n~Y5ggAyvib>d)6?)|E@{FIz?G3PVGLf7-;BxaP;c?7ddH$z zA+{~k^V=bZuXafOv!RPsE1GrR3J2TH9uB=Z67gok+u`V#}BR86hB1xl}H4v`F+mRfr zYhortD%@IGfh!JB(NUNSDh+qDz?4ztEgCz&bIG-Wg7w-ua4ChgQR_c+z8dT3<1?uX z*G(DKy_LTl*Ea!%v!RhpCXW1WJO6F`bgS-SB;Xw9#! z<*K}=#wVu9$`Yo|e!z-CPYH!nj7s9dEPr-E`DXUBu0n!xX~&|%#G=BeM?X@shQQMf zMvr2!y7p_gD5-!Lnm|a@z8Of^EKboZsTMk%5VsJEm>VsJ4W7Kv{<|#4f-qDE$D-W>gWT%z-!qXnDHhOvLk=?^a1*|0j z{pW{M0{#1VcR5;F!!fIlLVNh_Gj zbnW(_j?0c2q$EHIi@fSMR{OUKBcLr{Y&$hrM8XhPByyZaXy|dd&{hYQRJ9@Fn%h3p7*VQolBIV@Eq`=y%5BU~3RPa^$a?ixp^cCg z+}Q*X+CW9~TL29@OOng(#OAOd!)e$d%sr}^KBJ-?-X&|4HTmtemxmp?cT3uA?md4% zT8yZ0U;6Rg6JHy3fJae{6TMGS?ZUX6+gGTT{Q{)SI85$5FD{g-eR%O0KMpWPY`4@O zx!hen1*8^E(*}{m^V_?}(b5k3hYo=T+$&M32+B`}81~KKZhY;2H{7O-M@vbCzuX0n zW-&HXeyr1%I3$@ns-V1~Lb@wIpkmx|8I~ob1Of7i6BTNysEwI}=!nU%q7(V_^+d*G z7G;07m(CRTJup!`cdYi93r^+LY+`M*>aMuHJm(A8_O8C#A*$!Xvddgpjx5)?_EB*q zgE8o5O>e~9IiSC@WtZpF{4Bj2J5eZ>uUzY%TgWF7wdDE!fSQIAWCP)V{;HsU3ap?4 znRsiiDbtN7i9hapO;(|Ew>Ip2TZSvK9Z^N21%J?OiA_&eP1{(Pu_=%JjKy|HOardq ze?zK^K zA%sjF64*Wufad%H<) z^|t>e*h+Z1#l=5wHexzt9HNDNXgM=-OPWKd^5p!~%SIl>Fo&7BvNpbf8{NXmH)o{r zO=aBJ;meX1^{O%q;kqdw*5k!Y7%t_30 zy{nGRVc&5qt?dBwLs+^Sfp;f`YVMSB#C>z^a9@fpZ!xb|b-JEz1LBX7ci)V@W+kvQ89KWA0T~Lj$aCcfW#nD5bt&Y_< z-q{4ZXDqVg?|0o)j1%l0^_it0WF*LCn-+)c!2y5yS7aZIN$>0LqNnkujV*YVes(v$ zY@_-!Q;!ZyJ}Bg|G-~w@or&u0RO?vlt5*9~yeoPV_UWrO2J54b4#{D(D>jF(R88u2 zo#B^@iF_%S>{iXSol8jpmsZuJ?+;epg>k=$d`?GSegAVp3n$`GVDvK${N*#L_1`44 z{w0fL{2%)0|E+qgZtjX}itZz^KJt4Y;*8uSK}Ft38+3>j|K(PxIXXR-t4VopXo#9# zt|F{LWr-?34y`$nLBVV_*UEgA6AUI65dYIbqpNq9cl&uLJ0~L}<=ESlOm?Y-S@L*d z<7vt}`)TW#f%Rp$Q}6@3=j$7Tze@_uZO@aMn<|si{?S}~maII`VTjs&?}jQ4_cut9$)PEqMukwoXobzaKx^MV z2fQwl+;LSZ$qy%Tys0oo^K=jOw$!YwCv^ei4NBVauL)tN%=wz9M{uf{IB(BxK|lT*pFkmNK_1tV`nb%jH=a0~VNq2RCKY(rG7jz!-D^k)Ec)yS%17pE#o6&eY+ z^qN(hQT$}5F(=4lgNQhlxj?nB4N6ntUY6(?+R#B?W3hY_a*)hnr4PA|vJ<6p`K3Z5Hy z{{8(|ux~NLUW=!?9Qe&WXMTAkQnLXg(g=I@(VG3{HE13OaUT|DljyWXPs2FE@?`iU z4GQlM&Q=T<4&v@Fe<+TuXiZQT3G~vZ&^POfmI1K2h6t4eD}Gk5XFGpbj1n_g*{qmD6Xy z`6Vv|lLZtLmrnv*{Q%xxtcWVj3K4M%$bdBk_a&ar{{GWyu#ljM;dII;*jP;QH z#+^o-A4np{@|Mz+LphTD0`FTyxYq#wY)*&Ls5o{0z9yg2K+K7ZN>j1>N&;r+Z`vI| zDzG1LJZ+sE?m?>x{5LJx^)g&pGEpY=fQ-4}{x=ru;}FL$inHemOg%|R*ZXPodU}Kh zFEd5#+8rGq$Y<_?k-}r5zgQ3jRV=ooHiF|@z_#D4pKVEmn5CGV(9VKCyG|sT9nc=U zEoT67R`C->KY8Wp-fEcjjFm^;Cg(ls|*ABVHq8clBE(;~K^b+S>6uj70g? z&{XQ5U&!Z$SO7zfP+y^8XBbiu*Cv-yJG|l-oe*!s5$@Lh_KpxYL2sx`B|V=dETN>5K+C+CU~a_3cI8{vbu$TNVdGf15*>D zz@f{zIlorkY>TRh7mKuAlN9A0>N>SV`X)+bEHms=mfYTMWt_AJtz_h+JMmrgH?mZt zm=lfdF`t^J*XLg7v+iS)XZROygK=CS@CvUaJo&w2W!Wb@aa?~Drtf`JV^cCMjngVZ zv&xaIBEo8EYWuML+vxCpjjY^s1-ahXJzAV6hTw%ZIy!FjI}aJ+{rE&u#>rs)vzuxz z+$5z=7W?zH2>Eb32dvgHYZtCAf!=OLY-pb4>Ae79rd68E2LkVPj-|jFeyqtBCCwiW zkB@kO_(3wFq)7qwV}bA=zD!*@UhT`geq}ITo%@O(Z5Y80nEX~;0-8kO{oB6|(4fQh z);73T!>3@{ZobPwRv*W?7m0Ml9GmJBCJd&6E?hdj9lV= z4flNfsc(J*DyPv?RCOx!MSvk(M952PJ-G|JeVxWVjN~SNS6n-_Ge3Q;TGE;EQvZg86%wZ`MB zSMQua(i*R8a75!6$QRO^(o7sGoomb+Y{OMy;m~Oa`;P9Yqo>?bJAhqXxLr7_3g_n>f#UVtxG!^F#1+y@os6x(sg z^28bsQ@8rw%Gxk-stAEPRbv^}5sLe=VMbkc@Jjimqjvmd!3E7+QnL>|(^3!R} zD-l1l7*Amu@j+PWLGHXXaFG0Ct2Q=}5YNUxEQHCAU7gA$sSC<5OGylNnQUa>>l%sM zyu}z6i&({U@x^hln**o6r2s-(C-L50tQvz|zHTqW!ir?w&V23tuYEDJVV#5pE|OJu z7^R!A$iM$YCe?8n67l*J-okwfZ+ZTkGvZ)tVPfR;|3gyFjF)8V zyXXN=!*bpyRg9#~Bg1+UDYCt0 ztp4&?t1X0q>uz;ann$OrZs{5*r`(oNvw=$7O#rD|Wuv*wIi)4b zGtq4%BX+kkagv3F9Id6~-c+1&?zny%w5j&nk9SQfo0k4LhdSU_kWGW7axkfpgR`8* z!?UTG*Zi_baA1^0eda8S|@&F z{)Rad0kiLjB|=}XFJhD(S3ssKlveFFmkN{Vl^_nb!o5M!RC=m)V&v2%e?ZoRC@h3> zJ(?pvToFd`*Zc@HFPL#=otWKwtuuQ_dT-Hr{S%pQX<6dqVJ8;f(o)4~VM_kEQkMR+ zs1SCVi~k>M`u1u2xc}>#D!V&6nOOh-E$O&SzYrjJdZpaDv1!R-QGA141WjQe2s0J~ zQ;AXG)F+K#K8_5HVqRoRM%^EduqOnS(j2)|ctA6Q^=|s_WJYU;Z%5bHp08HPL`YF2 zR)Ad1z{zh`=sDs^&V}J z%$Z$!jd7BY5AkT?j`eqMs%!Gm@T8)4w3GYEX~IwgE~`d|@T{WYHkudy(47brgHXx& zBL1yFG6!!!VOSmDxBpefy2{L_u5yTwja&HA!mYA#wg#bc-m%~8aRR|~AvMnind@zs zy>wkShe5&*un^zvSOdlVu%kHsEo>@puMQ`b1}(|)l~E{5)f7gC=E$fP(FC2=F<^|A zxeIm?{EE!3sO!Gr7e{w)Dx(uU#3WrFZ>ibmKSQ1tY?*-Nh1TDHLe+k*;{Rp!Bmd_m zb#^kh`Y*8l|9Cz2e{;RL%_lg{#^Ar+NH|3z*Zye>!alpt{z;4dFAw^^H!6ING*EFc z_yqhr8d!;%nHX9AKhFQZBGrSzfzYCi%C!(Q5*~hX>)0N`vbhZ@N|i;_972WSx*>LH z87?en(;2_`{_JHF`Sv6Wlps;dCcj+8IJ8ca6`DsOQCMb3n# z3)_w%FuJ3>fjeOOtWyq)ag|PmgQbC-s}KRHG~enBcIwqIiGW8R8jFeBNY9|YswRY5 zjGUxdGgUD26wOpwM#8a!Nuqg68*dG@VM~SbOroL_On0N6QdT9?)NeB3@0FCC?Z|E0 z6TPZj(AsPtwCw>*{eDEE}Gby>0q{*lI+g2e&(YQrsY&uGM{O~}(oM@YWmb*F zA0^rr5~UD^qmNljq$F#ARXRZ1igP`MQx4aS6*MS;Ot(1L5jF2NJ;de!NujUYg$dr# z=TEL_zTj2@>ZZN(NYCeVX2==~=aT)R30gETO{G&GM4XN<+!&W&(WcDP%oL8PyIVUC zs5AvMgh6qr-2?^unB@mXK*Dbil^y-GTC+>&N5HkzXtozVf93m~xOUHn8`HpX=$_v2 z61H;Z1qK9o;>->tb8y%#4H)765W4E>TQ1o0PFj)uTOPEvv&}%(_mG0ISmyhnQV33Z$#&yd{ zc{>8V8XK$3u8}04CmAQ#I@XvtmB*s4t8va?-IY4@CN>;)mLb_4!&P3XSw4pA_NzDb zORn!blT-aHk1%Jpi>T~oGLuh{DB)JIGZ9KOsciWs2N7mM1JWM+lna4vkDL?Q)z_Ct z`!mi0jtr+4*L&N7jk&LodVO#6?_qRGVaucqVB8*us6i3BTa^^EI0x%EREQSXV@f!lak6Wf1cNZ8>*artIJ(ADO*=<-an`3zB4d*oO*8D1K!f z*A@P1bZCNtU=p!742MrAj%&5v%Xp_dSX@4YCw%F|%Dk=u|1BOmo)HsVz)nD5USa zR~??e61sO(;PR)iaxK{M%QM_rIua9C^4ppVS$qCT9j2%?*em?`4Z;4@>I(c%M&#cH z>4}*;ej<4cKkbCAjjDsyKS8rIm90O)Jjgyxj5^venBx&7B!xLmzxW3jhj7sR(^3Fz z84EY|p1NauwXUr;FfZjdaAfh%ivyp+^!jBjJuAaKa!yCq=?T_)R!>16?{~p)FQ3LDoMyG%hL#pR!f@P%*;#90rs_y z@9}@r1BmM-SJ#DeuqCQk=J?ixDSwL*wh|G#us;dd{H}3*-Y7Tv5m=bQJMcH+_S`zVtf;!0kt*(zwJ zs+kedTm!A}cMiM!qv(c$o5K%}Yd0|nOd0iLjus&;s0Acvoi-PFrWm?+q9f^FslxGi z6ywB`QpL$rJzWDg(4)C4+!2cLE}UPCTBLa*_=c#*$b2PWrRN46$y~yST3a2$7hEH= zNjux+wna^AzQ=KEa_5#9Ph=G1{S0#hh1L3hQ`@HrVnCx{!fw_a0N5xV(iPdKZ-HOM za)LdgK}1ww*C_>V7hbQnTzjURJL`S%`6nTHcgS+dB6b_;PY1FsrdE8(2K6FN>37!62j_cBlui{jO^$dPkGHV>pXvW0EiOA zqW`YaSUBWg_v^Y5tPJfWLcLpsA8T zG)!x>pKMpt!lv3&KV!-um= zKCir6`bEL_LCFx4Z5bAFXW$g3Cq`?Q%)3q0r852XI*Der*JNuKUZ`C{cCuu8R8nkt z%pnF>R$uY8L+D!V{s^9>IC+bmt<05h**>49R*#vpM*4i0qRB2uPbg8{{s#9yC;Z18 zD7|4m<9qneQ84uX|J&f-g8a|nFKFt34@Bt{CU`v(SYbbn95Q67*)_Esl_;v291s=9 z+#2F2apZU4Tq=x+?V}CjwD(P=U~d<=mfEFuyPB`Ey82V9G#Sk8H_Ob_RnP3s?)S_3 zr%}Pb?;lt_)Nf>@zX~D~TBr;-LS<1I##8z`;0ZCvI_QbXNh8Iv)$LS=*gHr;}dgb=w5$3k2la1keIm|=7<-JD>)U%=Avl0Vj@+&vxn zt-)`vJxJr88D&!}2^{GPXc^nmRf#}nb$4MMkBA21GzB`-Or`-3lq^O^svO7Vs~FdM zv`NvzyG+0T!P8l_&8gH|pzE{N(gv_tgDU7SWeiI-iHC#0Ai%Ixn4&nt{5y3(GQs)i z&uA;~_0shP$0Wh0VooIeyC|lak__#KVJfxa7*mYmZ22@(<^W}FdKjd*U1CqSjNKW% z*z$5$=t^+;Ui=MoDW~A7;)Mj%ibX1_p4gu>RC}Z_pl`U*{_z@+HN?AF{_W z?M_X@o%w8fgFIJ$fIzBeK=v#*`mtY$HC3tqw7q^GCT!P$I%=2N4FY7j9nG8aIm$c9 zeKTxVKN!UJ{#W)zxW|Q^K!3s;(*7Gbn;e@pQBCDS(I|Y0euK#dSQ_W^)sv5pa%<^o zyu}3d?Lx`)3-n5Sy9r#`I{+t6x%I%G(iewGbvor&I^{lhu-!#}*Q3^itvY(^UWXgvthH52zLy&T+B)Pw;5>4D6>74 zO_EBS)>l!zLTVkX@NDqyN2cXTwsUVao7$HcqV2%t$YzdAC&T)dwzExa3*kt9d(}al zA~M}=%2NVNUjZiO7c>04YH)sRelXJYpWSn^aC$|Ji|E13a^-v2MB!Nc*b+=KY7MCm zqIteKfNkONq}uM;PB?vvgQvfKLPMB8u5+Am=d#>g+o&Ysb>dX9EC8q?D$pJH!MTAqa=DS5$cb+;hEvjwVfF{4;M{5U&^_+r zvZdu_rildI!*|*A$TzJ&apQWV@p{!W`=?t(o0{?9y&vM)V)ycGSlI3`;ps(vf2PUq zX745#`cmT*ra7XECC0gKkpu2eyhFEUb?;4@X7weEnLjXj_F~?OzL1U1L0|s6M+kIhmi%`n5vvDALMagi4`wMc=JV{XiO+^ z?s9i7;GgrRW{Mx)d7rj)?(;|b-`iBNPqdwtt%32se@?w4<^KU&585_kZ=`Wy^oLu9 z?DQAh5z%q;UkP48jgMFHTf#mj?#z|=w= z(q6~17Vn}P)J3M?O)x))%a5+>TFW3No~TgP;f}K$#icBh;rSS+R|}l鯊%1Et zwk~hMkhq;MOw^Q5`7oC{CUUyTw9x>^%*FHx^qJw(LB+E0WBX@{Ghw;)6aA-KyYg8p z7XDveQOpEr;B4je@2~usI5BlFadedX^ma{b{ypd|RNYqo#~d*mj&y`^iojR}s%~vF z(H!u`yx68D1Tj(3(m;Q+Ma}s2n#;O~bcB1`lYk%Irx60&-nWIUBr2x&@}@76+*zJ5 ze&4?q8?m%L9c6h=J$WBzbiTf1Z-0Eb5$IZs>lvm$>1n_Mezp*qw_pr8<8$6f)5f<@ zyV#tzMCs51nTv_5ca`x`yfE5YA^*%O_H?;tWYdM_kHPubA%vy47i=9>Bq) zRQ&0UwLQHeswmB1yP)+BiR;S+Vc-5TX84KUA;8VY9}yEj0eESSO`7HQ4lO z4(CyA8y1G7_C;6kd4U3K-aNOK!sHE}KL_-^EDl(vB42P$2Km7$WGqNy=%fqB+ zSLdrlcbEH=T@W8V4(TgoXZ*G1_aq$K^@ek=TVhoKRjw;HyI&coln|uRr5mMOy2GXP zwr*F^Y|!Sjr2YQXX(Fp^*`Wk905K%$bd03R4(igl0&7IIm*#f`A!DCarW9$h$z`kYk9MjjqN&5-DsH@8xh63!fTNPxWsFQhNv z#|3RjnP$Thdb#Ys7M+v|>AHm0BVTw)EH}>x@_f4zca&3tXJhTZ8pO}aN?(dHo)44Z z_5j+YP=jMlFqwvf3lq!57-SAuRV2_gJ*wsR_!Y4Z(trO}0wmB9%f#jNDHPdQGHFR; zZXzS-$`;7DQ5vF~oSgP3bNV$6Z(rwo6W(U07b1n3UHqml>{=6&-4PALATsH@Bh^W? z)ob%oAPaiw{?9HfMzpGb)@Kys^J$CN{uf*HX?)z=g`J(uK1YO^8~s1(ZIbG%Et(|q z$D@_QqltVZu9Py4R0Ld8!U|#`5~^M=b>fnHthzKBRr=i+w@0Vr^l|W;=zFT#PJ?*a zbC}G#It}rQP^Ait^W&aa6B;+0gNvz4cWUMzpv(1gvfw-X4xJ2Sv;mt;zb2Tsn|kSS zo*U9N?I{=-;a-OybL4r;PolCfiaL=y@o9{%`>+&FI#D^uy#>)R@b^1ue&AKKwuI*` zx%+6r48EIX6nF4o;>)zhV_8(IEX})NGU6Vs(yslrx{5fII}o3SMHW7wGtK9oIO4OM&@@ECtXSICLcPXoS|{;=_yj>hh*%hP27yZwOmj4&Lh z*Nd@OMkd!aKReoqNOkp5cW*lC)&C$P?+H3*%8)6HcpBg&IhGP^77XPZpc%WKYLX$T zsSQ$|ntaVVOoRat$6lvZO(G-QM5s#N4j*|N_;8cc2v_k4n6zx9c1L4JL*83F-C1Cn zaJhd;>rHXB%%ZN=3_o3&Qd2YOxrK~&?1=UuN9QhL$~OY-Qyg&})#ez*8NpQW_*a&kD&ANjedxT0Ar z<6r{eaVz3`d~+N~vkMaV8{F?RBVemN(jD@S8qO~L{rUw#=2a$V(7rLE+kGUZ<%pdr z?$DP|Vg#gZ9S}w((O2NbxzQ^zTot=89!0^~hE{|c9q1hVzv0?YC5s42Yx($;hAp*E zyoGuRyphQY{Q2ee0Xx`1&lv(l-SeC$NEyS~8iil3_aNlnqF_G|;zt#F%1;J)jnPT& z@iU0S;wHJ2$f!juqEzPZeZkjcQ+Pa@eERSLKsWf=`{R@yv7AuRh&ALRTAy z8=g&nxsSJCe!QLchJ=}6|LshnXIK)SNd zRkJNiqHwKK{SO;N5m5wdL&qK`v|d?5<4!(FAsDxR>Ky#0#t$8XCMptvNo?|SY?d8b z`*8dVBlXTUanlh6n)!EHf2&PDG8sXNAt6~u-_1EjPI1|<=33T8 zEnA00E!`4Ave0d&VVh0e>)Dc}=FfAFxpsC1u9ATfQ`-Cu;mhc8Z>2;uyXtqpLb7(P zd2F9<3cXS} znMg?{&8_YFTGRQZEPU-XPq55%51}RJpw@LO_|)CFAt62-_!u_Uq$csc+7|3+TV_!h z+2a7Yh^5AA{q^m|=KSJL+w-EWDBc&I_I1vOr^}P8i?cKMhGy$CP0XKrQzCheG$}G# zuglf8*PAFO8%xop7KSwI8||liTaQ9NCAFarr~psQt)g*pC@9bORZ>m`_GA`_K@~&% zijH0z;T$fd;-Liw8%EKZas>BH8nYTqsK7F;>>@YsE=Rqo?_8}UO-S#|6~CAW0Oz1} z3F(1=+#wrBJh4H)9jTQ_$~@#9|Bc1Pd3rAIA_&vOpvvbgDJOM(yNPhJJq2%PCcMaI zrbe~toYzvkZYQ{ea(Wiyu#4WB#RRN%bMe=SOk!CbJZv^m?Flo5p{W8|0i3`hI3Np# zvCZqY%o258CI=SGb+A3yJe~JH^i{uU`#U#fvSC~rWTq+K`E%J@ zasU07&pB6A4w3b?d?q}2=0rA#SA7D`X+zg@&zm^iA*HVi z009#PUH<%lk4z~p^l0S{lCJk1Uxi=F4e_DwlfHA`X`rv(|JqWKAA5nH+u4Da+E_p+ zVmH@lg^n4ixs~*@gm_dgQ&eDmE1mnw5wBz9Yg?QdZwF|an67Xd*x!He)Gc8&2!urh z4_uXzbYz-aX)X1>&iUjGp;P1u8&7TID0bTH-jCL&Xk8b&;;6p2op_=y^m@Nq*0{#o!!A;wNAFG@0%Z9rHo zcJs?Th>Ny6+hI`+1XoU*ED$Yf@9f91m9Y=#N(HJP^Y@ZEYR6I?oM{>&Wq4|v0IB(p zqX#Z<_3X(&{H+{3Tr|sFy}~=bv+l=P;|sBz$wk-n^R`G3p0(p>p=5ahpaD7>r|>pm zv;V`_IR@tvZreIuv2EM7ZQHhO+qUgw#kOs%*ekY^n|=1#x9&c;Ro&I~{rG-#_3ZB1 z?|9}IFdbP}^DneP*T-JaoYHt~r@EfvnPE5EKUwIxjPbsr$% zfWW83pgWST7*B(o=kmo)74$8UU)v0{@4DI+ci&%=#90}!CZz|rnH+Mz=HN~97G3~@ z;v5(9_2%eca(9iu@J@aqaMS6*$TMw!S>H(b z4(*B!|H|8&EuB%mITr~O?vVEf%(Gr)6E=>H~1VR z&1YOXluJSG1!?TnT)_*YmJ*o_Q@om~(GdrhI{$Fsx_zrkupc#y{DK1WOUR>tk>ZE) ziOLoBkhZZ?0Uf}cm>GsA>Rd6V8@JF)J*EQlQ<=JD@m<)hyElXR0`pTku*3MU`HJn| zIf7$)RlK^pW-$87U;431;Ye4Ie+l~_B3*bH1>*yKzn23cH0u(i5pXV! z4K?{3oF7ZavmmtTq((wtml)m6i)8X6ot_mrE-QJCW}Yn!(3~aUHYG=^fA<^~`e3yc z-NWTb{gR;DOUcK#zPbN^D*e=2eR^_!(!RKkiwMW@@yYtEoOp4XjOGgzi`;=8 zi3`Ccw1%L*y(FDj=C7Ro-V?q)-%p?Ob2ZElu`eZ99n14-ZkEV#y5C+{Pq87Gu3&>g zFy~Wk7^6v*)4pF3@F@rE__k3ikx(hzN3@e*^0=KNA6|jC^B5nf(XaoQaZN?Xi}Rn3 z$8&m*KmWvPaUQ(V<#J+S&zO|8P-#!f%7G+n_%sXp9=J%Z4&9OkWXeuZN}ssgQ#Tcj z8p6ErJQJWZ+fXLCco=RN8D{W%+*kko*2-LEb))xcHwNl~Xmir>kmAxW?eW50Osw3# zki8Fl$#fvw*7rqd?%E?}ZX4`c5-R&w!Y0#EBbelVXSng+kUfeUiqofPehl}$ormli zg%r)}?%=?_pHb9`Cq9Z|B`L8b>(!+8HSX?`5+5mm81AFXfnAt1*R3F z%b2RPIacKAddx%JfQ8l{3U|vK@W7KB$CdLqn@wP^?azRks@x8z59#$Q*7q!KilY-P zHUbs(IFYRGG1{~@RF;Lqyho$~7^hNC`NL3kn^Td%A7dRgr_&`2k=t+}D-o9&C!y^? z6MsQ=tc3g0xkK(O%DzR9nbNB(r@L;1zQrs8mzx&4dz}?3KNYozOW5;=w18U6$G4U2 z#2^qRLT*Mo4bV1Oeo1PKQ2WQS2Y-hv&S|C7`xh6=Pj7MNLC5K-zokZ67S)C;(F0Dd zloDK2_o1$Fmza>EMj3X9je7e%Q`$39Dk~GoOj89-6q9|_WJlSl!!+*{R=tGp z8u|MuSwm^t7K^nUe+^0G3dkGZr3@(X+TL5eah)K^Tn zXEtHmR9UIaEYgD5Nhh(s*fcG_lh-mfy5iUF3xxpRZ0q3nZ=1qAtUa?(LnT9I&~uxX z`pV?+=|-Gl(kz?w!zIieXT}o}7@`QO>;u$Z!QB${a08_bW0_o@&9cjJUXzVyNGCm8 zm=W+$H!;_Kzp6WQqxUI;JlPY&`V}9C$8HZ^m?NvI*JT@~BM=()T()Ii#+*$y@lTZBkmMMda>7s#O(1YZR+zTG@&}!EXFG{ zEWPSDI5bFi;NT>Yj*FjH((=oe%t%xYmE~AGaOc4#9K_XsVpl<4SP@E!TgC0qpe1oi zNpxU2b0(lEMcoibQ-G^cxO?ySVW26HoBNa;n0}CWL*{k)oBu1>F18X061$SP{Gu67 z-v-Fa=Fl^u3lnGY^o5v)Bux}bNZ~ z5pL+7F_Esoun8^5>z8NFoIdb$sNS&xT8_|`GTe8zSXQzs4r^g0kZjg(b0bJvz`g<70u9Z3fQILX1Lj@;@+##bP|FAOl)U^9U>0rx zGi)M1(Hce)LAvQO-pW!MN$;#ZMX?VE(22lTlJrk#pB0FJNqVwC+*%${Gt#r_tH9I_ z;+#)#8cWAl?d@R+O+}@1A^hAR1s3UcW{G+>;X4utD2d9X(jF555}!TVN-hByV6t+A zdFR^aE@GNNgSxxixS2p=on4(+*+f<8xrwAObC)D5)4!z7)}mTpb7&ofF3u&9&wPS< zB62WHLGMhmrmOAgmJ+|c>qEWTD#jd~lHNgT0?t-p{T=~#EMcB| z=AoDKOL+qXCfk~F)-Rv**V}}gWFl>liXOl7Uec_8v)(S#av99PX1sQIVZ9eNLkhq$ zt|qu0b?GW_uo}TbU8!jYn8iJeIP)r@;!Ze_7mj{AUV$GEz6bDSDO=D!&C9!M@*S2! zfGyA|EPlXGMjkH6x7OMF?gKL7{GvGfED=Jte^p=91FpCu)#{whAMw`vSLa`K#atdN zThnL+7!ZNmP{rc=Z>%$meH;Qi1=m1E3Lq2D_O1-X5C;!I0L>zur@tPAC9*7Jeh)`;eec}1`nkRP(%iv-`N zZ@ip-g|7l6Hz%j%gcAM}6-nrC8oA$BkOTz^?dakvX?`^=ZkYh%vUE z9+&)K1UTK=ahYiaNn&G5nHUY5niLGus@p5E2@RwZufRvF{@$hW{;{3QhjvEHMvduO z#Wf-@oYU4ht?#uP{N3utVzV49mEc9>*TV_W2TVC`6+oI)zAjy$KJrr=*q##&kobiQ z1vNbya&OVjK`2pdRrM?LuK6BgrLN7H_3m z!qpNKg~87XgCwb#I=Q&0rI*l$wM!qTkXrx1ko5q-f;=R2fImRMwt5Qs{P*p^z@9ex z`2#v(qE&F%MXlHpdO#QEZyZftn4f05ab^f2vjxuFaat2}jke{j?5GrF=WYBR?gS(^ z9SBiNi}anzBDBRc+QqizTTQuJrzm^bNA~A{j%ugXP7McZqJ}65l10({wk++$=e8O{ zxWjG!Qp#5OmI#XRQQM?n6?1ztl6^D40hDJr?4$Wc&O_{*OfMfxe)V0=e{|N?J#fgE>j9jAajze$iN!*yeF%jJU#G1c@@rm zolGW!j?W6Q8pP=lkctNFdfgUMg92wlM4E$aks1??M$~WQfzzzXtS)wKrr2sJeCN4X zY(X^H_c^PzfcO8Bq(Q*p4c_v@F$Y8cHLrH$`pJ2}=#*8%JYdqsqnGqEdBQMpl!Ot04tUGSXTQdsX&GDtjbWD=prcCT9(+ z&UM%lW%Q3yrl1yiYs;LxzIy>2G}EPY6|sBhL&X&RAQrSAV4Tlh2nITR?{6xO9ujGu zr*)^E`>o!c=gT*_@6S&>0POxcXYNQd&HMw6<|#{eSute2C3{&h?Ah|cw56-AP^f8l zT^kvZY$YiH8j)sk7_=;gx)vx-PW`hbSBXJGCTkpt;ap(}G2GY=2bbjABU5)ty%G#x zAi07{Bjhv}>OD#5zh#$0w;-vvC@^}F! z#X$@)zIs1L^E;2xDAwEjaXhTBw2<{&JkF*`;c3<1U@A4MaLPe{M5DGGkL}#{cHL%* zYMG+-Fm0#qzPL#V)TvQVI|?_M>=zVJr9>(6ib*#z8q@mYKXDP`k&A4A};xMK0h=yrMp~JW{L?mE~ph&1Y1a#4%SO)@{ zK2juwynUOC)U*hVlJU17%llUxAJFuKZh3K0gU`aP)pc~bE~mM!i1mi!~LTf>1Wp< zuG+ahp^gH8g8-M$u{HUWh0m^9Rg@cQ{&DAO{PTMudV6c?ka7+AO& z746QylZ&Oj`1aqfu?l&zGtJnpEQOt;OAFq19MXTcI~`ZcoZmyMrIKDFRIDi`FH)w; z8+*8tdevMDv*VtQi|e}CnB_JWs>fhLOH-+Os2Lh!&)Oh2utl{*AwR)QVLS49iTp{6 z;|172Jl!Ml17unF+pd+Ff@jIE-{Oxv)5|pOm@CkHW?{l}b@1>Pe!l}VccX#xp@xgJ zyE<&ep$=*vT=}7vtvif0B?9xw_3Gej7mN*dOHdQPtW5kA5_zGD zpA4tV2*0E^OUimSsV#?Tg#oiQ>%4D@1F5@AHwT8Kgen$bSMHD3sXCkq8^(uo7CWk`mT zuslYq`6Yz;L%wJh$3l1%SZv#QnG3=NZ=BK4yzk#HAPbqXa92;3K5?0kn4TQ`%E%X} z&>Lbt!!QclYKd6+J7Nl@xv!uD%)*bY-;p`y^ZCC<%LEHUi$l5biu!sT3TGGSTPA21 zT8@B&a0lJHVn1I$I3I1I{W9fJAYc+8 zVj8>HvD}&O`TqU2AAb={?eT;0hyL(R{|h23=4fDSZKC32;wWxsVj`P z3J3{M$PwdH!ro*Cn!D&=jnFR>BNGR<<|I8CI@+@658Dy(lhqbhXfPTVecY@L8%`3Q z1Fux2w?2C3th60jI~%OC9BtpNF$QPqcG+Pz96qZJ71_`0o0w_q7|h&O>`6U+^BA&5 zXd5Zp1Xkw~>M%RixTm&OqpNl8Q+ue=92Op_>T~_9UON?ZM2c0aGm=^A4ejrXj3dV9 zhh_bCt-b9`uOX#cFLj!vhZ#lS8Tc47OH>*)y#{O9?AT~KR9LntM|#l#Dlm^8{nZdk zjMl#>ZM%#^nK2TPzLcKxqx24P7R1FPlBy7LSBrRvx>fE$9AJ;7{PQm~^LBX^k#6Zq zw*Z(zJC|`!6_)EFR}8|n8&&Rbj8y028~P~sFXBFRt+tmqH-S3<%N;C&WGH!f3{7cm zy_fCAb9@HqaXa1Y5vFbxWf%#zg6SI$C+Uz5=CTO}e|2fjWkZ;Dx|84Ow~bkI=LW+U zuq;KSv9VMboRvs9)}2PAO|b(JCEC_A0wq{uEj|3x@}*=bOd zwr{TgeCGG>HT<@Zeq8y}vTpwDg#UBvD)BEs@1KP$^3$sh&_joQPn{hjBXmLPJ{tC) z*HS`*2+VtJO{|e$mM^|qv1R*8i(m1`%)}g=SU#T#0KlTM2RSvYUc1fP+va|4;5}Bfz98UvDCpq7}+SMV&;nX zQw~N6qOX{P55{#LQkrZk(e5YGzr|(B;Q;ju;2a`q+S9bsEH@i1{_Y0;hWYn1-79jl z5c&bytD*k)GqrVcHn6t-7kinadiD>B{Tl`ZY@`g|b~pvHh5!gKP4({rp?D0aFd_cN zhHRo4dd5^S6ViN(>(28qZT6E>??aRhc($kP`>@<+lIKS5HdhjVU;>f7<4))E*5|g{ z&d1}D|vpuV^eRj5j|xx9nwaCxXFG?Qbjn~_WSy=N}P0W>MP zG-F%70lX5Xr$a)2i6?i|iMyM|;Jtf*hO?=Jxj12oz&>P=1#h~lf%#fc73M2_(SUM- zf&qnjS80|_Y0lDgl&I?*eMumUklLe_=Td!9G@eR*tcPOgIShJipp3{A10u(4eT~DY zHezEj8V+7m!knn7)W!-5QI3=IvC^as5+TW1@Ern@yX| z7Nn~xVx&fGSr+L%4iohtS3w^{-H1A_5=r&x8}R!YZvp<2T^YFvj8G_vm}5q;^UOJf ztl=X3iL;;^^a#`t{Ae-%5Oq{?M#s6Npj+L(n-*LMI-yMR{)qki!~{5z{&`-iL}lgW zxo+tnvICK=lImjV$Z|O_cYj_PlEYCzu-XBz&XC-JVxUh9;6*z4fuBG+H{voCC;`~GYV|hj%j_&I zDZCj>Q_0RCwFauYoVMiUSB+*Mx`tg)bWmM^SwMA+?lBg12QUF_x2b)b?qb88K-YUd z0dO}3k#QirBV<5%jL$#wlf!60dizu;tsp(7XLdI=eQs?P`tOZYMjVq&jE)qK*6B^$ zBe>VvH5TO>s>izhwJJ$<`a8fakTL!yM^Zfr2hV9`f}}VVUXK39p@G|xYRz{fTI+Yq z20d=)iwjuG9RB$%$^&8#(c0_j0t_C~^|n+c`Apu|x7~;#cS-s=X1|C*YxX3ailhg_|0`g!E&GZJEr?bh#Tpb8siR=JxWKc{#w7g zWznLwi;zLFmM1g8V5-P#RsM@iX>TK$xsWuujcsVR^7TQ@!+vCD<>Bk9tdCo7Mzgq5 zv8d>dK9x8C@Qoh01u@3h0X_`SZluTb@5o;{4{{eF!-4405x8X7hewZWpz z2qEi4UTiXTvsa(0X7kQH{3VMF>W|6;6iTrrYD2fMggFA&-CBEfSqPlQDxqsa>{e2M z(R5PJ7uOooFc|9GU0ELA%m4&4Ja#cQpNw8i8ACAoK6?-px+oBl_yKmenZut#Xumjz zk8p^OV2KY&?5MUwGrBOo?ki`Sxo#?-Q4gw*Sh0k`@ zFTaYK2;}%Zk-68`#5DXU$2#=%YL#S&MTN8bF+!J2VT6x^XBci6O)Q#JfW{YMz) zOBM>t2rSj)n#0a3cjvu}r|k3od6W(SN}V-cL?bi*Iz-8uOcCcsX0L>ZXjLqk zZu2uHq5B|Kt>e+=pPKu=1P@1r9WLgYFq_TNV1p9pu0erHGd!+bBp!qGi+~4A(RsYN@CyXNrC&hxGmW)u5m35OmWwX`I+0yByglO`}HC4nGE^_HUs^&A(uaM zKPj^=qI{&ayOq#z=p&pnx@@k&I1JI>cttJcu@Ihljt?6p^6{|ds`0MoQwp+I{3l6` zB<9S((RpLG^>=Kic`1LnhpW2=Gu!x`m~=y;A`Qk!-w`IN;S8S930#vBVMv2vCKi}u z6<-VPrU0AnE&vzwV(CFC0gnZYcpa-l5T0ZS$P6(?9AM;`Aj~XDvt;Jua=jIgF=Fm? zdp=M$>`phx%+Gu};;-&7T|B1AcC#L4@mW5SV_^1BRbo6;2PWe$r+npRV`yc;T1mo& z+~_?7rA+(Um&o@Tddl zL_hxvWk~a)yY}%j`Y+200D%9$bWHy&;(yj{jpi?Rtz{J66ANw)UyPOm;t6FzY3$hx zcn)Ir79nhFvNa7^a{SHN7XH*|Vlsx`CddPnA&Qvh8aNhEA;mPVv;Ah=k<*u!Zq^7 z<=xs*iQTQOMMcg|(NA_auh@x`3#_LFt=)}%SQppP{E>mu_LgquAWvh<>L7tf9+~rO znwUDS52u)OtY<~!d$;m9+87aO+&`#2ICl@Y>&F{jI=H(K+@3M1$rr=*H^dye#~TyD z!){#Pyfn+|ugUu}G;a~!&&0aqQ59U@UT3|_JuBlYUpT$2+11;}JBJ`{+lQN9T@QFY z5+`t;6(TS0F?OlBTE!@7D`8#URDNqx2t6`GZ{ZgXeS@v%-eJzZOHz18aS|svxII$a zZeFjrJ*$IwX$f-Rzr_G>xbu@euGl)B7pC&S+CmDJBg$BoV~jxSO#>y z33`bupN#LDoW0feZe0%q8un0rYN|eRAnwDHQ6e_)xBTbtoZtTA=Fvk){q}9Os~6mQ zKB80VI_&6iSq`LnK7*kfHZoeX6?WE}8yjuDn=2#JG$+;-TOA1%^=DnXx%w{b=w}tS zQbU3XxtOI8E(!%`64r2`zog;5<0b4i)xBmGP^jiDZ2%HNSxIf3@wKs~uk4%3Mxz;~ zts_S~E4>W+YwI<-*-$U8*^HKDEa8oLbmqGg?3vewnaNg%Mm)W=)lcC_J+1ov^u*N3 zXJ?!BrH-+wGYziJq2Y#vyry6Z>NPgkEk+Ke`^DvNRdb>Q2Nlr#v%O@<5hbflI6EKE z9dWc0-ORk^T}jP!nkJ1imyjdVX@GrjOs%cpgA8-c&FH&$(4od#x6Y&=LiJZPINVyW z0snY$8JW@>tc2}DlrD3StQmA0Twck~@>8dSix9CyQOALcREdxoM$Sw*l!}bXKq9&r zysMWR@%OY24@e`?+#xV2bk{T^C_xSo8v2ZI=lBI*l{RciPwuE>L5@uhz@{!l)rtVlWC>)6(G)1~n=Q|S!{E9~6*fdpa*n z!()-8EpTdj=zr_Lswi;#{TxbtH$8*G=UM`I+icz7sr_SdnHXrv=?iEOF1UL+*6O;% zPw>t^kbW9X@oEXx<97%lBm-9?O_7L!DeD)Me#rwE54t~UBu9VZ zl_I1tBB~>jm@bw0Aljz8! zXBB6ATG6iByKIxs!qr%pz%wgqbg(l{65DP4#v(vqhhL{0b#0C8mq`bnqZ1OwFV z7mlZZJFMACm>h9v^2J9+^_zc1=JjL#qM5ZHaThH&n zXPTsR8(+)cj&>Un{6v*z?@VTLr{TmZ@-fY%*o2G}*G}#!bmqpoo*Ay@U!JI^Q@7gj;Kg-HIrLj4}#ec4~D2~X6vo;ghep-@&yOivYP zC19L0D`jjKy1Yi-SGPAn94(768Tcf$urAf{)1)9W58P`6MA{YG%O?|07!g9(b`8PXG1B1Sh0?HQmeJtP0M$O$hI z{5G`&9XzYhh|y@qsF1GnHN|~^ru~HVf#)lOTSrv=S@DyR$UKQk zjdEPFDz{uHM&UM;=mG!xKvp;xAGHOBo~>_=WFTmh$chpC7c`~7?36h)7$fF~Ii}8q zF|YXxH-Z?d+Q+27Rs3X9S&K3N+)OBxMHn1u(vlrUC6ckBY@@jl+mgr#KQUKo#VeFm zFwNYgv0<%~Wn}KeLeD9e1$S>jhOq&(e*I@L<=I5b(?G(zpqI*WBqf|Zge0&aoDUsC zngMRA_Kt0>La+Erl=Uv_J^p(z=!?XHpenzn$%EA`JIq#yYF?JLDMYiPfM(&Csr#f{ zdd+LJL1by?xz|D8+(fgzRs~(N1k9DSyK@LJygwaYX8dZl0W!I&c^K?7)z{2is;OkE zd$VK-(uH#AUaZrp=1z;O*n=b?QJkxu`Xsw&7yrX0?(CX=I-C#T;yi8a<{E~?vr3W> zQrpPqOW2M+AnZ&p{hqmHZU-;Q(7?- zP8L|Q0RM~sB0w1w53f&Kd*y}ofx@c z5Y6B8qGel+uT1JMot$nT1!Tim6{>oZzJXdyA+4euOLME?5Fd_85Uk%#E*ln%y{u8Q z$|?|R@Hpb~yTVK-Yr_S#%NUy7EBfYGAg>b({J|5b+j-PBpPy$Ns`PaJin4JdRfOaS zE|<HjH%NuJgsd2wOlv>~y=np%=2)$M9LS|>P)zJ+Fei5vYo_N~B0XCn+GM76 z)Xz3tg*FRVFgIl9zpESgdpWAavvVViGlU8|UFY{{gVJskg*I!ZjWyk~OW-Td4(mZ6 zB&SQreAAMqwp}rjy`HsG({l2&q5Y52<@AULVAu~rWI$UbFuZs>Sc*x+XI<+ez%$U)|a^unjpiW0l0 zj1!K0(b6$8LOjzRqQ~K&dfbMIE=TF}XFAi)$+h}5SD3lo z%%Qd>p9se=VtQG{kQ;N`sI)G^u|DN#7{aoEd zkksYP%_X$Rq08);-s6o>CGJ<}v`qs%eYf+J%DQ^2k68C%nvikRsN?$ap--f+vCS`K z#&~)f7!N^;sdUXu54gl3L=LN>FB^tuK=y2e#|hWiWUls__n@L|>xH{%8lIJTd5`w? zSwZbnS;W~DawT4OwSJVdAylbY+u5S+ZH{4hAi2&}Iv~W(UvHg(1GTZRPz`@{SOqzy z(8g&Dz=$PfRV=6FgxN~zo+G8OoPI&d-thcGVR*_^(R8COTM@bq?fDwY{}WhsQS1AK zF6R1t8!RdFmfocpJ6?9Yv~;WYi~XPgs(|>{5})j!AR!voO7y9&cMPo#80A(`za@t>cx<0;qxM@S*m(jYP)dMXr*?q0E`oL;12}VAep179uEr8c<=D zr5?A*C{eJ`z9Ee;E$8)MECqatHkbHH z&Y+ho0B$31MIB-xm&;xyaFCtg<{m~M-QDbY)fQ>Q*Xibb~8ytxZQ?QMf9!%cV zU0_X1@b4d+Pg#R!`OJ~DOrQz3@cpiGy~XSKjZQQ|^4J1puvwKeScrH8o{bscBsowomu z^f12kTvje`yEI3eEXDHJ6L+O{Jv$HVj%IKb|J{IvD*l6IG8WUgDJ*UGz z3!C%>?=dlfSJ>4U88)V+`U-!9r^@AxJBx8R;)J4Fn@`~k>8>v0M9xp90OJElWP&R5 zM#v*vtT}*Gm1^)Bv!s72T3PB0yVIjJW)H7a)ilkAvoaH?)jjb`MP>2z{%Y?}83 zUIwBKn`-MSg)=?R)1Q0z3b>dHE^)D8LFs}6ASG1|daDly_^lOSy&zIIhm*HXm1?VS=_iacG);_I9c zUQH1>i#*?oPIwBMJkzi_*>HoUe}_4o>2(SHWzqQ=;TyhAHS;Enr7!#8;sdlty&(>d zl%5cjri8`2X^Ds`jnw7>A`X|bl=U8n+3LKLy(1dAu8`g@9=5iw$R0qk)w8Vh_Dt^U zIglK}sn^)W7aB(Q>HvrX=rxB z+*L)3DiqpQ_%~|m=44LcD4-bxO3OO*LPjsh%p(k?&jvLp0py57oMH|*IMa(<|{m1(0S|x)?R-mqJ=I;_YUZA>J z62v*eSK;5w!h8J+6Z2~oyGdZ68waWfy09?4fU&m7%u~zi?YPHPgK6LDwphgaYu%0j zurtw)AYOpYKgHBrkX189mlJ`q)w-f|6>IER{5Lk97%P~a-JyCRFjejW@L>n4vt6#hq;!|m;hNE||LK3nw1{bJOy+eBJjK=QqNjI;Q6;Rp5 z&035pZDUZ#%Oa;&_7x0T<7!RW`#YBOj}F380Bq?MjjEhrvlCATPdkCTTl+2efTX$k zH&0zR1n^`C3ef~^sXzJK-)52(T}uTG%OF8yDhT76L~|^+hZ2hiSM*QA9*D5odI1>& z9kV9jC~twA5MwyOx(lsGD_ggYmztXPD`2=_V|ks_FOx!_J8!zM zTzh^cc+=VNZ&(OdN=y4Juw)@8-85lwf_#VMN!Ed(eQiRiLB2^2e`4dp286h@v@`O%_b)Y~A; zv}r6U?zs&@uD_+(_4bwoy7*uozNvp?bXFoB8?l8yG0qsm1JYzIvB_OH4_2G*IIOwT zVl%HX1562vLVcxM_RG*~w_`FbIc!(T=3>r528#%mwwMK}uEhJ()3MEby zQQjzqjWkwfI~;Fuj(Lj=Ug0y`>~C7`w&wzjK(rPw+Hpd~EvQ-ufQOiB4OMpyUKJhw zqEt~jle9d7S~LI~$6Z->J~QJ{Vdn3!c}g9}*KG^Kzr^(7VI5Gk(mHLL{itj_hG?&K4Ws0+T4gLfi3eu$N=`s36geNC?c zm!~}vG6lx9Uf^5M;bWntF<-{p^bruy~f?sk9 zcETAPQZLoJ8JzMMg<-=ju4keY@SY%Wo?u9Gx=j&dfa6LIAB|IrbORLV1-H==Z1zCM zeZcOYpm5>U2fU7V*h;%n`8 zN95QhfD994={1*<2vKLCNF)feKOGk`R#K~G=;rfq}|)s20&MCa65 zUM?xF5!&e0lF%|U!#rD@I{~OsS_?=;s_MQ_b_s=PuWdC)q|UQ&ea)DMRh5>fpQjXe z%9#*x=7{iRCtBKT#H>#v%>77|{4_slZ)XCY{s3j_r{tdpvb#|r|sbS^dU1x70$eJMU!h{Y7Kd{dl}9&vxQl6Jt1a` zHQZrWyY0?!vqf@u-fxU_@+}u(%Wm>0I#KP48tiAPYY!TdW(o|KtVI|EUB9V`CBBNaBLVih7+yMVF|GSoIQD0Jfb{ z!OXq;(>Z?O`1gap(L~bUcp>Lc@Jl-})^=6P%<~~9ywY=$iu8pJ0m*hOPzr~q`23eX zgbs;VOxxENe0UMVeN*>uCn9Gk!4siN-e>x)pIKAbQz!G)TcqIJ0`JBBaX>1-4_XO_-HCS^vr2vjv#7KltDZdyQ{tlWh4$Gm zB>|O1cBDC)yG(sbnc*@w6e%e}r*|IhpXckx&;sQCwGdKH+3oSG-2)Bf#x`@<4ETAr z0My%7RFh6ZLiZ_;X6Mu1YmXx7C$lSZ^}1h;j`EZd6@%JNUe=btBE z%s=Xmo1Ps?8G`}9+6>iaB8bgjUdXT?=trMu|4yLX^m0Dg{m7rpKNJey|EwHI+nN1e zL^>qN%5Fg)dGs4DO~uwIdXImN)QJ*Jhpj7$fq_^`{3fwpztL@WBB}OwQ#Epo-mqMO zsM$UgpFiG&d#)lzEQ{3Q;)&zTw;SzGOah-Dpm{!q7<8*)Ti_;xvV2TYXa}=faXZy? z3y?~GY@kl)>G&EvEijk9y1S`*=zBJSB1iet>0;x1Ai)*`^{pj0JMs)KAM=@UyOGtO z3y0BouW$N&TnwU6!%zS%nIrnANvZF&vB1~P5_d`x-giHuG zPJ;>XkVoghm#kZXRf>qxxEix;2;D1CC~NrbO6NBX!`&_$iXwP~P*c($EVV|669kDO zKoTLZNF4Cskh!Jz5ga9uZ`3o%7Pv`d^;a=cXI|>y;zC3rYPFLQkF*nv(r>SQvD*## z(Vo%^9g`%XwS0t#94zPq;mYGLKu4LU3;txF26?V~A0xZbU4Lmy`)>SoQX^m7fd^*E z+%{R4eN!rIk~K)M&UEzxp9dbY;_I^c} zOc{wlIrN_P(PPqi51k_$>Lt|X6A^|CGYgKAmoI#Li?;Wq%q~q*L7ehZkUrMxW67Jl zhsb~+U?33QS>eqyN{(odAkbopo=Q$Az?L+NZW>j;#~@wCDX?=L5SI|OxI~7!Pli;e zELMFcZtJY3!|=Gr2L4>z8yQ-{To>(f80*#;6`4IAiqUw`=Pg$%C?#1 z_g@hIGerILSU>=P>z{gM|DS91A4cT@PEIB^hSop!uhMo#2G;+tQSpDO_6nOnPWSLU zS;a9m^DFMXR4?*X=}d7l;nXuHk&0|m`NQn%d?8|Ab3A9l9Jh5s120ibWBdB z$5YwsK3;wvp!Kn@)Qae{ef`0#NwlRpQ}k^r>yos_Ne1;xyKLO?4)t_G4eK~wkUS2A&@_;)K0-03XGBzU+5f+uMDxC z(s8!8!RvdC#@`~fx$r)TKdLD6fWEVdEYtV#{ncT-ZMX~eI#UeQ-+H(Z43vVn%Yj9X zLdu9>o%wnWdvzA-#d6Z~vzj-}V3FQ5;axDIZ;i(95IIU=GQ4WuU{tl-{gk!5{l4_d zvvb&uE{%!iFwpymz{wh?bKr1*qzeZb5f6e6m_ozRF&zux2mlK=v_(_s^R6b5lu?_W4W3#<$zeG~Pd)^!4tzhs}-Sx$FJP>)ZGF(hVTH|C3(U zs0PO&*h_ zNA-&qZpTP$$LtIgfiCn07}XDbK#HIXdmv8zdz4TY;ifNIH-0jy(gMSByG2EF~Th#eb_TueZC` zE?3I>UTMpKQ})=C;6p!?G)M6w^u*A57bD?2X`m3X^6;&4%i_m(uGJ3Z5h`nwxM<)H z$I5m?wN>O~8`BGnZ=y^p6;0+%_0K}Dcg|K;+fEi|qoBqvHj(M&aHGqNF48~XqhtU? z^ogwBzRlOfpAJ+Rw7IED8lRbTdBdyEK$gPUpUG}j-M42xDj_&qEAQEtbs>D#dRd7Y z<&TpSZ(quQDHiCFn&0xsrz~4`4tz!CdL8m~HxZM_agu@IrBpyeL1Ft}V$HX_ZqDPm z-f89)pjuEzGdq-PRu`b1m+qBGY{zr_>{6Ss>F|xHZlJj9dt5HD$u`1*WZe)qEIuDSR)%z+|n zatVlhQ?$w#XRS7xUrFE;Y8vMGhQS5*T{ZnY=q1P?w5g$OKJ#M&e??tAmPWHMj3xhS ziGxapy?kn@$~2%ZY;M8Bc@%$pkl%Rvj!?o%agBvpQ-Q61n9kznC4ttrRNQ4%GFR5u zyv%Yo9~yxQJWJSfj z?#HY$y=O~F|2pZs22pu|_&Ajd+D(Mt!nPUG{|1nlvP`=R#kKH zO*s$r_%ss5h1YO7k0bHJ2CXN)Yd6CHn~W!R=SqkWe=&nAZu(Q1G!xgcUilM@YVei@2@a`8he z9@pM`)VB*=e7-MWgLlXlc)t;fF&-AwM{E-EX}pViFn0I0CNw2bNEnN2dj!^4(^zS3 zobUm1uQnpqk_4q{pl*n06=TfK_C>UgurKFjRXsK_LEn};=79`TB12tv6KzwSu*-C8 z;=~ohDLZylHQ|Mpx-?yql>|e=vI1Z!epyUpAcDCp4T|*RV&X`Q$0ogNwy6mFALo^@ z9=&(9txO8V@E!@6^(W0{*~CT>+-MA~vnJULBxCTUW>X5>r7*eXYUT0B6+w@lzw%n> z_VjJ<2qf|(d6jYq2(x$(ZDf!yVkfnbvNmb5c|hhZ^2TV_LBz`9w!e_V*W_(MiA7|= z&EeIIkw*+$Xd!)j8<@_<}A5;~A_>3JT*kX^@}cDoLd>Qj<`Se^wdUa(j0dp+Tl8EptwBm{9OGsdFEq zM`!pjf(Lm(`$e3FLOjqA5LnN5o!}z{ zNf}rJuZh@yUtq&ErjHeGzX4(!luV!jB&;FAP|!R_QHYw#^Z1LwTePAKJ6X&IDNO#; z)#I@Xnnzyij~C@UH~X51JCgQeF0&hTXnuoElz#m{heZRexWc0k4<>0+ClX7%0 zEBqCCld1tD9Zwkr4{?Nor19#E5-YKfB8d?qgR82-Ow2^AuNevly2*tHA|sK!ybYkX zm-sLQH72P&{vEAW6+z~O5d0qd=xW~rua~5a?ymYFSD@8&gV)E5@RNNBAj^C99+Z5Z zR@Pq55mbCQbz+Mn$d_CMW<-+?TU960agEk1J<>d>0K=pF19yN))a~4>m^G&tc*xR+yMD*S=yip-q=H zIlredHpsJV8H(32@Zxc@bX6a21dUV95Th--8pE6C&3F>pk=yv$yd6@Haw;$v4+Fcb zRwn{Qo@0`7aPa2LQOP}j9v>sjOo5Kqvn|`FLizX zB+@-u4Lw|jsvz{p^>n8Vo8H2peIqJJnMN}A)q6%$Tmig7eu^}K2 zrh$X?T|ZMsoh{6pdw1G$_T<`Ds-G=jc;qcGdK4{?dN2-XxjDNbb(7pk|3JUVCU4y; z)?LXR>f+AAu)JEiti_Zy#z5{RgsC}R(@jl%9YZ>zu~hKQ*AxbvhC378-I@{~#%Y`Z zy=a=9YpewPIC+gkEUUwtUL7|RU7=!^Aa}Mk^6uxOgRGA#JXjWLsjFUnix|Mau{hDT z7mn*z1m5g`vP(#tjT0Zy4eAY(br&!RiiXE=ZI!{sE1#^#%x^Z7t1U)b<;%Y}Q9=5v z;wpDCEZ@OE36TWT=|gxigT@VaW9BvHS05;_P(#s z8zI4XFQys}q)<`tkX$WnSarn{3e!s}4(J!=Yf>+Y>cP3f;vr63f2{|S^`_pWc)^5_!R z*(x-fuBxL51@xe!lnDBKi}Br$c$BMZ3%f2Sa6kLabiBS{pq*yj;q|k(86x`PiC{p6 z_bxCW{>Q2BA8~Ggz&0jkrcU+-$ANBsOop*ms>34K9lNYil@}jC;?cYP(m^P}nR6FV zk(M%48Z&%2Rx$A&FhOEirEhY0(dn;-k(qkTU)sFQ`+-ih+s@A8g?r8Pw+}2;35WYf zi}VO`jS`p(tc)$X$a>-#WXoW!phhatC*$}|rk>|wUU71eUJG^$c6_jwX?iSHM@6__ zvV|6%U*$sSXJu9SX?2%M^kK|}a2QJ8AhF{fuXrHZxXsI~O zGKX45!K7p*MCPEQ=gp?eu&#AW*pR{lhQR##P_*{c_DjMGL|3T3-bSJ(o$|M{ytU}> zAV>wq*uE*qFo9KvnA^@juy{x<-u*#2NvkV={Ly}ysKYB-k`K3@K#^S1Bb$8Y#0L0# z`6IkSG&|Z$ODy|VLS+y5pFJx&8tvPmMd8c9FhCyiU8~k6FwkakUd^(_ml8`rnl>JS zZV){9G*)xBqPz^LDqRwyS6w86#D^~xP4($150M)SOZRe9sn=>V#aG0Iy(_^YcPpIz8QYM-#s+n% z@Jd?xQq?Xk6=<3xSY7XYP$$yd&Spu{A#uafiIfy8gRC`o0nk{ezEDjb=q_qRAlR1d zFq^*9Gn)yTG4b}R{!+3hWQ+u3GT~8nwl2S1lpw`s0X_qpxv)g+JIkVKl${sYf_nV~B>Em>M;RlqGb5WVil(89 zs=ld@|#;dq1*vQGz=7--Br-|l) zZ%Xh@v8>B7P?~}?Cg$q9_={59l%m~O&*a6TKsCMAzG&vD>k2WDzJ6!tc!V)+oxF;h zJH;apM=wO?r_+*#;ulohuP=E>^zon}a$NnlcQ{1$SO*i=jnGVcQa^>QOILc)e6;eNTI>os=eaJ{*^DE+~jc zS}TYeOykDmJ=6O%>m`i*>&pO_S;qMySJIyP=}4E&J%#1zju$RpVAkZbEl+p%?ZP^C z*$$2b4t%a(e+%>a>d_f_<JjxI#J1x;=hPd1zFPx=6T$;;X1TD*2(edZ3f46zaAoW>L53vS_J*N8TMB|n+;LD| zC=GkQPpyDY#Am4l49chDv*gojhRj_?63&&8#doW`INATAo(qY#{q}%nf@eTIXmtU< zdB<7YWfyCmBs|c)cK>1)v&M#!yNj#4d$~pVfDWQc_ke1?fw{T1Nce_b`v|Vp5ig(H zJvRD^+ps46^hLX;=e2!2e;w9y1D@!D$c@Jc&%%%IL=+xzw55&2?darw=9g~>P z9>?Kdc$r?6c$m%x2S$sdpPl>GQZ{rC9mPS63*qjCVa?OIBj!fW zm|g?>CVfGXNjOfcyqImXR_(tXS(F{FcoNzKvG5R$IgGaxC@)i(e+$ME}vPVIhd|mx2IIE+f zM?9opQHIVgBWu)^A|RzXw!^??S!x)SZOwZaJkGjc<_}2l^eSBm!eAJG9T>EC6I_sy z?bxzDIAn&K5*mX)$RQzDA?s)-no-XF(g*yl4%+GBf`##bDXJ==AQk*xmnatI;SsLp zP9XTHq5mmS=iWu~9ES>b%Q=1aMa|ya^vj$@qz9S!ih{T8_PD%Sf_QrNKwgrXw9ldm zHRVR98*{C?_XNpJn{abA!oix_mowRMu^2lV-LPi;0+?-F(>^5#OHX-fPED zCu^l7u3E%STI}c4{J2!)9SUlGP_@!d?5W^QJXOI-Ea`hFMKjR7TluLvzC-ozCPn1`Tpy z!vlv@_Z58ILX6>nDjTp-1LlFMx~-%GA`aJvG$?8*Ihn;mH37eK**rmOEwqegf-Ccx zrIX4;{c~RK>XuTXxYo5kMiWMy)!IC{*DHG@E$hx?RwP@+wuad(P1{@%tRkyJRqD)3 zMHHHZ4boqDn>-=DgR5VlhQTpfVy182Gk;A_S8A1-;U1RR>+$62>(MUx@Nox$vTjHq z%QR=j!6Gdyb5wu7y(YUktwMuW5<@jl?m4cv4BODiT5o8qVdC0MBqGr@-YBIwnpZAY znX9(_uQjP}JJ=!~Ve9#5I~rUnN|P_3D$LqZcvBnywYhjlMSFHm`;u9GPla{5QD7(7*6Tb3Svr8;(nuAd81q$*uq6HC_&~je*Ca7hP4sJp0av{M8480wF zxASi7Qv+~@2U%Nu1Ud;s-G4CTVWIPyx!sg&8ZG0Wq zG_}i3C(6_1>q3w!EH7$Kwq8uBp2F2N7}l65mk1p*9v0&+;th=_E-W)E;w}P(j⁢ zv5o9#E7!G0XmdzfsS{efPNi`1b44~SZ4Z8fuX!I}#8g+(wxzQwUT#Xb2(tbY1+EUhGKoT@KEU9Ktl>_0 z%bjDJg;#*gtJZv!-Zs`?^}v5eKmnbjqlvnSzE@_SP|LG_PJ6CYU+6zY6>92%E+ z=j@TZf-iW4(%U{lnYxQA;7Q!b;^brF8n0D>)`q5>|WDDXLrqYU_tKN2>=#@~OE7grMnNh?UOz-O~6 z6%rHy{#h9K0AT+lDC7q4{hw^|q6*Ry;;L%Q@)Ga}$60_q%D)rv(CtS$CQbpq9|y1e zRSrN4;$Jyl{m5bZw`$8TGvb}(LpY{-cQ)fcyJv7l3S52TLXVDsphtv&aPuDk1OzCA z4A^QtC(!11`IsNx_HnSy?>EKpHJWT^wmS~hc^p^zIIh@9f6U@I2 zC=Mve{j2^)mS#U$e{@Q?SO6%LDsXz@SY+=cK_QMmXBIU)j!$ajc-zLx3V60EXJ!qC zi<%2x8Q24YN+&8U@CIlN zrZkcT9yh%LrlGS9`G)KdP(@9Eo-AQz@8GEFWcb7U=a0H^ZVbLmz{+&M7W(nXJ4sN8 zJLR7eeK(K8`2-}j(T7JsO`L!+CvbueT%izanm-^A1Dn{`1Nw`9P?cq;7no+XfC`K(GO9?O^5zNIt4M+M8LM0=7Gz8UA@Z0N+lg+cX)NfazRu z5D)~HA^(u%w^cz+@2@_#S|u>GpB+j4KzQ^&Wcl9f z&hG#bCA(Yk0D&t&aJE^xME^&E-&xGHhXn%}psEIj641H+Nl-}boj;)Zt*t(4wZ5DN z@GXF$bL=&pBq-#vkTkh>7hl%K5|3 z{`Vn9b$iR-SoGENp}bn4;fR3>9sA%X2@1L3aE9yTra;Wb#_`xWwLSLdfu+PAu+o3| zGVnpzPr=ch{uuoHjtw7+_!L_2;knQ!DuDl0R`|%jr+}jFzXtrHIKc323?JO{l&;VF z*L1+}JU7%QJOg|5|Tc|D8fN zJORAg=_vsy{ak|o);@)Yh8Lkcg@$FG3k@ep36BRa^>~UmnRPziS>Z=`Jb2x*Q#`%A zU*i3&Vg?TluO@X0O;r2Jl6LKLUOVhSqg1*qOt^|8*c7 zo(298@+r$k_wQNGHv{|$tW(T8L+4_`FQ{kEW5Jgg{yf7ey4ss_(SNKfz(N9lx&a;< je(UuV8hP?p&}TPdm1I$XmG#(RzlD&B2izSj9sl%y5~4qc literal 0 HcmV?d00001 diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..3fa8f86 --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew new file mode 100755 index 0000000..1aa94a4 --- /dev/null +++ b/gradlew @@ -0,0 +1,249 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 0000000..6689b85 --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,92 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/settings.gradle.kts b/settings.gradle.kts new file mode 100644 index 0000000..0cb9913 --- /dev/null +++ b/settings.gradle.kts @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2022 Fraunhofer Institute for Software and Systems Engineering + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation + * + */ + +/* rootProject.name = "samples" */ + +pluginManagement { + repositories { + mavenCentral() + gradlePluginPortal() + } +} + +dependencyResolutionManagement { + repositories { + mavenCentral() + mavenLocal() + } +} + +include(":base") From 43788a4ffc9152ac430190fd32040da01fe3d0c1 Mon Sep 17 00:00:00 2001 From: Juan Calero Date: Thu, 18 Jan 2024 16:18:33 +0100 Subject: [PATCH 002/174] Build infrastructure adapted to DEIN Changed the gradle build system, not using EDC project conventions and using ours. The EDC extension dependency mechanisms is kept, of course. --- .gitignore | 3 +- base/build.gradle.kts | 18 +--- build-logic/build.gradle | 21 ++++ build-logic/settings.gradle | 8 ++ .../com.gmv.inesdata.edc-application.gradle | 12 +++ ...om.gmv.java-application-conventions.gradle | 19 ++++ .../com.gmv.java-common-conventions.gradle | 95 +++++++++++++++++++ .../com.gmv.java-library-conventions.gradle | 13 +++ build.gradle.kts | 60 ------------ gradle.properties | 5 +- settings.gradle.kts | 29 +----- 11 files changed, 179 insertions(+), 104 deletions(-) create mode 100644 build-logic/build.gradle create mode 100644 build-logic/settings.gradle create mode 100644 build-logic/src/main/groovy/com.gmv.inesdata.edc-application.gradle create mode 100644 build-logic/src/main/groovy/com.gmv.java-application-conventions.gradle create mode 100644 build-logic/src/main/groovy/com.gmv.java-common-conventions.gradle create mode 100644 build-logic/src/main/groovy/com.gmv.java-library-conventions.gradle delete mode 100644 build.gradle.kts diff --git a/.gitignore b/.gitignore index 1c93825..3958a43 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,8 @@ .gradle # Ignore Gradle build output directory -build +build/ +bin/ .idea .vs diff --git a/base/build.gradle.kts b/base/build.gradle.kts index d7d5628..bfd22e2 100644 --- a/base/build.gradle.kts +++ b/base/build.gradle.kts @@ -1,21 +1,7 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * Fraunhofer Institute for Software and Systems Engineering - added dependencies - * - */ + plugins { - `java-library` - id("application") + id("com.gmv.inesdata.edc-application") alias(libs.plugins.shadow) } diff --git a/build-logic/build.gradle b/build-logic/build.gradle new file mode 100644 index 0000000..6da5faf --- /dev/null +++ b/build-logic/build.gradle @@ -0,0 +1,21 @@ +/* + * This file was generated by the Gradle 'init' task. + * + * This project uses @Incubating APIs which are subject to change. + */ + +plugins { + // Support convention plugins written in Groovy. Convention plugins are build scripts in 'src/main' that automatically become available as plugins in the main build. + id 'groovy-gradle-plugin' +} + +repositories { + // Use the plugin portal to apply community plugins in convention plugins. + gradlePluginPortal() +} + +dependencies { + implementation "org.sonarsource.scanner.gradle:sonarqube-gradle-plugin:4.4.1.3373" + implementation "org.owasp:dependency-check-gradle:8.4.2" + implementation "pl.allegro.tech.build:axion-release-plugin:1.16.1" +} diff --git a/build-logic/settings.gradle b/build-logic/settings.gradle new file mode 100644 index 0000000..31e6f99 --- /dev/null +++ b/build-logic/settings.gradle @@ -0,0 +1,8 @@ +/* + * This file was generated by the Gradle 'init' task. + * + * This settings file is used to specify which projects to include in your build-logic build. + * This project uses @Incubating APIs which are subject to change. + */ + +rootProject.name = 'idsob-pkg-build-logic' diff --git a/build-logic/src/main/groovy/com.gmv.inesdata.edc-application.gradle b/build-logic/src/main/groovy/com.gmv.inesdata.edc-application.gradle new file mode 100644 index 0000000..ef708f4 --- /dev/null +++ b/build-logic/src/main/groovy/com.gmv.inesdata.edc-application.gradle @@ -0,0 +1,12 @@ +/* + * This file was generated by the Gradle 'init' task. + * + * This project uses @Incubating APIs which are subject to change. + */ + +plugins { + // Apply the common convention plugin for shared build configuration between library and application projects. + id 'com.gmv.java-application-conventions' + +} + diff --git a/build-logic/src/main/groovy/com.gmv.java-application-conventions.gradle b/build-logic/src/main/groovy/com.gmv.java-application-conventions.gradle new file mode 100644 index 0000000..e41072a --- /dev/null +++ b/build-logic/src/main/groovy/com.gmv.java-application-conventions.gradle @@ -0,0 +1,19 @@ +/* + * This file was generated by the Gradle 'init' task. + * + * This project uses @Incubating APIs which are subject to change. + */ + +plugins { + // Apply the common convention plugin for shared build configuration between library and application projects. + id 'com.gmv.java-common-conventions' + + // Apply the application plugin to add support for building a CLI application in Java. + id 'application' +} + +dependencies { + implementation "ch.qos.logback:logback-core:1.3.14" + implementation "ch.qos.logback:logback-classic:1.3.14" + testImplementation "org.slf4j:slf4j-api:2.0.4" +} diff --git a/build-logic/src/main/groovy/com.gmv.java-common-conventions.gradle b/build-logic/src/main/groovy/com.gmv.java-common-conventions.gradle new file mode 100644 index 0000000..d03614e --- /dev/null +++ b/build-logic/src/main/groovy/com.gmv.java-common-conventions.gradle @@ -0,0 +1,95 @@ +/* + * This file was generated by the Gradle 'init' task. + * + * This project uses @Incubating APIs which are subject to change. + */ + +plugins { + // Apply the java Plugin to add support for Java. + id 'java' + id 'pl.allegro.tech.build.axion-release' + id 'jacoco' + id "org.sonarqube" + id "org.owasp.dependencycheck" + id 'maven-publish' +} + +scmVersion { + tag { + prefix.set("${project.name}-v") + } +} +version = scmVersion.version + +repositories { + // mavenRepoUrl, mavenRepoUsername and mavenRepoPassword must be configured in ~/.gradle/gradle.properties, + // project gradle.properties, env vars or command line. + // See https://docs.gradle.org/current/userguide/declaring_repositories.html#sec:handling_credentials + maven { + name = 'mavenRepo' + url mavenRepoUrl + credentials(PasswordCredentials) + } + // As a fallback, use Maven Central for resolving dependencies. + mavenCentral() + // Tried to use composite builds, but there is no support in VSCode + mavenLocal() +} + +dependencies { + constraints { + // Define dependency versions as constraints + implementation 'org.apache.commons:commons-text:1.10.0' + } + +} + +testing { + suites { + // Configure the built-in test suite + test { + // Use JUnit Jupiter test framework + useJUnitJupiter('5.9.3') + } + } +} + +// Apply a specific Java toolchain to ease working on different environments. +java { + toolchain { + languageVersion = JavaLanguageVersion.of(17) + } +} + +sonar { + properties { + // Recommended way to use the plugin. Compilation must be ensured + // externally before calling the sonar task + property "sonar.gradle.skipCompile", "true" + } +} + + +dependencyCheck { + failBuildOnCVSS = 7 + analyzers { + // To avoid trying to analyze DLL files + assemblyEnabled=false + ossIndex { + // remote errors from the OSS Index (e.g. BAD GATEWAY, RATE LIMIT EXCEEDED) + // will result in warnings only instead of failing execution. + warnOnlyOnRemoteErrors = true + } + } +} + +publishing { + + repositories { + maven { + name = 'mavenPublishRepo' + credentials(PasswordCredentials) + url = version.endsWith('SNAPSHOT') ? mavenPublishRepoSnapshotsUrl : mavenPublishRepoReleasesUrl + } + } +} \ No newline at end of file diff --git a/build-logic/src/main/groovy/com.gmv.java-library-conventions.gradle b/build-logic/src/main/groovy/com.gmv.java-library-conventions.gradle new file mode 100644 index 0000000..f991193 --- /dev/null +++ b/build-logic/src/main/groovy/com.gmv.java-library-conventions.gradle @@ -0,0 +1,13 @@ +/* + * This file was generated by the Gradle 'init' task. + * + * This project uses @Incubating APIs which are subject to change. + */ + +plugins { + // Apply the common convention plugin for shared build configuration between library and application projects. + id 'com.gmv.java-common-conventions' + + // Apply the java-library plugin for API and implementation separation. + id 'java-library' +} diff --git a/build.gradle.kts b/build.gradle.kts deleted file mode 100644 index 83279cf..0000000 --- a/build.gradle.kts +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2022 Fraunhofer Institute for Software and Systems Engineering - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation - * - */ - - -plugins { - `java-library` -} - -repositories { - mavenCentral() -} - -buildscript { - dependencies { - classpath(libs.edc.build.plugin) - } -} - -val edcVersion = libs.versions.edc - -allprojects { - apply(plugin = "$group.edc-build") - - // configure which version of the annotation processor to use. defaults to the same version as the plugin - configure { - processorVersion.set(edcVersion) - outputDirectory.set(project.buildDir) - } - - configure { - versions { - // override default dependency versions here - metaModel.set(edcVersion) - } - publish.set(false) - } - - configure { - configFile = rootProject.file("resources/edc-checkstyle-config.xml") - configDirectory.set(rootProject.file("resources")) - } - - // EdcRuntimeExtension uses this to determine the runtime classpath of the module to run. - tasks.register("printClasspath") { - doLast { - println(sourceSets["main"].runtimeClasspath.asPath) - } - } -} \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index 858fd3f..172fb26 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1 +1,4 @@ -group=org.eclipse.edc +# Recommended way to use the plugin. Compilation must be ensured +# externally before calling the sonar task +# Included here to avoid annoying warnings in all tasks (happens in configure stage) +systemProp.sonar.gradle.skipCompile=true \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index 0cb9913..1fdaa82 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,31 +1,8 @@ -/* - * Copyright (c) 2022 Fraunhofer Institute for Software and Systems Engineering - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation - * - */ - -/* rootProject.name = "samples" */ - pluginManagement { - repositories { - mavenCentral() - gradlePluginPortal() - } + // Include 'plugins build' to define convention plugins. + includeBuild("build-logic") } -dependencyResolutionManagement { - repositories { - mavenCentral() - mavenLocal() - } -} +rootProject.name = "inesdata-connector" include(":base") From 677d26719544f9ac16ac39f4e710ba3efefbd4ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Tue, 23 Jan 2024 17:43:22 +0100 Subject: [PATCH 003/174] base connector components --- gradle/libs.versions.toml | 20 +++++- launchers/connector/build.gradle.kts | 62 ++++++++++++++++++ launchers/connector/resources/certs/cert.pem | 21 ++++++ launchers/connector/resources/certs/cert.pfx | Bin 0 -> 2477 bytes launchers/connector/resources/certs/key.pem | 28 ++++++++ .../consumer-configuration.properties | 19 ++++++ .../configuration/consumer-vault.properties | 1 + .../provider-configuration.properties | 19 ++++++ .../configuration/provider-vault.properties | 1 + .../register-data-plane-consumer.json | 17 +++++ .../register-data-plane-provider.json | 17 +++++ settings.gradle.kts | 1 + 12 files changed, 205 insertions(+), 1 deletion(-) create mode 100644 launchers/connector/build.gradle.kts create mode 100644 launchers/connector/resources/certs/cert.pem create mode 100644 launchers/connector/resources/certs/cert.pfx create mode 100644 launchers/connector/resources/certs/key.pem create mode 100644 launchers/connector/resources/configuration/consumer-configuration.properties create mode 100644 launchers/connector/resources/configuration/consumer-vault.properties create mode 100644 launchers/connector/resources/configuration/provider-configuration.properties create mode 100644 launchers/connector/resources/configuration/provider-vault.properties create mode 100644 launchers/connector/resources/dataplane/register-data-plane-consumer.json create mode 100644 launchers/connector/resources/dataplane/register-data-plane-provider.json diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 60b609f..e4f2c37 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -3,7 +3,7 @@ format.version = "1.1" [versions] assertj = "3.25.1" -edc = "0.4.1" +edc = "0.5.0" junit-pioneer = "2.2.0" jupiter = "5.10.1" rsApi = "3.1.0" @@ -15,7 +15,25 @@ edc-boot = { module = "org.eclipse.edc:boot", version.ref = "edc" } edc-build-plugin = { module = "org.eclipse.edc.edc-build:org.eclipse.edc.edc-build.gradle.plugin", version.ref = "edc" } edc-configuration-filesystem = { module = "org.eclipse.edc:configuration-filesystem", version.ref = "edc" } edc-connector-core = { module = "org.eclipse.edc:connector-core", version.ref = "edc" } +edc-control-plane-api = { module = "org.eclipse.edc:control-plane-api", version.ref = "edc" } +edc-control-plane-api-client = { module = "org.eclipse.edc:control-plane-api-client", version.ref = "edc" } +edc-control-plane-core = { module = "org.eclipse.edc:control-plane-core", version.ref = "edc" } +edc-data-plane-api = { module = "org.eclipse.edc:data-plane-api", version.ref = "edc" } +edc-data-plane-control-api = { module = "org.eclipse.edc:data-plane-control-api", version.ref = "edc" } +edc-data-plane-core = { module = "org.eclipse.edc:data-plane-core", version.ref = "edc" } +edc-data-plane-http = { module = "org.eclipse.edc:data-plane-http", version.ref = "edc" } +edc-data-plane-public-api = { module = "org.eclipse.edc:data-plane-public-api", version.ref = "edc" } +edc-data-plane-selector-api = { module = "org.eclipse.edc:data-plane-selector-api", version.ref = "edc" } +edc-data-plane-selector-core = { module = "org.eclipse.edc:data-plane-selector-core", version.ref = "edc" } +edc-dsp = { module = "org.eclipse.edc:dsp", version.ref = "edc" } edc-http = { module = "org.eclipse.edc:http", version.ref = "edc" } +edc-iam-mock = { module = "org.eclipse.edc:iam-mock", version.ref = "edc" } +edc-management-api = { module = "org.eclipse.edc:management-api", version.ref = "edc" } +edc-micrometer-core = { module = "org.eclipse.edc:micrometer-core", version.ref = "edc" } +edc-transfer-data-plane = { module = "org.eclipse.edc:transfer-data-plane", version.ref = "edc" } +edc-transfer-pull-http-receiver = { module = "org.eclipse.edc:transfer-pull-http-receiver", version.ref = "edc" } +edc-vault-filesystem = { module = "org.eclipse.edc:vault-filesystem", version.ref = "edc" } +edc-web-spi = { module = "org.eclipse.edc:web-spi", version.ref = "edc" } jakarta-rsApi = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version.ref = "rsApi" } junit-jupiter-api = { module = "org.junit.jupiter:junit-jupiter-api", version.ref = "jupiter" } junit-jupiter-engine = { module = "org.junit.jupiter:junit-jupiter-engine", version.ref = "jupiter" } diff --git a/launchers/connector/build.gradle.kts b/launchers/connector/build.gradle.kts new file mode 100644 index 0000000..8774cc2 --- /dev/null +++ b/launchers/connector/build.gradle.kts @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2020, 2021 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial API and implementation + * Fraunhofer Institute for Software and Systems Engineering - added dependencies + * + */ + +plugins { + `java-library` + id("application") + alias(libs.plugins.shadow) +} + +dependencies { + // Librerias base + implementation(libs.edc.boot) + implementation(libs.edc.connector.core) + + // Common libs + implementation(libs.edc.web.spi) + implementation(libs.edc.dsp) + implementation(libs.edc.management.api) + implementation(libs.edc.vault.filesystem) + + // Temporary libs + implementation(libs.edc.configuration.filesystem) + implementation(libs.edc.iam.mock) + + // Control Plane + implementation(libs.edc.control.plane.api.client) + implementation(libs.edc.control.plane.api) + implementation(libs.edc.control.plane.core) + + // Data Plane + implementation(libs.edc.data.plane.control.api) + implementation(libs.edc.data.plane.public.api) + implementation(libs.edc.data.plane.core) + implementation(libs.edc.data.plane.http) + implementation(libs.edc.data.plane.selector.api) + implementation(libs.edc.data.plane.selector.core) + + // Transferencia + implementation(libs.edc.transfer.pull.http.receiver) + implementation(libs.edc.transfer.data.plane) +} + +application { + mainClass.set("org.eclipse.edc.boot.system.runtime.BaseRuntime") +} + +tasks.withType { + mergeServiceFiles() + archiveFileName.set("connector-app.jar") +} diff --git a/launchers/connector/resources/certs/cert.pem b/launchers/connector/resources/certs/cert.pem new file mode 100644 index 0000000..c7dc26f --- /dev/null +++ b/launchers/connector/resources/certs/cert.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDazCCAlOgAwIBAgIUZ3/sZXYzW4PjmOXKrZn6WBmUJ+4wDQYJKoZIhvcNAQEL +BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMjAyMjMxNTA2MDNaFw0zMjAy +MjExNTA2MDNaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw +HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQDBl6XaJnXTL+6DWip3aBhU+MzmY4d1V9hbTm1tiZ3g +E0VbUrvGO3LoYaxpPv6zFmsg3uJv6JxVAde7EddidN0ITHB9cQNdAfdUJ5njmsGS +PbdQuOQTHw0aG7/QvTI/nsvfEE6e0lbV/0e7DHacZT/+OztBH1RwkG2ymM94Hf8H +I6x7q6yfRTAZOqeOMrPCYTcluAgE9NskoPvjX5qASakBtXISKIsOU84N0/2HDN3W +EGMXvoHUQu6vrij6BwiwxKaw1AKwWENKoga775bPXN3M+JTSaIKE7dZbKzvx0Zi0 +h5X+bxc3BJi3Z/CsUBCzE+Y0SFetOiYmyl/2YmnneYoVAgMBAAGjUzBRMB0GA1Ud +DgQWBBTvK1wVERwjni4B2vdH7KtEJeVWFzAfBgNVHSMEGDAWgBTvK1wVERwjni4B +2vdH7KtEJeVWFzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBn +QHiPA7OBYukHd9gS7c0HXE+fsWcS3GZeLqcHfQQnV3pte1vTmu9//IVW71wNCJ1/ +rySRyODPQoPehxEcyHwupNZSzXK//nPlTdSgjMfFxscvt1YndyQLQYCfyOJMixAe +Aqrb14GTFHUUrdor0PyElhkULjkOXUrSIsdBrfWrwLTkelE8NK3tb5ZG8KPzD9Jy ++NwEPPr9d+iHkUkM7EFWw/cl56wka9ryBb97RI7DqbO6/j6OXHMk4GByxKv7DSIR +IvF9/Dw20qytajtaHV0pluFcOBuFc0NfiDvCaQlbTsfjzbc6UmZWbOi9YOJl3VQ/ +g3h+15GuzbsSzOCOEYOT +-----END CERTIFICATE----- diff --git a/launchers/connector/resources/certs/cert.pfx b/launchers/connector/resources/certs/cert.pfx new file mode 100644 index 0000000000000000000000000000000000000000..7ac9c73e0eb6b5c2fa625d818e42d55bdafc8a2b GIT binary patch literal 2477 zcmY+^c{J3G8V7K*W@N}R7=$7FHnNl@TSGL2Fj;DNOC*GhE&IP}zqvy)xzv zQL;p;A(BR~v1ZAx(5-Xsz3;t$Jm)#*`#tCR@A+T>L=6}O!2$?>I6^7OB54~5Is~Es z2zMxexcDn}!U9m>;!u#NoRD=ju#8QFzm^2$y49g1TMv z=T@sTo@Iuph_A%yU=lxc7M{sJMHUT>F|}KXUgo$hFFS>A2!j7*6*i&bNm83Pw2;<1 zlD}V5KUbh&o|jOxsP+BiwXBG4&`tucGj8`-Hm@xYJ}`H^&q$4GmUu<1YCBvj`C$0i zkFsaL3jy!*E?2AUSW6Pe-Zjg?APSh+Uhq-Q)U@7jmt#8E>T$EH6!|pKz~;1w=$~nhB-?cONKvYWB(}r)QbC0-GdWIqew0w+G5Wq#m^rPtZ{$ z&t3^ej`3$~G<<2pp49+5+q{j;Rh)KZXRnVO3qF5>eHL=xcsdXrkEpgxwm0V8`ssk6Z8vc4Et<{a&YT958$yF6j}K( z4hN^-U5JL_{5>%{Y?3HMe0}$$xMM|0Q~;4aZqhSedSM+=_Eg(Xx&_xr&h4{kQ6=Zy zW@OQ7;(7v+Fweo zb}gS9>{w6uxhvyvlziQ9z-3d+n125oDs)1&%G zEwA3(y*}qIWtKDR8WA@gI9QkkDNyw1yM+2Q51Ffz7>M>4w=m^={|4pZ%S)yOfKw=S zz69`I8q8wN^N`NiDcpnDll_@y_Lt-uFCKP{LaIrnK#~AY;j~X%jH{UC%6?YFF(6`J zeO0vTRy6aht=iu+Gn1WZZ9${yya0`i>EAky_j=kWwxgJ@qBH-e;dkS@+_oyS3!Aiw z;flVSZ2Gwg5uLIPua4c7LXmk++;IQ)ShM>oI!v-Ny??463&2eN=kX~3>^KyFsr`zT zevKUb@P8QK0)c;dvHXjWh5yq5@>_>7Tkb5#{9e;<9iRa8RoC_4ISv`^=M(qQdQ#2V zIza+e3IWHj4>i1V?+l92n8{c(>>O7{qXrb~$} z8V6i+B42m1p07qSh%@E`sm?9n2@PvNYy1jX*e|%F=6kUedQy;Iln(yLlX1`aN#?b- zZwXNhKWgoMN~ESsM{=(WNU3B-P_1gdul?TFtyYb>%F_uP1wj4yp>Z!i6*k*Ul%Gz)g&^5L%mg;3Yei{>pW3}~9}Fie!CscOnc zJgrKau67c@sA=-R{3>m^F}gVEM{D)BD;Rfv&5G3Fy=WURVqP6QJxj82TTmc;y4Q1- zx|64G<-u>)G8wNcEv5P}fF>N`2APM^GsZsD9(GGFY8a1t-ibjD#{`UD_>(v-XlYzo zA;v?EMrJqVD#h=r6cyea-Ed8TH>Zq;HKlDnF#HM0hMbMSu#w}iv=RZVfG{~?FaJbD zMa^hLzlX||5&8d(&Eyl0P5RU*~LQSx?Wln6}c1 zjE%SIZYd-En@6!L4H>W9OOz7Uh%swV~-_xx9L37=5~uCx-Us8{Wfv z!)kuI6u*;E^(MO0p!ubmp{22ExfFD%2=4oC9Gi#0bMN>{7MOeNuCxxli^wSO z#`0sH5ncdKZJqBDq$!_D$m6M1M9n|sZbu%mtl$5t$fH@XWMBH^*&PYhQKlKIzN#AS zP+p19Vo>>tV1Zg+=2Mjn;E1Epntc)Ceohe56KASXPy5a+R<=Vut-5#zROAXC2-6Qq!-V_HxY5dt)>gm+(P2v_EXSK~h`}Bm!(8j7N zDp}fG?p^A7UZ|tg4P;$YSJ-!8MYi_%t?<-l^SUW^yH23Ug-IiYTF2H#qp7+WS4$R* zaAti?s5msEyR0#3X@J~V5>*x`)cC0CVNqh9tGDxxLcyCjv}KFV2j%^Du{Tt5oE7^+ z;-xx^OdO|Xz6kXWC+|Nr=h33y@q4B#HOS`(w#wQyP1IL;yd7dDd?TSkv`?1vgrI`tLtzxEjlPjW_Z(lo%ao_L5vmNUfyH1EaF{X=1S|^U nfC>)L0b!m6KEg{ Date: Thu, 25 Jan 2024 09:30:15 +0100 Subject: [PATCH 004/174] update config and add json demo operations --- .../consumer-configuration.properties | 1 - .../provider-configuration.properties | 1 - .../resources/operations/create-asset.json | 16 +++++++++++++++ .../create-contract-definition.json | 9 +++++++++ .../resources/operations/create-policy.json | 13 ++++++++++++ .../resources/operations/fetch-catalog.json | 7 +++++++ .../resources/operations/get-dataset.json | 7 +++++++ .../operations/negotiate-contract.json | 20 +++++++++++++++++++ .../resources/operations/start-transfer.json | 14 +++++++++++++ 9 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 launchers/connector/resources/operations/create-asset.json create mode 100644 launchers/connector/resources/operations/create-contract-definition.json create mode 100644 launchers/connector/resources/operations/create-policy.json create mode 100644 launchers/connector/resources/operations/fetch-catalog.json create mode 100644 launchers/connector/resources/operations/get-dataset.json create mode 100644 launchers/connector/resources/operations/negotiate-contract.json create mode 100644 launchers/connector/resources/operations/start-transfer.json diff --git a/launchers/connector/resources/configuration/consumer-configuration.properties b/launchers/connector/resources/configuration/consumer-configuration.properties index bb8b3a3..bb7df00 100644 --- a/launchers/connector/resources/configuration/consumer-configuration.properties +++ b/launchers/connector/resources/configuration/consumer-configuration.properties @@ -6,7 +6,6 @@ web.http.management.port=29193 web.http.management.path=/management web.http.protocol.port=29194 web.http.protocol.path=/protocol -edc.dpf.selector.url=http://localhost:29191/select edc.receiver.http.endpoint=http://localhost:4000/receiver/urn:connector:provider/callback edc.public.key.alias=public-key edc.transfer.dataplane.token.signer.privatekey.alias=1 diff --git a/launchers/connector/resources/configuration/provider-configuration.properties b/launchers/connector/resources/configuration/provider-configuration.properties index 270c51a..4679ff5 100644 --- a/launchers/connector/resources/configuration/provider-configuration.properties +++ b/launchers/connector/resources/configuration/provider-configuration.properties @@ -6,7 +6,6 @@ web.http.management.port=19193 web.http.management.path=/management web.http.protocol.port=19194 web.http.protocol.path=/protocol -edc.dpf.selector.url=http://localhost:19191/select edc.receiver.http.endpoint=http://localhost:4000/receiver/urn:connector:provider/callback edc.public.key.alias=public-key edc.transfer.dataplane.token.signer.privatekey.alias=1 diff --git a/launchers/connector/resources/operations/create-asset.json b/launchers/connector/resources/operations/create-asset.json new file mode 100644 index 0000000..018e9f6 --- /dev/null +++ b/launchers/connector/resources/operations/create-asset.json @@ -0,0 +1,16 @@ +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@id": "assetId", + "properties": { + "name": "product description", + "contenttype": "application/json" + }, + "dataAddress": { + "type": "HttpData", + "name": "Test asset", + "baseUrl": "https://jsonplaceholder.typicode.com/users", + "proxyPath": "true" + } +} diff --git a/launchers/connector/resources/operations/create-contract-definition.json b/launchers/connector/resources/operations/create-contract-definition.json new file mode 100644 index 0000000..5ee5c67 --- /dev/null +++ b/launchers/connector/resources/operations/create-contract-definition.json @@ -0,0 +1,9 @@ +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@id": "1", + "accessPolicyId": "aPolicy", + "contractPolicyId": "aPolicy", + "assetsSelector": [] +} diff --git a/launchers/connector/resources/operations/create-policy.json b/launchers/connector/resources/operations/create-policy.json new file mode 100644 index 0000000..5047f00 --- /dev/null +++ b/launchers/connector/resources/operations/create-policy.json @@ -0,0 +1,13 @@ +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/", + "odrl": "http://www.w3.org/ns/odrl/2/" + }, + "@id": "aPolicy", + "policy": { + "@type": "set", + "odrl:permission": [], + "odrl:prohibition": [], + "odrl:obligation": [] + } +} diff --git a/launchers/connector/resources/operations/fetch-catalog.json b/launchers/connector/resources/operations/fetch-catalog.json new file mode 100644 index 0000000..370b4f2 --- /dev/null +++ b/launchers/connector/resources/operations/fetch-catalog.json @@ -0,0 +1,7 @@ +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "counterPartyAddress": "http://localhost:19194/protocol", + "protocol": "dataspace-protocol-http" +} diff --git a/launchers/connector/resources/operations/get-dataset.json b/launchers/connector/resources/operations/get-dataset.json new file mode 100644 index 0000000..5286af0 --- /dev/null +++ b/launchers/connector/resources/operations/get-dataset.json @@ -0,0 +1,7 @@ +{ + "@context": { "@vocab": "https://w3id.org/edc/v0.0.1/ns/" }, + "@type": "DatasetRequest", + "@id": "assetId", + "counterPartyAddress": "http://localhost:19194/protocol", + "protocol": "dataspace-protocol-http" +} \ No newline at end of file diff --git a/launchers/connector/resources/operations/negotiate-contract.json b/launchers/connector/resources/operations/negotiate-contract.json new file mode 100644 index 0000000..fb1e2e9 --- /dev/null +++ b/launchers/connector/resources/operations/negotiate-contract.json @@ -0,0 +1,20 @@ +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "NegotiationInitiateRequestDto", + "connectorId": "provider", + "counterPartyAddress": "http://localhost:19194/protocol", + "consumerId": "consumer", + "providerId": "provider", + "protocol": "dataspace-protocol-http", + "policy": { + "@context": "http://www.w3.org/ns/odrl.jsonld", + "@id": "MQ==:YXNzZXRJZA==:OTFhZDFiMzUtZjk3MS00ZTVlLThhYWYtYTY3Yzk1YjQyY2Mw", + "@type": "Set", + "permission": [], + "prohibition": [], + "obligation": [], + "target": "assetId" + } +} diff --git a/launchers/connector/resources/operations/start-transfer.json b/launchers/connector/resources/operations/start-transfer.json new file mode 100644 index 0000000..8f2996f --- /dev/null +++ b/launchers/connector/resources/operations/start-transfer.json @@ -0,0 +1,14 @@ +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "TransferRequestDto", + "connectorId": "provider", + "counterPartyAddress": "http://localhost:19194/protocol", + "contractId": "49aea7b0-86b6-4c78-a09b-1b1faf205899", + "assetId": "assetId", + "protocol": "dataspace-protocol-http", + "dataDestination": { + "type": "HttpProxy" + } +} From 6b15ad46d4e78151cdc5b3cca68eeb436ef75a3b Mon Sep 17 00:00:00 2001 From: Juan Calero Date: Fri, 26 Jan 2024 09:37:00 +0100 Subject: [PATCH 005/174] Project name changed to inesdata It was taken from other project --- build-logic/settings.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-logic/settings.gradle b/build-logic/settings.gradle index 31e6f99..7b59ec3 100644 --- a/build-logic/settings.gradle +++ b/build-logic/settings.gradle @@ -5,4 +5,4 @@ * This project uses @Incubating APIs which are subject to change. */ -rootProject.name = 'idsob-pkg-build-logic' +rootProject.name = 'inesdata-connector-build-logic' From ad5d5b2b704e57c1c39d5d246e5358d4bd9b4387 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Wed, 31 Jan 2024 12:40:52 +0100 Subject: [PATCH 006/174] update connector --- launchers/connector/build.gradle.kts | 5 +++-- launchers/connector/resources/certs/store.pfx | Bin 0 -> 2627 bytes .../consumer-configuration.properties | 3 +++ .../provider-configuration.properties | 2 ++ 4 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 launchers/connector/resources/certs/store.pfx diff --git a/launchers/connector/build.gradle.kts b/launchers/connector/build.gradle.kts index 8774cc2..dc4a243 100644 --- a/launchers/connector/build.gradle.kts +++ b/launchers/connector/build.gradle.kts @@ -28,9 +28,9 @@ dependencies { implementation(libs.edc.web.spi) implementation(libs.edc.dsp) implementation(libs.edc.management.api) - implementation(libs.edc.vault.filesystem) - + // Temporary libs + implementation(libs.edc.vault.filesystem) implementation(libs.edc.configuration.filesystem) implementation(libs.edc.iam.mock) @@ -50,6 +50,7 @@ dependencies { // Transferencia implementation(libs.edc.transfer.pull.http.receiver) implementation(libs.edc.transfer.data.plane) + } application { diff --git a/launchers/connector/resources/certs/store.pfx b/launchers/connector/resources/certs/store.pfx new file mode 100644 index 0000000000000000000000000000000000000000..2de2c7e82f8beea06a48c177e19eab609ba66e02 GIT binary patch literal 2627 zcmai$XEYm%8^$FP#E6|zO2iCBs7owOo6&W1+k6P1w5bx0YpJex{AG;l?G1VYLc!Sv|J0&2*Fa0)Nj8N ze|OjN;htK$;p&cNV@A&FV#93VFz+x?UWtJokwI|7z2+h*($a%fBVbQ`p~TS*`Bj!Y zat!x#0w4UKAi6c1dUzn$LEe%D4M)o`P@3{<4G7=L6XZ@^9+tJM3Rm39piV@7P=r_r z@!ElV{XCkrx_KI_Fo=1v~Smfl!SoWPH zwS$$k$##;}s21Cfa}WhGxTY#m46op&e$A_6&O!8?=4#`+Q670K23@}UH=JqO~MoJin84>4Y?P^p$KtUbIkC|iPy@~Mgz=gK~5@KUd z*`*)QThQB0Tpi!@dQN3-aGEq@jiVg9S$0Pkl6GHnS`bk%zKm1|s*Q>Pv!G1E`84m_ z7yO79RmAj*vA;NRLq=zOl6;AWhWk{|)Cpc9pZ$1>iSal&$d0b@V2w$NVeQnirt9F@ z9O^DemU9p-G|4|GlVn}a6@Ah|&-%s1L?*@YvTgnL&ZdSTQ-JU3j>^}Urr8LA;27E> zxI>?%4B>99QoeEOEf_2zxZy0W44RGgc2N}9!bt)t(YJ%jU&u)I%m&HB+GYIZpsZF@KeP)G5Arx*$}% zj4PCer`mjRZ|-$Zw>K!TI&wf8DrwO);j2eq7xLvTEB{I|luM-T8Gg{?P$K%*n!&ma z)+i)B(1Y>B^X5l07D72i%Vh7jT=Oei+3@BHaM#ix8CvRkS|cZF?-2Zpb%} z_6D6Z$54WtLlf+^{>!CV=vjOa2)6Rq=~IKl5m7 zxic9$Vg`Cx7<@d42-Dd=D@h7)lqFcX3Y;6K+eA*oBpb0wah*?MUfAg&Z!{z!paaCH z>pu2WY6a?6zU`J(6$HL^Ht4M_Q;pHH`GZf{5gyIxof91SEstURjd{2HMuAvEeA|X@ z-%%o-QVFe8$Mml0s#TD`KIxs{5J(}o9qTPP8+ys^4|v9S?O-3PL)TPaAjaQ(KB@Za zz0{H|5B1jLu$2wDa$98}PK`Ez)hzNTxM+X>ram=O?mGSh(VNcC~E_|Ht= z10o;+y$$zyz$dMh{tHj$1`F|py7Z!2AhQK>SrSRrT_-1#(}`a>3P5S8^+$1gUVJts z&5MIg6~2tvOzSAOW`T@(S;xZ~!+{RwvJSS@3F;U!oSnwomqgBpZWqj z(cf2X)XS06d_G<-Q8nTEmEJ+2xY63K%yeDiQ~ay?woY5l-YMJYH=wMEoGIRcXw>I5 zZgfzogd;6m)2!Kxj=P+5W&D1QHdu!380WDzw{$LaB=^^OSk7vhp;=~N)g-gZ8@@r;5^pf9O6B zWvTt?&lSiSa@%d!)lRDdWU@X;^;mU)Z92}KSXj+7V>7AJn&fd_B-!mZJr*wt-@%o6 zYBy@|^^a7BF3$d?Y?vqYCdT2WPuSk|!`NNUOL#R8q1GKO>!ENhxrYD>odC%NuWo_Slh<+cev&4C{B>Wh?IB!oq zK#X&Fel=$kJS^Z3@;{+`)V8hu@qSLmLzCDn7Dqap)y>b9MK4nGLC$gQZ|V+s`rRn}Grj&}ek53|<|hytsMq?=uS*2^L!%P)01IJSJk+M%U zNYoW_s0c;0?8gbHwUCL?3Ax;_!`!EPW;YPi85xp7Z3myK80`bKj_K)HKf*uuseJz# zR_eH9r)Ju=I=lMAT}?I0K!3yO5UMfLqlitma*+~5@&5)agk1^-V#}lU%df^LtBLe% zFsI&Ka&%E+&qb-Dq)-gM-!C8lOaX$2Z@3i>wk-T9+c;wEEOKhrF#Tw7t)HzXPluZ- VX#f)fqPe=k6u9hv&ioH*{{|M4&8+|c literal 0 HcmV?d00001 diff --git a/launchers/connector/resources/configuration/consumer-configuration.properties b/launchers/connector/resources/configuration/consumer-configuration.properties index bb7df00..1dd7c83 100644 --- a/launchers/connector/resources/configuration/consumer-configuration.properties +++ b/launchers/connector/resources/configuration/consumer-configuration.properties @@ -11,8 +11,11 @@ edc.public.key.alias=public-key edc.transfer.dataplane.token.signer.privatekey.alias=1 edc.transfer.proxy.token.signer.privatekey.alias=1 edc.transfer.proxy.token.verifier.publickey.alias=public-key +edc.web.rest.cors.enabled=true +edc.web.rest.cors.headers=origin, content-type, accept, authorization, x-api-key web.http.public.port=29291 web.http.public.path=/public web.http.control.port=29192 web.http.control.path=/control edc.dataplane.token.validation.endpoint=http://localhost:29192/control/token +edc.catalog.cache.execution.period.seconds=120 diff --git a/launchers/connector/resources/configuration/provider-configuration.properties b/launchers/connector/resources/configuration/provider-configuration.properties index 4679ff5..93dcccd 100644 --- a/launchers/connector/resources/configuration/provider-configuration.properties +++ b/launchers/connector/resources/configuration/provider-configuration.properties @@ -11,6 +11,8 @@ edc.public.key.alias=public-key edc.transfer.dataplane.token.signer.privatekey.alias=1 edc.transfer.proxy.token.signer.privatekey.alias=1 edc.transfer.proxy.token.verifier.publickey.alias=public-key +edc.web.rest.cors.enabled=true +edc.web.rest.cors.headers=origin, content-type, accept, authorization, x-api-key web.http.public.port=19291 web.http.public.path=/public web.http.control.port=19192 From 682aec0540537d4908753ea7208cd5119f455580 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Wed, 31 Jan 2024 14:14:58 +0100 Subject: [PATCH 007/174] relocate resources for testing purposes --- launchers/connector/resources/certs/cert.pem | 21 ------------- launchers/connector/resources/certs/cert.pfx | Bin 2477 -> 0 bytes launchers/connector/resources/certs/key.pem | 28 ------------------ launchers/connector/resources/certs/store.pfx | Bin 2627 -> 0 bytes resources/certs/cert.pem | 19 ++++++++++++ resources/certs/key.pem | 28 ++++++++++++++++++ resources/certs/store.pfx | Bin 0 -> 2531 bytes .../consumer-configuration.properties | 0 .../configuration/consumer-vault.properties | 0 .../provider-configuration.properties | 0 .../configuration/provider-vault.properties | 0 .../register-data-plane-consumer.json | 0 .../register-data-plane-provider.json | 0 .../operations/create-asset.json | 0 .../create-contract-definition.json | 0 .../operations/create-policy.json | 0 .../operations/fetch-catalog.json | 0 .../operations/get-dataset.json | 0 .../operations/negotiate-contract.json | 0 .../operations/start-transfer.json | 0 20 files changed, 47 insertions(+), 49 deletions(-) delete mode 100644 launchers/connector/resources/certs/cert.pem delete mode 100644 launchers/connector/resources/certs/cert.pfx delete mode 100644 launchers/connector/resources/certs/key.pem delete mode 100644 launchers/connector/resources/certs/store.pfx create mode 100644 resources/certs/cert.pem create mode 100644 resources/certs/key.pem create mode 100644 resources/certs/store.pfx rename {launchers/connector/resources => resources}/configuration/consumer-configuration.properties (100%) rename {launchers/connector/resources => resources}/configuration/consumer-vault.properties (100%) rename {launchers/connector/resources => resources}/configuration/provider-configuration.properties (100%) rename {launchers/connector/resources => resources}/configuration/provider-vault.properties (100%) rename {launchers/connector/resources => resources}/dataplane/register-data-plane-consumer.json (100%) rename {launchers/connector/resources => resources}/dataplane/register-data-plane-provider.json (100%) rename {launchers/connector/resources => resources}/operations/create-asset.json (100%) rename {launchers/connector/resources => resources}/operations/create-contract-definition.json (100%) rename {launchers/connector/resources => resources}/operations/create-policy.json (100%) rename {launchers/connector/resources => resources}/operations/fetch-catalog.json (100%) rename {launchers/connector/resources => resources}/operations/get-dataset.json (100%) rename {launchers/connector/resources => resources}/operations/negotiate-contract.json (100%) rename {launchers/connector/resources => resources}/operations/start-transfer.json (100%) diff --git a/launchers/connector/resources/certs/cert.pem b/launchers/connector/resources/certs/cert.pem deleted file mode 100644 index c7dc26f..0000000 --- a/launchers/connector/resources/certs/cert.pem +++ /dev/null @@ -1,21 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDazCCAlOgAwIBAgIUZ3/sZXYzW4PjmOXKrZn6WBmUJ+4wDQYJKoZIhvcNAQEL -BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM -GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMjAyMjMxNTA2MDNaFw0zMjAy -MjExNTA2MDNaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw -HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB -AQUAA4IBDwAwggEKAoIBAQDBl6XaJnXTL+6DWip3aBhU+MzmY4d1V9hbTm1tiZ3g -E0VbUrvGO3LoYaxpPv6zFmsg3uJv6JxVAde7EddidN0ITHB9cQNdAfdUJ5njmsGS -PbdQuOQTHw0aG7/QvTI/nsvfEE6e0lbV/0e7DHacZT/+OztBH1RwkG2ymM94Hf8H -I6x7q6yfRTAZOqeOMrPCYTcluAgE9NskoPvjX5qASakBtXISKIsOU84N0/2HDN3W -EGMXvoHUQu6vrij6BwiwxKaw1AKwWENKoga775bPXN3M+JTSaIKE7dZbKzvx0Zi0 -h5X+bxc3BJi3Z/CsUBCzE+Y0SFetOiYmyl/2YmnneYoVAgMBAAGjUzBRMB0GA1Ud -DgQWBBTvK1wVERwjni4B2vdH7KtEJeVWFzAfBgNVHSMEGDAWgBTvK1wVERwjni4B -2vdH7KtEJeVWFzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBn -QHiPA7OBYukHd9gS7c0HXE+fsWcS3GZeLqcHfQQnV3pte1vTmu9//IVW71wNCJ1/ -rySRyODPQoPehxEcyHwupNZSzXK//nPlTdSgjMfFxscvt1YndyQLQYCfyOJMixAe -Aqrb14GTFHUUrdor0PyElhkULjkOXUrSIsdBrfWrwLTkelE8NK3tb5ZG8KPzD9Jy -+NwEPPr9d+iHkUkM7EFWw/cl56wka9ryBb97RI7DqbO6/j6OXHMk4GByxKv7DSIR -IvF9/Dw20qytajtaHV0pluFcOBuFc0NfiDvCaQlbTsfjzbc6UmZWbOi9YOJl3VQ/ -g3h+15GuzbsSzOCOEYOT ------END CERTIFICATE----- diff --git a/launchers/connector/resources/certs/cert.pfx b/launchers/connector/resources/certs/cert.pfx deleted file mode 100644 index 7ac9c73e0eb6b5c2fa625d818e42d55bdafc8a2b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2477 zcmY+^c{J3G8V7K*W@N}R7=$7FHnNl@TSGL2Fj;DNOC*GhE&IP}zqvy)xzv zQL;p;A(BR~v1ZAx(5-Xsz3;t$Jm)#*`#tCR@A+T>L=6}O!2$?>I6^7OB54~5Is~Es z2zMxexcDn}!U9m>;!u#NoRD=ju#8QFzm^2$y49g1TMv z=T@sTo@Iuph_A%yU=lxc7M{sJMHUT>F|}KXUgo$hFFS>A2!j7*6*i&bNm83Pw2;<1 zlD}V5KUbh&o|jOxsP+BiwXBG4&`tucGj8`-Hm@xYJ}`H^&q$4GmUu<1YCBvj`C$0i zkFsaL3jy!*E?2AUSW6Pe-Zjg?APSh+Uhq-Q)U@7jmt#8E>T$EH6!|pKz~;1w=$~nhB-?cONKvYWB(}r)QbC0-GdWIqew0w+G5Wq#m^rPtZ{$ z&t3^ej`3$~G<<2pp49+5+q{j;Rh)KZXRnVO3qF5>eHL=xcsdXrkEpgxwm0V8`ssk6Z8vc4Et<{a&YT958$yF6j}K( z4hN^-U5JL_{5>%{Y?3HMe0}$$xMM|0Q~;4aZqhSedSM+=_Eg(Xx&_xr&h4{kQ6=Zy zW@OQ7;(7v+Fweo zb}gS9>{w6uxhvyvlziQ9z-3d+n125oDs)1&%G zEwA3(y*}qIWtKDR8WA@gI9QkkDNyw1yM+2Q51Ffz7>M>4w=m^={|4pZ%S)yOfKw=S zz69`I8q8wN^N`NiDcpnDll_@y_Lt-uFCKP{LaIrnK#~AY;j~X%jH{UC%6?YFF(6`J zeO0vTRy6aht=iu+Gn1WZZ9${yya0`i>EAky_j=kWwxgJ@qBH-e;dkS@+_oyS3!Aiw z;flVSZ2Gwg5uLIPua4c7LXmk++;IQ)ShM>oI!v-Ny??463&2eN=kX~3>^KyFsr`zT zevKUb@P8QK0)c;dvHXjWh5yq5@>_>7Tkb5#{9e;<9iRa8RoC_4ISv`^=M(qQdQ#2V zIza+e3IWHj4>i1V?+l92n8{c(>>O7{qXrb~$} z8V6i+B42m1p07qSh%@E`sm?9n2@PvNYy1jX*e|%F=6kUedQy;Iln(yLlX1`aN#?b- zZwXNhKWgoMN~ESsM{=(WNU3B-P_1gdul?TFtyYb>%F_uP1wj4yp>Z!i6*k*Ul%Gz)g&^5L%mg;3Yei{>pW3}~9}Fie!CscOnc zJgrKau67c@sA=-R{3>m^F}gVEM{D)BD;Rfv&5G3Fy=WURVqP6QJxj82TTmc;y4Q1- zx|64G<-u>)G8wNcEv5P}fF>N`2APM^GsZsD9(GGFY8a1t-ibjD#{`UD_>(v-XlYzo zA;v?EMrJqVD#h=r6cyea-Ed8TH>Zq;HKlDnF#HM0hMbMSu#w}iv=RZVfG{~?FaJbD zMa^hLzlX||5&8d(&Eyl0P5RU*~LQSx?Wln6}c1 zjE%SIZYd-En@6!L4H>W9OOz7Uh%swV~-_xx9L37=5~uCx-Us8{Wfv z!)kuI6u*;E^(MO0p!ubmp{22ExfFD%2=4oC9Gi#0bMN>{7MOeNuCxxli^wSO z#`0sH5ncdKZJqBDq$!_D$m6M1M9n|sZbu%mtl$5t$fH@XWMBH^*&PYhQKlKIzN#AS zP+p19Vo>>tV1Zg+=2Mjn;E1Epntc)Ceohe56KASXPy5a+R<=Vut-5#zROAXC2-6Qq!-V_HxY5dt)>gm+(P2v_EXSK~h`}Bm!(8j7N zDp}fG?p^A7UZ|tg4P;$YSJ-!8MYi_%t?<-l^SUW^yH23Ug-IiYTF2H#qp7+WS4$R* zaAti?s5msEyR0#3X@J~V5>*x`)cC0CVNqh9tGDxxLcyCjv}KFV2j%^Du{Tt5oE7^+ z;-xx^OdO|Xz6kXWC+|Nr=h33y@q4B#HOS`(w#wQyP1IL;yd7dDd?TSkv`?1vgrI`tLtzxEjlPjW_Z(lo%ao_L5vmNUfyH1EaF{X=1S|^U nfC>)L0b!m6KEg{owOo6&W1+k6P1w5bx0YpJex{AG;l?G1VYLc!Sv|J0&2*Fa0)Nj8N ze|OjN;htK$;p&cNV@A&FV#93VFz+x?UWtJokwI|7z2+h*($a%fBVbQ`p~TS*`Bj!Y zat!x#0w4UKAi6c1dUzn$LEe%D4M)o`P@3{<4G7=L6XZ@^9+tJM3Rm39piV@7P=r_r z@!ElV{XCkrx_KI_Fo=1v~Smfl!SoWPH zwS$$k$##;}s21Cfa}WhGxTY#m46op&e$A_6&O!8?=4#`+Q670K23@}UH=JqO~MoJin84>4Y?P^p$KtUbIkC|iPy@~Mgz=gK~5@KUd z*`*)QThQB0Tpi!@dQN3-aGEq@jiVg9S$0Pkl6GHnS`bk%zKm1|s*Q>Pv!G1E`84m_ z7yO79RmAj*vA;NRLq=zOl6;AWhWk{|)Cpc9pZ$1>iSal&$d0b@V2w$NVeQnirt9F@ z9O^DemU9p-G|4|GlVn}a6@Ah|&-%s1L?*@YvTgnL&ZdSTQ-JU3j>^}Urr8LA;27E> zxI>?%4B>99QoeEOEf_2zxZy0W44RGgc2N}9!bt)t(YJ%jU&u)I%m&HB+GYIZpsZF@KeP)G5Arx*$}% zj4PCer`mjRZ|-$Zw>K!TI&wf8DrwO);j2eq7xLvTEB{I|luM-T8Gg{?P$K%*n!&ma z)+i)B(1Y>B^X5l07D72i%Vh7jT=Oei+3@BHaM#ix8CvRkS|cZF?-2Zpb%} z_6D6Z$54WtLlf+^{>!CV=vjOa2)6Rq=~IKl5m7 zxic9$Vg`Cx7<@d42-Dd=D@h7)lqFcX3Y;6K+eA*oBpb0wah*?MUfAg&Z!{z!paaCH z>pu2WY6a?6zU`J(6$HL^Ht4M_Q;pHH`GZf{5gyIxof91SEstURjd{2HMuAvEeA|X@ z-%%o-QVFe8$Mml0s#TD`KIxs{5J(}o9qTPP8+ys^4|v9S?O-3PL)TPaAjaQ(KB@Za zz0{H|5B1jLu$2wDa$98}PK`Ez)hzNTxM+X>ram=O?mGSh(VNcC~E_|Ht= z10o;+y$$zyz$dMh{tHj$1`F|py7Z!2AhQK>SrSRrT_-1#(}`a>3P5S8^+$1gUVJts z&5MIg6~2tvOzSAOW`T@(S;xZ~!+{RwvJSS@3F;U!oSnwomqgBpZWqj z(cf2X)XS06d_G<-Q8nTEmEJ+2xY63K%yeDiQ~ay?woY5l-YMJYH=wMEoGIRcXw>I5 zZgfzogd;6m)2!Kxj=P+5W&D1QHdu!380WDzw{$LaB=^^OSk7vhp;=~N)g-gZ8@@r;5^pf9O6B zWvTt?&lSiSa@%d!)lRDdWU@X;^;mU)Z92}KSXj+7V>7AJn&fd_B-!mZJr*wt-@%o6 zYBy@|^^a7BF3$d?Y?vqYCdT2WPuSk|!`NNUOL#R8q1GKO>!ENhxrYD>odC%NuWo_Slh<+cev&4C{B>Wh?IB!oq zK#X&Fel=$kJS^Z3@;{+`)V8hu@qSLmLzCDn7Dqap)y>b9MK4nGLC$gQZ|V+s`rRn}Grj&}ek53|<|hytsMq?=uS*2^L!%P)01IJSJk+M%U zNYoW_s0c;0?8gbHwUCL?3Ax;_!`!EPW;YPi85xp7Z3myK80`bKj_K)HKf*uuseJz# zR_eH9r)Ju=I=lMAT}?I0K!3yO5UMfLqlitma*+~5@&5)agk1^-V#}lU%df^LtBLe% zFsI&Ka&%E+&qb-Dq)-gM-!C8lOaX$2Z@3i>wk-T9+c;wEEOKhrF#Tw7t)HzXPluZ- VX#f)fqPe=k6u9hv&ioH*{{|M4&8+|c diff --git a/resources/certs/cert.pem b/resources/certs/cert.pem new file mode 100644 index 0000000..82a7f2e --- /dev/null +++ b/resources/certs/cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDETCCAfkCFBEOuzF+mRgrzrxVv70zNDUEv+nUMA0GCSqGSIb3DQEBCwUAMEUx +CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl +cm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjQwMTMxMTIyNTUyWhcNMjUwMTMwMTIy +NTUyWjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE +CgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC +AQ8AMIIBCgKCAQEAxHbsW755sB/5YwjPyk0xPJwyQkxkGV2SS4sQQCCz8KV/QoEi +lk/PzUqUsE2e7LqavWL5+FPSic79ZKEljeeNB1z5xoE2YnKEs55MLi/PgfLJHJ9M +bA42lJBYI37MeGv0kGkIQDIrtpDjBfSgGSINXeTjs0T/l5sXXPTfSRm4URWX3I+Q +O9ACYLb7Cz//G8HdYHp8MHNa38x4BeBku7cT7xgEUPfhu4LCnBW1u0pMbeT8A2M9 +zz/TIOJKwUeupacXjJ/tqHGLYMgBLCSRNK4zFJiGzhDbEZA6+NMUp660t7Su6M5i +x3WIffKlIcxi0bnk9slhjK5NynJ3H/uvbfYnCwIDAQABMA0GCSqGSIb3DQEBCwUA +A4IBAQBAttZGxh1k87S8OHA6L8fmu/PEtjdkJV3Cu7J3xTGfmSBVMTStl2NI4D4i +FJUFdTJjvEibEaQEETz2zJakwdp7JGG7ip7e/dKsP0wVMC63MZ8eeDzST7IzBLk6 +mjb+Co0+2p/9rHPBz1AFpHRcJ9jaORh8FlctfcxcBsr6dMptGtCZMzam26WwePxa +pNd+wBNFHuAXHyTzHl3NIq2Bd+56Roen5muZKv4xqI7iVBbVrejvVyXNDhrqh9QB +VushjclGdc2GSdCVpUhiv1srX59/yX5HBZG8HRQSgCXpcSiGIhiMmd1gaU/qrXa8 +kwmd6tdKUcAiLU7qXEJX4iBziobm +-----END CERTIFICATE----- diff --git a/resources/certs/key.pem b/resources/certs/key.pem new file mode 100644 index 0000000..75f7835 --- /dev/null +++ b/resources/certs/key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDEduxbvnmwH/lj +CM/KTTE8nDJCTGQZXZJLixBAILPwpX9CgSKWT8/NSpSwTZ7supq9Yvn4U9KJzv1k +oSWN540HXPnGgTZicoSznkwuL8+B8skcn0xsDjaUkFgjfsx4a/SQaQhAMiu2kOMF +9KAZIg1d5OOzRP+Xmxdc9N9JGbhRFZfcj5A70AJgtvsLP/8bwd1genwwc1rfzHgF +4GS7txPvGARQ9+G7gsKcFbW7Skxt5PwDYz3PP9Mg4krBR66lpxeMn+2ocYtgyAEs +JJE0rjMUmIbOENsRkDr40xSnrrS3tK7ozmLHdYh98qUhzGLRueT2yWGMrk3Kcncf ++69t9icLAgMBAAECggEAMrh/IdplgS5oDITQXGqcXQj4QryFFTVemQkwa1XmDqkz +VyCOpN67B8WK6I9JidVNMLq2TUGat8BME+g0kVOnybBbvb9LIpRtJRAnm771JBfh +ivWnXbg3qBgx4QwRzr8UNxoeqrApL+ts6PM0R/jOGaEHlhcoo2PDXtjLMoyk1K26 +JdXibZ8TjqS59MG9BB8j2/NiyaQxC46k8FPz5UXnLAVr8z8kzi0P/LfuZdFO3BjB +1Pgsgy7zme32nUCTJSQbCUKngY7KAwaeuPVji5QbkJ0Tk1tWop9IKg43RzqVp+Me +QZWDgIyeWMRkmKb6GXrl6HjuOFEymMQkPlRH7QPDcQKBgQDjOaYaPFQIgkkKAoMa +DGT0bv1RdPWithYHDTKpa6xMqfZ5/H+09i/+wVH1P8j2osylZOunsCVoIeeeTuv4 +VYC+VTOboOFdbzKD9gOyvZ4nUox+nT/1awK/ZBRcX1GrItZLs7QbrWHswT1VWTcO +ccj7wVIKs15yuBuV0w97qA1h0QKBgQDdWA2v1Yj0jgOn8leCSGj2LlzYObuEWEn4 +lCNn0b5ZABTqFpG6viNpalyTUHDXGYt3mZpEBpW2O+x7Bv9EO8+G9TtkW13NL+eD +4+4wftRx4LTuOlG5wAicNG0dvaobMZ3Ba/v+KCs2pQaC6pC0E12j5euPoTl6pUxL +0caANS72GwKBgA6Yu3Tf/z30sKB1/NoFhc61+ovrCYSEUfHuBR0hD5Z1LKI1eDa1 +weuJm6w3hr9hQeg2DXVeOWjxFKH0y+/N/lE7w+7xbMjFxeb4PNCUob5A7UmM7Hjl +gO2ujihI70QhHscKKj3E64978mazLng9Ev4pMw4A68ZO0WbUgiAdcY6hAoGBAJA6 +YDDDI4q3PwqgjwUFyd8gNr9uXxHNteWUIb30nePAeeBO9IaKfuihtdEICX8f1hEI +tcy9vbARO/auiIcdsgxun1S1dFnPbTwzuXFAM45AraTNu5Md3n5qau+GNuGhYvo2 +Mu1Zn07sGWIVFlVAiTPTDbt5gsq4Cw8ckgvYm9EhAoGAJEi0oDOOPS8skP71HsEm +fwMhp414LL7xFf4LZj8RL4+vDfzyNnbQ3/zpGKCzwO2MWAQx4GB9hNrX8meGLJXP +w5zMub2RLs0x4pZPuIQVIfcf6Ryuwx/kM3uZAj6lUm5LpfsWJNFZylC2ixuLTLgn +B1fcrLsMChYYsJvCfzElLSs= +-----END PRIVATE KEY----- diff --git a/resources/certs/store.pfx b/resources/certs/store.pfx new file mode 100644 index 0000000000000000000000000000000000000000..a1b3326e6fbdc4f2dd2b1e154952f054b7a78926 GIT binary patch literal 2531 zcmai$c{J1w7srhm!wh2^TMS{8HT}ktWh_}6YmHGFPbEdRtYhEOm=uZ16QKk##i0bkqj_NkD+jAHjkEOaa+{BnNsML{m?Irs_U+SY*K!`!P8DVoNG%aN7jSofN+up#*vM+%O%s5J}S);&!0&j zk#21iuC@07SzREBXqsJgJv(htkK<2`EM}&sHU=ALS6(f2wD4ZTShR&BG~X@}%3om4 zce&>TWhiuOpz38sUAuUVRzSIRWXpq$Zsi$&Bc$HU!L^@)w0R?$2yN0a1B{=c!GSLz7YD5L;7NBd`| z*T>3QdypFDr^LPNa`*E0Yc$hC*fT}J?B_ZAGty37E+21FYs*sN)v>8_4)eSBT z(#-z7&>M-@xRxmGf(Y)dPgySW#`rNn=+nGzZmt;#f#Fc=d5{l;=f;ueo9!f zjNPy5mrKuN!#`dZZ;5I;Qe6%8iU;-}0IdhP&rGO#+-1frt8rG zb>T5;F?gSk%hHdDP2J12z7>r+iusO+MzIKs6gb`F0FV(WxMX!8k3g?289htz zqC?1#o&+fQO?&H3c(l1TvE&hSLio$?vS34-%FbuTe>Ho}QTmElu_a22e5RjVGp68N zDeNynmlcT_7EG{XKa^Dc>(}MVH$A=aRfDWyOZqb%x-P#e<9rzKh6_zY6&+j8;to5G zi@i6^?P^1SO3^vZ1_~=_>~|sn#sxc$m+Tk=((z^+R8DN0T(ZNQ`Md)pa9KcluXBbm zQFu=BgXbqFa_l^1oqD3X2_1^i|-NI!V*4lDgNqBP@+p zhZZ)UW2&!MO4Vc~WKK^S*SL^fRU|sG4oWn0gM>N6o7J?ZD>tXJi}l)&b=q2}CprrqE?$!!d`k z6TP-czmkeY7yEqt)AG)!#ETzKE8p0>1A7sEUNiAqxGOy(TFCmAn^Ln`!%x#y8~wE@ zAL9(0**q;CjM!iWZq>Nn2i&vB#^ENx?j!!DJhQ966z4NHk7M3r7g!r0KN$aTILcu+ zxy!@wllkh%L!=f@j#nmHhv2VgPp6+*6(W|qsQCR@*lcNf{=N7mvc+pi@W-A6dzc1# z&*{9VQQB@*4i~l`4;3|L&HEE?wSzRiSN;@)e*UZSu{kgt{41qAz0T-1Z4tCLU4K5~N8h=a1KE9+e$h z3D^-%X`ToEZ(nmM8>;t_u8wnx)gew+L<^rrVyaHR>Vki(VH=z>%64rgd`4vGoVEba zQImejV&w*g`gxB@E)41}|7i&Al$sdtNmw&pZ{QuFV`YPeZHddUnRj>#;AR?@d-&I~ z^mF{ymVb2E>34dXS??A%D+6&8br(5LP3GNiECu%#*#s7{=Bq<}H$vYb*aa@B?<-jE zzf;u>x(JOP<^V`=)WRC&yy*fo>loZWNJdDd2JNMeuZ<)qS4QS&4e(QTBC>2L{U1vZKzyOFp?-4VIg$W{VBfUfx52KE?I!S7BQ*hhW)Lp@Y cm-U-U!sfrJ3I=2dTcuYl(!e~^`ft?!7wu7x_y7O^ literal 0 HcmV?d00001 diff --git a/launchers/connector/resources/configuration/consumer-configuration.properties b/resources/configuration/consumer-configuration.properties similarity index 100% rename from launchers/connector/resources/configuration/consumer-configuration.properties rename to resources/configuration/consumer-configuration.properties diff --git a/launchers/connector/resources/configuration/consumer-vault.properties b/resources/configuration/consumer-vault.properties similarity index 100% rename from launchers/connector/resources/configuration/consumer-vault.properties rename to resources/configuration/consumer-vault.properties diff --git a/launchers/connector/resources/configuration/provider-configuration.properties b/resources/configuration/provider-configuration.properties similarity index 100% rename from launchers/connector/resources/configuration/provider-configuration.properties rename to resources/configuration/provider-configuration.properties diff --git a/launchers/connector/resources/configuration/provider-vault.properties b/resources/configuration/provider-vault.properties similarity index 100% rename from launchers/connector/resources/configuration/provider-vault.properties rename to resources/configuration/provider-vault.properties diff --git a/launchers/connector/resources/dataplane/register-data-plane-consumer.json b/resources/dataplane/register-data-plane-consumer.json similarity index 100% rename from launchers/connector/resources/dataplane/register-data-plane-consumer.json rename to resources/dataplane/register-data-plane-consumer.json diff --git a/launchers/connector/resources/dataplane/register-data-plane-provider.json b/resources/dataplane/register-data-plane-provider.json similarity index 100% rename from launchers/connector/resources/dataplane/register-data-plane-provider.json rename to resources/dataplane/register-data-plane-provider.json diff --git a/launchers/connector/resources/operations/create-asset.json b/resources/operations/create-asset.json similarity index 100% rename from launchers/connector/resources/operations/create-asset.json rename to resources/operations/create-asset.json diff --git a/launchers/connector/resources/operations/create-contract-definition.json b/resources/operations/create-contract-definition.json similarity index 100% rename from launchers/connector/resources/operations/create-contract-definition.json rename to resources/operations/create-contract-definition.json diff --git a/launchers/connector/resources/operations/create-policy.json b/resources/operations/create-policy.json similarity index 100% rename from launchers/connector/resources/operations/create-policy.json rename to resources/operations/create-policy.json diff --git a/launchers/connector/resources/operations/fetch-catalog.json b/resources/operations/fetch-catalog.json similarity index 100% rename from launchers/connector/resources/operations/fetch-catalog.json rename to resources/operations/fetch-catalog.json diff --git a/launchers/connector/resources/operations/get-dataset.json b/resources/operations/get-dataset.json similarity index 100% rename from launchers/connector/resources/operations/get-dataset.json rename to resources/operations/get-dataset.json diff --git a/launchers/connector/resources/operations/negotiate-contract.json b/resources/operations/negotiate-contract.json similarity index 100% rename from launchers/connector/resources/operations/negotiate-contract.json rename to resources/operations/negotiate-contract.json diff --git a/launchers/connector/resources/operations/start-transfer.json b/resources/operations/start-transfer.json similarity index 100% rename from launchers/connector/resources/operations/start-transfer.json rename to resources/operations/start-transfer.json From bff1831e531e3a315e416f2d96c8a1c43cd9cd07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Mon, 5 Feb 2024 07:58:00 +0100 Subject: [PATCH 008/174] add postgresql persistence --- gradle/libs.versions.toml | 20 +++++++++++++++++++ launchers/connector/build.gradle.kts | 18 ++++++++++++++++- .../consumer-configuration.properties | 7 +++++++ .../provider-configuration.properties | 7 +++++++ 4 files changed, 51 insertions(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index e4f2c37..79b1f26 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -6,6 +6,7 @@ assertj = "3.25.1" edc = "0.5.0" junit-pioneer = "2.2.0" jupiter = "5.10.1" +postgres = "42.7.1" rsApi = "3.1.0" [libraries] @@ -40,5 +41,24 @@ junit-jupiter-engine = { module = "org.junit.jupiter:junit-jupiter-engine", vers junit-jupiter-params = { module = "org.junit.jupiter:junit-jupiter-params", version.ref = "jupiter" } junit-pioneer = { module = "org.junit-pioneer:junit-pioneer", version.ref = "junit-pioneer" } +#Policy monitor +edc-core-policy-monitor = { module = "org.eclipse.edc:policy-monitor-core", version.ref = "edc" } + +#SQL Extensions +edc-transaction-local = { module = "org.eclipse.edc:transaction-local", version.ref = "edc" } +edc-sql-core = { module = "org.eclipse.edc:sql-core", version.ref = "edc" } +edc-sql-lease = { module = "org.eclipse.edc:sql-lease", version.ref = "edc" } +edc-sql-pool = { module = "org.eclipse.edc:sql-pool-apache-commons", version.ref = "edc" } +edc-sql-asset-index = { module = "org.eclipse.edc:asset-index-sql", version.ref = "edc" } +edc-sql-contract-definition-store = { module = "org.eclipse.edc:contract-definition-store-sql", version.ref = "edc" } +edc-sql-contract-negotiation-store = { module = "org.eclipse.edc:contract-negotiation-store-sql", version.ref = "edc" } +edc-sql-policy-definition-store = { module = "org.eclipse.edc:policy-definition-store-sql", version.ref = "edc" } +edc-sql-transfer-process-store = { module = "org.eclipse.edc:transfer-process-store-sql", version.ref = "edc" } +edc-sql-data-plane-store = { module = "org.eclipse.edc:data-plane-store-sql", version.ref = "edc" } +edc-sql-policy-monitor-store = { module = "org.eclipse.edc:policy-monitor-store-sql", version.ref = "edc" } + +# no EDC references +postgres = { module = "org.postgresql:postgresql", version.ref = "postgres" } + [plugins] shadow = { id = "com.github.johnrengelman.shadow", version = "8.1.1" } diff --git a/launchers/connector/build.gradle.kts b/launchers/connector/build.gradle.kts index dc4a243..c4b789b 100644 --- a/launchers/connector/build.gradle.kts +++ b/launchers/connector/build.gradle.kts @@ -15,7 +15,7 @@ plugins { `java-library` - id("application") + id("com.gmv.inesdata.edc-application") alias(libs.plugins.shadow) } @@ -51,6 +51,22 @@ dependencies { implementation(libs.edc.transfer.pull.http.receiver) implementation(libs.edc.transfer.data.plane) + // Persistencia comun + implementation(libs.edc.sql.core) + implementation(libs.edc.sql.lease) + implementation(libs.edc.sql.pool) + // Persistencia control plane + implementation(libs.edc.sql.asset.index) + implementation(libs.edc.sql.contract.definition.store) + implementation(libs.edc.sql.contract.negotiation.store) + implementation(libs.edc.sql.policy.definition.store) + implementation(libs.edc.sql.transfer.process.store) + // Persistencia data plane + implementation(libs.edc.sql.data.plane.store) + + runtimeOnly(libs.edc.transaction.local) + runtimeOnly(libs.postgres) + } application { diff --git a/resources/configuration/consumer-configuration.properties b/resources/configuration/consumer-configuration.properties index 1dd7c83..fbe9771 100644 --- a/resources/configuration/consumer-configuration.properties +++ b/resources/configuration/consumer-configuration.properties @@ -19,3 +19,10 @@ web.http.control.port=29192 web.http.control.path=/control edc.dataplane.token.validation.endpoint=http://localhost:29192/control/token edc.catalog.cache.execution.period.seconds=120 + +edc.datasource.default.url=jdbc:postgresql://localhost:5433/inesdata +edc.datasource.default.user=inesdata +edc.datasource.default.password=inesdata +edc.datasource.default.pool.maxIdleConnections=10 +edc.datasource.default.pool.maxTotalConnections=10 +edc.datasource.default.pool.minIdleConnections=5 \ No newline at end of file diff --git a/resources/configuration/provider-configuration.properties b/resources/configuration/provider-configuration.properties index 93dcccd..4a1263d 100644 --- a/resources/configuration/provider-configuration.properties +++ b/resources/configuration/provider-configuration.properties @@ -18,3 +18,10 @@ web.http.public.path=/public web.http.control.port=19192 web.http.control.path=/control edc.dataplane.token.validation.endpoint=http://localhost:19192/control/token + +edc.datasource.default.url=jdbc:postgresql://localhost:5432/inesdata +edc.datasource.default.user=inesdata +edc.datasource.default.password=inesdata +edc.datasource.default.pool.maxIdleConnections=10 +edc.datasource.default.pool.maxTotalConnections=10 +edc.datasource.default.pool.minIdleConnections=5 \ No newline at end of file From 70e0e55449421be57520c5a3d08af4e04ceca057 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Mon, 5 Feb 2024 08:02:16 +0100 Subject: [PATCH 009/174] removed policy monitor libs from version file --- gradle/libs.versions.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 79b1f26..be54294 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -41,9 +41,6 @@ junit-jupiter-engine = { module = "org.junit.jupiter:junit-jupiter-engine", vers junit-jupiter-params = { module = "org.junit.jupiter:junit-jupiter-params", version.ref = "jupiter" } junit-pioneer = { module = "org.junit-pioneer:junit-pioneer", version.ref = "junit-pioneer" } -#Policy monitor -edc-core-policy-monitor = { module = "org.eclipse.edc:policy-monitor-core", version.ref = "edc" } - #SQL Extensions edc-transaction-local = { module = "org.eclipse.edc:transaction-local", version.ref = "edc" } edc-sql-core = { module = "org.eclipse.edc:sql-core", version.ref = "edc" } @@ -55,7 +52,6 @@ edc-sql-contract-negotiation-store = { module = "org.eclipse.edc:contract-negoti edc-sql-policy-definition-store = { module = "org.eclipse.edc:policy-definition-store-sql", version.ref = "edc" } edc-sql-transfer-process-store = { module = "org.eclipse.edc:transfer-process-store-sql", version.ref = "edc" } edc-sql-data-plane-store = { module = "org.eclipse.edc:data-plane-store-sql", version.ref = "edc" } -edc-sql-policy-monitor-store = { module = "org.eclipse.edc:policy-monitor-store-sql", version.ref = "edc" } # no EDC references postgres = { module = "org.postgresql:postgresql", version.ref = "postgres" } From 5fee3a25c853001a2958f56272b39a456c46b4a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Mon, 5 Feb 2024 14:28:59 +0100 Subject: [PATCH 010/174] fix MR comment and add needed SQL files --- .../consumer-configuration.properties | 2 +- .../provider-configuration.properties | 2 +- resources/sql/asset-schema.sql | 30 +++++++ resources/sql/contract-definition-schema.sql | 27 ++++++ resources/sql/contract-negotiation-schema.sql | 82 +++++++++++++++++++ resources/sql/data-plane-schema.sql | 40 +++++++++ resources/sql/policy-definition-schema.sql | 41 ++++++++++ resources/sql/transfer-process-schema.sql | 82 +++++++++++++++++++ 8 files changed, 304 insertions(+), 2 deletions(-) create mode 100644 resources/sql/asset-schema.sql create mode 100644 resources/sql/contract-definition-schema.sql create mode 100644 resources/sql/contract-negotiation-schema.sql create mode 100644 resources/sql/data-plane-schema.sql create mode 100644 resources/sql/policy-definition-schema.sql create mode 100644 resources/sql/transfer-process-schema.sql diff --git a/resources/configuration/consumer-configuration.properties b/resources/configuration/consumer-configuration.properties index fbe9771..01d6ae3 100644 --- a/resources/configuration/consumer-configuration.properties +++ b/resources/configuration/consumer-configuration.properties @@ -22,7 +22,7 @@ edc.catalog.cache.execution.period.seconds=120 edc.datasource.default.url=jdbc:postgresql://localhost:5433/inesdata edc.datasource.default.user=inesdata -edc.datasource.default.password=inesdata +edc.datasource.default.password= edc.datasource.default.pool.maxIdleConnections=10 edc.datasource.default.pool.maxTotalConnections=10 edc.datasource.default.pool.minIdleConnections=5 \ No newline at end of file diff --git a/resources/configuration/provider-configuration.properties b/resources/configuration/provider-configuration.properties index 4a1263d..cdd6a58 100644 --- a/resources/configuration/provider-configuration.properties +++ b/resources/configuration/provider-configuration.properties @@ -21,7 +21,7 @@ edc.dataplane.token.validation.endpoint=http://localhost:19192/control/token edc.datasource.default.url=jdbc:postgresql://localhost:5432/inesdata edc.datasource.default.user=inesdata -edc.datasource.default.password=inesdata +edc.datasource.default.password= edc.datasource.default.pool.maxIdleConnections=10 edc.datasource.default.pool.maxTotalConnections=10 edc.datasource.default.pool.minIdleConnections=5 \ No newline at end of file diff --git a/resources/sql/asset-schema.sql b/resources/sql/asset-schema.sql new file mode 100644 index 0000000..6274b5f --- /dev/null +++ b/resources/sql/asset-schema.sql @@ -0,0 +1,30 @@ +-- +-- Copyright (c) 2022 - 2023 Daimler TSS GmbH +-- +-- This program and the accompanying materials are made available under the +-- terms of the Apache License, Version 2.0 which is available at +-- https://www.apache.org/licenses/LICENSE-2.0 +-- +-- SPDX-License-Identifier: Apache-2.0 +-- +-- Contributors: +-- Daimler TSS GmbH - Initial SQL Query +-- Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - improvements +-- + +-- THIS SCHEMA HAS BEEN WRITTEN AND TESTED ONLY FOR POSTGRES + +-- table: edc_asset +CREATE TABLE IF NOT EXISTS edc_asset +( + asset_id VARCHAR NOT NULL, + created_at BIGINT NOT NULL, + properties JSON DEFAULT '{}', + private_properties JSON DEFAULT '{}', + data_address JSON DEFAULT '{}', + PRIMARY KEY (asset_id) +); + +COMMENT ON COLUMN edc_asset.properties IS 'Asset properties serialized as JSON'; +COMMENT ON COLUMN edc_asset.private_properties IS 'Asset private properties serialized as JSON'; +COMMENT ON COLUMN edc_asset.data_address IS 'Asset DataAddress serialized as JSON'; diff --git a/resources/sql/contract-definition-schema.sql b/resources/sql/contract-definition-schema.sql new file mode 100644 index 0000000..98a30b7 --- /dev/null +++ b/resources/sql/contract-definition-schema.sql @@ -0,0 +1,27 @@ +-- +-- Copyright (c) 2022 Daimler TSS GmbH +-- +-- This program and the accompanying materials are made available under the +-- terms of the Apache License, Version 2.0 which is available at +-- https://www.apache.org/licenses/LICENSE-2.0 +-- +-- SPDX-License-Identifier: Apache-2.0 +-- +-- Contributors: +-- Daimler TSS GmbH - Initial SQL Query +-- Microsoft Corporation - refactoring +-- SAP SE - add private properties to contract definition +-- + +-- table: edc_contract_definitions +-- only intended for and tested with H2 and Postgres! +CREATE TABLE IF NOT EXISTS edc_contract_definitions +( + created_at BIGINT NOT NULL, + contract_definition_id VARCHAR NOT NULL, + access_policy_id VARCHAR NOT NULL, + contract_policy_id VARCHAR NOT NULL, + assets_selector JSON NOT NULL, + private_properties JSON, + PRIMARY KEY (contract_definition_id) +); diff --git a/resources/sql/contract-negotiation-schema.sql b/resources/sql/contract-negotiation-schema.sql new file mode 100644 index 0000000..2d909ec --- /dev/null +++ b/resources/sql/contract-negotiation-schema.sql @@ -0,0 +1,82 @@ +-- Statements are designed for and tested with Postgres only! + +CREATE TABLE IF NOT EXISTS edc_lease +( + leased_by VARCHAR NOT NULL, + leased_at BIGINT, + lease_duration INTEGER DEFAULT 60000 NOT NULL, + lease_id VARCHAR NOT NULL + CONSTRAINT lease_pk + PRIMARY KEY +); + +COMMENT ON COLUMN edc_lease.leased_at IS 'posix timestamp of lease'; + +COMMENT ON COLUMN edc_lease.lease_duration IS 'duration of lease in milliseconds'; + + +CREATE UNIQUE INDEX IF NOT EXISTS lease_lease_id_uindex + ON edc_lease (lease_id); + + + +CREATE TABLE IF NOT EXISTS edc_contract_agreement +( + agr_id VARCHAR NOT NULL + CONSTRAINT contract_agreement_pk + PRIMARY KEY, + provider_agent_id VARCHAR, + consumer_agent_id VARCHAR, + signing_date BIGINT, + start_date BIGINT, + end_date INTEGER, + asset_id VARCHAR NOT NULL, + policy JSON +); + + +CREATE TABLE IF NOT EXISTS edc_contract_negotiation +( + id VARCHAR NOT NULL + CONSTRAINT contract_negotiation_pk + PRIMARY KEY, + created_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL, + correlation_id VARCHAR, + counterparty_id VARCHAR NOT NULL, + counterparty_address VARCHAR NOT NULL, + protocol VARCHAR NOT NULL, + type VARCHAR NOT NULL, + state INTEGER DEFAULT 0 NOT NULL, + state_count INTEGER DEFAULT 0, + state_timestamp BIGINT, + error_detail VARCHAR, + agreement_id VARCHAR + CONSTRAINT contract_negotiation_contract_agreement_id_fk + REFERENCES edc_contract_agreement, + contract_offers JSON, + callback_addresses JSON, + trace_context JSON, + pending BOOLEAN DEFAULT FALSE, + protocol_messages JSON, + lease_id VARCHAR + CONSTRAINT contract_negotiation_lease_lease_id_fk + REFERENCES edc_lease + ON DELETE SET NULL +); + +COMMENT ON COLUMN edc_contract_negotiation.agreement_id IS 'ContractAgreement serialized as JSON'; + +COMMENT ON COLUMN edc_contract_negotiation.contract_offers IS 'List serialized as JSON'; + +COMMENT ON COLUMN edc_contract_negotiation.trace_context IS 'Map serialized as JSON'; + + +CREATE INDEX IF NOT EXISTS contract_negotiation_correlationid_index + ON edc_contract_negotiation (correlation_id); + +CREATE UNIQUE INDEX IF NOT EXISTS contract_negotiation_id_uindex + ON edc_contract_negotiation (id); + +CREATE UNIQUE INDEX IF NOT EXISTS contract_agreement_id_uindex + ON edc_contract_agreement (agr_id); diff --git a/resources/sql/data-plane-schema.sql b/resources/sql/data-plane-schema.sql new file mode 100644 index 0000000..91769de --- /dev/null +++ b/resources/sql/data-plane-schema.sql @@ -0,0 +1,40 @@ +-- Statements are designed for and tested with Postgres only! + +CREATE TABLE IF NOT EXISTS edc_lease +( + leased_by VARCHAR NOT NULL, + leased_at BIGINT, + lease_duration INTEGER NOT NULL, + lease_id VARCHAR NOT NULL + CONSTRAINT lease_pk + PRIMARY KEY +); + +COMMENT ON COLUMN edc_lease.leased_at IS 'posix timestamp of lease'; +COMMENT ON COLUMN edc_lease.lease_duration IS 'duration of lease in milliseconds'; + +CREATE TABLE IF NOT EXISTS edc_data_plane +( + process_id VARCHAR NOT NULL PRIMARY KEY, + state INTEGER NOT NULL , + created_at BIGINT NOT NULL , + updated_at BIGINT NOT NULL , + state_count INTEGER DEFAULT 0 NOT NULL, + state_time_stamp BIGINT, + trace_context JSON, + error_detail VARCHAR, + callback_address VARCHAR, + trackable BOOLEAN, + lease_id VARCHAR + CONSTRAINT data_plane_lease_lease_id_fk + REFERENCES edc_lease + ON DELETE SET NULL, + source JSON, + destination JSON, + properties JSON +); + +COMMENT ON COLUMN edc_data_plane.trace_context IS 'Java Map serialized as JSON'; +COMMENT ON COLUMN edc_data_plane.source IS 'DataAddress serialized as JSON'; +COMMENT ON COLUMN edc_data_plane.destination IS 'DataAddress serialized as JSON'; +COMMENT ON COLUMN edc_data_plane.properties IS 'Java Map serialized as JSON'; diff --git a/resources/sql/policy-definition-schema.sql b/resources/sql/policy-definition-schema.sql new file mode 100644 index 0000000..d4ef812 --- /dev/null +++ b/resources/sql/policy-definition-schema.sql @@ -0,0 +1,41 @@ +-- +-- Copyright (c) 2022 ZF Friedrichshafen AG +-- +-- This program and the accompanying materials are made available under the +-- terms of the Apache License, Version 2.0 which is available at +-- https://www.apache.org/licenses/LICENSE-2.0 +-- +-- SPDX-License-Identifier: Apache-2.0 +-- +-- Contributors: +-- ZF Friedrichshafen AG - Initial SQL Query +-- + +-- Statements are designed for and tested with Postgres only! + +-- table: edc_policydefinitions +CREATE TABLE IF NOT EXISTS edc_policydefinitions +( + policy_id VARCHAR NOT NULL, + created_at BIGINT NOT NULL, + permissions JSON, + prohibitions JSON, + duties JSON, + extensible_properties JSON, + inherits_from VARCHAR, + assigner VARCHAR, + assignee VARCHAR, + target VARCHAR, + policy_type VARCHAR NOT NULL, + private_properties JSON, + PRIMARY KEY (policy_id) +); + +COMMENT ON COLUMN edc_policydefinitions.permissions IS 'Java List serialized as JSON'; +COMMENT ON COLUMN edc_policydefinitions.prohibitions IS 'Java List serialized as JSON'; +COMMENT ON COLUMN edc_policydefinitions.duties IS 'Java List serialized as JSON'; +COMMENT ON COLUMN edc_policydefinitions.extensible_properties IS 'Java Map serialized as JSON'; +COMMENT ON COLUMN edc_policydefinitions.policy_type IS 'Java PolicyType serialized as JSON'; + +CREATE UNIQUE INDEX IF NOT EXISTS edc_policydefinitions_id_uindex + ON edc_policydefinitions (policy_id); diff --git a/resources/sql/transfer-process-schema.sql b/resources/sql/transfer-process-schema.sql new file mode 100644 index 0000000..ba7d38e --- /dev/null +++ b/resources/sql/transfer-process-schema.sql @@ -0,0 +1,82 @@ +-- Statements are designed for and tested with Postgres only! + +CREATE TABLE IF NOT EXISTS edc_lease +( + leased_by VARCHAR NOT NULL, + leased_at BIGINT, + lease_duration INTEGER NOT NULL, + lease_id VARCHAR NOT NULL + CONSTRAINT lease_pk + PRIMARY KEY +); + +COMMENT ON COLUMN edc_lease.leased_at IS 'posix timestamp of lease'; + +COMMENT ON COLUMN edc_lease.lease_duration IS 'duration of lease in milliseconds'; + +CREATE TABLE IF NOT EXISTS edc_transfer_process +( + transferprocess_id VARCHAR NOT NULL + CONSTRAINT transfer_process_pk + PRIMARY KEY, + type VARCHAR NOT NULL, + state INTEGER NOT NULL, + state_count INTEGER DEFAULT 0 NOT NULL, + state_time_stamp BIGINT, + created_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL, + trace_context JSON, + error_detail VARCHAR, + resource_manifest JSON, + provisioned_resource_set JSON, + content_data_address JSON, + deprovisioned_resources JSON, + private_properties JSON, + callback_addresses JSON, + pending BOOLEAN DEFAULT FALSE, + transfer_type VARCHAR, + protocol_messages JSON, + lease_id VARCHAR + CONSTRAINT transfer_process_lease_lease_id_fk + REFERENCES edc_lease + ON DELETE SET NULL +); + +COMMENT ON COLUMN edc_transfer_process.trace_context IS 'Java Map serialized as JSON'; + +COMMENT ON COLUMN edc_transfer_process.resource_manifest IS 'java ResourceManifest serialized as JSON'; + +COMMENT ON COLUMN edc_transfer_process.provisioned_resource_set IS 'ProvisionedResourceSet serialized as JSON'; + +COMMENT ON COLUMN edc_transfer_process.content_data_address IS 'DataAddress serialized as JSON'; + +COMMENT ON COLUMN edc_transfer_process.deprovisioned_resources IS 'List of deprovisioned resources, serialized as JSON'; + + +CREATE UNIQUE INDEX IF NOT EXISTS transfer_process_id_uindex + ON edc_transfer_process (transferprocess_id); + +CREATE TABLE IF NOT EXISTS edc_data_request +( + datarequest_id VARCHAR NOT NULL + CONSTRAINT data_request_pk + PRIMARY KEY, + process_id VARCHAR NOT NULL, + connector_address VARCHAR NOT NULL, + protocol VARCHAR NOT NULL, + asset_id VARCHAR NOT NULL, + contract_id VARCHAR NOT NULL, + data_destination JSON NOT NULL, + transfer_process_id VARCHAR NOT NULL + CONSTRAINT data_request_transfer_process_id_fk + REFERENCES edc_transfer_process + ON UPDATE RESTRICT ON DELETE CASCADE +); + +COMMENT ON COLUMN edc_data_request.data_destination IS 'DataAddress serialized as JSON'; + +CREATE UNIQUE INDEX IF NOT EXISTS data_request_id_uindex + ON edc_data_request (datarequest_id); + +CREATE UNIQUE INDEX IF NOT EXISTS lease_lease_id_uindex + ON edc_lease (lease_id); From 6debe717b81f63139823ccef20e0fccd2bbc473d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Tue, 6 Feb 2024 13:44:33 +0100 Subject: [PATCH 011/174] fix wrong vault public certificate --- resources/configuration/consumer-vault.properties | 2 +- resources/configuration/provider-vault.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/configuration/consumer-vault.properties b/resources/configuration/consumer-vault.properties index 6ebdebd..63ddb09 100644 --- a/resources/configuration/consumer-vault.properties +++ b/resources/configuration/consumer-vault.properties @@ -1 +1 @@ -public-key=-----BEGIN CERTIFICATE-----\r\nMIIDazCCAlOgAwIBAgIUZ3/sZXYzW4PjmOXKrZn6WBmUJ+4wDQYJKoZIhvcNAQEL\r\nBQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM\r\nGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMjAyMjMxNTA2MDNaFw0zMjAy\r\nMjExNTA2MDNaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw\r\nHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB\r\nAQUAA4IBDwAwggEKAoIBAQDBl6XaJnXTL+6DWip3aBhU+MzmY4d1V9hbTm1tiZ3g\r\nE0VbUrvGO3LoYaxpPv6zFmsg3uJv6JxVAde7EddidN0ITHB9cQNdAfdUJ5njmsGS\r\nPbdQuOQTHw0aG7/QvTI/nsvfEE6e0lbV/0e7DHacZT/+OztBH1RwkG2ymM94Hf8H\r\nI6x7q6yfRTAZOqeOMrPCYTcluAgE9NskoPvjX5qASakBtXISKIsOU84N0/2HDN3W\r\nEGMXvoHUQu6vrij6BwiwxKaw1AKwWENKoga775bPXN3M+JTSaIKE7dZbKzvx0Zi0\r\nh5X+bxc3BJi3Z/CsUBCzE+Y0SFetOiYmyl/2YmnneYoVAgMBAAGjUzBRMB0GA1Ud\r\nDgQWBBTvK1wVERwjni4B2vdH7KtEJeVWFzAfBgNVHSMEGDAWgBTvK1wVERwjni4B\r\n2vdH7KtEJeVWFzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBn\r\nQHiPA7OBYukHd9gS7c0HXE+fsWcS3GZeLqcHfQQnV3pte1vTmu9//IVW71wNCJ1/\r\nrySRyODPQoPehxEcyHwupNZSzXK//nPlTdSgjMfFxscvt1YndyQLQYCfyOJMixAe\r\nAqrb14GTFHUUrdor0PyElhkULjkOXUrSIsdBrfWrwLTkelE8NK3tb5ZG8KPzD9Jy\r\n+NwEPPr9d+iHkUkM7EFWw/cl56wka9ryBb97RI7DqbO6/j6OXHMk4GByxKv7DSIR\r\nIvF9/Dw20qytajtaHV0pluFcOBuFc0NfiDvCaQlbTsfjzbc6UmZWbOi9YOJl3VQ/\r\ng3h+15GuzbsSzOCOEYOT\r\n-----END CERTIFICATE----- +public-key=-----BEGIN CERTIFICATE-----\r\nMIIDETCCAfkCFBEOuzF+mRgrzrxVv70zNDUEv+nUMA0GCSqGSIb3DQEBCwUAMEUx\r\nCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl\r\ncm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjQwMTMxMTIyNTUyWhcNMjUwMTMwMTIy\r\nNTUyWjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE\r\nCgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC\r\nAQ8AMIIBCgKCAQEAxHbsW755sB/5YwjPyk0xPJwyQkxkGV2SS4sQQCCz8KV/QoEi\r\nlk/PzUqUsE2e7LqavWL5+FPSic79ZKEljeeNB1z5xoE2YnKEs55MLi/PgfLJHJ9M\r\nbA42lJBYI37MeGv0kGkIQDIrtpDjBfSgGSINXeTjs0T/l5sXXPTfSRm4URWX3I+Q\r\nO9ACYLb7Cz//G8HdYHp8MHNa38x4BeBku7cT7xgEUPfhu4LCnBW1u0pMbeT8A2M9\r\nzz/TIOJKwUeupacXjJ/tqHGLYMgBLCSRNK4zFJiGzhDbEZA6+NMUp660t7Su6M5i\r\nx3WIffKlIcxi0bnk9slhjK5NynJ3H/uvbfYnCwIDAQABMA0GCSqGSIb3DQEBCwUA\r\nA4IBAQBAttZGxh1k87S8OHA6L8fmu/PEtjdkJV3Cu7J3xTGfmSBVMTStl2NI4D4i\r\nFJUFdTJjvEibEaQEETz2zJakwdp7JGG7ip7e/dKsP0wVMC63MZ8eeDzST7IzBLk6\r\nmjb+Co0+2p/9rHPBz1AFpHRcJ9jaORh8FlctfcxcBsr6dMptGtCZMzam26WwePxa\r\npNd+wBNFHuAXHyTzHl3NIq2Bd+56Roen5muZKv4xqI7iVBbVrejvVyXNDhrqh9QB\r\nVushjclGdc2GSdCVpUhiv1srX59/yX5HBZG8HRQSgCXpcSiGIhiMmd1gaU/qrXa8\r\nkwmd6tdKUcAiLU7qXEJX4iBziobm\r\n-----END CERTIFICATE----- diff --git a/resources/configuration/provider-vault.properties b/resources/configuration/provider-vault.properties index 6ebdebd..bb037d6 100644 --- a/resources/configuration/provider-vault.properties +++ b/resources/configuration/provider-vault.properties @@ -1 +1 @@ -public-key=-----BEGIN CERTIFICATE-----\r\nMIIDazCCAlOgAwIBAgIUZ3/sZXYzW4PjmOXKrZn6WBmUJ+4wDQYJKoZIhvcNAQEL\r\nBQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM\r\nGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMjAyMjMxNTA2MDNaFw0zMjAy\r\nMjExNTA2MDNaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw\r\nHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB\r\nAQUAA4IBDwAwggEKAoIBAQDBl6XaJnXTL+6DWip3aBhU+MzmY4d1V9hbTm1tiZ3g\r\nE0VbUrvGO3LoYaxpPv6zFmsg3uJv6JxVAde7EddidN0ITHB9cQNdAfdUJ5njmsGS\r\nPbdQuOQTHw0aG7/QvTI/nsvfEE6e0lbV/0e7DHacZT/+OztBH1RwkG2ymM94Hf8H\r\nI6x7q6yfRTAZOqeOMrPCYTcluAgE9NskoPvjX5qASakBtXISKIsOU84N0/2HDN3W\r\nEGMXvoHUQu6vrij6BwiwxKaw1AKwWENKoga775bPXN3M+JTSaIKE7dZbKzvx0Zi0\r\nh5X+bxc3BJi3Z/CsUBCzE+Y0SFetOiYmyl/2YmnneYoVAgMBAAGjUzBRMB0GA1Ud\r\nDgQWBBTvK1wVERwjni4B2vdH7KtEJeVWFzAfBgNVHSMEGDAWgBTvK1wVERwjni4B\r\n2vdH7KtEJeVWFzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBn\r\nQHiPA7OBYukHd9gS7c0HXE+fsWcS3GZeLqcHfQQnV3pte1vTmu9//IVW71wNCJ1/\r\nrySRyODPQoPehxEcyHwupNZSzXK//nPlTdSgjMfFxscvt1YndyQLQYCfyOJMixAe\r\nAqrb14GTFHUUrdor0PyElhkULjkOXUrSIsdBrfWrwLTkelE8NK3tb5ZG8KPzD9Jy\r\n+NwEPPr9d+iHkUkM7EFWw/cl56wka9ryBb97RI7DqbO6/j6OXHMk4GByxKv7DSIR\r\nIvF9/Dw20qytajtaHV0pluFcOBuFc0NfiDvCaQlbTsfjzbc6UmZWbOi9YOJl3VQ/\r\ng3h+15GuzbsSzOCOEYOT\r\n-----END CERTIFICATE----- +public-key=-----BEGIN CERTIFICATE-----\r\nMIIDETCCAfkCFBEOuzF+mRgrzrxVv70zNDUEv+nUMA0GCSqGSIb3DQEBCwUAMEUx\r\nCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl\r\ncm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjQwMTMxMTIyNTUyWhcNMjUwMTMwMTIy\r\nNTUyWjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE\r\nCgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC\r\nAQ8AMIIBCgKCAQEAxHbsW755sB/5YwjPyk0xPJwyQkxkGV2SS4sQQCCz8KV/QoEi\r\nlk/PzUqUsE2e7LqavWL5+FPSic79ZKEljeeNB1z5xoE2YnKEs55MLi/PgfLJHJ9M\r\nbA42lJBYI37MeGv0kGkIQDIrtpDjBfSgGSINXeTjs0T/l5sXXPTfSRm4URWX3I+Q\r\nO9ACYLb7Cz//G8HdYHp8MHNa38x4BeBku7cT7xgEUPfhu4LCnBW1u0pMbeT8A2M9\r\nzz/TIOJKwUeupacXjJ/tqHGLYMgBLCSRNK4zFJiGzhDbEZA6+NMUp660t7Su6M5i\r\nx3WIffKlIcxi0bnk9slhjK5NynJ3H/uvbfYnCwIDAQABMA0GCSqGSIb3DQEBCwUA\r\nA4IBAQBAttZGxh1k87S8OHA6L8fmu/PEtjdkJV3Cu7J3xTGfmSBVMTStl2NI4D4i\r\nFJUFdTJjvEibEaQEETz2zJakwdp7JGG7ip7e/dKsP0wVMC63MZ8eeDzST7IzBLk6\r\nmjb+Co0+2p/9rHPBz1AFpHRcJ9jaORh8FlctfcxcBsr6dMptGtCZMzam26WwePxa\r\npNd+wBNFHuAXHyTzHl3NIq2Bd+56Roen5muZKv4xqI7iVBbVrejvVyXNDhrqh9QB\r\nVushjclGdc2GSdCVpUhiv1srX59/yX5HBZG8HRQSgCXpcSiGIhiMmd1gaU/qrXa8\r\nkwmd6tdKUcAiLU7qXEJX4iBziobm\r\n-----END CERTIFICATE----- \ No newline at end of file From 9be3b5b342a7bf4e45f85f6656728c3d637b9756 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Tue, 6 Feb 2024 13:54:04 +0100 Subject: [PATCH 012/174] primeros cambios minio --- gradle/libs.versions.toml | 7 ++++++ launchers/connector/build.gradle.kts | 15 ------------ .../consumer-configuration.properties | 8 +++++-- .../provider-configuration.properties | 8 +++++-- .../register-data-plane-consumer.json | 6 +++-- .../register-data-plane-provider.json | 6 +++-- resources/operations/negotiate-contract.json | 2 +- resources/operations/start-transfer.json | 2 +- resources/operationss3/create-asset.json | 16 +++++++++++++ .../create-contract-definition.json | 9 ++++++++ resources/operationss3/create-policy.json | 13 +++++++++++ resources/operationss3/fetch-catalog.json | 7 ++++++ resources/operationss3/get-dataset.json | 7 ++++++ .../operationss3/negotiate-contract.json | 20 ++++++++++++++++ .../operationss3/start-transfer-proxy.json | 14 +++++++++++ resources/operationss3/start-transfer.json | 23 +++++++++++++++++++ 16 files changed, 138 insertions(+), 25 deletions(-) create mode 100644 resources/operationss3/create-asset.json create mode 100644 resources/operationss3/create-contract-definition.json create mode 100644 resources/operationss3/create-policy.json create mode 100644 resources/operationss3/fetch-catalog.json create mode 100644 resources/operationss3/get-dataset.json create mode 100644 resources/operationss3/negotiate-contract.json create mode 100644 resources/operationss3/start-transfer-proxy.json create mode 100644 resources/operationss3/start-transfer.json diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index be54294..9d4a78d 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -3,6 +3,7 @@ format.version = "1.1" [versions] assertj = "3.25.1" +aws = "2.23.12" edc = "0.5.0" junit-pioneer = "2.2.0" jupiter = "5.10.1" @@ -53,7 +54,13 @@ edc-sql-policy-definition-store = { module = "org.eclipse.edc:policy-definition- edc-sql-transfer-process-store = { module = "org.eclipse.edc:transfer-process-store-sql", version.ref = "edc" } edc-sql-data-plane-store = { module = "org.eclipse.edc:data-plane-store-sql", version.ref = "edc" } +# EDC aws s3 stuff +edc-aws-s3-core = { module = "org.eclipse.edc:aws-s3-core", version.ref = "edc" } +edc-data-plane-aws-s3 = { module = "org.eclipse.edc:data-plane-aws-s3", version.ref = "edc" } + # no EDC references +aws-s3 = { module = "software.amazon.awssdk:s3", version.ref = "aws" } +aws-s3-transfer = { module = "software.amazon.awssdk:s3-transfer-manager", version.ref = "aws" } postgres = { module = "org.postgresql:postgresql", version.ref = "postgres" } [plugins] diff --git a/launchers/connector/build.gradle.kts b/launchers/connector/build.gradle.kts index c4b789b..398d7e3 100644 --- a/launchers/connector/build.gradle.kts +++ b/launchers/connector/build.gradle.kts @@ -51,21 +51,6 @@ dependencies { implementation(libs.edc.transfer.pull.http.receiver) implementation(libs.edc.transfer.data.plane) - // Persistencia comun - implementation(libs.edc.sql.core) - implementation(libs.edc.sql.lease) - implementation(libs.edc.sql.pool) - // Persistencia control plane - implementation(libs.edc.sql.asset.index) - implementation(libs.edc.sql.contract.definition.store) - implementation(libs.edc.sql.contract.negotiation.store) - implementation(libs.edc.sql.policy.definition.store) - implementation(libs.edc.sql.transfer.process.store) - // Persistencia data plane - implementation(libs.edc.sql.data.plane.store) - - runtimeOnly(libs.edc.transaction.local) - runtimeOnly(libs.postgres) } diff --git a/resources/configuration/consumer-configuration.properties b/resources/configuration/consumer-configuration.properties index 01d6ae3..f387943 100644 --- a/resources/configuration/consumer-configuration.properties +++ b/resources/configuration/consumer-configuration.properties @@ -22,7 +22,11 @@ edc.catalog.cache.execution.period.seconds=120 edc.datasource.default.url=jdbc:postgresql://localhost:5433/inesdata edc.datasource.default.user=inesdata -edc.datasource.default.password= +edc.datasource.default.password=inesdata edc.datasource.default.pool.maxIdleConnections=10 edc.datasource.default.pool.maxTotalConnections=10 -edc.datasource.default.pool.minIdleConnections=5 \ No newline at end of file +edc.datasource.default.pool.minIdleConnections=5 + +edc.aws.access.key=wEfyrv2vH7Wj3ON03g8V +edc.aws.secret.access.key=JlHVGgSgeRkbs6WtKTugR4I5uU7oA1VRf8kBtrGd +edc.aws.endpoint.override=http://localhost:9010 diff --git a/resources/configuration/provider-configuration.properties b/resources/configuration/provider-configuration.properties index cdd6a58..b8ed938 100644 --- a/resources/configuration/provider-configuration.properties +++ b/resources/configuration/provider-configuration.properties @@ -21,7 +21,11 @@ edc.dataplane.token.validation.endpoint=http://localhost:19192/control/token edc.datasource.default.url=jdbc:postgresql://localhost:5432/inesdata edc.datasource.default.user=inesdata -edc.datasource.default.password= +edc.datasource.default.password=inesdata edc.datasource.default.pool.maxIdleConnections=10 edc.datasource.default.pool.maxTotalConnections=10 -edc.datasource.default.pool.minIdleConnections=5 \ No newline at end of file +edc.datasource.default.pool.minIdleConnections=5 + +edc.aws.access.key=DrJ6PmnJ3QtlxYowRSLi +edc.aws.secret.access.key=gjaVnJstoeV4bWLrJkmswKBMIZxY10PjZOJhp2qk +edc.aws.endpoint.override=http://localhost:9000 \ No newline at end of file diff --git a/resources/dataplane/register-data-plane-consumer.json b/resources/dataplane/register-data-plane-consumer.json index 7ca6ff1..cbae3bd 100644 --- a/resources/dataplane/register-data-plane-consumer.json +++ b/resources/dataplane/register-data-plane-consumer.json @@ -5,11 +5,13 @@ "@id": "http-pull-consumer-dataplane", "url": "http://localhost:29192/control/transfer", "allowedSourceTypes": [ - "HttpData" + "HttpData", + "AmazonS3" ], "allowedDestTypes": [ "HttpProxy", - "HttpData" + "HttpData", + "AmazonS3" ], "properties": { "https://w3id.org/edc/v0.0.1/ns/publicApiUrl/publicApiUrl": "http://localhost:29291/public/" diff --git a/resources/dataplane/register-data-plane-provider.json b/resources/dataplane/register-data-plane-provider.json index 0ee24de..e88a576 100644 --- a/resources/dataplane/register-data-plane-provider.json +++ b/resources/dataplane/register-data-plane-provider.json @@ -5,11 +5,13 @@ "@id": "http-pull-provider-dataplane", "url": "http://localhost:19192/control/transfer", "allowedSourceTypes": [ - "HttpData" + "HttpData", + "AmazonS3" ], "allowedDestTypes": [ "HttpProxy", - "HttpData" + "HttpData", + "AmazonS3" ], "properties": { "https://w3id.org/edc/v0.0.1/ns/publicApiUrl": "http://localhost:19291/public/" diff --git a/resources/operations/negotiate-contract.json b/resources/operations/negotiate-contract.json index fb1e2e9..fc096c4 100644 --- a/resources/operations/negotiate-contract.json +++ b/resources/operations/negotiate-contract.json @@ -10,7 +10,7 @@ "protocol": "dataspace-protocol-http", "policy": { "@context": "http://www.w3.org/ns/odrl.jsonld", - "@id": "MQ==:YXNzZXRJZA==:OTFhZDFiMzUtZjk3MS00ZTVlLThhYWYtYTY3Yzk1YjQyY2Mw", + "@id": "MQ==:YXNzZXRJZA==:NDRlN2FkNjktNmQyMy00ZmM5LTljM2EtYmRkN2Q0NDcxNzM3", "@type": "Set", "permission": [], "prohibition": [], diff --git a/resources/operations/start-transfer.json b/resources/operations/start-transfer.json index 8f2996f..5a094d2 100644 --- a/resources/operations/start-transfer.json +++ b/resources/operations/start-transfer.json @@ -5,7 +5,7 @@ "@type": "TransferRequestDto", "connectorId": "provider", "counterPartyAddress": "http://localhost:19194/protocol", - "contractId": "49aea7b0-86b6-4c78-a09b-1b1faf205899", + "contractId": "36568fc7-0fc8-4e11-a730-6da40ccdb93b", "assetId": "assetId", "protocol": "dataspace-protocol-http", "dataDestination": { diff --git a/resources/operationss3/create-asset.json b/resources/operationss3/create-asset.json new file mode 100644 index 0000000..28c83fa --- /dev/null +++ b/resources/operationss3/create-asset.json @@ -0,0 +1,16 @@ +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@id": "s3asset", + "properties": { + "name": "product description", + "contenttype": "application/json" + }, + "dataAddress": { + "type": "AmazonS3", + "region": "eu-south-2", + "keyName": "text-document-provider-dataset.txt", + "bucketName": "bucket-connector-1" + } +} diff --git a/resources/operationss3/create-contract-definition.json b/resources/operationss3/create-contract-definition.json new file mode 100644 index 0000000..50e16c2 --- /dev/null +++ b/resources/operationss3/create-contract-definition.json @@ -0,0 +1,9 @@ +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@id": "1", + "accessPolicyId": "s3Policy", + "contractPolicyId": "s3Policy", + "assetsSelector": [] +} diff --git a/resources/operationss3/create-policy.json b/resources/operationss3/create-policy.json new file mode 100644 index 0000000..187bf51 --- /dev/null +++ b/resources/operationss3/create-policy.json @@ -0,0 +1,13 @@ +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/", + "odrl": "http://www.w3.org/ns/odrl/2/" + }, + "@id": "s3Policy", + "policy": { + "@type": "set", + "odrl:permission": [], + "odrl:prohibition": [], + "odrl:obligation": [] + } +} diff --git a/resources/operationss3/fetch-catalog.json b/resources/operationss3/fetch-catalog.json new file mode 100644 index 0000000..370b4f2 --- /dev/null +++ b/resources/operationss3/fetch-catalog.json @@ -0,0 +1,7 @@ +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "counterPartyAddress": "http://localhost:19194/protocol", + "protocol": "dataspace-protocol-http" +} diff --git a/resources/operationss3/get-dataset.json b/resources/operationss3/get-dataset.json new file mode 100644 index 0000000..5286af0 --- /dev/null +++ b/resources/operationss3/get-dataset.json @@ -0,0 +1,7 @@ +{ + "@context": { "@vocab": "https://w3id.org/edc/v0.0.1/ns/" }, + "@type": "DatasetRequest", + "@id": "assetId", + "counterPartyAddress": "http://localhost:19194/protocol", + "protocol": "dataspace-protocol-http" +} \ No newline at end of file diff --git a/resources/operationss3/negotiate-contract.json b/resources/operationss3/negotiate-contract.json new file mode 100644 index 0000000..84e12fe --- /dev/null +++ b/resources/operationss3/negotiate-contract.json @@ -0,0 +1,20 @@ +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "NegotiationInitiateRequestDto", + "connectorId": "provider", + "counterPartyAddress": "http://localhost:19194/protocol", + "consumerId": "consumer", + "providerId": "provider", + "protocol": "dataspace-protocol-http", + "policy": { + "@context": "http://www.w3.org/ns/odrl.jsonld", + "@id": "MQ==:czNhc3NldA==:NDBkMjY1ZDAtNjljOS00ZWY1LTg4NzktM2QxNmVlYWU1NTZk", + "@type": "Set", + "permission": [], + "prohibition": [], + "obligation": [], + "target": "assetId" + } +} diff --git a/resources/operationss3/start-transfer-proxy.json b/resources/operationss3/start-transfer-proxy.json new file mode 100644 index 0000000..783a1d4 --- /dev/null +++ b/resources/operationss3/start-transfer-proxy.json @@ -0,0 +1,14 @@ +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "TransferRequestDto", + "connectorId": "provider", + "counterPartyAddress": "http://localhost:19194/protocol", + "contractId": "7c507b6a-828b-4948-a2b9-a7e4bcd392de", + "assetId": "s3asset", + "protocol": "dataspace-protocol-http", + "dataDestination": { + "type": "HttpProxy" + } +} diff --git a/resources/operationss3/start-transfer.json b/resources/operationss3/start-transfer.json new file mode 100644 index 0000000..a213935 --- /dev/null +++ b/resources/operationss3/start-transfer.json @@ -0,0 +1,23 @@ +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "TransferRequestDto", + "connectorId": "provider", + "counterPartyAddress": "http://localhost:19194/protocol", + "contractId": "94ef53b4-548c-47d3-bce3-e17f62820c5e", + "assetId": "s3asset", + "protocol": "dataspace-protocol-http", + "dataDestination": { + "type": "AmazonS3", + "region": "us-east-1", + "bucketName": "bucket-connector-2", + "endpointOverride": "http://localhost:9010", + "accessKeyId": "wEfyrv2vH7Wj3ON03g8V", + "secretAccessKey": "JlHVGgSgeRkbs6WtKTugR4I5uU7oA1VRf8kBtrGd" + }, + "transferType": { + "contentType": "application/octet-stream", + "isFinite": true + } +} From 213fe50fdbce2fe1c0f44021989ad595c6c0b7c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Tue, 6 Feb 2024 13:44:33 +0100 Subject: [PATCH 013/174] fix wrong vault public certificate --- resources/configuration/consumer-vault.properties | 2 +- resources/configuration/provider-vault.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/configuration/consumer-vault.properties b/resources/configuration/consumer-vault.properties index 6ebdebd..63ddb09 100644 --- a/resources/configuration/consumer-vault.properties +++ b/resources/configuration/consumer-vault.properties @@ -1 +1 @@ -public-key=-----BEGIN CERTIFICATE-----\r\nMIIDazCCAlOgAwIBAgIUZ3/sZXYzW4PjmOXKrZn6WBmUJ+4wDQYJKoZIhvcNAQEL\r\nBQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM\r\nGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMjAyMjMxNTA2MDNaFw0zMjAy\r\nMjExNTA2MDNaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw\r\nHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB\r\nAQUAA4IBDwAwggEKAoIBAQDBl6XaJnXTL+6DWip3aBhU+MzmY4d1V9hbTm1tiZ3g\r\nE0VbUrvGO3LoYaxpPv6zFmsg3uJv6JxVAde7EddidN0ITHB9cQNdAfdUJ5njmsGS\r\nPbdQuOQTHw0aG7/QvTI/nsvfEE6e0lbV/0e7DHacZT/+OztBH1RwkG2ymM94Hf8H\r\nI6x7q6yfRTAZOqeOMrPCYTcluAgE9NskoPvjX5qASakBtXISKIsOU84N0/2HDN3W\r\nEGMXvoHUQu6vrij6BwiwxKaw1AKwWENKoga775bPXN3M+JTSaIKE7dZbKzvx0Zi0\r\nh5X+bxc3BJi3Z/CsUBCzE+Y0SFetOiYmyl/2YmnneYoVAgMBAAGjUzBRMB0GA1Ud\r\nDgQWBBTvK1wVERwjni4B2vdH7KtEJeVWFzAfBgNVHSMEGDAWgBTvK1wVERwjni4B\r\n2vdH7KtEJeVWFzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBn\r\nQHiPA7OBYukHd9gS7c0HXE+fsWcS3GZeLqcHfQQnV3pte1vTmu9//IVW71wNCJ1/\r\nrySRyODPQoPehxEcyHwupNZSzXK//nPlTdSgjMfFxscvt1YndyQLQYCfyOJMixAe\r\nAqrb14GTFHUUrdor0PyElhkULjkOXUrSIsdBrfWrwLTkelE8NK3tb5ZG8KPzD9Jy\r\n+NwEPPr9d+iHkUkM7EFWw/cl56wka9ryBb97RI7DqbO6/j6OXHMk4GByxKv7DSIR\r\nIvF9/Dw20qytajtaHV0pluFcOBuFc0NfiDvCaQlbTsfjzbc6UmZWbOi9YOJl3VQ/\r\ng3h+15GuzbsSzOCOEYOT\r\n-----END CERTIFICATE----- +public-key=-----BEGIN CERTIFICATE-----\r\nMIIDETCCAfkCFBEOuzF+mRgrzrxVv70zNDUEv+nUMA0GCSqGSIb3DQEBCwUAMEUx\r\nCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl\r\ncm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjQwMTMxMTIyNTUyWhcNMjUwMTMwMTIy\r\nNTUyWjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE\r\nCgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC\r\nAQ8AMIIBCgKCAQEAxHbsW755sB/5YwjPyk0xPJwyQkxkGV2SS4sQQCCz8KV/QoEi\r\nlk/PzUqUsE2e7LqavWL5+FPSic79ZKEljeeNB1z5xoE2YnKEs55MLi/PgfLJHJ9M\r\nbA42lJBYI37MeGv0kGkIQDIrtpDjBfSgGSINXeTjs0T/l5sXXPTfSRm4URWX3I+Q\r\nO9ACYLb7Cz//G8HdYHp8MHNa38x4BeBku7cT7xgEUPfhu4LCnBW1u0pMbeT8A2M9\r\nzz/TIOJKwUeupacXjJ/tqHGLYMgBLCSRNK4zFJiGzhDbEZA6+NMUp660t7Su6M5i\r\nx3WIffKlIcxi0bnk9slhjK5NynJ3H/uvbfYnCwIDAQABMA0GCSqGSIb3DQEBCwUA\r\nA4IBAQBAttZGxh1k87S8OHA6L8fmu/PEtjdkJV3Cu7J3xTGfmSBVMTStl2NI4D4i\r\nFJUFdTJjvEibEaQEETz2zJakwdp7JGG7ip7e/dKsP0wVMC63MZ8eeDzST7IzBLk6\r\nmjb+Co0+2p/9rHPBz1AFpHRcJ9jaORh8FlctfcxcBsr6dMptGtCZMzam26WwePxa\r\npNd+wBNFHuAXHyTzHl3NIq2Bd+56Roen5muZKv4xqI7iVBbVrejvVyXNDhrqh9QB\r\nVushjclGdc2GSdCVpUhiv1srX59/yX5HBZG8HRQSgCXpcSiGIhiMmd1gaU/qrXa8\r\nkwmd6tdKUcAiLU7qXEJX4iBziobm\r\n-----END CERTIFICATE----- diff --git a/resources/configuration/provider-vault.properties b/resources/configuration/provider-vault.properties index 6ebdebd..bb037d6 100644 --- a/resources/configuration/provider-vault.properties +++ b/resources/configuration/provider-vault.properties @@ -1 +1 @@ -public-key=-----BEGIN CERTIFICATE-----\r\nMIIDazCCAlOgAwIBAgIUZ3/sZXYzW4PjmOXKrZn6WBmUJ+4wDQYJKoZIhvcNAQEL\r\nBQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM\r\nGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMjAyMjMxNTA2MDNaFw0zMjAy\r\nMjExNTA2MDNaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw\r\nHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB\r\nAQUAA4IBDwAwggEKAoIBAQDBl6XaJnXTL+6DWip3aBhU+MzmY4d1V9hbTm1tiZ3g\r\nE0VbUrvGO3LoYaxpPv6zFmsg3uJv6JxVAde7EddidN0ITHB9cQNdAfdUJ5njmsGS\r\nPbdQuOQTHw0aG7/QvTI/nsvfEE6e0lbV/0e7DHacZT/+OztBH1RwkG2ymM94Hf8H\r\nI6x7q6yfRTAZOqeOMrPCYTcluAgE9NskoPvjX5qASakBtXISKIsOU84N0/2HDN3W\r\nEGMXvoHUQu6vrij6BwiwxKaw1AKwWENKoga775bPXN3M+JTSaIKE7dZbKzvx0Zi0\r\nh5X+bxc3BJi3Z/CsUBCzE+Y0SFetOiYmyl/2YmnneYoVAgMBAAGjUzBRMB0GA1Ud\r\nDgQWBBTvK1wVERwjni4B2vdH7KtEJeVWFzAfBgNVHSMEGDAWgBTvK1wVERwjni4B\r\n2vdH7KtEJeVWFzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBn\r\nQHiPA7OBYukHd9gS7c0HXE+fsWcS3GZeLqcHfQQnV3pte1vTmu9//IVW71wNCJ1/\r\nrySRyODPQoPehxEcyHwupNZSzXK//nPlTdSgjMfFxscvt1YndyQLQYCfyOJMixAe\r\nAqrb14GTFHUUrdor0PyElhkULjkOXUrSIsdBrfWrwLTkelE8NK3tb5ZG8KPzD9Jy\r\n+NwEPPr9d+iHkUkM7EFWw/cl56wka9ryBb97RI7DqbO6/j6OXHMk4GByxKv7DSIR\r\nIvF9/Dw20qytajtaHV0pluFcOBuFc0NfiDvCaQlbTsfjzbc6UmZWbOi9YOJl3VQ/\r\ng3h+15GuzbsSzOCOEYOT\r\n-----END CERTIFICATE----- +public-key=-----BEGIN CERTIFICATE-----\r\nMIIDETCCAfkCFBEOuzF+mRgrzrxVv70zNDUEv+nUMA0GCSqGSIb3DQEBCwUAMEUx\r\nCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl\r\ncm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjQwMTMxMTIyNTUyWhcNMjUwMTMwMTIy\r\nNTUyWjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE\r\nCgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC\r\nAQ8AMIIBCgKCAQEAxHbsW755sB/5YwjPyk0xPJwyQkxkGV2SS4sQQCCz8KV/QoEi\r\nlk/PzUqUsE2e7LqavWL5+FPSic79ZKEljeeNB1z5xoE2YnKEs55MLi/PgfLJHJ9M\r\nbA42lJBYI37MeGv0kGkIQDIrtpDjBfSgGSINXeTjs0T/l5sXXPTfSRm4URWX3I+Q\r\nO9ACYLb7Cz//G8HdYHp8MHNa38x4BeBku7cT7xgEUPfhu4LCnBW1u0pMbeT8A2M9\r\nzz/TIOJKwUeupacXjJ/tqHGLYMgBLCSRNK4zFJiGzhDbEZA6+NMUp660t7Su6M5i\r\nx3WIffKlIcxi0bnk9slhjK5NynJ3H/uvbfYnCwIDAQABMA0GCSqGSIb3DQEBCwUA\r\nA4IBAQBAttZGxh1k87S8OHA6L8fmu/PEtjdkJV3Cu7J3xTGfmSBVMTStl2NI4D4i\r\nFJUFdTJjvEibEaQEETz2zJakwdp7JGG7ip7e/dKsP0wVMC63MZ8eeDzST7IzBLk6\r\nmjb+Co0+2p/9rHPBz1AFpHRcJ9jaORh8FlctfcxcBsr6dMptGtCZMzam26WwePxa\r\npNd+wBNFHuAXHyTzHl3NIq2Bd+56Roen5muZKv4xqI7iVBbVrejvVyXNDhrqh9QB\r\nVushjclGdc2GSdCVpUhiv1srX59/yX5HBZG8HRQSgCXpcSiGIhiMmd1gaU/qrXa8\r\nkwmd6tdKUcAiLU7qXEJX4iBziobm\r\n-----END CERTIFICATE----- \ No newline at end of file From ef6dee0e729bd7f7accf0db9c7223b7b4e6e2810 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Wed, 7 Feb 2024 11:39:33 +0100 Subject: [PATCH 014/174] connector and examples --- launchers/connector/build.gradle.kts | 21 ++++++++++++++++ .../consumer-configuration.properties | 7 ++---- .../configuration/consumer-vault.properties | 2 ++ .../provider-configuration.properties | 4 +-- .../configuration/provider-vault.properties | 4 ++- resources/operations/get-dataset.json | 7 ------ resources/operationss3/create-asset.json | 4 +-- resources/operationss3/get-dataset.json | 7 ------ .../operationss3/negotiate-contract.json | 4 +-- .../operationss3/start-transfer-proxy.json | 2 +- resources/operationss3/start-transfer.json | 25 +++++++++++-------- 11 files changed, 50 insertions(+), 37 deletions(-) delete mode 100644 resources/operations/get-dataset.json delete mode 100644 resources/operationss3/get-dataset.json diff --git a/launchers/connector/build.gradle.kts b/launchers/connector/build.gradle.kts index 398d7e3..84fee0e 100644 --- a/launchers/connector/build.gradle.kts +++ b/launchers/connector/build.gradle.kts @@ -51,6 +51,27 @@ dependencies { implementation(libs.edc.transfer.pull.http.receiver) implementation(libs.edc.transfer.data.plane) + // Persistencia comun + implementation(libs.edc.sql.core) + implementation(libs.edc.sql.lease) + implementation(libs.edc.sql.pool) + // Persistencia control plane + implementation(libs.edc.sql.asset.index) + implementation(libs.edc.sql.contract.definition.store) + implementation(libs.edc.sql.contract.negotiation.store) + implementation(libs.edc.sql.policy.definition.store) + implementation(libs.edc.sql.transfer.process.store) + // Persistencia data plane + implementation(libs.edc.sql.data.plane.store) + + // Persistencia de objetos + implementation(libs.edc.aws.s3.core) + implementation(libs.edc.data.plane.aws.s3) + //implementation(libs.edc.provision.aws.s3) + + runtimeOnly(libs.edc.transaction.local) + runtimeOnly(libs.postgres) + } diff --git a/resources/configuration/consumer-configuration.properties b/resources/configuration/consumer-configuration.properties index f387943..0157b43 100644 --- a/resources/configuration/consumer-configuration.properties +++ b/resources/configuration/consumer-configuration.properties @@ -23,10 +23,7 @@ edc.catalog.cache.execution.period.seconds=120 edc.datasource.default.url=jdbc:postgresql://localhost:5433/inesdata edc.datasource.default.user=inesdata edc.datasource.default.password=inesdata -edc.datasource.default.pool.maxIdleConnections=10 -edc.datasource.default.pool.maxTotalConnections=10 -edc.datasource.default.pool.minIdleConnections=5 -edc.aws.access.key=wEfyrv2vH7Wj3ON03g8V -edc.aws.secret.access.key=JlHVGgSgeRkbs6WtKTugR4I5uU7oA1VRf8kBtrGd +edc.aws.access.key=access-key +edc.aws.secret.access.key=secret-key edc.aws.endpoint.override=http://localhost:9010 diff --git a/resources/configuration/consumer-vault.properties b/resources/configuration/consumer-vault.properties index 63ddb09..b649f4e 100644 --- a/resources/configuration/consumer-vault.properties +++ b/resources/configuration/consumer-vault.properties @@ -1 +1,3 @@ public-key=-----BEGIN CERTIFICATE-----\r\nMIIDETCCAfkCFBEOuzF+mRgrzrxVv70zNDUEv+nUMA0GCSqGSIb3DQEBCwUAMEUx\r\nCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl\r\ncm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjQwMTMxMTIyNTUyWhcNMjUwMTMwMTIy\r\nNTUyWjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE\r\nCgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC\r\nAQ8AMIIBCgKCAQEAxHbsW755sB/5YwjPyk0xPJwyQkxkGV2SS4sQQCCz8KV/QoEi\r\nlk/PzUqUsE2e7LqavWL5+FPSic79ZKEljeeNB1z5xoE2YnKEs55MLi/PgfLJHJ9M\r\nbA42lJBYI37MeGv0kGkIQDIrtpDjBfSgGSINXeTjs0T/l5sXXPTfSRm4URWX3I+Q\r\nO9ACYLb7Cz//G8HdYHp8MHNa38x4BeBku7cT7xgEUPfhu4LCnBW1u0pMbeT8A2M9\r\nzz/TIOJKwUeupacXjJ/tqHGLYMgBLCSRNK4zFJiGzhDbEZA6+NMUp660t7Su6M5i\r\nx3WIffKlIcxi0bnk9slhjK5NynJ3H/uvbfYnCwIDAQABMA0GCSqGSIb3DQEBCwUA\r\nA4IBAQBAttZGxh1k87S8OHA6L8fmu/PEtjdkJV3Cu7J3xTGfmSBVMTStl2NI4D4i\r\nFJUFdTJjvEibEaQEETz2zJakwdp7JGG7ip7e/dKsP0wVMC63MZ8eeDzST7IzBLk6\r\nmjb+Co0+2p/9rHPBz1AFpHRcJ9jaORh8FlctfcxcBsr6dMptGtCZMzam26WwePxa\r\npNd+wBNFHuAXHyTzHl3NIq2Bd+56Roen5muZKv4xqI7iVBbVrejvVyXNDhrqh9QB\r\nVushjclGdc2GSdCVpUhiv1srX59/yX5HBZG8HRQSgCXpcSiGIhiMmd1gaU/qrXa8\r\nkwmd6tdKUcAiLU7qXEJX4iBziobm\r\n-----END CERTIFICATE----- +access-key=wEfyrv2vH7Wj3ON03g8V +secret-key=JlHVGgSgeRkbs6WtKTugR4I5uU7oA1VRf8kBtrGd diff --git a/resources/configuration/provider-configuration.properties b/resources/configuration/provider-configuration.properties index b8ed938..3b1b7a1 100644 --- a/resources/configuration/provider-configuration.properties +++ b/resources/configuration/provider-configuration.properties @@ -26,6 +26,6 @@ edc.datasource.default.pool.maxIdleConnections=10 edc.datasource.default.pool.maxTotalConnections=10 edc.datasource.default.pool.minIdleConnections=5 -edc.aws.access.key=DrJ6PmnJ3QtlxYowRSLi -edc.aws.secret.access.key=gjaVnJstoeV4bWLrJkmswKBMIZxY10PjZOJhp2qk +edc.aws.access.key=access-key +edc.aws.secret.access.key=secret-key edc.aws.endpoint.override=http://localhost:9000 \ No newline at end of file diff --git a/resources/configuration/provider-vault.properties b/resources/configuration/provider-vault.properties index bb037d6..09ae550 100644 --- a/resources/configuration/provider-vault.properties +++ b/resources/configuration/provider-vault.properties @@ -1 +1,3 @@ -public-key=-----BEGIN CERTIFICATE-----\r\nMIIDETCCAfkCFBEOuzF+mRgrzrxVv70zNDUEv+nUMA0GCSqGSIb3DQEBCwUAMEUx\r\nCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl\r\ncm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjQwMTMxMTIyNTUyWhcNMjUwMTMwMTIy\r\nNTUyWjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE\r\nCgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC\r\nAQ8AMIIBCgKCAQEAxHbsW755sB/5YwjPyk0xPJwyQkxkGV2SS4sQQCCz8KV/QoEi\r\nlk/PzUqUsE2e7LqavWL5+FPSic79ZKEljeeNB1z5xoE2YnKEs55MLi/PgfLJHJ9M\r\nbA42lJBYI37MeGv0kGkIQDIrtpDjBfSgGSINXeTjs0T/l5sXXPTfSRm4URWX3I+Q\r\nO9ACYLb7Cz//G8HdYHp8MHNa38x4BeBku7cT7xgEUPfhu4LCnBW1u0pMbeT8A2M9\r\nzz/TIOJKwUeupacXjJ/tqHGLYMgBLCSRNK4zFJiGzhDbEZA6+NMUp660t7Su6M5i\r\nx3WIffKlIcxi0bnk9slhjK5NynJ3H/uvbfYnCwIDAQABMA0GCSqGSIb3DQEBCwUA\r\nA4IBAQBAttZGxh1k87S8OHA6L8fmu/PEtjdkJV3Cu7J3xTGfmSBVMTStl2NI4D4i\r\nFJUFdTJjvEibEaQEETz2zJakwdp7JGG7ip7e/dKsP0wVMC63MZ8eeDzST7IzBLk6\r\nmjb+Co0+2p/9rHPBz1AFpHRcJ9jaORh8FlctfcxcBsr6dMptGtCZMzam26WwePxa\r\npNd+wBNFHuAXHyTzHl3NIq2Bd+56Roen5muZKv4xqI7iVBbVrejvVyXNDhrqh9QB\r\nVushjclGdc2GSdCVpUhiv1srX59/yX5HBZG8HRQSgCXpcSiGIhiMmd1gaU/qrXa8\r\nkwmd6tdKUcAiLU7qXEJX4iBziobm\r\n-----END CERTIFICATE----- \ No newline at end of file +public-key=-----BEGIN CERTIFICATE-----\r\nMIIDETCCAfkCFBEOuzF+mRgrzrxVv70zNDUEv+nUMA0GCSqGSIb3DQEBCwUAMEUx\r\nCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl\r\ncm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjQwMTMxMTIyNTUyWhcNMjUwMTMwMTIy\r\nNTUyWjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE\r\nCgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC\r\nAQ8AMIIBCgKCAQEAxHbsW755sB/5YwjPyk0xPJwyQkxkGV2SS4sQQCCz8KV/QoEi\r\nlk/PzUqUsE2e7LqavWL5+FPSic79ZKEljeeNB1z5xoE2YnKEs55MLi/PgfLJHJ9M\r\nbA42lJBYI37MeGv0kGkIQDIrtpDjBfSgGSINXeTjs0T/l5sXXPTfSRm4URWX3I+Q\r\nO9ACYLb7Cz//G8HdYHp8MHNa38x4BeBku7cT7xgEUPfhu4LCnBW1u0pMbeT8A2M9\r\nzz/TIOJKwUeupacXjJ/tqHGLYMgBLCSRNK4zFJiGzhDbEZA6+NMUp660t7Su6M5i\r\nx3WIffKlIcxi0bnk9slhjK5NynJ3H/uvbfYnCwIDAQABMA0GCSqGSIb3DQEBCwUA\r\nA4IBAQBAttZGxh1k87S8OHA6L8fmu/PEtjdkJV3Cu7J3xTGfmSBVMTStl2NI4D4i\r\nFJUFdTJjvEibEaQEETz2zJakwdp7JGG7ip7e/dKsP0wVMC63MZ8eeDzST7IzBLk6\r\nmjb+Co0+2p/9rHPBz1AFpHRcJ9jaORh8FlctfcxcBsr6dMptGtCZMzam26WwePxa\r\npNd+wBNFHuAXHyTzHl3NIq2Bd+56Roen5muZKv4xqI7iVBbVrejvVyXNDhrqh9QB\r\nVushjclGdc2GSdCVpUhiv1srX59/yX5HBZG8HRQSgCXpcSiGIhiMmd1gaU/qrXa8\r\nkwmd6tdKUcAiLU7qXEJX4iBziobm\r\n-----END CERTIFICATE----- +access-key=DrJ6PmnJ3QtlxYowRSLi +secret-key=gjaVnJstoeV4bWLrJkmswKBMIZxY10PjZOJhp2qk diff --git a/resources/operations/get-dataset.json b/resources/operations/get-dataset.json deleted file mode 100644 index 5286af0..0000000 --- a/resources/operations/get-dataset.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "@context": { "@vocab": "https://w3id.org/edc/v0.0.1/ns/" }, - "@type": "DatasetRequest", - "@id": "assetId", - "counterPartyAddress": "http://localhost:19194/protocol", - "protocol": "dataspace-protocol-http" -} \ No newline at end of file diff --git a/resources/operationss3/create-asset.json b/resources/operationss3/create-asset.json index 28c83fa..0983e80 100644 --- a/resources/operationss3/create-asset.json +++ b/resources/operationss3/create-asset.json @@ -5,11 +5,11 @@ "@id": "s3asset", "properties": { "name": "product description", - "contenttype": "application/json" + "contenttype": "text/plain" }, "dataAddress": { "type": "AmazonS3", - "region": "eu-south-2", + "region": "us-east-1", "keyName": "text-document-provider-dataset.txt", "bucketName": "bucket-connector-1" } diff --git a/resources/operationss3/get-dataset.json b/resources/operationss3/get-dataset.json deleted file mode 100644 index 5286af0..0000000 --- a/resources/operationss3/get-dataset.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "@context": { "@vocab": "https://w3id.org/edc/v0.0.1/ns/" }, - "@type": "DatasetRequest", - "@id": "assetId", - "counterPartyAddress": "http://localhost:19194/protocol", - "protocol": "dataspace-protocol-http" -} \ No newline at end of file diff --git a/resources/operationss3/negotiate-contract.json b/resources/operationss3/negotiate-contract.json index 84e12fe..78e65fd 100644 --- a/resources/operationss3/negotiate-contract.json +++ b/resources/operationss3/negotiate-contract.json @@ -10,11 +10,11 @@ "protocol": "dataspace-protocol-http", "policy": { "@context": "http://www.w3.org/ns/odrl.jsonld", - "@id": "MQ==:czNhc3NldA==:NDBkMjY1ZDAtNjljOS00ZWY1LTg4NzktM2QxNmVlYWU1NTZk", + "@id": "MQ==:czNhc3NldA==:ZDU2M2VkNjQtNGU4MS00YTk0LWI4ODctNzU5N2NiYjlmNDM0", "@type": "Set", "permission": [], "prohibition": [], "obligation": [], - "target": "assetId" + "target": "s3asset" } } diff --git a/resources/operationss3/start-transfer-proxy.json b/resources/operationss3/start-transfer-proxy.json index 783a1d4..f869a43 100644 --- a/resources/operationss3/start-transfer-proxy.json +++ b/resources/operationss3/start-transfer-proxy.json @@ -5,7 +5,7 @@ "@type": "TransferRequestDto", "connectorId": "provider", "counterPartyAddress": "http://localhost:19194/protocol", - "contractId": "7c507b6a-828b-4948-a2b9-a7e4bcd392de", + "contractId": "af9c225e-8a19-442d-abde-8c8d0af9f044", "assetId": "s3asset", "protocol": "dataspace-protocol-http", "dataDestination": { diff --git a/resources/operationss3/start-transfer.json b/resources/operationss3/start-transfer.json index a213935..a9f77d8 100644 --- a/resources/operationss3/start-transfer.json +++ b/resources/operationss3/start-transfer.json @@ -3,21 +3,26 @@ "@vocab": "https://w3id.org/edc/v0.0.1/ns/" }, "@type": "TransferRequestDto", - "connectorId": "provider", "counterPartyAddress": "http://localhost:19194/protocol", - "contractId": "94ef53b4-548c-47d3-bce3-e17f62820c5e", - "assetId": "s3asset", "protocol": "dataspace-protocol-http", + "connectorId": "consumer", + "assetId": "s3asset", + "contractId": "af9c225e-8a19-442d-abde-8c8d0af9f044", "dataDestination": { - "type": "AmazonS3", - "region": "us-east-1", - "bucketName": "bucket-connector-2", + "properties": { + "type": "AmazonS3", + "region": "us-east-1", + "bucketName": "bucket-connector-2", + "folderName": "descargas", + "accessKeyId": "ymapc4WGhWrgaywwSyq1", + "secretAccessKey": "wxwo6F4WTTi9q5Gts7Ve57TnxO29p1waJua6xpvT" + }, "endpointOverride": "http://localhost:9010", - "accessKeyId": "wEfyrv2vH7Wj3ON03g8V", - "secretAccessKey": "JlHVGgSgeRkbs6WtKTugR4I5uU7oA1VRf8kBtrGd" + "type": "AmazonS3" }, + "managedResources": true, "transferType": { - "contentType": "application/octet-stream", + "contentType": "text/plain", "isFinite": true } -} +} \ No newline at end of file From 0fa79309e8222d99c8db069efd4c7457bf62382d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Fri, 9 Feb 2024 12:42:52 +0100 Subject: [PATCH 015/174] update config --- resources/configuration/consumer-configuration.properties | 8 +++++--- resources/configuration/provider-configuration.properties | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/resources/configuration/consumer-configuration.properties b/resources/configuration/consumer-configuration.properties index 0157b43..4625fed 100644 --- a/resources/configuration/consumer-configuration.properties +++ b/resources/configuration/consumer-configuration.properties @@ -6,7 +6,7 @@ web.http.management.port=29193 web.http.management.path=/management web.http.protocol.port=29194 web.http.protocol.path=/protocol -edc.receiver.http.endpoint=http://localhost:4000/receiver/urn:connector:provider/callback +edc.receiver.http.endpoint=http://localhost:4001/receiver/urn:connector:provider/callback edc.public.key.alias=public-key edc.transfer.dataplane.token.signer.privatekey.alias=1 edc.transfer.proxy.token.signer.privatekey.alias=1 @@ -18,11 +18,13 @@ web.http.public.path=/public web.http.control.port=29192 web.http.control.path=/control edc.dataplane.token.validation.endpoint=http://localhost:29192/control/token -edc.catalog.cache.execution.period.seconds=120 edc.datasource.default.url=jdbc:postgresql://localhost:5433/inesdata edc.datasource.default.user=inesdata -edc.datasource.default.password=inesdata +edc.datasource.default.password= +edc.datasource.default.pool.maxIdleConnections=10 +edc.datasource.default.pool.maxTotalConnections=10 +edc.datasource.default.pool.minIdleConnections=5 edc.aws.access.key=access-key edc.aws.secret.access.key=secret-key diff --git a/resources/configuration/provider-configuration.properties b/resources/configuration/provider-configuration.properties index 3b1b7a1..1fbe8d5 100644 --- a/resources/configuration/provider-configuration.properties +++ b/resources/configuration/provider-configuration.properties @@ -21,7 +21,7 @@ edc.dataplane.token.validation.endpoint=http://localhost:19192/control/token edc.datasource.default.url=jdbc:postgresql://localhost:5432/inesdata edc.datasource.default.user=inesdata -edc.datasource.default.password=inesdata +edc.datasource.default.password= edc.datasource.default.pool.maxIdleConnections=10 edc.datasource.default.pool.maxTotalConnections=10 edc.datasource.default.pool.minIdleConnections=5 From b3ae9fca8b144643b4e0c8cc2311fa0063e8aec5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Wed, 14 Feb 2024 07:33:14 +0100 Subject: [PATCH 016/174] change config values --- resources/configuration/consumer-configuration.properties | 2 +- resources/configuration/consumer-vault.properties | 4 ++-- resources/configuration/provider-configuration.properties | 2 +- resources/configuration/provider-vault.properties | 4 ++-- resources/operations/create-asset.json | 2 +- resources/operationss3/start-transfer-proxy.json | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/resources/configuration/consumer-configuration.properties b/resources/configuration/consumer-configuration.properties index 4625fed..179737d 100644 --- a/resources/configuration/consumer-configuration.properties +++ b/resources/configuration/consumer-configuration.properties @@ -21,7 +21,7 @@ edc.dataplane.token.validation.endpoint=http://localhost:29192/control/token edc.datasource.default.url=jdbc:postgresql://localhost:5433/inesdata edc.datasource.default.user=inesdata -edc.datasource.default.password= +edc.datasource.default.password= edc.datasource.default.pool.maxIdleConnections=10 edc.datasource.default.pool.maxTotalConnections=10 edc.datasource.default.pool.minIdleConnections=5 diff --git a/resources/configuration/consumer-vault.properties b/resources/configuration/consumer-vault.properties index b649f4e..d91e3e6 100644 --- a/resources/configuration/consumer-vault.properties +++ b/resources/configuration/consumer-vault.properties @@ -1,3 +1,3 @@ public-key=-----BEGIN CERTIFICATE-----\r\nMIIDETCCAfkCFBEOuzF+mRgrzrxVv70zNDUEv+nUMA0GCSqGSIb3DQEBCwUAMEUx\r\nCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl\r\ncm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjQwMTMxMTIyNTUyWhcNMjUwMTMwMTIy\r\nNTUyWjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE\r\nCgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC\r\nAQ8AMIIBCgKCAQEAxHbsW755sB/5YwjPyk0xPJwyQkxkGV2SS4sQQCCz8KV/QoEi\r\nlk/PzUqUsE2e7LqavWL5+FPSic79ZKEljeeNB1z5xoE2YnKEs55MLi/PgfLJHJ9M\r\nbA42lJBYI37MeGv0kGkIQDIrtpDjBfSgGSINXeTjs0T/l5sXXPTfSRm4URWX3I+Q\r\nO9ACYLb7Cz//G8HdYHp8MHNa38x4BeBku7cT7xgEUPfhu4LCnBW1u0pMbeT8A2M9\r\nzz/TIOJKwUeupacXjJ/tqHGLYMgBLCSRNK4zFJiGzhDbEZA6+NMUp660t7Su6M5i\r\nx3WIffKlIcxi0bnk9slhjK5NynJ3H/uvbfYnCwIDAQABMA0GCSqGSIb3DQEBCwUA\r\nA4IBAQBAttZGxh1k87S8OHA6L8fmu/PEtjdkJV3Cu7J3xTGfmSBVMTStl2NI4D4i\r\nFJUFdTJjvEibEaQEETz2zJakwdp7JGG7ip7e/dKsP0wVMC63MZ8eeDzST7IzBLk6\r\nmjb+Co0+2p/9rHPBz1AFpHRcJ9jaORh8FlctfcxcBsr6dMptGtCZMzam26WwePxa\r\npNd+wBNFHuAXHyTzHl3NIq2Bd+56Roen5muZKv4xqI7iVBbVrejvVyXNDhrqh9QB\r\nVushjclGdc2GSdCVpUhiv1srX59/yX5HBZG8HRQSgCXpcSiGIhiMmd1gaU/qrXa8\r\nkwmd6tdKUcAiLU7qXEJX4iBziobm\r\n-----END CERTIFICATE----- -access-key=wEfyrv2vH7Wj3ON03g8V -secret-key=JlHVGgSgeRkbs6WtKTugR4I5uU7oA1VRf8kBtrGd +access-key= +secret-key= diff --git a/resources/configuration/provider-configuration.properties b/resources/configuration/provider-configuration.properties index 1fbe8d5..a162530 100644 --- a/resources/configuration/provider-configuration.properties +++ b/resources/configuration/provider-configuration.properties @@ -21,7 +21,7 @@ edc.dataplane.token.validation.endpoint=http://localhost:19192/control/token edc.datasource.default.url=jdbc:postgresql://localhost:5432/inesdata edc.datasource.default.user=inesdata -edc.datasource.default.password= +edc.datasource.default.password= edc.datasource.default.pool.maxIdleConnections=10 edc.datasource.default.pool.maxTotalConnections=10 edc.datasource.default.pool.minIdleConnections=5 diff --git a/resources/configuration/provider-vault.properties b/resources/configuration/provider-vault.properties index 09ae550..d91e3e6 100644 --- a/resources/configuration/provider-vault.properties +++ b/resources/configuration/provider-vault.properties @@ -1,3 +1,3 @@ public-key=-----BEGIN CERTIFICATE-----\r\nMIIDETCCAfkCFBEOuzF+mRgrzrxVv70zNDUEv+nUMA0GCSqGSIb3DQEBCwUAMEUx\r\nCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl\r\ncm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjQwMTMxMTIyNTUyWhcNMjUwMTMwMTIy\r\nNTUyWjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE\r\nCgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC\r\nAQ8AMIIBCgKCAQEAxHbsW755sB/5YwjPyk0xPJwyQkxkGV2SS4sQQCCz8KV/QoEi\r\nlk/PzUqUsE2e7LqavWL5+FPSic79ZKEljeeNB1z5xoE2YnKEs55MLi/PgfLJHJ9M\r\nbA42lJBYI37MeGv0kGkIQDIrtpDjBfSgGSINXeTjs0T/l5sXXPTfSRm4URWX3I+Q\r\nO9ACYLb7Cz//G8HdYHp8MHNa38x4BeBku7cT7xgEUPfhu4LCnBW1u0pMbeT8A2M9\r\nzz/TIOJKwUeupacXjJ/tqHGLYMgBLCSRNK4zFJiGzhDbEZA6+NMUp660t7Su6M5i\r\nx3WIffKlIcxi0bnk9slhjK5NynJ3H/uvbfYnCwIDAQABMA0GCSqGSIb3DQEBCwUA\r\nA4IBAQBAttZGxh1k87S8OHA6L8fmu/PEtjdkJV3Cu7J3xTGfmSBVMTStl2NI4D4i\r\nFJUFdTJjvEibEaQEETz2zJakwdp7JGG7ip7e/dKsP0wVMC63MZ8eeDzST7IzBLk6\r\nmjb+Co0+2p/9rHPBz1AFpHRcJ9jaORh8FlctfcxcBsr6dMptGtCZMzam26WwePxa\r\npNd+wBNFHuAXHyTzHl3NIq2Bd+56Roen5muZKv4xqI7iVBbVrejvVyXNDhrqh9QB\r\nVushjclGdc2GSdCVpUhiv1srX59/yX5HBZG8HRQSgCXpcSiGIhiMmd1gaU/qrXa8\r\nkwmd6tdKUcAiLU7qXEJX4iBziobm\r\n-----END CERTIFICATE----- -access-key=DrJ6PmnJ3QtlxYowRSLi -secret-key=gjaVnJstoeV4bWLrJkmswKBMIZxY10PjZOJhp2qk +access-key= +secret-key= diff --git a/resources/operations/create-asset.json b/resources/operations/create-asset.json index 018e9f6..1931546 100644 --- a/resources/operations/create-asset.json +++ b/resources/operations/create-asset.json @@ -9,7 +9,7 @@ }, "dataAddress": { "type": "HttpData", - "name": "Test asset", + "name": "JSON place holder users", "baseUrl": "https://jsonplaceholder.typicode.com/users", "proxyPath": "true" } diff --git a/resources/operationss3/start-transfer-proxy.json b/resources/operationss3/start-transfer-proxy.json index f869a43..65228c3 100644 --- a/resources/operationss3/start-transfer-proxy.json +++ b/resources/operationss3/start-transfer-proxy.json @@ -5,7 +5,7 @@ "@type": "TransferRequestDto", "connectorId": "provider", "counterPartyAddress": "http://localhost:19194/protocol", - "contractId": "af9c225e-8a19-442d-abde-8c8d0af9f044", + "contractId": "f2e5f10c-050f-41de-ac60-a3936cb535cf", "assetId": "s3asset", "protocol": "dataspace-protocol-http", "dataDestination": { From 7320b6c69a327f68e3999719f2a0f804c695766e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Wed, 14 Feb 2024 07:38:06 +0100 Subject: [PATCH 017/174] update README --- README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 00efc1e..2bda640 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,15 @@ Dataspaces connector for the InesData project, based on the EDC dataspaces frame - Compile: ``` - gradle base:build + ./gradlew launchers:connector:build ``` - Launch: ``` - java -jar base/build/libs/connector-app.jar + java -jar launchers/connector/build/libs/connector-app.jar ``` + + Example: + ``` + java -Dedc.keystore=resources/certs/store.pfx -Dedc.keystore.password= -Dedc.vault=resources/configuration/provider-vault.properties -Dedc.fs.config=resources/configuration/provider-configuration.properties -jar launchers/connector/build/libs/connector-app.jar + ``` \ No newline at end of file From dfed398957a971281715a9cdd2865b3844ab8108 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Wed, 14 Feb 2024 07:51:20 +0100 Subject: [PATCH 018/174] remove commented line --- launchers/connector/build.gradle.kts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/launchers/connector/build.gradle.kts b/launchers/connector/build.gradle.kts index 84fee0e..2752be8 100644 --- a/launchers/connector/build.gradle.kts +++ b/launchers/connector/build.gradle.kts @@ -67,8 +67,7 @@ dependencies { // Persistencia de objetos implementation(libs.edc.aws.s3.core) implementation(libs.edc.data.plane.aws.s3) - //implementation(libs.edc.provision.aws.s3) - + runtimeOnly(libs.edc.transaction.local) runtimeOnly(libs.postgres) From eebbe1cbd730614eae12cca32deec36adab1ae34 Mon Sep 17 00:00:00 2001 From: Juan Calero Date: Thu, 15 Feb 2024 12:50:16 +0100 Subject: [PATCH 019/174] Inocuous change to test external pipeline --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 00efc1e..26a5b7e 100644 --- a/README.md +++ b/README.md @@ -13,3 +13,5 @@ Dataspaces connector for the InesData project, based on the EDC dataspaces frame ``` java -jar base/build/libs/connector-app.jar ``` + + From 2ad068cc1d682f6665317b21cb32fbf2c5ea8c06 Mon Sep 17 00:00:00 2001 From: Juan Calero Date: Thu, 15 Feb 2024 12:55:25 +0100 Subject: [PATCH 020/174] Another inocuous change --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 26a5b7e..de5b40e 100644 --- a/README.md +++ b/README.md @@ -14,4 +14,3 @@ Dataspaces connector for the InesData project, based on the EDC dataspaces frame java -jar base/build/libs/connector-app.jar ``` - From 04be0bafe11ec3f527ad468838c32df47f97be45 Mon Sep 17 00:00:00 2001 From: Juan Calero Date: Thu, 15 Feb 2024 13:38:55 +0100 Subject: [PATCH 021/174] Tha latest inocuous change, I promise --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index de5b40e..26a5b7e 100644 --- a/README.md +++ b/README.md @@ -14,3 +14,4 @@ Dataspaces connector for the InesData project, based on the EDC dataspaces frame java -jar base/build/libs/connector-app.jar ``` + From 1bee787c216e08e3fcd720fa40e8e4ed83442fed Mon Sep 17 00:00:00 2001 From: Juan Calero Date: Mon, 19 Feb 2024 18:36:46 +0100 Subject: [PATCH 022/174] Small correction in README --- README.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/README.md b/README.md index 2bda640..a95d77c 100644 --- a/README.md +++ b/README.md @@ -10,11 +10,8 @@ Dataspaces connector for the InesData project, based on the EDC dataspaces frame ``` - Launch: - ``` - java -jar launchers/connector/build/libs/connector-app.jar - ``` Example: ``` java -Dedc.keystore=resources/certs/store.pfx -Dedc.keystore.password= -Dedc.vault=resources/configuration/provider-vault.properties -Dedc.fs.config=resources/configuration/provider-configuration.properties -jar launchers/connector/build/libs/connector-app.jar - ``` \ No newline at end of file + ``` From f60821520960f5a025b6083c02114dff85d21b58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Tue, 20 Feb 2024 11:02:00 +0100 Subject: [PATCH 023/174] containerizing inesdata connector --- README.md | 20 +++++++++++ base/build.gradle.kts | 20 ----------- docker/Dockerfile | 33 ++++++++++++++++++ gradle/libs.versions.toml | 3 ++ launchers/connector/build.gradle.kts | 4 ++- resources/certs/cert.pem | 19 ---------- resources/certs/key.pem | 28 --------------- resources/certs/store.pfx | Bin 2531 -> 0 bytes .../consumer-configuration.properties | 31 ---------------- .../configuration/consumer-vault.properties | 3 -- .../provider-configuration.properties | 31 ---------------- .../configuration/provider-vault.properties | 3 -- .../register-data-plane-consumer.json | 19 ---------- .../register-data-plane-provider.json | 19 ---------- resources/operations/create-asset.json | 16 --------- .../create-contract-definition.json | 9 ----- resources/operations/create-policy.json | 13 ------- resources/operations/fetch-catalog.json | 7 ---- resources/operations/negotiate-contract.json | 20 ----------- resources/operations/start-transfer.json | 14 -------- resources/operationss3/create-asset.json | 16 --------- .../create-contract-definition.json | 9 ----- resources/operationss3/create-policy.json | 13 ------- resources/operationss3/fetch-catalog.json | 7 ---- .../operationss3/negotiate-contract.json | 20 ----------- .../operationss3/start-transfer-proxy.json | 14 -------- resources/operationss3/start-transfer.json | 28 --------------- settings.gradle.kts | 1 - 28 files changed, 59 insertions(+), 361 deletions(-) delete mode 100644 base/build.gradle.kts create mode 100644 docker/Dockerfile delete mode 100644 resources/certs/cert.pem delete mode 100644 resources/certs/key.pem delete mode 100644 resources/certs/store.pfx delete mode 100644 resources/configuration/consumer-configuration.properties delete mode 100644 resources/configuration/consumer-vault.properties delete mode 100644 resources/configuration/provider-configuration.properties delete mode 100644 resources/configuration/provider-vault.properties delete mode 100644 resources/dataplane/register-data-plane-consumer.json delete mode 100644 resources/dataplane/register-data-plane-provider.json delete mode 100644 resources/operations/create-asset.json delete mode 100644 resources/operations/create-contract-definition.json delete mode 100644 resources/operations/create-policy.json delete mode 100644 resources/operations/fetch-catalog.json delete mode 100644 resources/operations/negotiate-contract.json delete mode 100644 resources/operations/start-transfer.json delete mode 100644 resources/operationss3/create-asset.json delete mode 100644 resources/operationss3/create-contract-definition.json delete mode 100644 resources/operationss3/create-policy.json delete mode 100644 resources/operationss3/fetch-catalog.json delete mode 100644 resources/operationss3/negotiate-contract.json delete mode 100644 resources/operationss3/start-transfer-proxy.json delete mode 100644 resources/operationss3/start-transfer.json diff --git a/README.md b/README.md index a95d77c..aaaf921 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,28 @@ Dataspaces connector for the InesData project, based on the EDC dataspaces frame ``` - Launch: + ``` + java -jar launchers/connector/build/libs/connector-app.jar + ``` Example: ``` java -Dedc.keystore=resources/certs/store.pfx -Dedc.keystore.password= -Dedc.vault=resources/configuration/provider-vault.properties -Dedc.fs.config=resources/configuration/provider-configuration.properties -jar launchers/connector/build/libs/connector-app.jar ``` + +## Containerizing the connector + +The connector is prepared to be executed inside a container. The following steps must be followed to generate the connector: +- Compile the connector: + ``` + ./gradlew launchers:connector:build + ``` + +- Create a local Docker image:: + ``` + docker build --tag inesdata/connector:0.1 --build-arg CONNECTOR_JAR=./launchers/connector/build/libs/connector-app.jar -f docker/Dockerfile . + ``` + +## Database + +The `resources/sql` folder contains all the required schemes to be applied in the connector database. diff --git a/base/build.gradle.kts b/base/build.gradle.kts deleted file mode 100644 index bfd22e2..0000000 --- a/base/build.gradle.kts +++ /dev/null @@ -1,20 +0,0 @@ - - -plugins { - id("com.gmv.inesdata.edc-application") - alias(libs.plugins.shadow) -} - -dependencies { - implementation(libs.edc.boot) - implementation(libs.edc.connector.core) -} - -application { - mainClass.set("org.eclipse.edc.boot.system.runtime.BaseRuntime") -} - -tasks.withType { - mergeServiceFiles() - archiveFileName.set("connector-app.jar") -} diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..9342cdd --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,33 @@ +FROM openjdk:17-jdk-alpine +ARG CONNECTOR_JAR + +# System env vars +ENV USER_NAME=connectoruser +ENV USER_UID=200 +ENV JVM_ARGS="" +ENV CONNECTOR_HOME=/opt/connector +ENV CONNECTOR_RESOURCES=${CONNECTOR_HOME}/resources + +# Add the user who executes the connector. The user belongs to the group with gui 0 +RUN adduser -G root -s /bin/sh -D -u ${USER_UID} ${USER_NAME} + +# Create the connector and resources directories +RUN mkdir -p ${CONNECTOR_HOME} +RUN mkdir -p ${CONNECTOR_RESOURCES} + +# Work with the minimum rights +RUN chown -R ${USER_NAME}:0 ${CONNECTOR_HOME} \ + && chmod -R g=u ${CONNECTOR_HOME} + +# User by default +USER ${USER_NAME} + +# Set the workdir +WORKDIR ${CONNECTOR_HOME} + +# Copy connector +COPY $CONNECTOR_JAR connector.jar + +# Use "exec" for Kubernetes graceful termination (SIGINT) to reach JVM. +ENTRYPOINT [ "sh", "-c", \ + "exec java $JVM_ARGS -jar connector.jar"] diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 9d4a78d..adf4848 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -58,6 +58,9 @@ edc-sql-data-plane-store = { module = "org.eclipse.edc:data-plane-store-sql", ve edc-aws-s3-core = { module = "org.eclipse.edc:aws-s3-core", version.ref = "edc" } edc-data-plane-aws-s3 = { module = "org.eclipse.edc:data-plane-aws-s3", version.ref = "edc" } +# EDC Observability +edc-observability-api = { module = "org.eclipse.edc:api-observability", version.ref = "edc" } + # no EDC references aws-s3 = { module = "software.amazon.awssdk:s3", version.ref = "aws" } aws-s3-transfer = { module = "software.amazon.awssdk:s3-transfer-manager", version.ref = "aws" } diff --git a/launchers/connector/build.gradle.kts b/launchers/connector/build.gradle.kts index 2752be8..99d560c 100644 --- a/launchers/connector/build.gradle.kts +++ b/launchers/connector/build.gradle.kts @@ -67,11 +67,13 @@ dependencies { // Persistencia de objetos implementation(libs.edc.aws.s3.core) implementation(libs.edc.data.plane.aws.s3) + + // Observability + implementation(libs.edc.observability.api) runtimeOnly(libs.edc.transaction.local) runtimeOnly(libs.postgres) - } application { diff --git a/resources/certs/cert.pem b/resources/certs/cert.pem deleted file mode 100644 index 82a7f2e..0000000 --- a/resources/certs/cert.pem +++ /dev/null @@ -1,19 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDETCCAfkCFBEOuzF+mRgrzrxVv70zNDUEv+nUMA0GCSqGSIb3DQEBCwUAMEUx -CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl -cm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjQwMTMxMTIyNTUyWhcNMjUwMTMwMTIy -NTUyWjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE -CgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC -AQ8AMIIBCgKCAQEAxHbsW755sB/5YwjPyk0xPJwyQkxkGV2SS4sQQCCz8KV/QoEi -lk/PzUqUsE2e7LqavWL5+FPSic79ZKEljeeNB1z5xoE2YnKEs55MLi/PgfLJHJ9M -bA42lJBYI37MeGv0kGkIQDIrtpDjBfSgGSINXeTjs0T/l5sXXPTfSRm4URWX3I+Q -O9ACYLb7Cz//G8HdYHp8MHNa38x4BeBku7cT7xgEUPfhu4LCnBW1u0pMbeT8A2M9 -zz/TIOJKwUeupacXjJ/tqHGLYMgBLCSRNK4zFJiGzhDbEZA6+NMUp660t7Su6M5i -x3WIffKlIcxi0bnk9slhjK5NynJ3H/uvbfYnCwIDAQABMA0GCSqGSIb3DQEBCwUA -A4IBAQBAttZGxh1k87S8OHA6L8fmu/PEtjdkJV3Cu7J3xTGfmSBVMTStl2NI4D4i -FJUFdTJjvEibEaQEETz2zJakwdp7JGG7ip7e/dKsP0wVMC63MZ8eeDzST7IzBLk6 -mjb+Co0+2p/9rHPBz1AFpHRcJ9jaORh8FlctfcxcBsr6dMptGtCZMzam26WwePxa -pNd+wBNFHuAXHyTzHl3NIq2Bd+56Roen5muZKv4xqI7iVBbVrejvVyXNDhrqh9QB -VushjclGdc2GSdCVpUhiv1srX59/yX5HBZG8HRQSgCXpcSiGIhiMmd1gaU/qrXa8 -kwmd6tdKUcAiLU7qXEJX4iBziobm ------END CERTIFICATE----- diff --git a/resources/certs/key.pem b/resources/certs/key.pem deleted file mode 100644 index 75f7835..0000000 --- a/resources/certs/key.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDEduxbvnmwH/lj -CM/KTTE8nDJCTGQZXZJLixBAILPwpX9CgSKWT8/NSpSwTZ7supq9Yvn4U9KJzv1k -oSWN540HXPnGgTZicoSznkwuL8+B8skcn0xsDjaUkFgjfsx4a/SQaQhAMiu2kOMF -9KAZIg1d5OOzRP+Xmxdc9N9JGbhRFZfcj5A70AJgtvsLP/8bwd1genwwc1rfzHgF -4GS7txPvGARQ9+G7gsKcFbW7Skxt5PwDYz3PP9Mg4krBR66lpxeMn+2ocYtgyAEs -JJE0rjMUmIbOENsRkDr40xSnrrS3tK7ozmLHdYh98qUhzGLRueT2yWGMrk3Kcncf -+69t9icLAgMBAAECggEAMrh/IdplgS5oDITQXGqcXQj4QryFFTVemQkwa1XmDqkz -VyCOpN67B8WK6I9JidVNMLq2TUGat8BME+g0kVOnybBbvb9LIpRtJRAnm771JBfh -ivWnXbg3qBgx4QwRzr8UNxoeqrApL+ts6PM0R/jOGaEHlhcoo2PDXtjLMoyk1K26 -JdXibZ8TjqS59MG9BB8j2/NiyaQxC46k8FPz5UXnLAVr8z8kzi0P/LfuZdFO3BjB -1Pgsgy7zme32nUCTJSQbCUKngY7KAwaeuPVji5QbkJ0Tk1tWop9IKg43RzqVp+Me -QZWDgIyeWMRkmKb6GXrl6HjuOFEymMQkPlRH7QPDcQKBgQDjOaYaPFQIgkkKAoMa -DGT0bv1RdPWithYHDTKpa6xMqfZ5/H+09i/+wVH1P8j2osylZOunsCVoIeeeTuv4 -VYC+VTOboOFdbzKD9gOyvZ4nUox+nT/1awK/ZBRcX1GrItZLs7QbrWHswT1VWTcO -ccj7wVIKs15yuBuV0w97qA1h0QKBgQDdWA2v1Yj0jgOn8leCSGj2LlzYObuEWEn4 -lCNn0b5ZABTqFpG6viNpalyTUHDXGYt3mZpEBpW2O+x7Bv9EO8+G9TtkW13NL+eD -4+4wftRx4LTuOlG5wAicNG0dvaobMZ3Ba/v+KCs2pQaC6pC0E12j5euPoTl6pUxL -0caANS72GwKBgA6Yu3Tf/z30sKB1/NoFhc61+ovrCYSEUfHuBR0hD5Z1LKI1eDa1 -weuJm6w3hr9hQeg2DXVeOWjxFKH0y+/N/lE7w+7xbMjFxeb4PNCUob5A7UmM7Hjl -gO2ujihI70QhHscKKj3E64978mazLng9Ev4pMw4A68ZO0WbUgiAdcY6hAoGBAJA6 -YDDDI4q3PwqgjwUFyd8gNr9uXxHNteWUIb30nePAeeBO9IaKfuihtdEICX8f1hEI -tcy9vbARO/auiIcdsgxun1S1dFnPbTwzuXFAM45AraTNu5Md3n5qau+GNuGhYvo2 -Mu1Zn07sGWIVFlVAiTPTDbt5gsq4Cw8ckgvYm9EhAoGAJEi0oDOOPS8skP71HsEm -fwMhp414LL7xFf4LZj8RL4+vDfzyNnbQ3/zpGKCzwO2MWAQx4GB9hNrX8meGLJXP -w5zMub2RLs0x4pZPuIQVIfcf6Ryuwx/kM3uZAj6lUm5LpfsWJNFZylC2ixuLTLgn -B1fcrLsMChYYsJvCfzElLSs= ------END PRIVATE KEY----- diff --git a/resources/certs/store.pfx b/resources/certs/store.pfx deleted file mode 100644 index a1b3326e6fbdc4f2dd2b1e154952f054b7a78926..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2531 zcmai$c{J1w7srhm!wh2^TMS{8HT}ktWh_}6YmHGFPbEdRtYhEOm=uZ16QKk##i0bkqj_NkD+jAHjkEOaa+{BnNsML{m?Irs_U+SY*K!`!P8DVoNG%aN7jSofN+up#*vM+%O%s5J}S);&!0&j zk#21iuC@07SzREBXqsJgJv(htkK<2`EM}&sHU=ALS6(f2wD4ZTShR&BG~X@}%3om4 zce&>TWhiuOpz38sUAuUVRzSIRWXpq$Zsi$&Bc$HU!L^@)w0R?$2yN0a1B{=c!GSLz7YD5L;7NBd`| z*T>3QdypFDr^LPNa`*E0Yc$hC*fT}J?B_ZAGty37E+21FYs*sN)v>8_4)eSBT z(#-z7&>M-@xRxmGf(Y)dPgySW#`rNn=+nGzZmt;#f#Fc=d5{l;=f;ueo9!f zjNPy5mrKuN!#`dZZ;5I;Qe6%8iU;-}0IdhP&rGO#+-1frt8rG zb>T5;F?gSk%hHdDP2J12z7>r+iusO+MzIKs6gb`F0FV(WxMX!8k3g?289htz zqC?1#o&+fQO?&H3c(l1TvE&hSLio$?vS34-%FbuTe>Ho}QTmElu_a22e5RjVGp68N zDeNynmlcT_7EG{XKa^Dc>(}MVH$A=aRfDWyOZqb%x-P#e<9rzKh6_zY6&+j8;to5G zi@i6^?P^1SO3^vZ1_~=_>~|sn#sxc$m+Tk=((z^+R8DN0T(ZNQ`Md)pa9KcluXBbm zQFu=BgXbqFa_l^1oqD3X2_1^i|-NI!V*4lDgNqBP@+p zhZZ)UW2&!MO4Vc~WKK^S*SL^fRU|sG4oWn0gM>N6o7J?ZD>tXJi}l)&b=q2}CprrqE?$!!d`k z6TP-czmkeY7yEqt)AG)!#ETzKE8p0>1A7sEUNiAqxGOy(TFCmAn^Ln`!%x#y8~wE@ zAL9(0**q;CjM!iWZq>Nn2i&vB#^ENx?j!!DJhQ966z4NHk7M3r7g!r0KN$aTILcu+ zxy!@wllkh%L!=f@j#nmHhv2VgPp6+*6(W|qsQCR@*lcNf{=N7mvc+pi@W-A6dzc1# z&*{9VQQB@*4i~l`4;3|L&HEE?wSzRiSN;@)e*UZSu{kgt{41qAz0T-1Z4tCLU4K5~N8h=a1KE9+e$h z3D^-%X`ToEZ(nmM8>;t_u8wnx)gew+L<^rrVyaHR>Vki(VH=z>%64rgd`4vGoVEba zQImejV&w*g`gxB@E)41}|7i&Al$sdtNmw&pZ{QuFV`YPeZHddUnRj>#;AR?@d-&I~ z^mF{ymVb2E>34dXS??A%D+6&8br(5LP3GNiECu%#*#s7{=Bq<}H$vYb*aa@B?<-jE zzf;u>x(JOP<^V`=)WRC&yy*fo>loZWNJdDd2JNMeuZ<)qS4QS&4e(QTBC>2L{U1vZKzyOFp?-4VIg$W{VBfUfx52KE?I!S7BQ*hhW)Lp@Y cm-U-U!sfrJ3I=2dTcuYl(!e~^`ft?!7wu7x_y7O^ diff --git a/resources/configuration/consumer-configuration.properties b/resources/configuration/consumer-configuration.properties deleted file mode 100644 index 179737d..0000000 --- a/resources/configuration/consumer-configuration.properties +++ /dev/null @@ -1,31 +0,0 @@ -edc.participant.id=consumer -edc.dsp.callback.address=http://localhost:29194/protocol -web.http.port=29191 -web.http.path=/api -web.http.management.port=29193 -web.http.management.path=/management -web.http.protocol.port=29194 -web.http.protocol.path=/protocol -edc.receiver.http.endpoint=http://localhost:4001/receiver/urn:connector:provider/callback -edc.public.key.alias=public-key -edc.transfer.dataplane.token.signer.privatekey.alias=1 -edc.transfer.proxy.token.signer.privatekey.alias=1 -edc.transfer.proxy.token.verifier.publickey.alias=public-key -edc.web.rest.cors.enabled=true -edc.web.rest.cors.headers=origin, content-type, accept, authorization, x-api-key -web.http.public.port=29291 -web.http.public.path=/public -web.http.control.port=29192 -web.http.control.path=/control -edc.dataplane.token.validation.endpoint=http://localhost:29192/control/token - -edc.datasource.default.url=jdbc:postgresql://localhost:5433/inesdata -edc.datasource.default.user=inesdata -edc.datasource.default.password= -edc.datasource.default.pool.maxIdleConnections=10 -edc.datasource.default.pool.maxTotalConnections=10 -edc.datasource.default.pool.minIdleConnections=5 - -edc.aws.access.key=access-key -edc.aws.secret.access.key=secret-key -edc.aws.endpoint.override=http://localhost:9010 diff --git a/resources/configuration/consumer-vault.properties b/resources/configuration/consumer-vault.properties deleted file mode 100644 index d91e3e6..0000000 --- a/resources/configuration/consumer-vault.properties +++ /dev/null @@ -1,3 +0,0 @@ -public-key=-----BEGIN CERTIFICATE-----\r\nMIIDETCCAfkCFBEOuzF+mRgrzrxVv70zNDUEv+nUMA0GCSqGSIb3DQEBCwUAMEUx\r\nCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl\r\ncm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjQwMTMxMTIyNTUyWhcNMjUwMTMwMTIy\r\nNTUyWjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE\r\nCgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC\r\nAQ8AMIIBCgKCAQEAxHbsW755sB/5YwjPyk0xPJwyQkxkGV2SS4sQQCCz8KV/QoEi\r\nlk/PzUqUsE2e7LqavWL5+FPSic79ZKEljeeNB1z5xoE2YnKEs55MLi/PgfLJHJ9M\r\nbA42lJBYI37MeGv0kGkIQDIrtpDjBfSgGSINXeTjs0T/l5sXXPTfSRm4URWX3I+Q\r\nO9ACYLb7Cz//G8HdYHp8MHNa38x4BeBku7cT7xgEUPfhu4LCnBW1u0pMbeT8A2M9\r\nzz/TIOJKwUeupacXjJ/tqHGLYMgBLCSRNK4zFJiGzhDbEZA6+NMUp660t7Su6M5i\r\nx3WIffKlIcxi0bnk9slhjK5NynJ3H/uvbfYnCwIDAQABMA0GCSqGSIb3DQEBCwUA\r\nA4IBAQBAttZGxh1k87S8OHA6L8fmu/PEtjdkJV3Cu7J3xTGfmSBVMTStl2NI4D4i\r\nFJUFdTJjvEibEaQEETz2zJakwdp7JGG7ip7e/dKsP0wVMC63MZ8eeDzST7IzBLk6\r\nmjb+Co0+2p/9rHPBz1AFpHRcJ9jaORh8FlctfcxcBsr6dMptGtCZMzam26WwePxa\r\npNd+wBNFHuAXHyTzHl3NIq2Bd+56Roen5muZKv4xqI7iVBbVrejvVyXNDhrqh9QB\r\nVushjclGdc2GSdCVpUhiv1srX59/yX5HBZG8HRQSgCXpcSiGIhiMmd1gaU/qrXa8\r\nkwmd6tdKUcAiLU7qXEJX4iBziobm\r\n-----END CERTIFICATE----- -access-key= -secret-key= diff --git a/resources/configuration/provider-configuration.properties b/resources/configuration/provider-configuration.properties deleted file mode 100644 index a162530..0000000 --- a/resources/configuration/provider-configuration.properties +++ /dev/null @@ -1,31 +0,0 @@ -edc.participant.id=provider -edc.dsp.callback.address=http://localhost:19194/protocol -web.http.port=19191 -web.http.path=/api -web.http.management.port=19193 -web.http.management.path=/management -web.http.protocol.port=19194 -web.http.protocol.path=/protocol -edc.receiver.http.endpoint=http://localhost:4000/receiver/urn:connector:provider/callback -edc.public.key.alias=public-key -edc.transfer.dataplane.token.signer.privatekey.alias=1 -edc.transfer.proxy.token.signer.privatekey.alias=1 -edc.transfer.proxy.token.verifier.publickey.alias=public-key -edc.web.rest.cors.enabled=true -edc.web.rest.cors.headers=origin, content-type, accept, authorization, x-api-key -web.http.public.port=19291 -web.http.public.path=/public -web.http.control.port=19192 -web.http.control.path=/control -edc.dataplane.token.validation.endpoint=http://localhost:19192/control/token - -edc.datasource.default.url=jdbc:postgresql://localhost:5432/inesdata -edc.datasource.default.user=inesdata -edc.datasource.default.password= -edc.datasource.default.pool.maxIdleConnections=10 -edc.datasource.default.pool.maxTotalConnections=10 -edc.datasource.default.pool.minIdleConnections=5 - -edc.aws.access.key=access-key -edc.aws.secret.access.key=secret-key -edc.aws.endpoint.override=http://localhost:9000 \ No newline at end of file diff --git a/resources/configuration/provider-vault.properties b/resources/configuration/provider-vault.properties deleted file mode 100644 index d91e3e6..0000000 --- a/resources/configuration/provider-vault.properties +++ /dev/null @@ -1,3 +0,0 @@ -public-key=-----BEGIN CERTIFICATE-----\r\nMIIDETCCAfkCFBEOuzF+mRgrzrxVv70zNDUEv+nUMA0GCSqGSIb3DQEBCwUAMEUx\r\nCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl\r\ncm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjQwMTMxMTIyNTUyWhcNMjUwMTMwMTIy\r\nNTUyWjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE\r\nCgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC\r\nAQ8AMIIBCgKCAQEAxHbsW755sB/5YwjPyk0xPJwyQkxkGV2SS4sQQCCz8KV/QoEi\r\nlk/PzUqUsE2e7LqavWL5+FPSic79ZKEljeeNB1z5xoE2YnKEs55MLi/PgfLJHJ9M\r\nbA42lJBYI37MeGv0kGkIQDIrtpDjBfSgGSINXeTjs0T/l5sXXPTfSRm4URWX3I+Q\r\nO9ACYLb7Cz//G8HdYHp8MHNa38x4BeBku7cT7xgEUPfhu4LCnBW1u0pMbeT8A2M9\r\nzz/TIOJKwUeupacXjJ/tqHGLYMgBLCSRNK4zFJiGzhDbEZA6+NMUp660t7Su6M5i\r\nx3WIffKlIcxi0bnk9slhjK5NynJ3H/uvbfYnCwIDAQABMA0GCSqGSIb3DQEBCwUA\r\nA4IBAQBAttZGxh1k87S8OHA6L8fmu/PEtjdkJV3Cu7J3xTGfmSBVMTStl2NI4D4i\r\nFJUFdTJjvEibEaQEETz2zJakwdp7JGG7ip7e/dKsP0wVMC63MZ8eeDzST7IzBLk6\r\nmjb+Co0+2p/9rHPBz1AFpHRcJ9jaORh8FlctfcxcBsr6dMptGtCZMzam26WwePxa\r\npNd+wBNFHuAXHyTzHl3NIq2Bd+56Roen5muZKv4xqI7iVBbVrejvVyXNDhrqh9QB\r\nVushjclGdc2GSdCVpUhiv1srX59/yX5HBZG8HRQSgCXpcSiGIhiMmd1gaU/qrXa8\r\nkwmd6tdKUcAiLU7qXEJX4iBziobm\r\n-----END CERTIFICATE----- -access-key= -secret-key= diff --git a/resources/dataplane/register-data-plane-consumer.json b/resources/dataplane/register-data-plane-consumer.json deleted file mode 100644 index cbae3bd..0000000 --- a/resources/dataplane/register-data-plane-consumer.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "@context": { - "@vocab": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@id": "http-pull-consumer-dataplane", - "url": "http://localhost:29192/control/transfer", - "allowedSourceTypes": [ - "HttpData", - "AmazonS3" - ], - "allowedDestTypes": [ - "HttpProxy", - "HttpData", - "AmazonS3" - ], - "properties": { - "https://w3id.org/edc/v0.0.1/ns/publicApiUrl/publicApiUrl": "http://localhost:29291/public/" - } -} diff --git a/resources/dataplane/register-data-plane-provider.json b/resources/dataplane/register-data-plane-provider.json deleted file mode 100644 index e88a576..0000000 --- a/resources/dataplane/register-data-plane-provider.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "@context": { - "@vocab": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@id": "http-pull-provider-dataplane", - "url": "http://localhost:19192/control/transfer", - "allowedSourceTypes": [ - "HttpData", - "AmazonS3" - ], - "allowedDestTypes": [ - "HttpProxy", - "HttpData", - "AmazonS3" - ], - "properties": { - "https://w3id.org/edc/v0.0.1/ns/publicApiUrl": "http://localhost:19291/public/" - } -} diff --git a/resources/operations/create-asset.json b/resources/operations/create-asset.json deleted file mode 100644 index 1931546..0000000 --- a/resources/operations/create-asset.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "@context": { - "@vocab": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@id": "assetId", - "properties": { - "name": "product description", - "contenttype": "application/json" - }, - "dataAddress": { - "type": "HttpData", - "name": "JSON place holder users", - "baseUrl": "https://jsonplaceholder.typicode.com/users", - "proxyPath": "true" - } -} diff --git a/resources/operations/create-contract-definition.json b/resources/operations/create-contract-definition.json deleted file mode 100644 index 5ee5c67..0000000 --- a/resources/operations/create-contract-definition.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "@context": { - "@vocab": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@id": "1", - "accessPolicyId": "aPolicy", - "contractPolicyId": "aPolicy", - "assetsSelector": [] -} diff --git a/resources/operations/create-policy.json b/resources/operations/create-policy.json deleted file mode 100644 index 5047f00..0000000 --- a/resources/operations/create-policy.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "@context": { - "@vocab": "https://w3id.org/edc/v0.0.1/ns/", - "odrl": "http://www.w3.org/ns/odrl/2/" - }, - "@id": "aPolicy", - "policy": { - "@type": "set", - "odrl:permission": [], - "odrl:prohibition": [], - "odrl:obligation": [] - } -} diff --git a/resources/operations/fetch-catalog.json b/resources/operations/fetch-catalog.json deleted file mode 100644 index 370b4f2..0000000 --- a/resources/operations/fetch-catalog.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "@context": { - "@vocab": "https://w3id.org/edc/v0.0.1/ns/" - }, - "counterPartyAddress": "http://localhost:19194/protocol", - "protocol": "dataspace-protocol-http" -} diff --git a/resources/operations/negotiate-contract.json b/resources/operations/negotiate-contract.json deleted file mode 100644 index fc096c4..0000000 --- a/resources/operations/negotiate-contract.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "@context": { - "@vocab": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@type": "NegotiationInitiateRequestDto", - "connectorId": "provider", - "counterPartyAddress": "http://localhost:19194/protocol", - "consumerId": "consumer", - "providerId": "provider", - "protocol": "dataspace-protocol-http", - "policy": { - "@context": "http://www.w3.org/ns/odrl.jsonld", - "@id": "MQ==:YXNzZXRJZA==:NDRlN2FkNjktNmQyMy00ZmM5LTljM2EtYmRkN2Q0NDcxNzM3", - "@type": "Set", - "permission": [], - "prohibition": [], - "obligation": [], - "target": "assetId" - } -} diff --git a/resources/operations/start-transfer.json b/resources/operations/start-transfer.json deleted file mode 100644 index 5a094d2..0000000 --- a/resources/operations/start-transfer.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "@context": { - "@vocab": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@type": "TransferRequestDto", - "connectorId": "provider", - "counterPartyAddress": "http://localhost:19194/protocol", - "contractId": "36568fc7-0fc8-4e11-a730-6da40ccdb93b", - "assetId": "assetId", - "protocol": "dataspace-protocol-http", - "dataDestination": { - "type": "HttpProxy" - } -} diff --git a/resources/operationss3/create-asset.json b/resources/operationss3/create-asset.json deleted file mode 100644 index 0983e80..0000000 --- a/resources/operationss3/create-asset.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "@context": { - "@vocab": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@id": "s3asset", - "properties": { - "name": "product description", - "contenttype": "text/plain" - }, - "dataAddress": { - "type": "AmazonS3", - "region": "us-east-1", - "keyName": "text-document-provider-dataset.txt", - "bucketName": "bucket-connector-1" - } -} diff --git a/resources/operationss3/create-contract-definition.json b/resources/operationss3/create-contract-definition.json deleted file mode 100644 index 50e16c2..0000000 --- a/resources/operationss3/create-contract-definition.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "@context": { - "@vocab": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@id": "1", - "accessPolicyId": "s3Policy", - "contractPolicyId": "s3Policy", - "assetsSelector": [] -} diff --git a/resources/operationss3/create-policy.json b/resources/operationss3/create-policy.json deleted file mode 100644 index 187bf51..0000000 --- a/resources/operationss3/create-policy.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "@context": { - "@vocab": "https://w3id.org/edc/v0.0.1/ns/", - "odrl": "http://www.w3.org/ns/odrl/2/" - }, - "@id": "s3Policy", - "policy": { - "@type": "set", - "odrl:permission": [], - "odrl:prohibition": [], - "odrl:obligation": [] - } -} diff --git a/resources/operationss3/fetch-catalog.json b/resources/operationss3/fetch-catalog.json deleted file mode 100644 index 370b4f2..0000000 --- a/resources/operationss3/fetch-catalog.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "@context": { - "@vocab": "https://w3id.org/edc/v0.0.1/ns/" - }, - "counterPartyAddress": "http://localhost:19194/protocol", - "protocol": "dataspace-protocol-http" -} diff --git a/resources/operationss3/negotiate-contract.json b/resources/operationss3/negotiate-contract.json deleted file mode 100644 index 78e65fd..0000000 --- a/resources/operationss3/negotiate-contract.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "@context": { - "@vocab": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@type": "NegotiationInitiateRequestDto", - "connectorId": "provider", - "counterPartyAddress": "http://localhost:19194/protocol", - "consumerId": "consumer", - "providerId": "provider", - "protocol": "dataspace-protocol-http", - "policy": { - "@context": "http://www.w3.org/ns/odrl.jsonld", - "@id": "MQ==:czNhc3NldA==:ZDU2M2VkNjQtNGU4MS00YTk0LWI4ODctNzU5N2NiYjlmNDM0", - "@type": "Set", - "permission": [], - "prohibition": [], - "obligation": [], - "target": "s3asset" - } -} diff --git a/resources/operationss3/start-transfer-proxy.json b/resources/operationss3/start-transfer-proxy.json deleted file mode 100644 index 65228c3..0000000 --- a/resources/operationss3/start-transfer-proxy.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "@context": { - "@vocab": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@type": "TransferRequestDto", - "connectorId": "provider", - "counterPartyAddress": "http://localhost:19194/protocol", - "contractId": "f2e5f10c-050f-41de-ac60-a3936cb535cf", - "assetId": "s3asset", - "protocol": "dataspace-protocol-http", - "dataDestination": { - "type": "HttpProxy" - } -} diff --git a/resources/operationss3/start-transfer.json b/resources/operationss3/start-transfer.json deleted file mode 100644 index a9f77d8..0000000 --- a/resources/operationss3/start-transfer.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "@context": { - "@vocab": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@type": "TransferRequestDto", - "counterPartyAddress": "http://localhost:19194/protocol", - "protocol": "dataspace-protocol-http", - "connectorId": "consumer", - "assetId": "s3asset", - "contractId": "af9c225e-8a19-442d-abde-8c8d0af9f044", - "dataDestination": { - "properties": { - "type": "AmazonS3", - "region": "us-east-1", - "bucketName": "bucket-connector-2", - "folderName": "descargas", - "accessKeyId": "ymapc4WGhWrgaywwSyq1", - "secretAccessKey": "wxwo6F4WTTi9q5Gts7Ve57TnxO29p1waJua6xpvT" - }, - "endpointOverride": "http://localhost:9010", - "type": "AmazonS3" - }, - "managedResources": true, - "transferType": { - "contentType": "text/plain", - "isFinite": true - } -} \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index 49000a3..ca5bc36 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -5,5 +5,4 @@ pluginManagement { rootProject.name = "inesdata-connector" -include(":base") include(":launchers:connector") From f98ff32a1327a0254d31888924d033adeece9a2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Mon, 26 Feb 2024 11:00:10 +0100 Subject: [PATCH 024/174] update docker image and user creation --- docker/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 9342cdd..1c4a733 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,15 +1,15 @@ -FROM openjdk:17-jdk-alpine +FROM eclipse-temurin:17-jre ARG CONNECTOR_JAR # System env vars +ENV USER_GROUP=connectorgroup ENV USER_NAME=connectoruser -ENV USER_UID=200 ENV JVM_ARGS="" ENV CONNECTOR_HOME=/opt/connector ENV CONNECTOR_RESOURCES=${CONNECTOR_HOME}/resources # Add the user who executes the connector. The user belongs to the group with gui 0 -RUN adduser -G root -s /bin/sh -D -u ${USER_UID} ${USER_NAME} +RUN addgroup ${USER_GROUP} && adduser --no-create-home --disabled-password --ingroup ${USER_GROUP} ${USER_NAME} # Create the connector and resources directories RUN mkdir -p ${CONNECTOR_HOME} From 33c6fe0bea538666b5e8a83496f026522f7017dd Mon Sep 17 00:00:00 2001 From: Juan Calero Date: Tue, 27 Feb 2024 10:48:04 +0100 Subject: [PATCH 025/174] Empty commit to trigger CI From 2ab5344155e4e7fe8f713beea35a597b08f2caea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Tue, 27 Feb 2024 13:10:09 +0100 Subject: [PATCH 026/174] adding federated catalog into inesdata connector --- .../participants-from-configuration/README.md | 13 ++++ .../build.gradle.kts | 14 ++++ .../ConfigurationTargetNodeDirectory.java | 73 +++++++++++++++++++ ...romConfigurationParticipantsExtension.java | 44 +++++++++++ ...rg.eclipse.edc.spi.system.ServiceExtension | 1 + .../ConfigurationTargetNodeDirectoryTest.java | 64 ++++++++++++++++ gradle/libs.versions.toml | 28 +++++-- launchers/connector/build.gradle.kts | 6 ++ settings.gradle.kts | 4 + 9 files changed, 239 insertions(+), 8 deletions(-) create mode 100644 extensions/participants-from-configuration/README.md create mode 100644 extensions/participants-from-configuration/build.gradle.kts create mode 100644 extensions/participants-from-configuration/src/main/java/org/upm/inesdata/catalog/ConfigurationTargetNodeDirectory.java create mode 100644 extensions/participants-from-configuration/src/main/java/org/upm/inesdata/catalog/FromConfigurationParticipantsExtension.java create mode 100644 extensions/participants-from-configuration/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension create mode 100644 extensions/participants-from-configuration/src/test/java/org/upm/inesdata/catalog/ConfigurationTargetNodeDirectoryTest.java diff --git a/extensions/participants-from-configuration/README.md b/extensions/participants-from-configuration/README.md new file mode 100644 index 0000000..820af44 --- /dev/null +++ b/extensions/participants-from-configuration/README.md @@ -0,0 +1,13 @@ +# InesData Connector + +This extension provides the capability to obtain the list of participants of a data space in order to allow the federated catalog to obtain their data offerings. + +## Configuration + +Exemplary configuration: + +```properties +edc.catalog.configuration.participant.list=100210;connector-company-sample;http://targe-url-cs:9194/protocol|561349;connector-company-example;http://targe-url-ce:9194/protocol +``` + +The `edc.catalog.configuration.participant.list` will be used by the federated catalog to retrieve the list of participants whose catalog will be obtained for federating it. \ No newline at end of file diff --git a/extensions/participants-from-configuration/build.gradle.kts b/extensions/participants-from-configuration/build.gradle.kts new file mode 100644 index 0000000..49480f1 --- /dev/null +++ b/extensions/participants-from-configuration/build.gradle.kts @@ -0,0 +1,14 @@ +plugins { + `java-library` + id("com.gmv.inesdata.edc-application") +} + +dependencies { + implementation(libs.edc.federated.catalog.spi) + implementation(libs.edc.federated.catalog.core) + + testImplementation(libs.edc.core.junit) + testImplementation(libs.junit.jupiter.api) + testImplementation(libs.assertj) + testImplementation(libs.mockito.core) +} diff --git a/extensions/participants-from-configuration/src/main/java/org/upm/inesdata/catalog/ConfigurationTargetNodeDirectory.java b/extensions/participants-from-configuration/src/main/java/org/upm/inesdata/catalog/ConfigurationTargetNodeDirectory.java new file mode 100644 index 0000000..e835787 --- /dev/null +++ b/extensions/participants-from-configuration/src/main/java/org/upm/inesdata/catalog/ConfigurationTargetNodeDirectory.java @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial API and implementation + * + */ + +package org.upm.inesdata.catalog; + +import org.eclipse.edc.crawler.spi.TargetNode; +import org.eclipse.edc.crawler.spi.TargetNodeDirectory; +import org.eclipse.edc.spi.monitor.Monitor; + +import java.util.ArrayList; +import java.util.List; + +import static java.lang.String.format; + +/** + * Federated cache directory using a string with a list of TargetNode items. + */ +public class ConfigurationTargetNodeDirectory implements TargetNodeDirectory { + + public static final List SUPPORTED_PROTOCOLS = List.of("dataspace-protocol-http"); + public static final String PARTICIPANT_SEPARATOR = "\\|"; + public static final String PARTICIPANT_INFO_SEPARATOR = ";"; + + private final Monitor monitor; + private final List participantList; + + /** + * Constructs {@link ConfigurationTargetNodeDirectory} + * + * @param monitor monitor + * @param rawParticipantList RegistrationService API client. + */ + public ConfigurationTargetNodeDirectory(Monitor monitor, String rawParticipantList) { + this.monitor = monitor; + this.participantList = processRawParticipantList(rawParticipantList); + } + + @Override + public List getAll() { + return participantList; + } + + @Override + public void insert(TargetNode targetNode) { + throw new UnsupportedOperationException(); + } + + private List processRawParticipantList(String rawParticipantList) { + var participants = new ArrayList(); + try { + String[] rawParticipants = rawParticipantList.split(ConfigurationTargetNodeDirectory.PARTICIPANT_SEPARATOR); + for (String rawParticipant : rawParticipants) { + String[] participant = rawParticipant.split(ConfigurationTargetNodeDirectory.PARTICIPANT_INFO_SEPARATOR); + participants.add(new TargetNode(participant[1], participant[0], participant[2], ConfigurationTargetNodeDirectory.SUPPORTED_PROTOCOLS)); + } + } catch (Exception ex) { + monitor.severe(format("Error processing the list of participants. The expected format is: id;name;targetUrl|id;name;targetUrl and the provided list is '%s'. The exception is: %s", rawParticipantList, ex.getMessage())); + } + monitor.warning(format("Obtained a total of %s participants", participants.size())); + return participants; + } +} diff --git a/extensions/participants-from-configuration/src/main/java/org/upm/inesdata/catalog/FromConfigurationParticipantsExtension.java b/extensions/participants-from-configuration/src/main/java/org/upm/inesdata/catalog/FromConfigurationParticipantsExtension.java new file mode 100644 index 0000000..d6620a3 --- /dev/null +++ b/extensions/participants-from-configuration/src/main/java/org/upm/inesdata/catalog/FromConfigurationParticipantsExtension.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial API and implementation + * + */ + +package org.upm.inesdata.catalog; + +import org.eclipse.edc.crawler.spi.TargetNodeDirectory; +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.runtime.metamodel.annotation.Provider; +import org.eclipse.edc.runtime.metamodel.annotation.Setting; +import org.eclipse.edc.spi.monitor.Monitor; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; + + +/** + * Extension to set up federated target node directory using a configuration variable. + */ +public class FromConfigurationParticipantsExtension implements ServiceExtension { + + @Setting + public static final String INESDATA_FC_PARTICIPANT_LIST = "edc.catalog.configuration.participant.list"; + + @Provider + public TargetNodeDirectory federatedCacheNodeDirectory(ServiceExtensionContext context) { + var monitor = context.getMonitor(); + var participantList = context.getConfig().getString(INESDATA_FC_PARTICIPANT_LIST); + monitor.debug("Retrieved the participant list from configuration: " + participantList); + + return new ConfigurationTargetNodeDirectory(monitor, participantList); + } +} + + diff --git a/extensions/participants-from-configuration/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/participants-from-configuration/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 0000000..84f5e03 --- /dev/null +++ b/extensions/participants-from-configuration/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1 @@ +org.upm.inesdata.catalog.FromConfigurationParticipantsExtension diff --git a/extensions/participants-from-configuration/src/test/java/org/upm/inesdata/catalog/ConfigurationTargetNodeDirectoryTest.java b/extensions/participants-from-configuration/src/test/java/org/upm/inesdata/catalog/ConfigurationTargetNodeDirectoryTest.java new file mode 100644 index 0000000..c610a6c --- /dev/null +++ b/extensions/participants-from-configuration/src/test/java/org/upm/inesdata/catalog/ConfigurationTargetNodeDirectoryTest.java @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial API and implementation + * + */ + +package org.upm.inesdata.catalog; + +import org.eclipse.edc.spi.monitor.Monitor; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +/** + * Federated cache directory using Registration Service as backend. + */ +public class ConfigurationTargetNodeDirectoryTest { + + private final Monitor monitor = mock(); + + @Test + void verify_withParticipants() { + ConfigurationTargetNodeDirectory configurationTargetNodeDirectory = new ConfigurationTargetNodeDirectory(monitor, "id;name;http://targetUrl:19194/protocol"); + assertThat(configurationTargetNodeDirectory.getAll().size()).isEqualTo(1); + } + + @Test + void verify_withCorrectParticipantValues() { + ConfigurationTargetNodeDirectory configurationTargetNodeDirectory = new ConfigurationTargetNodeDirectory(monitor, "id;name;http://targetUrl:19194/protocol"); + assertThat(configurationTargetNodeDirectory.getAll().get(0).id()).isEqualTo("id"); + assertThat(configurationTargetNodeDirectory.getAll().get(0).name()).isEqualTo("name"); + assertThat(configurationTargetNodeDirectory.getAll().get(0).targetUrl()).isEqualTo("http://targetUrl:19194/protocol"); + } + + @Test + void verify_withMultipleParticipants() { + ConfigurationTargetNodeDirectory configurationTargetNodeDirectory = new ConfigurationTargetNodeDirectory(monitor, "id1;name1;http://targetUrl1:19194/protocol|id2;name2;http://targetUrl2:19194/protocol|id3;name3;http://targetUrl3:19194/protocol"); + assertThat(configurationTargetNodeDirectory.getAll().size()).isEqualTo(3); + assertThat(configurationTargetNodeDirectory.getAll().get(0).id()).isEqualTo("id1"); + assertThat(configurationTargetNodeDirectory.getAll().get(1).id()).isEqualTo("id2"); + assertThat(configurationTargetNodeDirectory.getAll().get(2).id()).isEqualTo("id3"); + } + + @Test + void verify_noParticipants() { + ConfigurationTargetNodeDirectory configurationTargetNodeDirectory = new ConfigurationTargetNodeDirectory(monitor, ""); + assertThat(configurationTargetNodeDirectory.getAll().size()).isEqualTo(0); + } + + @Test + void verify_wrongParticipantListFormat() { + ConfigurationTargetNodeDirectory configurationTargetNodeDirectory = new ConfigurationTargetNodeDirectory(monitor, "id.name.http://url/id1.name1.http://url2"); + assertThat(configurationTargetNodeDirectory.getAll().size()).isEqualTo(0); + } +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index adf4848..e0f715d 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -4,14 +4,14 @@ format.version = "1.1" [versions] assertj = "3.25.1" aws = "2.23.12" -edc = "0.5.0" +edc = "0.5.1" junit-pioneer = "2.2.0" jupiter = "5.10.1" +mockito = "5.2.0" postgres = "42.7.1" rsApi = "3.1.0" [libraries] -assertj = { module = "org.assertj:assertj-core", version.ref = "assertj" } edc-api-core = { module = "org.eclipse.edc:api-core", version.ref = "edc" } edc-boot = { module = "org.eclipse.edc:boot", version.ref = "edc" } edc-build-plugin = { module = "org.eclipse.edc.edc-build:org.eclipse.edc.edc-build.gradle.plugin", version.ref = "edc" } @@ -37,12 +37,8 @@ edc-transfer-pull-http-receiver = { module = "org.eclipse.edc:transfer-pull-http edc-vault-filesystem = { module = "org.eclipse.edc:vault-filesystem", version.ref = "edc" } edc-web-spi = { module = "org.eclipse.edc:web-spi", version.ref = "edc" } jakarta-rsApi = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version.ref = "rsApi" } -junit-jupiter-api = { module = "org.junit.jupiter:junit-jupiter-api", version.ref = "jupiter" } -junit-jupiter-engine = { module = "org.junit.jupiter:junit-jupiter-engine", version.ref = "jupiter" } -junit-jupiter-params = { module = "org.junit.jupiter:junit-jupiter-params", version.ref = "jupiter" } -junit-pioneer = { module = "org.junit-pioneer:junit-pioneer", version.ref = "junit-pioneer" } -#SQL Extensions +# SQL Extensions edc-transaction-local = { module = "org.eclipse.edc:transaction-local", version.ref = "edc" } edc-sql-core = { module = "org.eclipse.edc:sql-core", version.ref = "edc" } edc-sql-lease = { module = "org.eclipse.edc:sql-lease", version.ref = "edc" } @@ -61,10 +57,26 @@ edc-data-plane-aws-s3 = { module = "org.eclipse.edc:data-plane-aws-s3", version. # EDC Observability edc-observability-api = { module = "org.eclipse.edc:api-observability", version.ref = "edc" } -# no EDC references +# EDC Federated Catalog +edc-federated-catalog-api = { module = "org.eclipse.edc:federated-catalog-api", version.ref = "edc" } +edc-federated-catalog-core = { module = "org.eclipse.edc:federated-catalog-core", version.ref = "edc" } +edc-federated-catalog-spi = { module = "org.eclipse.edc:federated-catalog-spi", version.ref = "edc" } + +# EDC testing +edc-core-junit = { module = "org.eclipse.edc:junit", version.ref = "edc" } + +# No EDC references aws-s3 = { module = "software.amazon.awssdk:s3", version.ref = "aws" } aws-s3-transfer = { module = "software.amazon.awssdk:s3-transfer-manager", version.ref = "aws" } postgres = { module = "org.postgresql:postgresql", version.ref = "postgres" } +# No EDC testing references +assertj = { module = "org.assertj:assertj-core", version.ref = "assertj" } +junit-jupiter-api = { module = "org.junit.jupiter:junit-jupiter-api", version.ref = "jupiter" } +junit-jupiter-engine = { module = "org.junit.jupiter:junit-jupiter-engine", version.ref = "jupiter" } +junit-jupiter-params = { module = "org.junit.jupiter:junit-jupiter-params", version.ref = "jupiter" } +junit-pioneer = { module = "org.junit-pioneer:junit-pioneer", version.ref = "junit-pioneer" } +mockito-core = { module = "org.mockito:mockito-core", version.ref = "mockito" } + [plugins] shadow = { id = "com.github.johnrengelman.shadow", version = "8.1.1" } diff --git a/launchers/connector/build.gradle.kts b/launchers/connector/build.gradle.kts index 99d560c..4736ee3 100644 --- a/launchers/connector/build.gradle.kts +++ b/launchers/connector/build.gradle.kts @@ -70,6 +70,12 @@ dependencies { // Observability implementation(libs.edc.observability.api) + + // Federated Catalog + implementation(project(":extensions:participants-from-configuration")) + implementation(libs.edc.federated.catalog.spi) + implementation(libs.edc.federated.catalog.core) + implementation(libs.edc.federated.catalog.api) runtimeOnly(libs.edc.transaction.local) runtimeOnly(libs.postgres) diff --git a/settings.gradle.kts b/settings.gradle.kts index ca5bc36..0816edd 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -5,4 +5,8 @@ pluginManagement { rootProject.name = "inesdata-connector" +// Extensions +include(":extensions:participants-from-configuration") + +// Connector include(":launchers:connector") From 8c32f675d778a2e38eec19ad761a6d23f16e14a6 Mon Sep 17 00:00:00 2001 From: Juan Calero Date: Fri, 1 Mar 2024 17:05:06 +0100 Subject: [PATCH 027/174] Modified the way of storing and parsing the particpant info Reading each parameter as different properties, taking advantage of the rich API in the `Config` class --- .../ConfigurationTargetNodeDirectory.java | 73 ------------------- ...romConfigurationParticipantsExtension.java | 40 ++++------ .../catalog/ParticipantConfiguration.java | 32 ++++++++ .../catalog/ParticipantConfigurationTest.java | 41 +++++++++++ 4 files changed, 86 insertions(+), 100 deletions(-) delete mode 100644 extensions/participants-from-configuration/src/main/java/org/upm/inesdata/catalog/ConfigurationTargetNodeDirectory.java create mode 100644 extensions/participants-from-configuration/src/main/java/org/upm/inesdata/catalog/ParticipantConfiguration.java create mode 100644 extensions/participants-from-configuration/src/test/java/org/upm/inesdata/catalog/ParticipantConfigurationTest.java diff --git a/extensions/participants-from-configuration/src/main/java/org/upm/inesdata/catalog/ConfigurationTargetNodeDirectory.java b/extensions/participants-from-configuration/src/main/java/org/upm/inesdata/catalog/ConfigurationTargetNodeDirectory.java deleted file mode 100644 index e835787..0000000 --- a/extensions/participants-from-configuration/src/main/java/org/upm/inesdata/catalog/ConfigurationTargetNodeDirectory.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.upm.inesdata.catalog; - -import org.eclipse.edc.crawler.spi.TargetNode; -import org.eclipse.edc.crawler.spi.TargetNodeDirectory; -import org.eclipse.edc.spi.monitor.Monitor; - -import java.util.ArrayList; -import java.util.List; - -import static java.lang.String.format; - -/** - * Federated cache directory using a string with a list of TargetNode items. - */ -public class ConfigurationTargetNodeDirectory implements TargetNodeDirectory { - - public static final List SUPPORTED_PROTOCOLS = List.of("dataspace-protocol-http"); - public static final String PARTICIPANT_SEPARATOR = "\\|"; - public static final String PARTICIPANT_INFO_SEPARATOR = ";"; - - private final Monitor monitor; - private final List participantList; - - /** - * Constructs {@link ConfigurationTargetNodeDirectory} - * - * @param monitor monitor - * @param rawParticipantList RegistrationService API client. - */ - public ConfigurationTargetNodeDirectory(Monitor monitor, String rawParticipantList) { - this.monitor = monitor; - this.participantList = processRawParticipantList(rawParticipantList); - } - - @Override - public List getAll() { - return participantList; - } - - @Override - public void insert(TargetNode targetNode) { - throw new UnsupportedOperationException(); - } - - private List processRawParticipantList(String rawParticipantList) { - var participants = new ArrayList(); - try { - String[] rawParticipants = rawParticipantList.split(ConfigurationTargetNodeDirectory.PARTICIPANT_SEPARATOR); - for (String rawParticipant : rawParticipants) { - String[] participant = rawParticipant.split(ConfigurationTargetNodeDirectory.PARTICIPANT_INFO_SEPARATOR); - participants.add(new TargetNode(participant[1], participant[0], participant[2], ConfigurationTargetNodeDirectory.SUPPORTED_PROTOCOLS)); - } - } catch (Exception ex) { - monitor.severe(format("Error processing the list of participants. The expected format is: id;name;targetUrl|id;name;targetUrl and the provided list is '%s'. The exception is: %s", rawParticipantList, ex.getMessage())); - } - monitor.warning(format("Obtained a total of %s participants", participants.size())); - return participants; - } -} diff --git a/extensions/participants-from-configuration/src/main/java/org/upm/inesdata/catalog/FromConfigurationParticipantsExtension.java b/extensions/participants-from-configuration/src/main/java/org/upm/inesdata/catalog/FromConfigurationParticipantsExtension.java index d6620a3..7c750dc 100644 --- a/extensions/participants-from-configuration/src/main/java/org/upm/inesdata/catalog/FromConfigurationParticipantsExtension.java +++ b/extensions/participants-from-configuration/src/main/java/org/upm/inesdata/catalog/FromConfigurationParticipantsExtension.java @@ -1,24 +1,8 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - package org.upm.inesdata.catalog; +import org.eclipse.edc.catalog.directory.InMemoryNodeDirectory; import org.eclipse.edc.crawler.spi.TargetNodeDirectory; -import org.eclipse.edc.runtime.metamodel.annotation.Inject; import org.eclipse.edc.runtime.metamodel.annotation.Provider; -import org.eclipse.edc.runtime.metamodel.annotation.Setting; -import org.eclipse.edc.spi.monitor.Monitor; import org.eclipse.edc.spi.system.ServiceExtension; import org.eclipse.edc.spi.system.ServiceExtensionContext; @@ -28,17 +12,19 @@ */ public class FromConfigurationParticipantsExtension implements ServiceExtension { - @Setting - public static final String INESDATA_FC_PARTICIPANT_LIST = "edc.catalog.configuration.participant.list"; - @Provider public TargetNodeDirectory federatedCacheNodeDirectory(ServiceExtensionContext context) { - var monitor = context.getMonitor(); - var participantList = context.getConfig().getString(INESDATA_FC_PARTICIPANT_LIST); - monitor.debug("Retrieved the participant list from configuration: " + participantList); - - return new ConfigurationTargetNodeDirectory(monitor, participantList); - } -} + var participantConfig = new ParticipantConfiguration(); + + // Maybe is much clerarer with a classic for loop + // InMemoryNodeDirectory dir = participantConfig.getTargetNodes(context.getConfig()).stream() + // .collect(InMemoryNodeDirectory::new, (x, y) -> x.insert(y), null); + var dir = new InMemoryNodeDirectory(); + for (var target : participantConfig.getTargetNodes(context.getConfig())) { + dir.insert(target); + } + return dir; + } +} diff --git a/extensions/participants-from-configuration/src/main/java/org/upm/inesdata/catalog/ParticipantConfiguration.java b/extensions/participants-from-configuration/src/main/java/org/upm/inesdata/catalog/ParticipantConfiguration.java new file mode 100644 index 0000000..ce93710 --- /dev/null +++ b/extensions/participants-from-configuration/src/main/java/org/upm/inesdata/catalog/ParticipantConfiguration.java @@ -0,0 +1,32 @@ +package org.upm.inesdata.catalog; + +import java.util.List; +import java.util.stream.Collectors; + +import org.eclipse.edc.crawler.spi.TargetNode; +import org.eclipse.edc.runtime.metamodel.annotation.Setting; +import org.eclipse.edc.spi.system.configuration.Config; + +public class ParticipantConfiguration { + + public static final List SUPPORTED_PROTOCOLS = List.of("dataspace-protocol-http"); + + @Setting + public static final String INESDATA_FC_PARTICIPANT_LIST = "edc.catalog.configuration.participant"; + + public List getTargetNodes(Config baseConfig) { + var participantsConfig = baseConfig.getConfig(INESDATA_FC_PARTICIPANT_LIST); + + // A partition returns a stream of all the groups formed by the properties sharing the next fragment in the key + return participantsConfig.partition().map(conf -> toTargetNode(conf)) + .collect(Collectors.toList()); + + } + + private TargetNode toTargetNode(Config participantConf) { + return new TargetNode( + participantConf.getString("name"), + participantConf.getString("id"), + participantConf.getString("targetUrl"), SUPPORTED_PROTOCOLS ); + } +} diff --git a/extensions/participants-from-configuration/src/test/java/org/upm/inesdata/catalog/ParticipantConfigurationTest.java b/extensions/participants-from-configuration/src/test/java/org/upm/inesdata/catalog/ParticipantConfigurationTest.java new file mode 100644 index 0000000..e91ba6e --- /dev/null +++ b/extensions/participants-from-configuration/src/test/java/org/upm/inesdata/catalog/ParticipantConfigurationTest.java @@ -0,0 +1,41 @@ +package org.upm.inesdata.catalog; + +import static java.util.Map.entry; +import static org.junit.Assert.assertTrue; + +import java.util.List; +import java.util.Map; + +import org.eclipse.edc.crawler.spi.TargetNode; +import org.eclipse.edc.spi.system.configuration.ConfigFactory; +import org.junit.Test; + +public class ParticipantConfigurationTest { + + @Test + public void testParticipantsRetrieval() { + + Map properties = Map.ofEntries( + entry("edc.catalog.configuration.participant.1.name", "Paco"), + entry("edc.catalog.configuration.participant.1.id", "paco"), + entry("edc.catalog.configuration.participant.1.targetUrl", "http://paco.com"), + entry("edc.catalog.configuration.participant.2.name", "Pepe"), + entry("edc.catalog.configuration.participant.2.id", "pepe"), + entry("edc.catalog.configuration.participant.2.targetUrl", "http://pepe.com"), + entry("edc.catalog.configuration.participant.3.name", "Pedro"), + entry("edc.catalog.configuration.participant.3.id", "pedro"), + entry("edc.catalog.configuration.participant.3.targetUrl", "http://pedro.com"), + entry("edc.catalog.configuration.other", "Other 1"), + entry("edc.catalog.configuration.mother", "Other Mother")); + + ParticipantConfiguration partConfiguration = new ParticipantConfiguration(); + var targets = partConfiguration.getTargetNodes(ConfigFactory.fromMap(properties)); + + assertTrue(targets.containsAll(List.of( + new TargetNode("Paco", "paco", "http://paco.com", ParticipantConfiguration.SUPPORTED_PROTOCOLS), + new TargetNode("Pepe", "pepe", "http://pepe.com", ParticipantConfiguration.SUPPORTED_PROTOCOLS), + new TargetNode("Pedro", "pedro", "http://pedro.com", ParticipantConfiguration.SUPPORTED_PROTOCOLS)))); + + } + +} From 69795c6157c7fb6490a0ed21c1e861182bdc665c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Mon, 4 Mar 2024 10:50:23 +0100 Subject: [PATCH 028/174] control errors --- ...romConfigurationParticipantsExtension.java | 12 ++-- .../catalog/ParticipantConfiguration.java | 45 +++++++++++-- .../ConfigurationTargetNodeDirectoryTest.java | 64 ------------------ .../catalog/ParticipantConfigurationTest.java | 66 +++++++++++++------ 4 files changed, 94 insertions(+), 93 deletions(-) delete mode 100644 extensions/participants-from-configuration/src/test/java/org/upm/inesdata/catalog/ConfigurationTargetNodeDirectoryTest.java diff --git a/extensions/participants-from-configuration/src/main/java/org/upm/inesdata/catalog/FromConfigurationParticipantsExtension.java b/extensions/participants-from-configuration/src/main/java/org/upm/inesdata/catalog/FromConfigurationParticipantsExtension.java index 7c750dc..22394cc 100644 --- a/extensions/participants-from-configuration/src/main/java/org/upm/inesdata/catalog/FromConfigurationParticipantsExtension.java +++ b/extensions/participants-from-configuration/src/main/java/org/upm/inesdata/catalog/FromConfigurationParticipantsExtension.java @@ -15,15 +15,15 @@ public class FromConfigurationParticipantsExtension implements ServiceExtension @Provider public TargetNodeDirectory federatedCacheNodeDirectory(ServiceExtensionContext context) { - var participantConfig = new ParticipantConfiguration(); - - // Maybe is much clerarer with a classic for loop - // InMemoryNodeDirectory dir = participantConfig.getTargetNodes(context.getConfig()).stream() - // .collect(InMemoryNodeDirectory::new, (x, y) -> x.insert(y), null); + var monitor = context.getMonitor(); + var participantConfig = new ParticipantConfiguration(monitor); var dir = new InMemoryNodeDirectory(); for (var target : participantConfig.getTargetNodes(context.getConfig())) { - dir.insert(target); + // skipping null target nodes + if (target != null){ + dir.insert(target); + } } return dir; } diff --git a/extensions/participants-from-configuration/src/main/java/org/upm/inesdata/catalog/ParticipantConfiguration.java b/extensions/participants-from-configuration/src/main/java/org/upm/inesdata/catalog/ParticipantConfiguration.java index ce93710..573b467 100644 --- a/extensions/participants-from-configuration/src/main/java/org/upm/inesdata/catalog/ParticipantConfiguration.java +++ b/extensions/participants-from-configuration/src/main/java/org/upm/inesdata/catalog/ParticipantConfiguration.java @@ -2,11 +2,17 @@ import java.util.List; import java.util.stream.Collectors; +import org.eclipse.edc.spi.monitor.Monitor; import org.eclipse.edc.crawler.spi.TargetNode; import org.eclipse.edc.runtime.metamodel.annotation.Setting; import org.eclipse.edc.spi.system.configuration.Config; +import static java.lang.String.format; + +/** + * Handles participants in configuration and transforms them into TargetNodes + */ public class ParticipantConfiguration { public static final List SUPPORTED_PROTOCOLS = List.of("dataspace-protocol-http"); @@ -14,6 +20,24 @@ public class ParticipantConfiguration { @Setting public static final String INESDATA_FC_PARTICIPANT_LIST = "edc.catalog.configuration.participant"; + private final Monitor monitor; + + /** + * Constructs {@link ConfigurationTargetNodeDirectory} + * + * @param monitor monitor + * @param rawParticipantList RegistrationService API client. + */ + public ParticipantConfiguration(Monitor monitor) { + this.monitor = monitor; + } + + /** + * Retrieve TargetNodes from configuration + * + * @param baseConfig EDC Configuration + * @return list of TargetNodes from configuration + */ public List getTargetNodes(Config baseConfig) { var participantsConfig = baseConfig.getConfig(INESDATA_FC_PARTICIPANT_LIST); @@ -24,9 +48,22 @@ public List getTargetNodes(Config baseConfig) { } private TargetNode toTargetNode(Config participantConf) { - return new TargetNode( - participantConf.getString("name"), - participantConf.getString("id"), - participantConf.getString("targetUrl"), SUPPORTED_PROTOCOLS ); + TargetNode targetNode = null; + + if (! participantConf.getString("name", "").isEmpty() && + ! participantConf.getString("id", "").isEmpty() && + ! participantConf.getString("targetUrl", "").isEmpty()) { + targetNode = new TargetNode( + participantConf.getString("name"), + participantConf.getString("id"), + participantConf.getString("targetUrl"), + SUPPORTED_PROTOCOLS ); + } else { + monitor.severe(format("Error processing participant, this participant is skipped. A participant must contain non empty values for id, name and targetUrl. Provided values are id '%s', name '%s' and targetUrl '%s'.", + participantConf.getString("name", ""), + participantConf.getString("id", ""), + participantConf.getString("targetUrl", ""))); + } + return targetNode; } } diff --git a/extensions/participants-from-configuration/src/test/java/org/upm/inesdata/catalog/ConfigurationTargetNodeDirectoryTest.java b/extensions/participants-from-configuration/src/test/java/org/upm/inesdata/catalog/ConfigurationTargetNodeDirectoryTest.java deleted file mode 100644 index c610a6c..0000000 --- a/extensions/participants-from-configuration/src/test/java/org/upm/inesdata/catalog/ConfigurationTargetNodeDirectoryTest.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.upm.inesdata.catalog; - -import org.eclipse.edc.spi.monitor.Monitor; -import org.junit.jupiter.api.Test; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.mock; - -/** - * Federated cache directory using Registration Service as backend. - */ -public class ConfigurationTargetNodeDirectoryTest { - - private final Monitor monitor = mock(); - - @Test - void verify_withParticipants() { - ConfigurationTargetNodeDirectory configurationTargetNodeDirectory = new ConfigurationTargetNodeDirectory(monitor, "id;name;http://targetUrl:19194/protocol"); - assertThat(configurationTargetNodeDirectory.getAll().size()).isEqualTo(1); - } - - @Test - void verify_withCorrectParticipantValues() { - ConfigurationTargetNodeDirectory configurationTargetNodeDirectory = new ConfigurationTargetNodeDirectory(monitor, "id;name;http://targetUrl:19194/protocol"); - assertThat(configurationTargetNodeDirectory.getAll().get(0).id()).isEqualTo("id"); - assertThat(configurationTargetNodeDirectory.getAll().get(0).name()).isEqualTo("name"); - assertThat(configurationTargetNodeDirectory.getAll().get(0).targetUrl()).isEqualTo("http://targetUrl:19194/protocol"); - } - - @Test - void verify_withMultipleParticipants() { - ConfigurationTargetNodeDirectory configurationTargetNodeDirectory = new ConfigurationTargetNodeDirectory(monitor, "id1;name1;http://targetUrl1:19194/protocol|id2;name2;http://targetUrl2:19194/protocol|id3;name3;http://targetUrl3:19194/protocol"); - assertThat(configurationTargetNodeDirectory.getAll().size()).isEqualTo(3); - assertThat(configurationTargetNodeDirectory.getAll().get(0).id()).isEqualTo("id1"); - assertThat(configurationTargetNodeDirectory.getAll().get(1).id()).isEqualTo("id2"); - assertThat(configurationTargetNodeDirectory.getAll().get(2).id()).isEqualTo("id3"); - } - - @Test - void verify_noParticipants() { - ConfigurationTargetNodeDirectory configurationTargetNodeDirectory = new ConfigurationTargetNodeDirectory(monitor, ""); - assertThat(configurationTargetNodeDirectory.getAll().size()).isEqualTo(0); - } - - @Test - void verify_wrongParticipantListFormat() { - ConfigurationTargetNodeDirectory configurationTargetNodeDirectory = new ConfigurationTargetNodeDirectory(monitor, "id.name.http://url/id1.name1.http://url2"); - assertThat(configurationTargetNodeDirectory.getAll().size()).isEqualTo(0); - } -} diff --git a/extensions/participants-from-configuration/src/test/java/org/upm/inesdata/catalog/ParticipantConfigurationTest.java b/extensions/participants-from-configuration/src/test/java/org/upm/inesdata/catalog/ParticipantConfigurationTest.java index e91ba6e..c992cc5 100644 --- a/extensions/participants-from-configuration/src/test/java/org/upm/inesdata/catalog/ParticipantConfigurationTest.java +++ b/extensions/participants-from-configuration/src/test/java/org/upm/inesdata/catalog/ParticipantConfigurationTest.java @@ -1,41 +1,69 @@ package org.upm.inesdata.catalog; import static java.util.Map.entry; -import static org.junit.Assert.assertTrue; +import static org.assertj.core.api.Assertions.assertThat; +//import static org.junit.Assert.assertTrue; import java.util.List; import java.util.Map; import org.eclipse.edc.crawler.spi.TargetNode; +import org.eclipse.edc.spi.monitor.Monitor; import org.eclipse.edc.spi.system.configuration.ConfigFactory; -import org.junit.Test; +import org.junit.jupiter.api.Test; + +import static org.mockito.Mockito.mock; public class ParticipantConfigurationTest { + private final Monitor monitor = mock(); + @Test public void testParticipantsRetrieval() { Map properties = Map.ofEntries( - entry("edc.catalog.configuration.participant.1.name", "Paco"), - entry("edc.catalog.configuration.participant.1.id", "paco"), - entry("edc.catalog.configuration.participant.1.targetUrl", "http://paco.com"), - entry("edc.catalog.configuration.participant.2.name", "Pepe"), - entry("edc.catalog.configuration.participant.2.id", "pepe"), - entry("edc.catalog.configuration.participant.2.targetUrl", "http://pepe.com"), - entry("edc.catalog.configuration.participant.3.name", "Pedro"), - entry("edc.catalog.configuration.participant.3.id", "pedro"), - entry("edc.catalog.configuration.participant.3.targetUrl", "http://pedro.com"), - entry("edc.catalog.configuration.other", "Other 1"), - entry("edc.catalog.configuration.mother", "Other Mother")); - - ParticipantConfiguration partConfiguration = new ParticipantConfiguration(); + entry("edc.catalog.configuration.participant.1.name", "connector-c1"), + entry("edc.catalog.configuration.participant.1.id", "cnc1"), + entry("edc.catalog.configuration.participant.1.targetUrl", "http://connector-c1.com"), + entry("edc.catalog.configuration.participant.2.name", "connector-c2"), + entry("edc.catalog.configuration.participant.2.id", "cnc2"), + entry("edc.catalog.configuration.participant.2.targetUrl", "http://connector-c2.com"), + entry("edc.catalog.configuration.participant.3.name", "connector-c3"), + entry("edc.catalog.configuration.participant.3.id", "cnc3"), + entry("edc.catalog.configuration.participant.3.targetUrl", "http://connector-c3.com"), + entry("edc.catalog.configuration.specs", "inesdata connector"), + entry("edc.catalog.configuration.version", "1.0.0")); + + ParticipantConfiguration partConfiguration = new ParticipantConfiguration(monitor); var targets = partConfiguration.getTargetNodes(ConfigFactory.fromMap(properties)); - assertTrue(targets.containsAll(List.of( - new TargetNode("Paco", "paco", "http://paco.com", ParticipantConfiguration.SUPPORTED_PROTOCOLS), - new TargetNode("Pepe", "pepe", "http://pepe.com", ParticipantConfiguration.SUPPORTED_PROTOCOLS), - new TargetNode("Pedro", "pedro", "http://pedro.com", ParticipantConfiguration.SUPPORTED_PROTOCOLS)))); + assertThat(targets).containsAll(List.of( + new TargetNode("connector-c1", "cnc1", "http://connector-c1.com", ParticipantConfiguration.SUPPORTED_PROTOCOLS), + new TargetNode("connector-c2", "cnc2", "http://connector-c2.com", ParticipantConfiguration.SUPPORTED_PROTOCOLS), + new TargetNode("connector-c3", "cnc3", "http://connector-c3.com", ParticipantConfiguration.SUPPORTED_PROTOCOLS))); + + assertThat(targets.size()).isEqualTo(3); + } + + @Test + void verify_noParticipants() { + Map properties = Map.ofEntries(); + ParticipantConfiguration partConfiguration = new ParticipantConfiguration(monitor); + var targets = partConfiguration.getTargetNodes(ConfigFactory.fromMap(properties)); + + assertThat(targets.size()).isEqualTo(0); + } + + @Test + void verify_wrongParticipantListFormat() { + Map properties = Map.ofEntries( + entry("edc.catalog.configuration.participant.1.name", "connector-c1"), + entry("edc.catalog.configuration.participant.1.id", "cnc1")); + + ParticipantConfiguration partConfiguration = new ParticipantConfiguration(monitor); + var targets = partConfiguration.getTargetNodes(ConfigFactory.fromMap(properties)); + assertThat(targets.get(targets.size() -1)).isNull(); } } From c3aa12a022605576f3ababde93dd6c6ff698a420 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Tue, 5 Mar 2024 09:20:36 +0100 Subject: [PATCH 029/174] change readme text --- extensions/participants-from-configuration/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/participants-from-configuration/README.md b/extensions/participants-from-configuration/README.md index 820af44..7e6d0bd 100644 --- a/extensions/participants-from-configuration/README.md +++ b/extensions/participants-from-configuration/README.md @@ -4,7 +4,7 @@ This extension provides the capability to obtain the list of participants of a d ## Configuration -Exemplary configuration: +Example configuration: ```properties edc.catalog.configuration.participant.list=100210;connector-company-sample;http://targe-url-cs:9194/protocol|561349;connector-company-example;http://targe-url-ce:9194/protocol From 1fed5d986eacdc49c6c36f66962649b416343872 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Wed, 13 Mar 2024 09:20:44 +0100 Subject: [PATCH 030/174] keycloak deps --- gradle/libs.versions.toml | 1 + launchers/connector/build.gradle.kts | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index e0f715d..71b7c0d 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -30,6 +30,7 @@ edc-data-plane-selector-core = { module = "org.eclipse.edc:data-plane-selector-c edc-dsp = { module = "org.eclipse.edc:dsp", version.ref = "edc" } edc-http = { module = "org.eclipse.edc:http", version.ref = "edc" } edc-iam-mock = { module = "org.eclipse.edc:iam-mock", version.ref = "edc" } +edc-iam-oauth2-service = { module = "org.eclipse.edc:oauth2-service", version.ref = "edc" } edc-management-api = { module = "org.eclipse.edc:management-api", version.ref = "edc" } edc-micrometer-core = { module = "org.eclipse.edc:micrometer-core", version.ref = "edc" } edc-transfer-data-plane = { module = "org.eclipse.edc:transfer-data-plane", version.ref = "edc" } diff --git a/launchers/connector/build.gradle.kts b/launchers/connector/build.gradle.kts index 4736ee3..a4ce2c8 100644 --- a/launchers/connector/build.gradle.kts +++ b/launchers/connector/build.gradle.kts @@ -32,8 +32,7 @@ dependencies { // Temporary libs implementation(libs.edc.vault.filesystem) implementation(libs.edc.configuration.filesystem) - implementation(libs.edc.iam.mock) - + // Control Plane implementation(libs.edc.control.plane.api.client) implementation(libs.edc.control.plane.api) @@ -68,6 +67,9 @@ dependencies { implementation(libs.edc.aws.s3.core) implementation(libs.edc.data.plane.aws.s3) + // IAM Identity + implementation(libs.edc.iam.oauth2.service) + // Observability implementation(libs.edc.observability.api) From a3f96b292d74a5223989d2539e507f6ea3d760f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Tue, 19 Mar 2024 07:43:58 +0100 Subject: [PATCH 031/174] fix scope errors using SNAPSHOT version --- .../src/main/groovy/com.gmv.java-common-conventions.gradle | 4 ++++ gradle/libs.versions.toml | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/build-logic/src/main/groovy/com.gmv.java-common-conventions.gradle b/build-logic/src/main/groovy/com.gmv.java-common-conventions.gradle index d03614e..1f00a3c 100644 --- a/build-logic/src/main/groovy/com.gmv.java-common-conventions.gradle +++ b/build-logic/src/main/groovy/com.gmv.java-common-conventions.gradle @@ -34,6 +34,10 @@ repositories { mavenCentral() // Tried to use composite builds, but there is no support in VSCode mavenLocal() + // SNAPSHOTS to use version with corrected known bugs + maven { + url = uri("https://oss.sonatype.org/content/repositories/snapshots/") + } } dependencies { diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 71b7c0d..93044c9 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -5,6 +5,7 @@ format.version = "1.1" assertj = "3.25.1" aws = "2.23.12" edc = "0.5.1" +edcSnapshot = "0.5.2-SNAPSHOT" junit-pioneer = "2.2.0" jupiter = "5.10.1" mockito = "5.2.0" @@ -30,7 +31,7 @@ edc-data-plane-selector-core = { module = "org.eclipse.edc:data-plane-selector-c edc-dsp = { module = "org.eclipse.edc:dsp", version.ref = "edc" } edc-http = { module = "org.eclipse.edc:http", version.ref = "edc" } edc-iam-mock = { module = "org.eclipse.edc:iam-mock", version.ref = "edc" } -edc-iam-oauth2-service = { module = "org.eclipse.edc:oauth2-service", version.ref = "edc" } +edc-iam-oauth2-service = { module = "org.eclipse.edc:oauth2-service", version.ref = "edcSnapshot" } edc-management-api = { module = "org.eclipse.edc:management-api", version.ref = "edc" } edc-micrometer-core = { module = "org.eclipse.edc:micrometer-core", version.ref = "edc" } edc-transfer-data-plane = { module = "org.eclipse.edc:transfer-data-plane", version.ref = "edc" } From 168ec2049b9cb369f135e5f971ec479168e44b3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Tue, 19 Mar 2024 08:19:23 +0100 Subject: [PATCH 032/174] use same edc version in all components --- gradle/libs.versions.toml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 93044c9..76dc33e 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -4,8 +4,7 @@ format.version = "1.1" [versions] assertj = "3.25.1" aws = "2.23.12" -edc = "0.5.1" -edcSnapshot = "0.5.2-SNAPSHOT" +edc = "0.5.2-SNAPSHOT" junit-pioneer = "2.2.0" jupiter = "5.10.1" mockito = "5.2.0" @@ -31,7 +30,7 @@ edc-data-plane-selector-core = { module = "org.eclipse.edc:data-plane-selector-c edc-dsp = { module = "org.eclipse.edc:dsp", version.ref = "edc" } edc-http = { module = "org.eclipse.edc:http", version.ref = "edc" } edc-iam-mock = { module = "org.eclipse.edc:iam-mock", version.ref = "edc" } -edc-iam-oauth2-service = { module = "org.eclipse.edc:oauth2-service", version.ref = "edcSnapshot" } +edc-iam-oauth2-service = { module = "org.eclipse.edc:oauth2-service", version.ref = "edc" } edc-management-api = { module = "org.eclipse.edc:management-api", version.ref = "edc" } edc-micrometer-core = { module = "org.eclipse.edc:micrometer-core", version.ref = "edc" } edc-transfer-data-plane = { module = "org.eclipse.edc:transfer-data-plane", version.ref = "edc" } From 2c14b92fbaae1011a1bbee3b717f1393c5b13c4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Wed, 20 Mar 2024 18:57:34 +0100 Subject: [PATCH 033/174] authorization based on JWT tokens --- extensions/auth-oauth2-jwt/README.md | 2 + extensions/auth-oauth2-jwt/build.gradle.kts | 17 +++ .../Oauth2JwtAuthenticationExtension.java | 50 ++++++++ .../auth/Oauth2JwtAuthenticationService.java | 92 +++++++++++++ ...rg.eclipse.edc.spi.system.ServiceExtension | 1 + .../Oauth2JwtAuthenticationExtensionTest.java | 46 +++++++ .../Oauth2JwtAuthenticationServiceTest.java | 121 ++++++++++++++++++ gradle/libs.versions.toml | 3 +- launchers/connector/build.gradle.kts | 1 + settings.gradle.kts | 1 + 10 files changed, 333 insertions(+), 1 deletion(-) create mode 100644 extensions/auth-oauth2-jwt/README.md create mode 100644 extensions/auth-oauth2-jwt/build.gradle.kts create mode 100644 extensions/auth-oauth2-jwt/src/main/java/org/upm/inesdata/auth/Oauth2JwtAuthenticationExtension.java create mode 100644 extensions/auth-oauth2-jwt/src/main/java/org/upm/inesdata/auth/Oauth2JwtAuthenticationService.java create mode 100644 extensions/auth-oauth2-jwt/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension create mode 100644 extensions/auth-oauth2-jwt/src/test/java/org/upm/inesdata/auth/Oauth2JwtAuthenticationExtensionTest.java create mode 100644 extensions/auth-oauth2-jwt/src/test/java/org/upm/inesdata/auth/Oauth2JwtAuthenticationServiceTest.java diff --git a/extensions/auth-oauth2-jwt/README.md b/extensions/auth-oauth2-jwt/README.md new file mode 100644 index 0000000..fec082b --- /dev/null +++ b/extensions/auth-oauth2-jwt/README.md @@ -0,0 +1,2 @@ +# Oauth2 JWT Token Authentication Service + diff --git a/extensions/auth-oauth2-jwt/build.gradle.kts b/extensions/auth-oauth2-jwt/build.gradle.kts new file mode 100644 index 0000000..61a391e --- /dev/null +++ b/extensions/auth-oauth2-jwt/build.gradle.kts @@ -0,0 +1,17 @@ +plugins { + `java-library` + id("com.gmv.inesdata.edc-application") +} + +dependencies { + api(libs.edc.auth.spi) + implementation(libs.edc.iam.oauth2.core) + + implementation(libs.jakarta.rsApi) + + testImplementation(libs.edc.core.junit) + testImplementation(libs.assertj) + testImplementation(libs.mockito.core) +} + + diff --git a/extensions/auth-oauth2-jwt/src/main/java/org/upm/inesdata/auth/Oauth2JwtAuthenticationExtension.java b/extensions/auth-oauth2-jwt/src/main/java/org/upm/inesdata/auth/Oauth2JwtAuthenticationExtension.java new file mode 100644 index 0000000..d0ec2a3 --- /dev/null +++ b/extensions/auth-oauth2-jwt/src/main/java/org/upm/inesdata/auth/Oauth2JwtAuthenticationExtension.java @@ -0,0 +1,50 @@ +package org.upm.inesdata.auth; + +import org.eclipse.edc.api.auth.spi.AuthenticationService; +import org.eclipse.edc.runtime.metamodel.annotation.Extension; +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.runtime.metamodel.annotation.Provides; +import org.eclipse.edc.runtime.metamodel.annotation.Setting; +import org.eclipse.edc.spi.iam.IdentityService; +import org.eclipse.edc.spi.security.Vault; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * Extension that registers an AuthenticationService that uses Oauth2 JWT Tokens + */ +@Provides(AuthenticationService.class) +@Extension(value = Oauth2JwtAuthenticationExtension.NAME) +public class Oauth2JwtAuthenticationExtension implements ServiceExtension { + + public static final String NAME = "Oauth2 JWT Authentication"; + @Setting + private static final String AUTH_SETTING_ALLOWEDROLES = "edc.api.auth.oauth2.allowedRoles"; + @Setting + private static final String ROLE_PROPERTY_NAME = "role"; + + @Inject + private IdentityService identityService; + @Inject + private Vault vault; + + @Override + public String name() { + return NAME; + } + + @Override + public void initialize(ServiceExtensionContext context) { + // Get participant ID + String participantId = context.getParticipantId(); + + // Retrieve list of allowed roles + var rolesConfig = context.getConfig(AUTH_SETTING_ALLOWEDROLES); + List allowedRoles = rolesConfig.partition().map(conf -> conf.getString(ROLE_PROPERTY_NAME)).collect(Collectors.toList()); + + context.registerService(AuthenticationService.class, new Oauth2JwtAuthenticationService(identityService, participantId, allowedRoles)); + } +} diff --git a/extensions/auth-oauth2-jwt/src/main/java/org/upm/inesdata/auth/Oauth2JwtAuthenticationService.java b/extensions/auth-oauth2-jwt/src/main/java/org/upm/inesdata/auth/Oauth2JwtAuthenticationService.java new file mode 100644 index 0000000..33d39d3 --- /dev/null +++ b/extensions/auth-oauth2-jwt/src/main/java/org/upm/inesdata/auth/Oauth2JwtAuthenticationService.java @@ -0,0 +1,92 @@ +package org.upm.inesdata.auth; + +import jakarta.ws.rs.core.HttpHeaders; +import org.eclipse.edc.api.auth.spi.AuthenticationService; +import org.eclipse.edc.spi.iam.IdentityService; +import org.eclipse.edc.spi.iam.TokenRepresentation; +import org.eclipse.edc.web.spi.exception.AuthenticationFailedException; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * AuthenticationService for access control based on JWT tokens + */ +public class Oauth2JwtAuthenticationService implements AuthenticationService { + + private static final String REALM_ACCESS_CLAIM_NAME = "realm_access"; + private static final String ROLES_NAME = "roles"; + private static final String BEARER_PREFIX = "Bearer "; + + private static final String AUTHORIZATION_HEADER_ERROR = HttpHeaders.AUTHORIZATION + " header not found"; + private static final String BEARER_PREFIX_ERROR = "Bearer token not found"; + + + private final IdentityService identityService; + private final String participantId; + private final List allowedRoles; + + public Oauth2JwtAuthenticationService(IdentityService identityService, String participantId, List allowedRoles) { + this.identityService = identityService; + this.participantId = participantId; + this.allowedRoles = allowedRoles; + } + + /** + * Checks whether a particular request is authorized based on the "Authorization" header. + * The header must contains a valid JWT token with the correct connector configuration and an allowed role + * + * @param headers the headers + * @throws IllegalArgumentException The map of headers did not contain the "Authorization" header + * @return whether it is a valid JWT token or not + */ + @Override + public boolean isAuthenticated(Map> headers) { + + Objects.requireNonNull(headers, "headers"); + + // Get the Authorization header + var apiKey = headers.keySet().stream() + .filter(k -> k.equalsIgnoreCase(HttpHeaders.AUTHORIZATION)) + .map(headers::get) + .findFirst(); + + return apiKey.map(this::checkTokenValid).orElseThrow(() -> new AuthenticationFailedException(AUTHORIZATION_HEADER_ERROR)); + } + + private boolean checkTokenValid(List tokenKeys) { + return tokenKeys.size() == 1 && tokenKeys.stream().allMatch(this::isJwtTokenValid); + } + + @SuppressWarnings("unchecked") + private boolean isJwtTokenValid(String jwtToken) { + boolean valid = false; + + if (!jwtToken.startsWith(BEARER_PREFIX)) { + throw new AuthenticationFailedException(BEARER_PREFIX_ERROR); + } + + var tokenRepresentation = TokenRepresentation.Builder.newInstance().token(jwtToken.replace(BEARER_PREFIX, "")).build(); + var tokenValidation = identityService.verifyJwtToken(tokenRepresentation, null); +System.out.println("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"); + if (!tokenValidation.failed()) { + System.out.println("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX1"); + var claimToken = tokenValidation.getContent(); + System.out.println("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX2"); + var realmAccess = (Map) claimToken.getClaim(REALM_ACCESS_CLAIM_NAME); + System.out.println("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX3"); + var roles = (List) realmAccess.get(ROLES_NAME); + System.out.println("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX4"); + + if (roles.contains(participantId) && roles.stream().anyMatch(allowedRoles::contains)) { + System.out.println("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX5"); + valid = true; + } + System.out.println("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX6"); + } + System.out.println("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX7"); + + return valid; + } +} diff --git a/extensions/auth-oauth2-jwt/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/auth-oauth2-jwt/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 0000000..6fe5b7f --- /dev/null +++ b/extensions/auth-oauth2-jwt/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1 @@ +org.upm.inesdata.auth.Oauth2JwtAuthenticationExtension diff --git a/extensions/auth-oauth2-jwt/src/test/java/org/upm/inesdata/auth/Oauth2JwtAuthenticationExtensionTest.java b/extensions/auth-oauth2-jwt/src/test/java/org/upm/inesdata/auth/Oauth2JwtAuthenticationExtensionTest.java new file mode 100644 index 0000000..8729a1f --- /dev/null +++ b/extensions/auth-oauth2-jwt/src/test/java/org/upm/inesdata/auth/Oauth2JwtAuthenticationExtensionTest.java @@ -0,0 +1,46 @@ +package org.upm.inesdata.auth; + +import org.eclipse.edc.junit.extensions.DependencyInjectionExtension; +import org.eclipse.edc.spi.iam.IdentityService; +import org.eclipse.edc.spi.system.ServiceExtensionContext; +import org.eclipse.edc.spi.system.configuration.ConfigFactory; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +import static org.assertj.core.api.Assertions.entry; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.Map; + +@ExtendWith(DependencyInjectionExtension.class) +public class Oauth2JwtAuthenticationExtensionTest { + + private static final String AUTH_SETTING_ALLOWEDROLES = "edc.api.auth.oauth2.allowedRoles"; + + private final IdentityService identityService = mock(); + + @BeforeEach + void setup(ServiceExtensionContext context) { + context.registerService(IdentityService.class, identityService); + } + + @Test + public void testPrimaryMethod_loadKeyFromVault(ServiceExtensionContext context, Oauth2JwtAuthenticationExtension extension) { + Map configuration = Map.ofEntries( + entry("1.role", "manager"), + entry("2.role", "admin")); + + when(context.getConfig(AUTH_SETTING_ALLOWEDROLES)).thenReturn(ConfigFactory.fromMap(configuration)); + + extension.initialize(context); + + verify(context) + .getConfig(AUTH_SETTING_ALLOWEDROLES); + } + + + +} diff --git a/extensions/auth-oauth2-jwt/src/test/java/org/upm/inesdata/auth/Oauth2JwtAuthenticationServiceTest.java b/extensions/auth-oauth2-jwt/src/test/java/org/upm/inesdata/auth/Oauth2JwtAuthenticationServiceTest.java new file mode 100644 index 0000000..774650f --- /dev/null +++ b/extensions/auth-oauth2-jwt/src/test/java/org/upm/inesdata/auth/Oauth2JwtAuthenticationServiceTest.java @@ -0,0 +1,121 @@ +package org.upm.inesdata.auth; + +import org.eclipse.edc.spi.iam.ClaimToken; +import org.eclipse.edc.spi.iam.IdentityService; +import org.eclipse.edc.spi.iam.TokenRepresentation; +import org.eclipse.edc.spi.result.Result; +import org.eclipse.edc.web.spi.exception.AuthenticationFailedException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import jakarta.ws.rs.core.HttpHeaders; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class Oauth2JwtAuthenticationServiceTest { + + private static final String TEST_CONNECTOR_ID = "connector-id"; + private static final List TEST_ALLOWED_ROLES = Arrays.asList("admin", "manager"); + private final IdentityService identityService = mock(); + private Oauth2JwtAuthenticationService service; + + @BeforeEach + void setUp() { + service = new Oauth2JwtAuthenticationService(identityService, TEST_CONNECTOR_ID, TEST_ALLOWED_ROLES); + } + + @Test + void isAuthorized_headerNotPresent_throwsException() { + var map = Map.of("header1", List.of("val1, val2"), + "header2", List.of("anotherval1", "anotherval2")); + assertThatThrownBy(() -> service.isAuthenticated(map)).isInstanceOf(AuthenticationFailedException.class).hasMessage(HttpHeaders.AUTHORIZATION + " header not found"); + } + + @Test + void isAuthorized_headersEmpty_throwsException() { + Map> map = Collections.emptyMap(); + assertThatThrownBy(() -> service.isAuthenticated(map)).isInstanceOf(AuthenticationFailedException.class).hasMessage(HttpHeaders.AUTHORIZATION + " header not found"); + } + + @Test + void isAuthorized_headersNull_throwsException() { + assertThatThrownBy(() -> service.isAuthenticated(null)).isInstanceOf(NullPointerException.class); + } + + @Test + void isAuthorized_authorizationNotBearer() { + var map = Map.of(HttpHeaders.AUTHORIZATION, List.of("no-bearer-token")); + assertThatThrownBy(() -> service.isAuthenticated(map)).isInstanceOf(AuthenticationFailedException.class).hasMessage("Bearer token not found"); + } + + @Test + void isAuthorized_correctAuthorized() { + var map = Map.of(HttpHeaders.AUTHORIZATION, List.of("Bearer valid-token-key")); + + // Mock verification token result + List roles = Arrays.asList("user", "manager", "connector-id"); + Map> rolesClaim = new HashMap<>(); + rolesClaim.put("roles", roles); + var tokenBuilder = ClaimToken.Builder.newInstance(); + tokenBuilder.claim("realm_access", rolesClaim); + var claimToken = tokenBuilder.build(); + + when(identityService.verifyJwtToken(any(TokenRepresentation.class), isNull())).thenReturn(Result.success(claimToken)); + + assertThat(service.isAuthenticated(map)).isTrue(); + } + + @Test + void isAuthorized_incorrectConnectorIdAuthorized() { + var map = Map.of(HttpHeaders.AUTHORIZATION, List.of("Bearer invalid-token-key")); + + // Mock verification token result + List roles = Arrays.asList("user", "manager", "connector-id-2"); + Map> rolesClaim = new HashMap<>(); + rolesClaim.put("roles", roles); + var tokenBuilder = ClaimToken.Builder.newInstance(); + tokenBuilder.claim("realm_access", rolesClaim); + var claimToken = tokenBuilder.build(); + + when(identityService.verifyJwtToken(any(TokenRepresentation.class), isNull())).thenReturn(Result.success(claimToken)); + + assertThat(service.isAuthenticated(map)).isFalse(); + } + + @Test + void isAuthorized_incorrectRolesAuthorized() { + var map = Map.of(HttpHeaders.AUTHORIZATION, List.of("Bearer invalid-token-key")); + + // Mock verification token result + List roles = Arrays.asList("user", "connector-id"); + Map> rolesClaim = new HashMap<>(); + rolesClaim.put("roles", roles); + var tokenBuilder = ClaimToken.Builder.newInstance(); + tokenBuilder.claim("realm_access", rolesClaim); + var claimToken = tokenBuilder.build(); + + when(identityService.verifyJwtToken(any(TokenRepresentation.class), isNull())).thenReturn(Result.success(claimToken)); + + assertThat(service.isAuthenticated(map)).isFalse(); + } + + @Test + void isAuthorized_badTokenValidation() { + var map = Map.of(HttpHeaders.AUTHORIZATION, List.of("Bearer invalid-token-key")); + + when(identityService.verifyJwtToken(any(TokenRepresentation.class), isNull())).thenReturn(Result.failure("Expired token")); + + assertThat(service.isAuthenticated(map)).isFalse(); + } +} \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 76dc33e..3bb73ec 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -13,6 +13,7 @@ rsApi = "3.1.0" [libraries] edc-api-core = { module = "org.eclipse.edc:api-core", version.ref = "edc" } +edc-auth-spi = { module = "org.eclipse.edc:api-core", version.ref = "edc" } edc-boot = { module = "org.eclipse.edc:boot", version.ref = "edc" } edc-build-plugin = { module = "org.eclipse.edc.edc-build:org.eclipse.edc.edc-build.gradle.plugin", version.ref = "edc" } edc-configuration-filesystem = { module = "org.eclipse.edc:configuration-filesystem", version.ref = "edc" } @@ -29,7 +30,7 @@ edc-data-plane-selector-api = { module = "org.eclipse.edc:data-plane-selector-ap edc-data-plane-selector-core = { module = "org.eclipse.edc:data-plane-selector-core", version.ref = "edc" } edc-dsp = { module = "org.eclipse.edc:dsp", version.ref = "edc" } edc-http = { module = "org.eclipse.edc:http", version.ref = "edc" } -edc-iam-mock = { module = "org.eclipse.edc:iam-mock", version.ref = "edc" } +edc-iam-oauth2-core = { module = "org.eclipse.edc:oauth2-core", version.ref = "edc" } edc-iam-oauth2-service = { module = "org.eclipse.edc:oauth2-service", version.ref = "edc" } edc-management-api = { module = "org.eclipse.edc:management-api", version.ref = "edc" } edc-micrometer-core = { module = "org.eclipse.edc:micrometer-core", version.ref = "edc" } diff --git a/launchers/connector/build.gradle.kts b/launchers/connector/build.gradle.kts index a4ce2c8..672d6ba 100644 --- a/launchers/connector/build.gradle.kts +++ b/launchers/connector/build.gradle.kts @@ -69,6 +69,7 @@ dependencies { // IAM Identity implementation(libs.edc.iam.oauth2.service) + implementation(project(":extensions:auth-oauth2-jwt")) // Observability implementation(libs.edc.observability.api) diff --git a/settings.gradle.kts b/settings.gradle.kts index 0816edd..cb5f866 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -7,6 +7,7 @@ rootProject.name = "inesdata-connector" // Extensions include(":extensions:participants-from-configuration") +include(":extensions:auth-oauth2-jwt") // Connector include(":launchers:connector") From 3acf753c6bb3dc5cd6ed29694816dbe9c479de82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Thu, 21 Mar 2024 07:31:53 +0100 Subject: [PATCH 034/174] auth-oauth2-jwt docs and fixes --- extensions/auth-oauth2-jwt/README.md | 12 ++++++++++++ .../auth/Oauth2JwtAuthenticationService.java | 8 -------- .../Oauth2JwtAuthenticationServiceTest.java | 18 +++++++++--------- .../participants-from-configuration/README.md | 9 +++++++-- 4 files changed, 28 insertions(+), 19 deletions(-) diff --git a/extensions/auth-oauth2-jwt/README.md b/extensions/auth-oauth2-jwt/README.md index fec082b..8aa93aa 100644 --- a/extensions/auth-oauth2-jwt/README.md +++ b/extensions/auth-oauth2-jwt/README.md @@ -1,2 +1,14 @@ # Oauth2 JWT Token Authentication Service +This extension provides the capability to authorizate the request to the connector management API. The extension will access the Bearer token provided in the Authorization header and validate that it is a valid JWT-encoded bearer token. It is necessary to have the `org.eclipse.edc:oauth2-core` extension correctly configured. + +## Configuration + +Example configuration: + +```properties +edc.api.auth.oauth2.allowedRoles.1.role=connector-admin +edc.api.auth.oauth2.allowedRoles.2.role=connector-management +``` + +The `edc.api.auth.oauth2.allowedRoles` will be used by the federated catalog to retrieve the list of allowed roles that can perform requests on the managemente API connector. \ No newline at end of file diff --git a/extensions/auth-oauth2-jwt/src/main/java/org/upm/inesdata/auth/Oauth2JwtAuthenticationService.java b/extensions/auth-oauth2-jwt/src/main/java/org/upm/inesdata/auth/Oauth2JwtAuthenticationService.java index 33d39d3..82fb8b7 100644 --- a/extensions/auth-oauth2-jwt/src/main/java/org/upm/inesdata/auth/Oauth2JwtAuthenticationService.java +++ b/extensions/auth-oauth2-jwt/src/main/java/org/upm/inesdata/auth/Oauth2JwtAuthenticationService.java @@ -69,23 +69,15 @@ private boolean isJwtTokenValid(String jwtToken) { var tokenRepresentation = TokenRepresentation.Builder.newInstance().token(jwtToken.replace(BEARER_PREFIX, "")).build(); var tokenValidation = identityService.verifyJwtToken(tokenRepresentation, null); -System.out.println("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"); if (!tokenValidation.failed()) { - System.out.println("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX1"); var claimToken = tokenValidation.getContent(); - System.out.println("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX2"); var realmAccess = (Map) claimToken.getClaim(REALM_ACCESS_CLAIM_NAME); - System.out.println("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX3"); var roles = (List) realmAccess.get(ROLES_NAME); - System.out.println("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX4"); if (roles.contains(participantId) && roles.stream().anyMatch(allowedRoles::contains)) { - System.out.println("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX5"); valid = true; } - System.out.println("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX6"); } - System.out.println("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX7"); return valid; } diff --git a/extensions/auth-oauth2-jwt/src/test/java/org/upm/inesdata/auth/Oauth2JwtAuthenticationServiceTest.java b/extensions/auth-oauth2-jwt/src/test/java/org/upm/inesdata/auth/Oauth2JwtAuthenticationServiceTest.java index 774650f..8608d17 100644 --- a/extensions/auth-oauth2-jwt/src/test/java/org/upm/inesdata/auth/Oauth2JwtAuthenticationServiceTest.java +++ b/extensions/auth-oauth2-jwt/src/test/java/org/upm/inesdata/auth/Oauth2JwtAuthenticationServiceTest.java @@ -36,31 +36,31 @@ void setUp() { } @Test - void isAuthorized_headerNotPresent_throwsException() { + void isAuthenticated_headerNotPresent_throwsException() { var map = Map.of("header1", List.of("val1, val2"), - "header2", List.of("anotherval1", "anotherval2")); + "header2", List.of("anotherval1", "anotherval2")); assertThatThrownBy(() -> service.isAuthenticated(map)).isInstanceOf(AuthenticationFailedException.class).hasMessage(HttpHeaders.AUTHORIZATION + " header not found"); } @Test - void isAuthorized_headersEmpty_throwsException() { + void isAuthenticated_headersEmpty_throwsException() { Map> map = Collections.emptyMap(); assertThatThrownBy(() -> service.isAuthenticated(map)).isInstanceOf(AuthenticationFailedException.class).hasMessage(HttpHeaders.AUTHORIZATION + " header not found"); } @Test - void isAuthorized_headersNull_throwsException() { + void isAuthenticated_headersNull_throwsException() { assertThatThrownBy(() -> service.isAuthenticated(null)).isInstanceOf(NullPointerException.class); } @Test - void isAuthorized_authorizationNotBearer() { + void isAuthenticated_authorizationNoBearerToken() { var map = Map.of(HttpHeaders.AUTHORIZATION, List.of("no-bearer-token")); assertThatThrownBy(() -> service.isAuthenticated(map)).isInstanceOf(AuthenticationFailedException.class).hasMessage("Bearer token not found"); } @Test - void isAuthorized_correctAuthorized() { + void iisAuthenticated_correctAuthorized() { var map = Map.of(HttpHeaders.AUTHORIZATION, List.of("Bearer valid-token-key")); // Mock verification token result @@ -77,7 +77,7 @@ void isAuthorized_correctAuthorized() { } @Test - void isAuthorized_incorrectConnectorIdAuthorized() { + void isAuthenticated_incorrectConnectorIdAuthorized() { var map = Map.of(HttpHeaders.AUTHORIZATION, List.of("Bearer invalid-token-key")); // Mock verification token result @@ -94,7 +94,7 @@ void isAuthorized_incorrectConnectorIdAuthorized() { } @Test - void isAuthorized_incorrectRolesAuthorized() { + void isAuthenticated_incorrectRolesAuthorized() { var map = Map.of(HttpHeaders.AUTHORIZATION, List.of("Bearer invalid-token-key")); // Mock verification token result @@ -111,7 +111,7 @@ void isAuthorized_incorrectRolesAuthorized() { } @Test - void isAuthorized_badTokenValidation() { + void isAuthenticated_badTokenValidation() { var map = Map.of(HttpHeaders.AUTHORIZATION, List.of("Bearer invalid-token-key")); when(identityService.verifyJwtToken(any(TokenRepresentation.class), isNull())).thenReturn(Result.failure("Expired token")); diff --git a/extensions/participants-from-configuration/README.md b/extensions/participants-from-configuration/README.md index 7e6d0bd..f4e21ab 100644 --- a/extensions/participants-from-configuration/README.md +++ b/extensions/participants-from-configuration/README.md @@ -7,7 +7,12 @@ This extension provides the capability to obtain the list of participants of a d Example configuration: ```properties -edc.catalog.configuration.participant.list=100210;connector-company-sample;http://targe-url-cs:9194/protocol|561349;connector-company-example;http://targe-url-ce:9194/protocol +edc.catalog.configuration.participant.1.name = connector-c1 +edc.catalog.configuration.participant.1.id = connector-c1 +edc.catalog.configuration.participant.1.targetUrl = http://localhost:19194/protocol +edc.catalog.configuration.participant.2.name = connector-c2 +edc.catalog.configuration.participant.2.id = connector-c2 +edc.catalog.configuration.participant.2.targetUrl = http://localhost:29194/protocol ``` -The `edc.catalog.configuration.participant.list` will be used by the federated catalog to retrieve the list of participants whose catalog will be obtained for federating it. \ No newline at end of file +The `edc.catalog.configuration.participant` will be used by the federated catalog to retrieve the list of participants whose catalog will be obtained for federating it. \ No newline at end of file From ae0eb0262141a90bcfba12c3dcca8c1d84c4661b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Thu, 21 Mar 2024 07:34:56 +0100 Subject: [PATCH 035/174] improve comment --- launchers/connector/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launchers/connector/build.gradle.kts b/launchers/connector/build.gradle.kts index 672d6ba..0e75408 100644 --- a/launchers/connector/build.gradle.kts +++ b/launchers/connector/build.gradle.kts @@ -67,7 +67,7 @@ dependencies { implementation(libs.edc.aws.s3.core) implementation(libs.edc.data.plane.aws.s3) - // IAM Identity + // IAM Identity and authorization implementation(libs.edc.iam.oauth2.service) implementation(project(":extensions:auth-oauth2-jwt")) From 2b8b2460fa017b13b16bd8123d43ae77e20d52ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Thu, 21 Mar 2024 07:37:40 +0100 Subject: [PATCH 036/174] remove double colon in Readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index aaaf921..7213fba 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ The connector is prepared to be executed inside a container. The following steps ./gradlew launchers:connector:build ``` -- Create a local Docker image:: +- Create a local Docker image: ``` docker build --tag inesdata/connector:0.1 --build-arg CONNECTOR_JAR=./launchers/connector/build/libs/connector-app.jar -f docker/Dockerfile . ``` From fc623e58244a98c63f2fd4fe84e8befddfd42a9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Thu, 21 Mar 2024 08:04:49 +0100 Subject: [PATCH 037/174] improve extension documentation --- extensions/auth-oauth2-jwt/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/extensions/auth-oauth2-jwt/README.md b/extensions/auth-oauth2-jwt/README.md index 8aa93aa..51bb140 100644 --- a/extensions/auth-oauth2-jwt/README.md +++ b/extensions/auth-oauth2-jwt/README.md @@ -2,6 +2,10 @@ This extension provides the capability to authorizate the request to the connector management API. The extension will access the Bearer token provided in the Authorization header and validate that it is a valid JWT-encoded bearer token. It is necessary to have the `org.eclipse.edc:oauth2-core` extension correctly configured. +To authorize a user, the roles of the provided JWT token must contain: +- a valid role from those configured in `allowedRoles` +- a role with the `connector name` + ## Configuration Example configuration: From 6cbd7cb1367cba6999f3be1d384d98e3a84cfae7 Mon Sep 17 00:00:00 2001 From: Juan Calero Date: Thu, 21 Mar 2024 12:00:02 +0100 Subject: [PATCH 038/174] Improving build scripts --- .../com.gmv.java-common-conventions.gradle | 22 +++++++++++-------- launchers/connector/build.gradle.kts | 2 +- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/build-logic/src/main/groovy/com.gmv.java-common-conventions.gradle b/build-logic/src/main/groovy/com.gmv.java-common-conventions.gradle index d03614e..70d9f9f 100644 --- a/build-logic/src/main/groovy/com.gmv.java-common-conventions.gradle +++ b/build-logic/src/main/groovy/com.gmv.java-common-conventions.gradle @@ -22,13 +22,15 @@ scmVersion { version = scmVersion.version repositories { - // mavenRepoUrl, mavenRepoUsername and mavenRepoPassword must be configured in ~/.gradle/gradle.properties, + // If using an org repo, mavenRepoUrl, mavenRepoUsername and mavenRepoPassword must be configured in ~/.gradle/gradle.properties, // project gradle.properties, env vars or command line. // See https://docs.gradle.org/current/userguide/declaring_repositories.html#sec:handling_credentials - maven { - name = 'mavenRepo' - url mavenRepoUrl - credentials(PasswordCredentials) + if (project.hasProperty('mavenRepoUrl')) { + maven { + name = 'mavenRepo' + url mavenRepoUrl + credentials(PasswordCredentials) + } } // As a fallback, use Maven Central for resolving dependencies. mavenCentral() @@ -86,10 +88,12 @@ dependencyCheck { publishing { repositories { - maven { - name = 'mavenPublishRepo' - credentials(PasswordCredentials) - url = version.endsWith('SNAPSHOT') ? mavenPublishRepoSnapshotsUrl : mavenPublishRepoReleasesUrl + if (project.hasProperty('mavenPublishRepoSnapshotsUrl')) { + maven { + name = 'mavenPublishRepo' + credentials(PasswordCredentials) + url = version.endsWith('SNAPSHOT') ? mavenPublishRepoSnapshotsUrl : mavenPublishRepoReleasesUrl + } } } } \ No newline at end of file diff --git a/launchers/connector/build.gradle.kts b/launchers/connector/build.gradle.kts index 4736ee3..b7b80cb 100644 --- a/launchers/connector/build.gradle.kts +++ b/launchers/connector/build.gradle.kts @@ -22,7 +22,7 @@ plugins { dependencies { // Librerias base implementation(libs.edc.boot) - implementation(libs.edc.connector.core) + implementation(libs.edc.connector.core) // Common libs implementation(libs.edc.web.spi) From 8aa59f42e8fa7dc72537fd26204b9cf189a2e8c8 Mon Sep 17 00:00:00 2001 From: Juan Calero Date: Fri, 22 Mar 2024 10:15:36 +0100 Subject: [PATCH 039/174] Managed vulnerabilities after dependency-check report --- .../com.gmv.java-common-conventions.gradle | 1 + gradle/libs.versions.toml | 2 +- suppressions.xml | 25 +++++++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 suppressions.xml diff --git a/build-logic/src/main/groovy/com.gmv.java-common-conventions.gradle b/build-logic/src/main/groovy/com.gmv.java-common-conventions.gradle index 70d9f9f..418545c 100644 --- a/build-logic/src/main/groovy/com.gmv.java-common-conventions.gradle +++ b/build-logic/src/main/groovy/com.gmv.java-common-conventions.gradle @@ -74,6 +74,7 @@ sonar { dependencyCheck { failBuildOnCVSS = 7 + suppressionFile "suppressions.xml" analyzers { // To avoid trying to analyze DLL files assemblyEnabled=false diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index e0f715d..63ee3d3 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -8,7 +8,7 @@ edc = "0.5.1" junit-pioneer = "2.2.0" jupiter = "5.10.1" mockito = "5.2.0" -postgres = "42.7.1" +postgres = "42.7.2" rsApi = "3.1.0" [libraries] diff --git a/suppressions.xml b/suppressions.xml new file mode 100644 index 0000000..0aebd28 --- /dev/null +++ b/suppressions.xml @@ -0,0 +1,25 @@ + + + + + ^pkg:maven/org\.eclipse\.edc/.*sql.*@.*$ + cpe:/a:www-sql_project:www-sql + + + + ^pkg:maven/org\.eclipse\.edc/jetty\-core@.*$ + cpe:/a:eclipse:jetty + + + + ^pkg:maven/org\.eclipse\.jetty\.toolchain/jetty\-jakarta\-websocket\-api@.*$ + cpe:/a:jetty:jetty + cpe:/a:eclipse:jetty + + \ No newline at end of file From 7264cb1fbd433e0bfa6efd8f15bdaf9e4e0d445f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Mon, 1 Apr 2024 08:01:58 +0200 Subject: [PATCH 040/174] update edc version to 0.6.0 --- gradle/libs.versions.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 3bb73ec..56758af 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -2,13 +2,13 @@ format.version = "1.1" [versions] -assertj = "3.25.1" +assertj = "3.25.3" aws = "2.23.12" -edc = "0.5.2-SNAPSHOT" +edc = "0.6.0" junit-pioneer = "2.2.0" -jupiter = "5.10.1" +jupiter = "5.10.2" mockito = "5.2.0" -postgres = "42.7.1" +postgres = "42.7.2" rsApi = "3.1.0" [libraries] From 0981de3754969c0abe569306a6620c3ec55bc77e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Alconada?= Date: Wed, 3 Apr 2024 07:49:52 +0200 Subject: [PATCH 041/174] update sql files required by edc 0.6.0 --- resources/sql/data-plane-schema.sql | 4 ++-- resources/sql/transfer-process-schema.sql | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/resources/sql/data-plane-schema.sql b/resources/sql/data-plane-schema.sql index 91769de..a7adac0 100644 --- a/resources/sql/data-plane-schema.sql +++ b/resources/sql/data-plane-schema.sql @@ -24,14 +24,14 @@ CREATE TABLE IF NOT EXISTS edc_data_plane trace_context JSON, error_detail VARCHAR, callback_address VARCHAR, - trackable BOOLEAN, lease_id VARCHAR CONSTRAINT data_plane_lease_lease_id_fk REFERENCES edc_lease ON DELETE SET NULL, source JSON, destination JSON, - properties JSON + properties JSON, + flow_type VARCHAR ); COMMENT ON COLUMN edc_data_plane.trace_context IS 'Java Map serialized as JSON'; diff --git a/resources/sql/transfer-process-schema.sql b/resources/sql/transfer-process-schema.sql index ba7d38e..2a91b18 100644 --- a/resources/sql/transfer-process-schema.sql +++ b/resources/sql/transfer-process-schema.sql @@ -36,6 +36,7 @@ CREATE TABLE IF NOT EXISTS edc_transfer_process pending BOOLEAN DEFAULT FALSE, transfer_type VARCHAR, protocol_messages JSON, + data_plane_id VARCHAR, lease_id VARCHAR CONSTRAINT transfer_process_lease_lease_id_fk REFERENCES edc_lease From 3f27f49e2f3270465e5b39104d548db3d41d3c4e Mon Sep 17 00:00:00 2001 From: Juan Calero Date: Thu, 18 Apr 2024 13:07:56 +0200 Subject: [PATCH 042/174] Trying to take logs under control Added a custom Monitor to lower the logging level to INFO. Unfortunately jetty logging is completely independent to this, and also logs as DEBUG by default. Finally solved it by taking advantage of logback, which was already included in the project and controls the SLF4J logging facade used by jetty. Decided to add code in the launcher itself. I think it's clean and straightforward. Some things don't make sense as extensions. --- launchers/connector/build.gradle.kts | 14 -------------- .../monitor/ConnectorMonitorExtension.java | 15 +++++++++++++++ .../org.eclipse.edc.spi.monitor.MonitorExtension | 1 + .../connector/src/main/resources/logback.xml | 11 +++++++++++ 4 files changed, 27 insertions(+), 14 deletions(-) create mode 100644 launchers/connector/src/main/java/org/upm/inesdata/monitor/ConnectorMonitorExtension.java create mode 100644 launchers/connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.monitor.MonitorExtension create mode 100644 launchers/connector/src/main/resources/logback.xml diff --git a/launchers/connector/build.gradle.kts b/launchers/connector/build.gradle.kts index 013064b..7a14370 100644 --- a/launchers/connector/build.gradle.kts +++ b/launchers/connector/build.gradle.kts @@ -1,17 +1,3 @@ -/* - * Copyright (c) 2020, 2021 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * Fraunhofer Institute for Software and Systems Engineering - added dependencies - * - */ plugins { `java-library` diff --git a/launchers/connector/src/main/java/org/upm/inesdata/monitor/ConnectorMonitorExtension.java b/launchers/connector/src/main/java/org/upm/inesdata/monitor/ConnectorMonitorExtension.java new file mode 100644 index 0000000..98021d6 --- /dev/null +++ b/launchers/connector/src/main/java/org/upm/inesdata/monitor/ConnectorMonitorExtension.java @@ -0,0 +1,15 @@ +package org.upm.inesdata.monitor; + +import org.eclipse.edc.spi.monitor.ConsoleMonitor; +import org.eclipse.edc.spi.monitor.ConsoleMonitor.Level; +import org.eclipse.edc.spi.monitor.Monitor; +import org.eclipse.edc.spi.system.MonitorExtension; + +public class ConnectorMonitorExtension implements MonitorExtension { + + @Override + public Monitor getMonitor() { + + return new ConsoleMonitor("", Level.INFO); + } +} \ No newline at end of file diff --git a/launchers/connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.monitor.MonitorExtension b/launchers/connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.monitor.MonitorExtension new file mode 100644 index 0000000..48bca39 --- /dev/null +++ b/launchers/connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.monitor.MonitorExtension @@ -0,0 +1 @@ +org.upm.inesdata.monitor.ConnectorMonitorExtension diff --git a/launchers/connector/src/main/resources/logback.xml b/launchers/connector/src/main/resources/logback.xml new file mode 100644 index 0000000..9d6afdf --- /dev/null +++ b/launchers/connector/src/main/resources/logback.xml @@ -0,0 +1,11 @@ + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + \ No newline at end of file From 7b6aa7726fc959381cbf60011cc79b14d5eef379 Mon Sep 17 00:00:00 2001 From: Juan Calero Date: Tue, 23 Apr 2024 15:07:58 +0200 Subject: [PATCH 043/174] Monitor implementation logging to SLF4J In this way, both Jetty and EDC can be controled through logback. Right now, every monitor log goes through the same SLF4J logger. Possible future improvement. --- .../monitor/ConnectorMonitorExtension.java | 4 +- .../upm/inesdata/monitor/Slf4jMonitor.java | 84 +++++++++++++++++++ ...g.eclipse.edc.spi.system.MonitorExtension} | 0 .../connector/src/main/resources/logback.xml | 24 ++++-- 4 files changed, 100 insertions(+), 12 deletions(-) create mode 100644 launchers/connector/src/main/java/org/upm/inesdata/monitor/Slf4jMonitor.java rename launchers/connector/src/main/resources/META-INF/services/{org.eclipse.edc.spi.monitor.MonitorExtension => org.eclipse.edc.spi.system.MonitorExtension} (100%) diff --git a/launchers/connector/src/main/java/org/upm/inesdata/monitor/ConnectorMonitorExtension.java b/launchers/connector/src/main/java/org/upm/inesdata/monitor/ConnectorMonitorExtension.java index 98021d6..f18cab4 100644 --- a/launchers/connector/src/main/java/org/upm/inesdata/monitor/ConnectorMonitorExtension.java +++ b/launchers/connector/src/main/java/org/upm/inesdata/monitor/ConnectorMonitorExtension.java @@ -1,7 +1,5 @@ package org.upm.inesdata.monitor; -import org.eclipse.edc.spi.monitor.ConsoleMonitor; -import org.eclipse.edc.spi.monitor.ConsoleMonitor.Level; import org.eclipse.edc.spi.monitor.Monitor; import org.eclipse.edc.spi.system.MonitorExtension; @@ -10,6 +8,6 @@ public class ConnectorMonitorExtension implements MonitorExtension { @Override public Monitor getMonitor() { - return new ConsoleMonitor("", Level.INFO); + return new Slf4jMonitor(); } } \ No newline at end of file diff --git a/launchers/connector/src/main/java/org/upm/inesdata/monitor/Slf4jMonitor.java b/launchers/connector/src/main/java/org/upm/inesdata/monitor/Slf4jMonitor.java new file mode 100644 index 0000000..1a3fa4e --- /dev/null +++ b/launchers/connector/src/main/java/org/upm/inesdata/monitor/Slf4jMonitor.java @@ -0,0 +1,84 @@ +package org.upm.inesdata.monitor; + +import java.util.function.Supplier; + +import org.eclipse.edc.spi.monitor.Monitor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class Slf4jMonitor implements Monitor { + + private static Logger logger = LoggerFactory.getLogger(Slf4jMonitor.class); + + @Override + public void debug(Supplier supplier, Throwable... errors) { + debug(supplier.get(), errors); + } + + @Override + public void debug(String message, Throwable... errors) { + if (errors.length == 0) { + logger.debug(message); + } + else { + for (Throwable error : errors) { + logger.debug(message, error); + } + } + } + + @Override + public void info(Supplier supplier, Throwable... errors) { + info(supplier.get(), errors); + } + + @Override + public void info(String message, Throwable... errors) { + if (errors.length == 0) { + logger.info("Yepah! " + message); + } + else { + for (Throwable error : errors) { + logger.info(message, error); + } + } + } + + @Override + public void severe(Supplier supplier, Throwable... errors) { + severe(supplier.get(), errors); + } + + @Override + public void severe(String message, Throwable... errors) { + if (errors.length == 0) { + logger.error(message); + } + else { + for (Throwable error : errors) { + logger.error(message, error); + } + } + } + + @Override + public void warning(Supplier supplier, Throwable... errors) { + warning(supplier.get(), errors); + } + + @Override + public void warning(String message, Throwable... errors) { + if (errors.length == 0) { + logger.warn(message); + } + else { + for (Throwable error : errors) { + logger.warn(message, error); + } + } + } + + + + +} diff --git a/launchers/connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.monitor.MonitorExtension b/launchers/connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.MonitorExtension similarity index 100% rename from launchers/connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.monitor.MonitorExtension rename to launchers/connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.MonitorExtension diff --git a/launchers/connector/src/main/resources/logback.xml b/launchers/connector/src/main/resources/logback.xml index 9d6afdf..0b3cd48 100644 --- a/launchers/connector/src/main/resources/logback.xml +++ b/launchers/connector/src/main/resources/logback.xml @@ -1,11 +1,17 @@ + + + - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - - + + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} -%kvp- %msg%n + + + + + + \ No newline at end of file From 572188394ac426f086d084a9fac80680fc5641d6 Mon Sep 17 00:00:00 2001 From: Juan Calero Date: Wed, 24 Apr 2024 15:45:31 +0200 Subject: [PATCH 044/174] Excluding bouncycastle dependency to avoid vulnerability Should be fixed in next versions of EDC. The exclusion can be removed then --- extensions/auth-oauth2-jwt/build.gradle.kts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/extensions/auth-oauth2-jwt/build.gradle.kts b/extensions/auth-oauth2-jwt/build.gradle.kts index 61a391e..e4724df 100644 --- a/extensions/auth-oauth2-jwt/build.gradle.kts +++ b/extensions/auth-oauth2-jwt/build.gradle.kts @@ -5,7 +5,12 @@ plugins { dependencies { api(libs.edc.auth.spi) - implementation(libs.edc.iam.oauth2.core) + // Temporal workaround to solve a vulnerability. Remove when using edc version > 0.6.0 + implementation(libs.edc.iam.oauth2.core) { + exclude("org.bouncycastle", "bcprov-jdk18on") + } + implementation("org.bouncycastle:bcprov-jdk18on:1.78") + implementation(libs.jakarta.rsApi) From 6f74412bc7ca0b0f0c0ac6a94dd815fb8cf48ad3 Mon Sep 17 00:00:00 2001 From: Juan Calero Date: Wed, 24 Apr 2024 16:30:56 +0200 Subject: [PATCH 045/174] Fixed vulnerability in all modules Fixed CVE-2024-29857, CVE-2024-30171, CVE-2024-30172 in participants-from-configuration and the connector launcher also --- .../participants-from-configuration/build.gradle.kts | 6 +++++- launchers/connector/build.gradle.kts | 7 +++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/extensions/participants-from-configuration/build.gradle.kts b/extensions/participants-from-configuration/build.gradle.kts index 49480f1..c7cfa5e 100644 --- a/extensions/participants-from-configuration/build.gradle.kts +++ b/extensions/participants-from-configuration/build.gradle.kts @@ -5,7 +5,11 @@ plugins { dependencies { implementation(libs.edc.federated.catalog.spi) - implementation(libs.edc.federated.catalog.core) + // Temporal workaround to solve a vulnerability. Remove when using edc version > 0.6.0 + implementation(libs.edc.federated.catalog.core) { + exclude("org.bouncycastle", "bcprov-jdk18on") + } + implementation("org.bouncycastle:bcprov-jdk18on:1.78") testImplementation(libs.edc.core.junit) testImplementation(libs.junit.jupiter.api) diff --git a/launchers/connector/build.gradle.kts b/launchers/connector/build.gradle.kts index 7a14370..53f106d 100644 --- a/launchers/connector/build.gradle.kts +++ b/launchers/connector/build.gradle.kts @@ -5,6 +5,13 @@ plugins { alias(libs.plugins.shadow) } +// Temporal workaround to solve a vulnerability. Remove when using edc version > 0.6.0 +configurations { + "compileClasspath" { + resolutionStrategy.force("org.bouncycastle:bcprov-jdk18on:1.78") + } +} + dependencies { // Librerias base implementation(libs.edc.boot) From 121a54c8dba0a610ff97233d49dbc8a791247895 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Fri, 26 Apr 2024 08:46:44 +0200 Subject: [PATCH 046/174] update edc versions to 0.6.2 --- extensions/auth-oauth2-jwt/build.gradle.kts | 9 ++------- .../participants-from-configuration/build.gradle.kts | 6 +----- gradle/libs.versions.toml | 4 ++-- launchers/connector/build.gradle.kts | 7 ------- 4 files changed, 5 insertions(+), 21 deletions(-) diff --git a/extensions/auth-oauth2-jwt/build.gradle.kts b/extensions/auth-oauth2-jwt/build.gradle.kts index e4724df..9261f24 100644 --- a/extensions/auth-oauth2-jwt/build.gradle.kts +++ b/extensions/auth-oauth2-jwt/build.gradle.kts @@ -5,13 +5,8 @@ plugins { dependencies { api(libs.edc.auth.spi) - // Temporal workaround to solve a vulnerability. Remove when using edc version > 0.6.0 - implementation(libs.edc.iam.oauth2.core) { - exclude("org.bouncycastle", "bcprov-jdk18on") - } - implementation("org.bouncycastle:bcprov-jdk18on:1.78") - - + + implementation(libs.edc.iam.oauth2.core) implementation(libs.jakarta.rsApi) testImplementation(libs.edc.core.junit) diff --git a/extensions/participants-from-configuration/build.gradle.kts b/extensions/participants-from-configuration/build.gradle.kts index c7cfa5e..49480f1 100644 --- a/extensions/participants-from-configuration/build.gradle.kts +++ b/extensions/participants-from-configuration/build.gradle.kts @@ -5,11 +5,7 @@ plugins { dependencies { implementation(libs.edc.federated.catalog.spi) - // Temporal workaround to solve a vulnerability. Remove when using edc version > 0.6.0 - implementation(libs.edc.federated.catalog.core) { - exclude("org.bouncycastle", "bcprov-jdk18on") - } - implementation("org.bouncycastle:bcprov-jdk18on:1.78") + implementation(libs.edc.federated.catalog.core) testImplementation(libs.edc.core.junit) testImplementation(libs.junit.jupiter.api) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 56758af..2134b91 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -4,11 +4,11 @@ format.version = "1.1" [versions] assertj = "3.25.3" aws = "2.23.12" -edc = "0.6.0" +edc = "0.6.2" junit-pioneer = "2.2.0" jupiter = "5.10.2" mockito = "5.2.0" -postgres = "42.7.2" +postgres = "42.7.3" rsApi = "3.1.0" [libraries] diff --git a/launchers/connector/build.gradle.kts b/launchers/connector/build.gradle.kts index 53f106d..7a14370 100644 --- a/launchers/connector/build.gradle.kts +++ b/launchers/connector/build.gradle.kts @@ -5,13 +5,6 @@ plugins { alias(libs.plugins.shadow) } -// Temporal workaround to solve a vulnerability. Remove when using edc version > 0.6.0 -configurations { - "compileClasspath" { - resolutionStrategy.force("org.bouncycastle:bcprov-jdk18on:1.78") - } -} - dependencies { // Librerias base implementation(libs.edc.boot) From e9adc5305ade7a983918f930e0777d850d4ace11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Fri, 26 Apr 2024 09:23:04 +0200 Subject: [PATCH 047/174] include separated extension for edr_entry --- gradle/libs.versions.toml | 1 + launchers/connector/build.gradle.kts | 1 + resources/sql/edr-index-schema.sql | 9 ++++++ resources/sql/transfer-process-schema.sql | 36 +++++++---------------- 4 files changed, 21 insertions(+), 26 deletions(-) create mode 100644 resources/sql/edr-index-schema.sql diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 2134b91..f1c9346 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -43,6 +43,7 @@ jakarta-rsApi = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version.ref = "rsA # SQL Extensions edc-transaction-local = { module = "org.eclipse.edc:transaction-local", version.ref = "edc" } edc-sql-core = { module = "org.eclipse.edc:sql-core", version.ref = "edc" } +edc-sql-edr = { module = "org.eclipse.edc:edr-index-sql", version.ref = "edc" } edc-sql-lease = { module = "org.eclipse.edc:sql-lease", version.ref = "edc" } edc-sql-pool = { module = "org.eclipse.edc:sql-pool-apache-commons", version.ref = "edc" } edc-sql-asset-index = { module = "org.eclipse.edc:asset-index-sql", version.ref = "edc" } diff --git a/launchers/connector/build.gradle.kts b/launchers/connector/build.gradle.kts index 7a14370..4aa2578 100644 --- a/launchers/connector/build.gradle.kts +++ b/launchers/connector/build.gradle.kts @@ -38,6 +38,7 @@ dependencies { // Persistencia comun implementation(libs.edc.sql.core) + implementation(libs.edc.sql.edr) implementation(libs.edc.sql.lease) implementation(libs.edc.sql.pool) // Persistencia control plane diff --git a/resources/sql/edr-index-schema.sql b/resources/sql/edr-index-schema.sql new file mode 100644 index 0000000..4ec968c --- /dev/null +++ b/resources/sql/edr-index-schema.sql @@ -0,0 +1,9 @@ +CREATE TABLE IF NOT EXISTS edc_edr_entry +( + transfer_process_id VARCHAR NOT NULL PRIMARY KEY, + agreement_id VARCHAR NOT NULL, + asset_id VARCHAR NOT NULL, + provider_id VARCHAR NOT NULL, + contract_negotiation_id VARCHAR, + created_at BIGINT NOT NULL +); \ No newline at end of file diff --git a/resources/sql/transfer-process-schema.sql b/resources/sql/transfer-process-schema.sql index 2a91b18..f2292c7 100644 --- a/resources/sql/transfer-process-schema.sql +++ b/resources/sql/transfer-process-schema.sql @@ -37,10 +37,16 @@ CREATE TABLE IF NOT EXISTS edc_transfer_process transfer_type VARCHAR, protocol_messages JSON, data_plane_id VARCHAR, + correlation_id VARCHAR, + counter_party_address VARCHAR, + protocol VARCHAR, + asset_id VARCHAR, + contract_id VARCHAR, + data_destination JSON, lease_id VARCHAR - CONSTRAINT transfer_process_lease_lease_id_fk - REFERENCES edc_lease - ON DELETE SET NULL + CONSTRAINT transfer_process_lease_lease_id_fk + REFERENCES edc_lease + ON DELETE SET NULL ); COMMENT ON COLUMN edc_transfer_process.trace_context IS 'Java Map serialized as JSON'; @@ -57,27 +63,5 @@ COMMENT ON COLUMN edc_transfer_process.deprovisioned_resources IS 'List of depro CREATE UNIQUE INDEX IF NOT EXISTS transfer_process_id_uindex ON edc_transfer_process (transferprocess_id); -CREATE TABLE IF NOT EXISTS edc_data_request -( - datarequest_id VARCHAR NOT NULL - CONSTRAINT data_request_pk - PRIMARY KEY, - process_id VARCHAR NOT NULL, - connector_address VARCHAR NOT NULL, - protocol VARCHAR NOT NULL, - asset_id VARCHAR NOT NULL, - contract_id VARCHAR NOT NULL, - data_destination JSON NOT NULL, - transfer_process_id VARCHAR NOT NULL - CONSTRAINT data_request_transfer_process_id_fk - REFERENCES edc_transfer_process - ON UPDATE RESTRICT ON DELETE CASCADE -); - -COMMENT ON COLUMN edc_data_request.data_destination IS 'DataAddress serialized as JSON'; - -CREATE UNIQUE INDEX IF NOT EXISTS data_request_id_uindex - ON edc_data_request (datarequest_id); - CREATE UNIQUE INDEX IF NOT EXISTS lease_lease_id_uindex - ON edc_lease (lease_id); + ON edc_lease (lease_id); \ No newline at end of file From e6420ca27469e1d51655fad935948e3849c52dcb Mon Sep 17 00:00:00 2001 From: Juan Calero Date: Fri, 3 May 2024 13:50:48 +0200 Subject: [PATCH 048/174] Creating a github action to publish a contianer image --- .github/workflows/build-publish.yml | 52 +++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .github/workflows/build-publish.yml diff --git a/.github/workflows/build-publish.yml b/.github/workflows/build-publish.yml new file mode 100644 index 0000000..91d43db --- /dev/null +++ b/.github/workflows/build-publish.yml @@ -0,0 +1,52 @@ +# +name: Demo Push + +on: + push: + branches: + - master + - develop + - feature/pipeline-* + tags: + - v* +env: + IMAGE_NAME: ${{ github.repository }} +# +jobs: + # This pushes the image to GitHub Packages. + push: + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - uses: actions/checkout@v4 + + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 + + - name: Run build with Gradle Wrapper + run: ./gradlew build + + - name: Build image + run: docker build . --file docker/Dockerfile --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}" --build-arg CONNECTOR_JAR=./launchers/connector/build/libs/connector-app.jar + + - name: Log in to registry + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + # + - name: Push image + run: | + IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME + + # This changes all uppercase characters to lowercase. + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + # This strips the git ref prefix from the version. + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + # This strips the "v" prefix from the tag name. + [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') + # This uses the Docker `latest` tag convention. + [ "$VERSION" == "main" ] && VERSION=latest + echo IMAGE_ID=$IMAGE_ID + echo VERSION=$VERSION + docker tag $IMAGE_NAME $IMAGE_ID:$VERSION + docker push $IMAGE_ID:$VERSION From bfeee0aa5caa76bc9dbb2ca1f28af76664260344 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Wed, 8 May 2024 11:55:23 +0200 Subject: [PATCH 049/174] vocabulary api management --- README.md | 2 +- extensions/vocabulary-api/build.gradle.kts | 24 +++ .../vocabulary/VocabularyApiExtension.java | 120 +++++++++++++ .../vocabulary/controller/VocabularyApi.java | 133 +++++++++++++++ .../controller/VocabularyApiController.java | 119 +++++++++++++ .../service/VocabularyServiceImpl.java | 70 ++++++++ .../storage/InMemoryVocabularyIndex.java | 107 ++++++++++++ .../JsonObjectFromVocabularyTransformer.java | 46 +++++ .../JsonObjectToVocabularyTransformer.java | 39 +++++ .../validator/MandatoryJsonField.java | 55 ++++++ .../validator/VocabularyValidator.java | 27 +++ ...rg.eclipse.edc.spi.system.ServiceExtension | 1 + extensions/vocabulary-index-sql/README.md | 30 ++++ .../vocabulary-index-sql/build.gradle.kts | 17 ++ extensions/vocabulary-index-sql/docs/er.puml | 9 + .../vocabulary-index-sql/docs/schema.sql | 11 ++ .../sql/index/SqlVocabularyIndex.java | 157 ++++++++++++++++++ .../SqlVocabularyIndexServiceExtension.java | 58 +++++++ .../schema/BaseSqlDialectStatements.java | 68 ++++++++ .../index/schema/VocabularyStatements.java | 79 +++++++++ .../postgres/PostgresDialectStatements.java | 17 ++ .../schema/postgres/VocabularyMapping.java | 20 +++ ...rg.eclipse.edc.spi.system.ServiceExtension | 1 + gradle/libs.versions.toml | 17 +- launchers/connector/build.gradle.kts | 4 + resources/sql/vocabulary-schema.sql | 11 ++ settings.gradle.kts | 5 + spi/vocabulary-spi/README.md | 1 + spi/vocabulary-spi/build.gradle.kts | 8 + .../spi/vocabulary/VocabularyIndex.java | 61 +++++++ .../spi/vocabulary/VocabularyService.java | 53 ++++++ .../spi/vocabulary/domain/Vocabulary.java | 100 +++++++++++ .../inesdata/spi/vocabulary/package-info.java | 18 ++ 33 files changed, 1486 insertions(+), 2 deletions(-) create mode 100644 extensions/vocabulary-api/build.gradle.kts create mode 100644 extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/VocabularyApiExtension.java create mode 100644 extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/controller/VocabularyApi.java create mode 100644 extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/controller/VocabularyApiController.java create mode 100644 extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/service/VocabularyServiceImpl.java create mode 100644 extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/storage/InMemoryVocabularyIndex.java create mode 100644 extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectFromVocabularyTransformer.java create mode 100644 extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectToVocabularyTransformer.java create mode 100644 extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/validator/MandatoryJsonField.java create mode 100644 extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/validator/VocabularyValidator.java create mode 100644 extensions/vocabulary-api/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension create mode 100644 extensions/vocabulary-index-sql/README.md create mode 100644 extensions/vocabulary-index-sql/build.gradle.kts create mode 100644 extensions/vocabulary-index-sql/docs/er.puml create mode 100644 extensions/vocabulary-index-sql/docs/schema.sql create mode 100644 extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/SqlVocabularyIndex.java create mode 100644 extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/SqlVocabularyIndexServiceExtension.java create mode 100644 extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/BaseSqlDialectStatements.java create mode 100644 extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/VocabularyStatements.java create mode 100644 extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/postgres/PostgresDialectStatements.java create mode 100644 extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/postgres/VocabularyMapping.java create mode 100644 extensions/vocabulary-index-sql/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension create mode 100644 resources/sql/vocabulary-schema.sql create mode 100644 spi/vocabulary-spi/README.md create mode 100644 spi/vocabulary-spi/build.gradle.kts create mode 100644 spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/VocabularyIndex.java create mode 100644 spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/VocabularyService.java create mode 100644 spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/domain/Vocabulary.java create mode 100644 spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/package-info.java diff --git a/README.md b/README.md index 7213fba..34b6405 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ The connector is prepared to be executed inside a container. The following steps - Create a local Docker image: ``` - docker build --tag inesdata/connector:0.1 --build-arg CONNECTOR_JAR=./launchers/connector/build/libs/connector-app.jar -f docker/Dockerfile . + docker build --tag inesdata/connector:0.2.0 --build-arg CONNECTOR_JAR=./launchers/connector/build/libs/connector-app.jar -f docker/Dockerfile . ``` ## Database diff --git a/extensions/vocabulary-api/build.gradle.kts b/extensions/vocabulary-api/build.gradle.kts new file mode 100644 index 0000000..910d678 --- /dev/null +++ b/extensions/vocabulary-api/build.gradle.kts @@ -0,0 +1,24 @@ +plugins { + `java-library` + id("com.gmv.inesdata.edc-application") +} + +dependencies { + api(project(":spi:vocabulary-spi")) + api(libs.edc.spi.core) + implementation(libs.edc.spi.transform) + implementation(libs.edc.web.spi) + + implementation(libs.edc.connector.core) + implementation(libs.edc.api.core) + implementation(libs.edc.lib.util) + implementation(libs.edc.lib.transform) + implementation(libs.edc.dsp.api.configuration) + implementation(libs.edc.api.management.config) + implementation(libs.edc.transaction.spi) + implementation(libs.edc.lib.validator) + implementation(libs.edc.validator.spi) + implementation(libs.swagger.annotations.jakarta) + runtimeOnly(libs.edc.spi.jsonld) + runtimeOnly(libs.edc.json.ld.lib) +} diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/VocabularyApiExtension.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/VocabularyApiExtension.java new file mode 100644 index 0000000..09b3e91 --- /dev/null +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/VocabularyApiExtension.java @@ -0,0 +1,120 @@ +package org.upm.inesdata.vocabulary; + +import jakarta.json.Json; +import org.eclipse.edc.connector.api.management.configuration.ManagementApiConfiguration; +import org.eclipse.edc.runtime.metamodel.annotation.Extension; +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.runtime.metamodel.annotation.Provider; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; +import org.eclipse.edc.spi.system.health.HealthCheckResult; +import org.eclipse.edc.spi.system.health.HealthCheckService; +import org.eclipse.edc.spi.types.TypeManager; +import org.eclipse.edc.transaction.spi.TransactionContext; +import org.eclipse.edc.transform.spi.TypeTransformerRegistry; +import org.eclipse.edc.validator.spi.JsonObjectValidatorRegistry; +import org.eclipse.edc.web.spi.WebService; +import org.upm.inesdata.spi.vocabulary.VocabularyIndex; +import org.upm.inesdata.spi.vocabulary.VocabularyService; +import org.upm.inesdata.vocabulary.controller.VocabularyApiController; +import org.upm.inesdata.vocabulary.service.VocabularyServiceImpl; +import org.upm.inesdata.vocabulary.storage.InMemoryVocabularyIndex; +import org.upm.inesdata.vocabulary.transformer.JsonObjectFromVocabularyTransformer; +import org.upm.inesdata.vocabulary.transformer.JsonObjectToVocabularyTransformer; +import org.upm.inesdata.vocabulary.validator.VocabularyValidator; + +import static org.eclipse.edc.spi.constants.CoreConstants.JSON_LD; +import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.EDC_VOCABULARY_TYPE; + +import java.util.Map; + +/** + * Extension that provides an API for managing vocabularies + */ +@Extension(value = VocabularyApiExtension.NAME) +public class VocabularyApiExtension implements ServiceExtension { + + public static final String NAME = "Vocabulary API Extension"; + private InMemoryVocabularyIndex defaultVocabularyIndex; + + @Inject + private WebService webService; + + @Inject + private VocabularyIndex vocabularyIndex; + + @Inject(required = false) + private HealthCheckService healthCheckService; + + @Inject + private ManagementApiConfiguration config; + + @Inject + private TypeManager typeManager; + + @Inject + private TransactionContext transactionContext; + + @Inject + private TypeTransformerRegistry transformerRegistry; + + @Inject + private JsonObjectValidatorRegistry validator; + + @Override + public String name() { + return NAME; + } + + /** + * Provides a default vocabularyService implementation + */ + @Provider(isDefault = true) + public VocabularyService vocabularyService() { + return new VocabularyServiceImpl(vocabularyIndex, transactionContext); + } + + /** + * Provides a default in memory vocabularyIndex + */ + @Provider(isDefault = true) + public VocabularyIndex defaultVocabularyIndex() { + return getVocabularyIndex(); + } + + /** + * Initializes the service + */ + @Override + public void initialize(ServiceExtensionContext context) { + var monitor = context.getMonitor(); + + var managementApiTransformerRegistry = transformerRegistry.forContext("management-api"); + + var factory = Json.createBuilderFactory(Map.of()); + var jsonLdMapper = typeManager.getMapper(JSON_LD); + managementApiTransformerRegistry.register(new JsonObjectFromVocabularyTransformer(factory, jsonLdMapper)); + managementApiTransformerRegistry.register(new JsonObjectToVocabularyTransformer()); + + validator.register(EDC_VOCABULARY_TYPE, VocabularyValidator.instance()); + var vocabularyApiController = new VocabularyApiController(this.vocabularyService(), managementApiTransformerRegistry, monitor, validator); + webService.registerResource(config.getContextAlias(), vocabularyApiController); + + // contribute to the liveness probe + if (healthCheckService != null) { + var successResult = HealthCheckResult.Builder.newInstance().component("FCC Query API").build(); + healthCheckService.addReadinessProvider(() -> successResult); + healthCheckService.addLivenessProvider(() -> successResult); + } + } + + /** + * Creates a InMemoryVocabularyIndex if not exists + */ + private InMemoryVocabularyIndex getVocabularyIndex() { + if (defaultVocabularyIndex == null) { + defaultVocabularyIndex = new InMemoryVocabularyIndex(); + } + return defaultVocabularyIndex; + } +} diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/controller/VocabularyApi.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/controller/VocabularyApi.java new file mode 100644 index 0000000..acb5a62 --- /dev/null +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/controller/VocabularyApi.java @@ -0,0 +1,133 @@ +package org.upm.inesdata.vocabulary.controller; + +import io.swagger.v3.oas.annotations.OpenAPIDefinition; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.info.Info; +import io.swagger.v3.oas.annotations.media.ArraySchema; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.json.JsonArray; +import jakarta.json.JsonObject; +import org.eclipse.edc.api.model.ApiCoreSchema; +import org.eclipse.edc.connector.controlplane.contract.spi.types.offer.ContractOffer; + +import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.ID; +import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.TYPE; +import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.EDC_VOCABULARY_TYPE; + +/** + * Controller for managing {@link Vocabulary} objects. + */ +@OpenAPIDefinition( + info = @Info(description = "Manages the connector vocabularies.", + title = "Vocabulary API", version = "1")) +@Tag(name = "Vocabulary") +public interface VocabularyApi { + + /** + * Get all the vocabularies stored in the vocabularies index. No filters are required due + * to the limited number of vocabularies that each data space will manage. + * + * @return list of vocabularies + */ + @Operation(description = "Obtains all vocabularies", + responses = { + @ApiResponse(responseCode = "200", description = "A list of vocabularies", + content = @Content(array = @ArraySchema(schema = @Schema(implementation = ContractOffer.class)))) + } + ) + JsonArray getVocabularies(); + + /** + * Retrieves the {@link Vocabulary} with the given ID + * + * @param id id of the vocabulary + * @return JsonObject with the vocabulary information + */ + @Operation(description = "Gets a vocabulary with the given ID", + responses = { + @ApiResponse(responseCode = "200", description = "The vocabulary", + content = @Content(schema = @Schema(implementation = VocabularyOutputSchema.class))), + @ApiResponse(responseCode = "400", description = "Request was malformed, e.g. id was null", + content = @Content(array = @ArraySchema(schema = @Schema(implementation = ApiCoreSchema.ApiErrorDetailSchema.class)))), + @ApiResponse(responseCode = "404", description = "A vocabulary with the given ID does not exist", + content = @Content(array = @ArraySchema(schema = @Schema(implementation = ApiCoreSchema.ApiErrorDetailSchema.class)))) + } + ) + JsonObject getVocabulary(String id); + + /** + * Creates a new vocabulary + * + * @param vocabulary the vocabulary + * @return JsonObject with the created vocabulary + */ + @Operation(description = "Creates a new vocabulary", + requestBody = @RequestBody(content = @Content(schema = @Schema(implementation = VocabularyOutputSchema.class))), + responses = { + @ApiResponse(responseCode = "200", description = "Vocabulary was created successfully", + content = @Content(schema = @Schema(implementation = ApiCoreSchema.IdResponseSchema.class))), + @ApiResponse(responseCode = "400", description = "Request body was malformed", + content = @Content(array = @ArraySchema(schema = @Schema(implementation = ApiCoreSchema.ApiErrorDetailSchema.class)))), + @ApiResponse(responseCode = "409", description = "Could not create vocabulary, because a vocabulary with that ID already exists", + content = @Content(array = @ArraySchema(schema = @Schema(implementation = ApiCoreSchema.ApiErrorDetailSchema.class)))) } + ) + JsonObject createVocabulary(JsonObject vocabylary); + + /** + * Updates a vocabulary + * + * @param vocabulary the vocabulary to be updated + * @return JsonObject with the updated vocabulary + */ + @Operation(description = "Updates a vocabulary with the given ID if it exists. If the vocabulary is not found, no further action is taken.", + requestBody = @RequestBody(content = @Content(schema = @Schema(implementation = VocabularyOutputSchema.class))), + responses = { + @ApiResponse(responseCode = "204", description = "Vocabulary was updated successfully"), + @ApiResponse(responseCode = "404", description = "Vocabulary could not be updated, because it does not exist."), + @ApiResponse(responseCode = "400", description = "Request was malformed, e.g. id was null", + content = @Content(array = @ArraySchema(schema = @Schema(implementation = ApiCoreSchema.ApiErrorDetailSchema.class)))), + }) + void updateVocabulary(JsonObject vocabulary); + + /** + * Removes the {@link Vocabulary} with the given ID + * + * @param id id of the vocabulary + * @return JsonObject with the updated vocabulary + */ + @Operation(description = "Removes a vocabulary with the given ID if possible", + responses = { + @ApiResponse(responseCode = "204", description = "Vocabulary was deleted successfully"), + @ApiResponse(responseCode = "400", description = "Request was malformed, e.g. id was null", + content = @Content(array = @ArraySchema(schema = @Schema(implementation = ApiCoreSchema.ApiErrorDetailSchema.class)))), + @ApiResponse(responseCode = "404", description = "Vocabulary could not be removed, because it does not exist.", + content = @Content(array = @ArraySchema(schema = @Schema(implementation = ApiCoreSchema.ApiErrorDetailSchema.class)))) + }) + void removeVocabulary(String id); + + /** + * Vocabulary output + */ + @ArraySchema() + @Schema(name = "VocabularyOutput", example = VocabularyOutputSchema.VOCABULARY_OUTPUT_EXAMPLE) + record VocabularyOutputSchema( + @Schema(name = ID) + String id, + @Schema(name = TYPE, example = EDC_VOCABULARY_TYPE) + String name, + String jsonSchema + ) { + public static final String VOCABULARY_OUTPUT_EXAMPLE = """ + { + "@id": "vocabularyId", + "name": "vocabulary name", + "jsonSchema": "{ \\"title\\": \\"vocabulary\\", \\"type\\": \\"object\\", \\"properties\\": { \\"name\\": { \\"type\\": \\"string\\", \\"title\\": \\"Name\\" }, \\"dct:keyword\\": { \\"type\\": \\"array\\", \\"title\\": \\"Keywords\\", \\"items\\": { \\"type\\": \\"string\\" } } }, \\"required\\": [ \\"name\\" ], \\"@context\\": { \\"dct\\": \\"http:\\/\\/purl.org\\/dc\\/terms\\/\" } }" + } + """; + } + +} diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/controller/VocabularyApiController.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/controller/VocabularyApiController.java new file mode 100644 index 0000000..48bc1d6 --- /dev/null +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/controller/VocabularyApiController.java @@ -0,0 +1,119 @@ +package org.upm.inesdata.vocabulary.controller; + +import jakarta.json.JsonArray; +import jakarta.json.JsonObject; +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.PUT; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; + +import org.eclipse.edc.spi.EdcException; +import org.eclipse.edc.spi.monitor.Monitor; +import org.eclipse.edc.spi.result.Result; +import org.eclipse.edc.transform.spi.TypeTransformerRegistry; +import org.eclipse.edc.validator.spi.JsonObjectValidatorRegistry; +import org.eclipse.edc.web.spi.exception.InvalidRequestException; +import org.eclipse.edc.web.spi.exception.ObjectNotFoundException; +import org.eclipse.edc.web.spi.exception.ValidationFailureException; +import org.upm.inesdata.spi.vocabulary.VocabularyService; +import org.upm.inesdata.spi.vocabulary.domain.Vocabulary; + +import static jakarta.json.stream.JsonCollectors.toJsonArray; +import static java.util.Optional.of; +import static org.eclipse.edc.web.spi.exception.ServiceResultHandler.exceptionMapper; +import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.EDC_VOCABULARY_TYPE; + +import org.eclipse.edc.api.model.IdResponse; + +/** + * Implementation of the controller for {@link Vocabulary} managing. + */ +@Consumes({MediaType.APPLICATION_JSON}) +@Produces({MediaType.APPLICATION_JSON}) +@Path("/vocabularies") +public class VocabularyApiController implements VocabularyApi { + private final TypeTransformerRegistry transformerRegistry; + private final VocabularyService service; + private final Monitor monitor; + private final JsonObjectValidatorRegistry validator; + + /** + * Constructor + */ + public VocabularyApiController(VocabularyService service, TypeTransformerRegistry transformerRegistry, + Monitor monitor, JsonObjectValidatorRegistry validator) { + this.transformerRegistry = transformerRegistry; + this.service = service; + this.monitor = monitor; + this.validator = validator; + } + + @POST + @Override + public JsonObject createVocabulary(JsonObject vocabularyJson) { + validator.validate(EDC_VOCABULARY_TYPE, vocabularyJson).orElseThrow(ValidationFailureException::new); + + var vocabulary = transformerRegistry.transform(vocabularyJson, Vocabulary.class) + .orElseThrow(InvalidRequestException::new); + + var idResponse = service.create(vocabulary) + .map(a -> IdResponse.Builder.newInstance() + .id(a.getId()) + .createdAt(a.getCreatedAt()) + .build()) + .orElseThrow(exceptionMapper(Vocabulary.class, vocabulary.getId())); + + return transformerRegistry.transform(idResponse, JsonObject.class) + .orElseThrow(f -> new EdcException(f.getFailureDetail())); + } + + @POST + @Path("/request") + @Override + public JsonArray getVocabularies() { + return service.search().getContent().stream() + .map(it -> transformerRegistry.transform(it, JsonObject.class)) + .peek(r -> r.onFailure(f -> monitor.warning(f.getFailureDetail()))) + .filter(Result::succeeded) + .map(Result::getContent) + .collect(toJsonArray()); + } + + @GET + @Path("{id}") + @Override + public JsonObject getVocabulary(@PathParam("id") String id) { + var vocabulary = of(id) + .map(it -> service.findById(id)) + .orElseThrow(() -> new ObjectNotFoundException(Vocabulary.class, id)); + + return transformerRegistry.transform(vocabulary, JsonObject.class) + .orElseThrow(f -> new EdcException(f.getFailureDetail())); + + } + + @DELETE + @Path("{id}") + @Override + public void removeVocabulary(@PathParam("id") String id) { + service.delete(id).orElseThrow(exceptionMapper(Vocabulary.class, id)); + } + + @PUT + @Override + public void updateVocabulary(JsonObject vocabularyJson) { + validator.validate(EDC_VOCABULARY_TYPE, vocabularyJson).orElseThrow(ValidationFailureException::new); + + var vocabularyResult = transformerRegistry.transform(vocabularyJson, Vocabulary.class) + .orElseThrow(InvalidRequestException::new); + + service.update(vocabularyResult) + .orElseThrow(exceptionMapper(Vocabulary.class, vocabularyResult.getId())); + } + +} diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/service/VocabularyServiceImpl.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/service/VocabularyServiceImpl.java new file mode 100644 index 0000000..e72ed5c --- /dev/null +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/service/VocabularyServiceImpl.java @@ -0,0 +1,70 @@ +package org.upm.inesdata.vocabulary.service; + +import org.eclipse.edc.spi.result.ServiceResult; +import org.eclipse.edc.transaction.spi.TransactionContext; + +import org.upm.inesdata.spi.vocabulary.VocabularyIndex; +import org.upm.inesdata.spi.vocabulary.VocabularyService; +import org.upm.inesdata.spi.vocabulary.domain.Vocabulary; + +import java.util.List; + +/** + * Implementation of the {@link VocabularyService} interface + */ +public class VocabularyServiceImpl implements VocabularyService { + + private final VocabularyIndex index; + private final TransactionContext transactionContext; + + /** + * Constructor + */ + public VocabularyServiceImpl(VocabularyIndex index, TransactionContext transactionContext) { + this.index = index; + + this.transactionContext = transactionContext; + } + + @Override + public Vocabulary findById(String vocabularyId) { + return transactionContext.execute(() -> index.findById(vocabularyId)); + } + + @Override + public ServiceResult> search() { + return transactionContext.execute(() -> { + try (var stream = index.allVocabularies()) { + return ServiceResult.success(stream.toList()); + } + }); + } + + @Override + public ServiceResult create(Vocabulary vocabulary) { + return transactionContext.execute(() -> { + var createResult = index.create(vocabulary); + if (createResult.succeeded()) { + return ServiceResult.success(vocabulary); + } + return ServiceResult.fromFailure(createResult); + }); + } + + @Override + public ServiceResult delete(String vocabularyId) { + return transactionContext.execute(() -> { + var deleted = index.deleteById(vocabularyId); + return ServiceResult.from(deleted); + }); + } + + @Override + public ServiceResult update(Vocabulary vocabulary) { + return transactionContext.execute(() -> { + var updatedVocabulary = index.updateVocabulary(vocabulary); + return ServiceResult.from(updatedVocabulary); + }); + } + +} diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/storage/InMemoryVocabularyIndex.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/storage/InMemoryVocabularyIndex.java new file mode 100644 index 0000000..1ce1afb --- /dev/null +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/storage/InMemoryVocabularyIndex.java @@ -0,0 +1,107 @@ +package org.upm.inesdata.vocabulary.storage; + +import org.eclipse.edc.spi.result.StoreResult; +import org.upm.inesdata.spi.vocabulary.domain.Vocabulary; +import org.upm.inesdata.spi.vocabulary.VocabularyIndex; + +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Stream; + +import static java.lang.String.format; + +/** + * An ephemeral vocabulary index + */ +public class InMemoryVocabularyIndex implements VocabularyIndex { + private final Map cache = new ConcurrentHashMap<>(); + + private final ReentrantReadWriteLock lock; + + /** + * Constructor + */ + public InMemoryVocabularyIndex() { + lock = new ReentrantReadWriteLock(true); + } + + @Override + public Stream allVocabularies() { + lock.readLock().lock(); + try { + return cache.values().stream(); + } finally { + lock.readLock().unlock(); + } + } + + @Override + public Vocabulary findById(String vocabularyId) { + lock.readLock().lock(); + try { + return cache.values().stream() + .filter(vocabulary -> vocabulary.getId().equals(vocabularyId)) + .findFirst() + .orElse(null); + } finally { + lock.readLock().unlock(); + } + } + + @Override + public StoreResult create(Vocabulary vocabulary) { + lock.writeLock().lock(); + try { + var id = vocabulary.getId(); + if (cache.containsKey(id)) { + return StoreResult.alreadyExists(format(VOCABULARY_EXISTS_TEMPLATE, id)); + } + Objects.requireNonNull(vocabulary, "vocabulary"); + Objects.requireNonNull(id, "vocabulary.getId()"); + cache.put(id, vocabulary); + } finally { + lock.writeLock().unlock(); + } + return StoreResult.success(); + } + + @Override + public StoreResult deleteById(String vocabularyId) { + lock.writeLock().lock(); + try { + return Optional.ofNullable(delete(vocabularyId)) + .map(StoreResult::success) + .orElse(StoreResult.notFound(format(VOCABULARY_NOT_FOUND_TEMPLATE, vocabularyId))); + } finally { + lock.writeLock().unlock(); + } + } + + @Override + public StoreResult updateVocabulary(Vocabulary vocabulary) { + lock.writeLock().lock(); + try { + var id = vocabulary.getId(); + Objects.requireNonNull(vocabulary, "vocabulary"); + Objects.requireNonNull(id, "vocabularyId"); + if (cache.containsKey(id)) { + cache.put(id, vocabulary); + return StoreResult.success(vocabulary); + } + return StoreResult.notFound(format(VOCABULARY_NOT_FOUND_TEMPLATE, id)); + } finally { + lock.writeLock().unlock(); + } + } + + /** + * Remove a vocabulary from cache based on its id + */ + private Vocabulary delete(String vocabularyId) { + return cache.remove(vocabularyId); + } + +} diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectFromVocabularyTransformer.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectFromVocabularyTransformer.java new file mode 100644 index 0000000..92599d5 --- /dev/null +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectFromVocabularyTransformer.java @@ -0,0 +1,46 @@ +package org.upm.inesdata.vocabulary.transformer; + +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.json.JsonBuilderFactory; +import jakarta.json.JsonObject; + +import org.eclipse.edc.jsonld.spi.transformer.AbstractJsonLdTransformer; +import org.eclipse.edc.transform.spi.TransformerContext; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.upm.inesdata.spi.vocabulary.domain.Vocabulary; + +import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.ID; +import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.TYPE; + +import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.EDC_VOCABULARY_TYPE; +import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_JSON_SCHEMA; +import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_NAME; + +/** + * Creates a JsonObject from a {@link Vocabulary} + */ +public class JsonObjectFromVocabularyTransformer extends AbstractJsonLdTransformer { + private final ObjectMapper mapper; + private final JsonBuilderFactory jsonFactory; + + /** + * Constructor + */ + public JsonObjectFromVocabularyTransformer(JsonBuilderFactory jsonFactory, ObjectMapper jsonLdMapper) { + super(Vocabulary.class, JsonObject.class); + this.jsonFactory = jsonFactory; + this.mapper = jsonLdMapper; + } + + @Override + public @Nullable JsonObject transform(@NotNull Vocabulary vocabulary, @NotNull TransformerContext context) { + var builder = jsonFactory.createObjectBuilder() + .add(ID, vocabulary.getId()) + .add(TYPE, EDC_VOCABULARY_TYPE) + .add(PROPERTY_NAME, vocabulary.getName()) + .add(PROPERTY_JSON_SCHEMA, vocabulary.getJsonSchema()); + + return builder.build(); + } +} diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectToVocabularyTransformer.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectToVocabularyTransformer.java new file mode 100644 index 0000000..be99616 --- /dev/null +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectToVocabularyTransformer.java @@ -0,0 +1,39 @@ +package org.upm.inesdata.vocabulary.transformer; + +import jakarta.json.JsonObject; +import org.eclipse.edc.jsonld.spi.transformer.AbstractJsonLdTransformer; +import org.eclipse.edc.transform.spi.TransformerContext; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.upm.inesdata.spi.vocabulary.domain.Vocabulary; + +import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_JSON_SCHEMA; +import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_NAME; + +/** + * Converts from an {@link Vocabulary} as a {@link JsonObject} in JSON-LD expanded form to an {@link Vocabulary}. + */ +public class JsonObjectToVocabularyTransformer extends AbstractJsonLdTransformer { + + /** + * Constructor + */ + public JsonObjectToVocabularyTransformer() { + super(JsonObject.class, Vocabulary.class); + } + + @Override + public @Nullable Vocabulary transform(@NotNull JsonObject jsonObject, @NotNull TransformerContext context) { + var builder = Vocabulary.Builder.newInstance() + .id(nodeId(jsonObject)); + + visitProperties(jsonObject, key -> switch (key) { + case PROPERTY_NAME -> value -> builder.name(transformString(value, context)); + case PROPERTY_JSON_SCHEMA -> value -> builder.jsonSchema(transformString(value, context)); + default -> doNothing(); + }); + + return builderResult(builder::build, context); + } + +} diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/validator/MandatoryJsonField.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/validator/MandatoryJsonField.java new file mode 100644 index 0000000..02dfd40 --- /dev/null +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/validator/MandatoryJsonField.java @@ -0,0 +1,55 @@ +package org.upm.inesdata.vocabulary.validator; + +import jakarta.json.Json; +import jakarta.json.JsonObject; +import jakarta.json.JsonReader; +import jakarta.json.stream.JsonParsingException; +import java.io.StringReader; +import org.eclipse.edc.validator.jsonobject.JsonLdPath; +import org.eclipse.edc.validator.spi.ValidationResult; +import org.eclipse.edc.validator.spi.Validator; + +import java.util.Optional; + +import static java.lang.String.format; +import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.VALUE; +import static org.eclipse.edc.validator.spi.Violation.violation; + +/** + * Verifies that a @value is present and is a Json. + */ +public class MandatoryJsonField implements Validator { + private final JsonLdPath path; + + /** + * Constructor + */ + public MandatoryJsonField(JsonLdPath path) { + this.path = path; + } + + @Override + public ValidationResult validate(JsonObject input) { + return Optional.ofNullable(input.getJsonArray(path.last())) + .filter(it -> !it.isEmpty()) + .map(it -> it.getJsonObject(0)) + .map(it -> it.getString(VALUE)) + .map(this::validateJSON) + .orElseGet(() -> ValidationResult.failure(violation(format("mandatory value '%s'", path), path.toString()))); + } + + /** + * Checks whether a string contains a valid json or not + */ + private ValidationResult validateJSON(String value) { + try { + try (JsonReader jsonReader = Json.createReader(new StringReader(value))) { + jsonReader.read(); + } + } catch (JsonParsingException e) { + return ValidationResult.failure(violation(format("content '%s' should be a valid Json", path), path.toString())); + } + return ValidationResult.success(); + } + +} diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/validator/VocabularyValidator.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/validator/VocabularyValidator.java new file mode 100644 index 0000000..1b1753c --- /dev/null +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/validator/VocabularyValidator.java @@ -0,0 +1,27 @@ +package org.upm.inesdata.vocabulary.validator; + +import jakarta.json.JsonObject; +import org.eclipse.edc.validator.jsonobject.JsonObjectValidator; +import org.eclipse.edc.validator.jsonobject.validators.MandatoryObject; +import org.eclipse.edc.validator.jsonobject.validators.OptionalIdNotBlank; +import org.eclipse.edc.validator.spi.Validator; + +import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_JSON_SCHEMA; +import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_NAME; + +/** + * Validator for Vocabulary + */ +public class VocabularyValidator { + + /** + * Defines the rules that a Vocabulary must comply with + */ + public static Validator instance() { + return JsonObjectValidator.newValidator() + .verifyId(OptionalIdNotBlank::new) + .verify(PROPERTY_NAME, MandatoryObject::new) + .verify(PROPERTY_JSON_SCHEMA, MandatoryJsonField::new) + .build(); + } +} diff --git a/extensions/vocabulary-api/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/vocabulary-api/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 0000000..8570889 --- /dev/null +++ b/extensions/vocabulary-api/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1 @@ +org.upm.inesdata.vocabulary.VocabularyApiExtension \ No newline at end of file diff --git a/extensions/vocabulary-index-sql/README.md b/extensions/vocabulary-index-sql/README.md new file mode 100644 index 0000000..2e8e55c --- /dev/null +++ b/extensions/vocabulary-index-sql/README.md @@ -0,0 +1,30 @@ +# SQL Vocabulary + +Provides SQL persistence for vocabularies. + +## Prerequisites + +Please apply this [schema](docs/schema.sql) to your SQL database. + +## Entity Diagram + +![ER Diagram](//www.plantuml.com/plantuml/png/SoWkIImgAStDuKhDAyaigLH8JKcEByjFJamgpKaigbIevb9Gq5B8JB5IA2ufoinBLx2n2V2simEBvYNcfiB4mG9PnVbvmSaPgRc9ACB9HQc99QafZYLM2ZdvO35TNQvQBeVKl1IWnG00) + + +## Configuration + +| Key | Description | Mandatory | +|:---|:---|---| +| edc.datasource.vocabulary.name | Datasource used to store vocabularies | X | diff --git a/extensions/vocabulary-index-sql/build.gradle.kts b/extensions/vocabulary-index-sql/build.gradle.kts new file mode 100644 index 0000000..f71e25e --- /dev/null +++ b/extensions/vocabulary-index-sql/build.gradle.kts @@ -0,0 +1,17 @@ +plugins { + `java-library` + id("com.gmv.inesdata.edc-application") +} + +dependencies { + api(project(":spi:vocabulary-spi")) + implementation(project(":extensions:vocabulary-api")) + api(libs.edc.spi.core) + api(libs.edc.transaction.spi) + implementation(libs.edc.transaction.spi) + implementation(libs.edc.transaction.datasource.spi) + implementation(libs.edc.sql.core) + implementation(libs.edc.lib.util) +} + + diff --git a/extensions/vocabulary-index-sql/docs/er.puml b/extensions/vocabulary-index-sql/docs/er.puml new file mode 100644 index 0000000..f081ea7 --- /dev/null +++ b/extensions/vocabulary-index-sql/docs/er.puml @@ -0,0 +1,9 @@ +@startuml +entity edc_vocabulary { + * id: string <> + * name: string + * jsonSchema: string + * createdAt: long + -- +} +@enduml \ No newline at end of file diff --git a/extensions/vocabulary-index-sql/docs/schema.sql b/extensions/vocabulary-index-sql/docs/schema.sql new file mode 100644 index 0000000..773df12 --- /dev/null +++ b/extensions/vocabulary-index-sql/docs/schema.sql @@ -0,0 +1,11 @@ +-- table: edc_vocabulary +CREATE TABLE IF NOT EXISTS edc_vocabulary +( + id VARCHAR NOT NULL, + created_at BIGINT NOT NULL, + json_schema JSON DEFAULT '{}', + vocabulary_name VARCHAR NOT NULL, + PRIMARY KEY (id) +); + +COMMENT ON COLUMN edc_vocabulary.json_schema IS 'JSON Schema with the vocabulary'; diff --git a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/SqlVocabularyIndex.java b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/SqlVocabularyIndex.java new file mode 100644 index 0000000..e2207c4 --- /dev/null +++ b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/SqlVocabularyIndex.java @@ -0,0 +1,157 @@ +package org.upm.inesdata.vocabulary.sql.index; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.eclipse.edc.spi.persistence.EdcPersistenceException; +import org.eclipse.edc.spi.query.QuerySpec; +import org.eclipse.edc.spi.result.StoreResult; +import org.eclipse.edc.sql.QueryExecutor; +import org.eclipse.edc.sql.store.AbstractSqlStore; +import org.eclipse.edc.transaction.datasource.spi.DataSourceRegistry; +import org.eclipse.edc.transaction.spi.TransactionContext; +import org.jetbrains.annotations.Nullable; + +import org.upm.inesdata.spi.vocabulary.domain.Vocabulary; +import org.upm.inesdata.vocabulary.sql.index.schema.VocabularyStatements; +import org.upm.inesdata.spi.vocabulary.VocabularyIndex; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Objects; +import java.util.stream.Stream; + +import static java.lang.String.format; +import static org.eclipse.edc.spi.query.Criterion.criterion; + +/** + * Implementation of the VocabularyIndes with SQL databases + */ +public class SqlVocabularyIndex extends AbstractSqlStore implements VocabularyIndex { + + private final VocabularyStatements vocabularyStatements; + + public SqlVocabularyIndex(DataSourceRegistry dataSourceRegistry, + String dataSourceName, + TransactionContext transactionContext, + ObjectMapper objectMapper, + VocabularyStatements vocabularyStatements, + QueryExecutor queryExecutor) { + super(dataSourceRegistry, dataSourceName, transactionContext, objectMapper, queryExecutor); + this.vocabularyStatements = Objects.requireNonNull(vocabularyStatements); + } + + @Override + public Stream allVocabularies() { + return transactionContext.execute(() -> { + try { + var statement = vocabularyStatements.createQuery(new QuerySpec()); + return queryExecutor.query(getConnection(), true, this::mapVocabulary, statement.getQueryAsString(), statement.getParameters()); + } catch (SQLException e) { + throw new EdcPersistenceException(e); + } + }); + } + + @Override + public @Nullable Vocabulary findById(String vocabularyId) { + Objects.requireNonNull(vocabularyId); + + try (var connection = getConnection()) { + var querySpec = QuerySpec.Builder.newInstance().filter(criterion("id", "=", vocabularyId)).build(); + var statement = vocabularyStatements.createQuery(querySpec); + return queryExecutor.query(connection, true, this::mapVocabulary, statement.getQueryAsString(), statement.getParameters()) + .findFirst().orElse(null); + } catch (SQLException e) { + throw new EdcPersistenceException(e); + } + } + + @Override + public StoreResult create(Vocabulary vocabulary) { + Objects.requireNonNull(vocabulary); + + var vocabularyId = vocabulary.getId(); + return transactionContext.execute(() -> { + try (var connection = getConnection()) { + if (existsById(vocabularyId, connection)) { + var msg = format(VocabularyIndex.VOCABULARY_NOT_FOUND_TEMPLATE, vocabularyId); + return StoreResult.alreadyExists(msg); + } + + queryExecutor.execute(connection, vocabularyStatements.getInsertVocabularyTemplate(), + vocabularyId, + vocabulary.getCreatedAt(), + vocabulary.getName(), + toJson(vocabulary.getJsonSchema()) + ); + + return StoreResult.success(); + } catch (Exception e) { + throw new EdcPersistenceException(e); + } + }); + } + + @Override + public StoreResult deleteById(String vocabularyId) { + Objects.requireNonNull(vocabularyId); + + return transactionContext.execute(() -> { + try (var connection = getConnection()) { + var vocabulary = findById(vocabularyId); + if (vocabulary == null) { + return StoreResult.notFound(format(VocabularyIndex.VOCABULARY_NOT_FOUND_TEMPLATE, vocabularyId)); + } + + queryExecutor.execute(connection, vocabularyStatements.getDeleteVocabularyByIdTemplate(), vocabularyId); + + return StoreResult.success(vocabulary); + } catch (Exception e) { + throw new EdcPersistenceException(e.getMessage(), e); + } + }); + } + + @Override + public StoreResult updateVocabulary(Vocabulary vocabulary) { + return transactionContext.execute(() -> { + try (var connection = getConnection()) { + var vocabularyId = vocabulary.getId(); + if (existsById(vocabularyId, connection)) { + queryExecutor.execute(connection, vocabularyStatements.getUpdateVocabularyTemplate(), + vocabulary.getName(), + toJson(vocabulary.getJsonSchema()), + vocabularyId + ); + + return StoreResult.success(vocabulary); + } + return StoreResult.notFound(format(VocabularyIndex.VOCABULARY_NOT_FOUND_TEMPLATE, vocabularyId)); + + } catch (Exception e) { + throw new EdcPersistenceException(e); + } + }); + } + + private int mapRowCount(ResultSet resultSet) throws SQLException { + return resultSet.getInt(vocabularyStatements.getCountVariableName()); + } + + private boolean existsById(String vocabularyId, Connection connection) { + var sql = vocabularyStatements.getCountVocabularyByIdClause(); + try (var stream = queryExecutor.query(connection, false, this::mapRowCount, sql, vocabularyId)) { + return stream.findFirst().orElse(0) > 0; + } + } + + private Vocabulary mapVocabulary(ResultSet resultSet) throws SQLException { + return Vocabulary.Builder.newInstance() + .id(resultSet.getString(vocabularyStatements.getVocabularyIdColumn())) + .createdAt(resultSet.getLong(vocabularyStatements.getCreatedAtColumn())) + .name(resultSet.getString(vocabularyStatements.getNameColumn())) + .jsonSchema(resultSet.getString(vocabularyStatements.getJsonSchemaColumn())) + .build(); + } + +} diff --git a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/SqlVocabularyIndexServiceExtension.java b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/SqlVocabularyIndexServiceExtension.java new file mode 100644 index 0000000..5ad7b15 --- /dev/null +++ b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/SqlVocabularyIndexServiceExtension.java @@ -0,0 +1,58 @@ +package org.upm.inesdata.vocabulary.sql.index; + +import org.eclipse.edc.runtime.metamodel.annotation.Extension; +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.runtime.metamodel.annotation.Provides; +import org.eclipse.edc.runtime.metamodel.annotation.Setting; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; +import org.eclipse.edc.spi.types.TypeManager; +import org.eclipse.edc.sql.QueryExecutor; +import org.eclipse.edc.transaction.datasource.spi.DataSourceRegistry; +import org.eclipse.edc.transaction.spi.TransactionContext; +import org.upm.inesdata.spi.vocabulary.VocabularyIndex; +import org.upm.inesdata.vocabulary.sql.index.schema.VocabularyStatements; +import org.upm.inesdata.vocabulary.sql.index.schema.postgres.PostgresDialectStatements; + +/** + * Extension that stores vocabylaries in SQL databases + */ +@Provides({ VocabularyIndex.class }) +@Extension(value = "SQL vocabulary index") +public class SqlVocabularyIndexServiceExtension implements ServiceExtension { + + /** + * Name of the vocabulary datasource. + */ + @Setting(required = true) + public static final String DATASOURCE_SETTING_NAME = "edc.datasource.vocabulary.name"; + + @Inject + private DataSourceRegistry dataSourceRegistry; + + @Inject + private TransactionContext transactionContext; + + @Inject(required = false) + private VocabularyStatements dialect; + + @Inject + private TypeManager typeManager; + + @Inject + private QueryExecutor queryExecutor; + + @Override + public void initialize(ServiceExtensionContext context) { + var dataSourceName = context.getConfig().getString(DATASOURCE_SETTING_NAME, DataSourceRegistry.DEFAULT_DATASOURCE); + + var sqlVocabularyLoader = new SqlVocabularyIndex(dataSourceRegistry, dataSourceName, transactionContext, typeManager.getMapper(), + getDialect(), queryExecutor); + + context.registerService(VocabularyIndex.class, sqlVocabularyLoader); + } + + private VocabularyStatements getDialect() { + return dialect != null ? dialect : new PostgresDialectStatements(); + } +} diff --git a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/BaseSqlDialectStatements.java b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/BaseSqlDialectStatements.java new file mode 100644 index 0000000..19e2fda --- /dev/null +++ b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/BaseSqlDialectStatements.java @@ -0,0 +1,68 @@ +package org.upm.inesdata.vocabulary.sql.index.schema; + +import org.upm.inesdata.vocabulary.sql.index.schema.postgres.VocabularyMapping; +import org.eclipse.edc.spi.query.QuerySpec; +import org.eclipse.edc.sql.translation.SqlOperatorTranslator; +import org.eclipse.edc.sql.translation.SqlQueryStatement; + +import static java.lang.String.format; + +/** + * Manages Vocabularies using specific SQL queries + */ +public class BaseSqlDialectStatements implements VocabularyStatements { + + protected final SqlOperatorTranslator operatorTranslator; + + public BaseSqlDialectStatements(SqlOperatorTranslator operatorTranslator) { + this.operatorTranslator = operatorTranslator; + } + + @Override + public String getInsertVocabularyTemplate() { + return executeStatement() + .column(getVocabularyIdColumn()) + .column(getCreatedAtColumn()) + .column(getNameColumn()) + .jsonColumn(getJsonSchemaColumn()) + .insertInto(getVocabularyTable()); + } + + @Override + public String getUpdateVocabularyTemplate() { + return executeStatement() + .column(getNameColumn()) + .jsonColumn(getJsonSchemaColumn()) + .update(getVocabularyTable(), getVocabularyIdColumn()); + } + + @Override + public String getCountVocabularyByIdClause() { + return format("SELECT COUNT(*) AS %s FROM %s WHERE %s = ?", + getCountVariableName(), + getVocabularyTable(), + getVocabularyIdColumn()); + } + + @Override + public String getSelectVocabularyTemplate() { + return format("SELECT * FROM %s AS a", getVocabularyTable()); + } + + @Override + public String getDeleteVocabularyByIdTemplate() { + return executeStatement() + .delete(getVocabularyTable(), getVocabularyIdColumn()); + } + + @Override + public String getCountVariableName() { + return "COUNT"; + } + + @Override + public SqlQueryStatement createQuery(QuerySpec querySpec) { + return new SqlQueryStatement(getSelectVocabularyTemplate(), querySpec, new VocabularyMapping(this), operatorTranslator); + } + +} diff --git a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/VocabularyStatements.java b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/VocabularyStatements.java new file mode 100644 index 0000000..ff59798 --- /dev/null +++ b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/VocabularyStatements.java @@ -0,0 +1,79 @@ +package org.upm.inesdata.vocabulary.sql.index.schema; + +import org.eclipse.edc.runtime.metamodel.annotation.ExtensionPoint; +import org.eclipse.edc.spi.query.QuerySpec; +import org.eclipse.edc.sql.statement.SqlStatements; +import org.eclipse.edc.sql.translation.SqlQueryStatement; + +/** + * Defines queries used by the SqlVocabularyIndexServiceExtension. + */ +@ExtensionPoint +public interface VocabularyStatements extends SqlStatements { + + /** + * The vocabulary table name. + */ + default String getVocabularyTable() { + return "edc_vocabulary"; + } + + /** + * The vocabulary table ID column. + */ + default String getVocabularyIdColumn() { + return "id"; + } + + default String getNameColumn() { + return "name"; + } + + default String getJsonSchemaColumn() { + return "json_schema"; + } + + default String getCreatedAtColumn() { + return "created_at"; + } + + + /** + * INSERT clause for vocabularys. + */ + String getInsertVocabularyTemplate(); + + /** + * UPDATE clause for vocabularys. + */ + String getUpdateVocabularyTemplate(); + + /** + * SELECT COUNT clause for vocabularys. + */ + String getCountVocabularyByIdClause(); + + /** + * SELECT clause for all vocabularys. + */ + String getSelectVocabularyTemplate(); + + /** + * DELETE clause for vocabularys. + */ + String getDeleteVocabularyByIdTemplate(); + + /** + * The COUNT variable used in SELECT COUNT queries. + */ + String getCountVariableName(); + + /** + * Generates a SQL query using sub-select statements out of the query spec. + * + * @param query query to be executed + * @return A {@link SqlQueryStatement} that contains the SQL and statement parameters + */ + SqlQueryStatement createQuery(QuerySpec query); + +} diff --git a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/postgres/PostgresDialectStatements.java b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/postgres/PostgresDialectStatements.java new file mode 100644 index 0000000..4a076f2 --- /dev/null +++ b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/postgres/PostgresDialectStatements.java @@ -0,0 +1,17 @@ +package org.upm.inesdata.vocabulary.sql.index.schema.postgres; + +import org.upm.inesdata.vocabulary.sql.index.schema.BaseSqlDialectStatements; +import org.eclipse.edc.sql.dialect.PostgresDialect; +import org.eclipse.edc.sql.translation.PostgresqlOperatorTranslator; + +public class PostgresDialectStatements extends BaseSqlDialectStatements { + + public PostgresDialectStatements() { + super(new PostgresqlOperatorTranslator()); + } + + @Override + public String getFormatAsJsonOperator() { + return PostgresDialect.getJsonCastOperator(); + } +} diff --git a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/postgres/VocabularyMapping.java b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/postgres/VocabularyMapping.java new file mode 100644 index 0000000..9df69b4 --- /dev/null +++ b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/postgres/VocabularyMapping.java @@ -0,0 +1,20 @@ +package org.upm.inesdata.vocabulary.sql.index.schema.postgres; + +import org.eclipse.edc.sql.translation.JsonFieldTranslator; +import org.eclipse.edc.sql.translation.TranslationMapping; +import org.upm.inesdata.vocabulary.sql.index.schema.VocabularyStatements; + +/** + * Maps fields of a {@link org.eclipse.edc.spi.types.domain.vocabulary.Vocabulary} onto the + * corresponding SQL schema (= column names) enabling access through Postgres JSON operators where applicable + */ +public class VocabularyMapping extends TranslationMapping { + + public VocabularyMapping(VocabularyStatements statements) { + add("id", statements.getVocabularyIdColumn()); + add("createdAt", statements.getCreatedAtColumn()); + add("name", statements.getNameColumn()); + add("json_schema", new JsonFieldTranslator(statements.getJsonSchemaColumn())); + } + +} diff --git a/extensions/vocabulary-index-sql/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/vocabulary-index-sql/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 0000000..897b043 --- /dev/null +++ b/extensions/vocabulary-index-sql/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1 @@ +org.upm.inesdata.vocabulary.sql.index.SqlVocabularyIndexServiceExtension \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index f1c9346..fdee51f 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -10,9 +10,11 @@ jupiter = "5.10.2" mockito = "5.2.0" postgres = "42.7.3" rsApi = "3.1.0" +swagger-annotations-jakarta = "2.2.20" [libraries] edc-api-core = { module = "org.eclipse.edc:api-core", version.ref = "edc" } +edc-api-management-config = { module = "org.eclipse.edc:management-api-configuration", version.ref = "edc" } edc-auth-spi = { module = "org.eclipse.edc:api-core", version.ref = "edc" } edc-boot = { module = "org.eclipse.edc:boot", version.ref = "edc" } edc-build-plugin = { module = "org.eclipse.edc.edc-build:org.eclipse.edc.edc-build.gradle.plugin", version.ref = "edc" } @@ -29,19 +31,31 @@ edc-data-plane-public-api = { module = "org.eclipse.edc:data-plane-public-api", edc-data-plane-selector-api = { module = "org.eclipse.edc:data-plane-selector-api", version.ref = "edc" } edc-data-plane-selector-core = { module = "org.eclipse.edc:data-plane-selector-core", version.ref = "edc" } edc-dsp = { module = "org.eclipse.edc:dsp", version.ref = "edc" } +edc-dsp-api-configuration = { module = "org.eclipse.edc:dsp-http-api-configuration", version.ref = "edc" } edc-http = { module = "org.eclipse.edc:http", version.ref = "edc" } edc-iam-oauth2-core = { module = "org.eclipse.edc:oauth2-core", version.ref = "edc" } edc-iam-oauth2-service = { module = "org.eclipse.edc:oauth2-service", version.ref = "edc" } +edc-json-ld-lib = { module = "org.eclipse.edc:json-ld-lib", version.ref = "edc" } edc-management-api = { module = "org.eclipse.edc:management-api", version.ref = "edc" } edc-micrometer-core = { module = "org.eclipse.edc:micrometer-core", version.ref = "edc" } +edc-spi-core = { module = "org.eclipse.edc:core-spi", version.ref = "edc" } +edc-spi-jsonld = { module = "org.eclipse.edc:json-ld-spi", version.ref = "edc" } +edc-spi-transform = { module = "org.eclipse.edc:transform-spi", version.ref = "edc" } +edc-transaction-local = { module = "org.eclipse.edc:transaction-local", version.ref = "edc" } +edc-transaction-datasource-spi = { module = "org.eclipse.edc:transaction-datasource-spi", version.ref = "edc" } +edc-transaction-spi = { module = "org.eclipse.edc:transaction-spi", version.ref = "edc" } edc-transfer-data-plane = { module = "org.eclipse.edc:transfer-data-plane", version.ref = "edc" } edc-transfer-pull-http-receiver = { module = "org.eclipse.edc:transfer-pull-http-receiver", version.ref = "edc" } edc-vault-filesystem = { module = "org.eclipse.edc:vault-filesystem", version.ref = "edc" } +edc-validator-spi = { module = "org.eclipse.edc:validator-spi", version.ref = "edc" } edc-web-spi = { module = "org.eclipse.edc:web-spi", version.ref = "edc" } jakarta-rsApi = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version.ref = "rsApi" } +edc-lib-transform = { module = "org.eclipse.edc:transform-lib", version.ref = "edc" } +edc-lib-util = { module = "org.eclipse.edc:util-lib", version.ref = "edc" } +edc-lib-validator = { module = "org.eclipse.edc:validator-lib", version.ref = "edc" } + # SQL Extensions -edc-transaction-local = { module = "org.eclipse.edc:transaction-local", version.ref = "edc" } edc-sql-core = { module = "org.eclipse.edc:sql-core", version.ref = "edc" } edc-sql-edr = { module = "org.eclipse.edc:edr-index-sql", version.ref = "edc" } edc-sql-lease = { module = "org.eclipse.edc:sql-lease", version.ref = "edc" } @@ -72,6 +86,7 @@ edc-core-junit = { module = "org.eclipse.edc:junit", version.ref = "edc" } aws-s3 = { module = "software.amazon.awssdk:s3", version.ref = "aws" } aws-s3-transfer = { module = "software.amazon.awssdk:s3-transfer-manager", version.ref = "aws" } postgres = { module = "org.postgresql:postgresql", version.ref = "postgres" } +swagger-annotations-jakarta = { module = "io.swagger.core.v3:swagger-annotations-jakarta", version.ref = "swagger-annotations-jakarta" } # No EDC testing references assertj = { module = "org.assertj:assertj-core", version.ref = "assertj" } diff --git a/launchers/connector/build.gradle.kts b/launchers/connector/build.gradle.kts index 4aa2578..726afe4 100644 --- a/launchers/connector/build.gradle.kts +++ b/launchers/connector/build.gradle.kts @@ -36,6 +36,9 @@ dependencies { implementation(libs.edc.transfer.pull.http.receiver) implementation(libs.edc.transfer.data.plane) + // Vocabularios + implementation(project(":extensions:vocabulary-api")) + // Persistencia comun implementation(libs.edc.sql.core) implementation(libs.edc.sql.edr) @@ -47,6 +50,7 @@ dependencies { implementation(libs.edc.sql.contract.negotiation.store) implementation(libs.edc.sql.policy.definition.store) implementation(libs.edc.sql.transfer.process.store) + implementation(project(":extensions:vocabulary-index-sql")) // Persistencia data plane implementation(libs.edc.sql.data.plane.store) diff --git a/resources/sql/vocabulary-schema.sql b/resources/sql/vocabulary-schema.sql new file mode 100644 index 0000000..773df12 --- /dev/null +++ b/resources/sql/vocabulary-schema.sql @@ -0,0 +1,11 @@ +-- table: edc_vocabulary +CREATE TABLE IF NOT EXISTS edc_vocabulary +( + id VARCHAR NOT NULL, + created_at BIGINT NOT NULL, + json_schema JSON DEFAULT '{}', + vocabulary_name VARCHAR NOT NULL, + PRIMARY KEY (id) +); + +COMMENT ON COLUMN edc_vocabulary.json_schema IS 'JSON Schema with the vocabulary'; diff --git a/settings.gradle.kts b/settings.gradle.kts index cb5f866..2375bba 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -5,9 +5,14 @@ pluginManagement { rootProject.name = "inesdata-connector" +// SPI +include(":spi:vocabulary-spi") + // Extensions include(":extensions:participants-from-configuration") include(":extensions:auth-oauth2-jwt") +include(":extensions:vocabulary-api") +include(":extensions:vocabulary-index-sql") // Connector include(":launchers:connector") diff --git a/spi/vocabulary-spi/README.md b/spi/vocabulary-spi/README.md new file mode 100644 index 0000000..a17be64 --- /dev/null +++ b/spi/vocabulary-spi/README.md @@ -0,0 +1 @@ +This module contains extension points and interfaces specifically for the Inesdata Vocabulary feature. \ No newline at end of file diff --git a/spi/vocabulary-spi/build.gradle.kts b/spi/vocabulary-spi/build.gradle.kts new file mode 100644 index 0000000..5f7ff12 --- /dev/null +++ b/spi/vocabulary-spi/build.gradle.kts @@ -0,0 +1,8 @@ +plugins { + `java-library` + id("com.gmv.inesdata.edc-application") +} + +dependencies { + api(libs.edc.spi.core) +} diff --git a/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/VocabularyIndex.java b/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/VocabularyIndex.java new file mode 100644 index 0000000..4bf45b4 --- /dev/null +++ b/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/VocabularyIndex.java @@ -0,0 +1,61 @@ +package org.upm.inesdata.spi.vocabulary; + +import org.upm.inesdata.spi.vocabulary.domain.Vocabulary; +import org.eclipse.edc.runtime.metamodel.annotation.ExtensionPoint; +import org.eclipse.edc.spi.persistence.EdcPersistenceException; +import org.eclipse.edc.spi.result.StoreResult; + +import java.util.stream.Stream; + +/** + * Datastore interface for {@link Vocabulary} objects. + */ +@ExtensionPoint +public interface VocabularyIndex { + + String VOCABULARY_EXISTS_TEMPLATE = "Vocabulary with ID %s already exists"; + String VOCABULARY_NOT_FOUND_TEMPLATE = "Vocabulary with ID %s not found"; + + /** + * Finds all stored vocabularies + * + * @return A potentially empty collection of {@link Vocabulary}, never null. + */ + Stream allVocabularies(); + + /** + * Fetches the {@link Vocabulary} with the given ID from the metadata backend. + * + * @param vocabularyId A String that represents the Vocabulary ID, in most cases this will be a UUID. + * @return The {@link Vocabulary} if one was found, or null otherwise. + * @throws NullPointerException If {@code vocabularyId} was null or empty. + */ + Vocabulary findById(String vocabularyId); + + /** + * Stores a {@link Vocabulary} in the vocabulary index, if no vocabulary with the same ID already exists. + * Implementors must ensure that it's stored in a transactional way. + * + * @param vocabulary The {@link Vocabulary} to store + * @return {@link StoreResult#success()} if the objects were stored, {@link StoreResult#alreadyExists(String)} when an object with the same ID already exists. + */ + StoreResult create(Vocabulary vocabulary); + + /** + * Deletes a vocabulary if it exists. + * + * @param vocabularyId Id of the vocabulary to be deleted. + * @return {@link StoreResult#success(Object)} if the object was deleted, {@link StoreResult#notFound(String)} when an object with that ID was not found. + * @throws EdcPersistenceException if something goes wrong. + */ + StoreResult deleteById(String vocabularyId); + + /** + * Updates a vocabulary with the content from the given {@link Vocabulary}. If the vocabulary is not found, no further database interaction takes place. + * + * @param vocabulary The Vocabulary containing the new values. ID will be ignored. + * @return {@link StoreResult#success(Object)} if the object was updated, {@link StoreResult#notFound(String)} when an object with that ID was not found. + */ + StoreResult updateVocabulary(Vocabulary vocabulary); + +} diff --git a/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/VocabularyService.java b/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/VocabularyService.java new file mode 100644 index 0000000..fccf217 --- /dev/null +++ b/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/VocabularyService.java @@ -0,0 +1,53 @@ +package org.upm.inesdata.spi.vocabulary; + +import org.upm.inesdata.spi.vocabulary.domain.Vocabulary; +import org.eclipse.edc.spi.result.ServiceResult; + +import java.util.List; + +/** + * Service interface for {@link Vocabulary} objects. + */ +public interface VocabularyService { + + /** + * Returns a vocabulary by its id + * + * @param vocabularyId id of the vocabulary + * @return the vocabulary, null if it's not found + */ + Vocabulary findById(String vocabularyId); + + /** + * Get Vocabularys + * + * @return the collection of vocabularys stored + */ + ServiceResult> search(); + + /** + * Create a vocabulary + * + * @param vocabulary the vocabulary + * @return successful result if the vocabulary is created correctly, failure otherwise + */ + ServiceResult create(Vocabulary vocabulary); + + /** + * Delete a vocabulary + * + * @param vocabularyId the id of the vocabulary to be deleted + * @return successful result if the vocabulary is deleted correctly, failure otherwise + */ + ServiceResult delete(String vocabularyId); + + /** + * Updates a vocabulary. If the vocabulary does not yet exist, {@link ServiceResult#notFound(String)} will be returned. + * + * @param vocabulary The content of the Vocabulary. Note that {@link Vocabulary#getId()} will be ignored, rather the separately supplied ID is used + * @return successful if updated, a failure otherwise. + */ + ServiceResult update(Vocabulary vocabulary); + +} + \ No newline at end of file diff --git a/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/domain/Vocabulary.java b/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/domain/Vocabulary.java new file mode 100644 index 0000000..b2f25ae --- /dev/null +++ b/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/domain/Vocabulary.java @@ -0,0 +1,100 @@ +package org.upm.inesdata.spi.vocabulary.domain; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; +import org.eclipse.edc.spi.entity.Entity; + +import java.util.Objects; +import java.util.UUID; + +import static org.eclipse.edc.spi.constants.CoreConstants.EDC_NAMESPACE; + + +/** + * The {@link Vocabulary} contains the vocabylary information. + */ +@JsonDeserialize(builder = Vocabulary.Builder.class) +public class Vocabulary extends Entity { + + public static final String PROPERTY_ID = EDC_NAMESPACE + "id"; + public static final String PROPERTY_NAME = EDC_NAMESPACE + "name"; + public static final String PROPERTY_JSON_SCHEMA = EDC_NAMESPACE + "jsonSchema"; + public static final String EDC_VOCABULARY_TYPE = EDC_NAMESPACE + "Vocabulary"; + + + private String name; + private String jsonSchema; + + private Vocabulary() { + } + + public String getName() { + return name; + } + + public String getJsonSchema() { + return jsonSchema; + } + + public Builder toBuilder() { + return Vocabulary.Builder.newInstance() + .id(id) + .name(name) + .jsonSchema(jsonSchema) + .createdAt(createdAt); + } + + @JsonPOJOBuilder(withPrefix = "") + public static class Builder extends Entity.Builder { + + protected Builder(Vocabulary vocabulary) { + super(vocabulary); + } + + @JsonCreator + public static Builder newInstance() { + return new Builder(new Vocabulary()); + } + + @Override + public Builder id(String id) { + entity.id = id; + return self(); + } + + public Builder name(String name) { + entity.name = name; + return self(); + } + + public Builder jsonSchema(String jsonSchema) { + Objects.requireNonNull(jsonSchema); + entity.jsonSchema = jsonSchema; + return self(); + } + + @Override + public Builder createdAt(long value) { + entity.createdAt = value; + return self(); + } + + @Override + public Builder self() { + return this; + } + + @Override + public Vocabulary build() { + super.build(); + + if (entity.getId() == null) { + id(UUID.randomUUID().toString()); + } + + return entity; + } + } + +} diff --git a/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/package-info.java b/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/package-info.java new file mode 100644 index 0000000..69fc60f --- /dev/null +++ b/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/package-info.java @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2020 - 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial API and implementation + * + */ + +@Spi("Vocabulary services") +package org.upm.inesdata.spi.vocabulary; + +import org.eclipse.edc.runtime.metamodel.annotation.Spi; From fc50dec2f20deca76e9578069ca654d547c34e59 Mon Sep 17 00:00:00 2001 From: ppel Date: Mon, 13 May 2024 10:25:09 +0200 Subject: [PATCH 050/174] Changed edc version to 0.6.3 (with dependencies) --- gradle/libs.versions.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index fdee51f..9ec91d0 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -4,13 +4,13 @@ format.version = "1.1" [versions] assertj = "3.25.3" aws = "2.23.12" -edc = "0.6.2" +edc = "0.6.3" junit-pioneer = "2.2.0" jupiter = "5.10.2" mockito = "5.2.0" postgres = "42.7.3" -rsApi = "3.1.0" -swagger-annotations-jakarta = "2.2.20" +rsApi = "4.0.0" +swagger-annotations-jakarta = "2.2.21" [libraries] edc-api-core = { module = "org.eclipse.edc:api-core", version.ref = "edc" } From 9647d4af824bdc9cef8209be271c49477b59a9cf Mon Sep 17 00:00:00 2001 From: ppel Date: Mon, 13 May 2024 13:54:25 +0200 Subject: [PATCH 051/174] Rerun pipeline From 28b16a20abbd039160a52a4c187e4cd9018671e6 Mon Sep 17 00:00:00 2001 From: Juan Calero Date: Mon, 13 May 2024 14:24:26 +0200 Subject: [PATCH 052/174] Improved supressions file to avoid a recurrent false positive --- suppressions.xml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/suppressions.xml b/suppressions.xml index 0aebd28..97081c8 100644 --- a/suppressions.xml +++ b/suppressions.xml @@ -2,9 +2,10 @@ - ^pkg:maven/org\.eclipse\.edc/.*sql.*@.*$ + ^pkg:maven/.*sql.*@.*$ cpe:/a:www-sql_project:www-sql @@ -22,4 +23,4 @@ cpe:/a:jetty:jetty cpe:/a:eclipse:jetty - \ No newline at end of file + From c4450cc13d9126f965f1864c7ecf66ee459aa4d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Tue, 14 May 2024 08:01:30 +0200 Subject: [PATCH 053/174] update documentation and vocabulary SQL files --- extensions/vocabulary-api/README.md | 19 +++++++++++++++++++ .../vocabulary-index-sql/docs/schema.sql | 4 ++-- .../index/schema/VocabularyStatements.java | 2 +- resources/sql/vocabulary-schema.sql | 4 ++-- 4 files changed, 24 insertions(+), 5 deletions(-) create mode 100644 extensions/vocabulary-api/README.md diff --git a/extensions/vocabulary-api/README.md b/extensions/vocabulary-api/README.md new file mode 100644 index 0000000..6fbc14d --- /dev/null +++ b/extensions/vocabulary-api/README.md @@ -0,0 +1,19 @@ +# Vocabulary API + +Provides a management API for work with vocabularies. This API expands the functionality of the control-plane management API to be able to handle Vocabulary entities. + + +## Vocabulary entity + +An example of a Vocabulary entity is shown below. + +``` +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@id": "vocabularyId", + "name": "Vocabulary name", + "jsonSchema": "{ \"title\": \"vocabulary\", \"type\": \"object\", \"properties\": { \"name\": { \"type\": \"string\", \"title\": \"Name\" }, \"keyword\": { \"type\": \"array\", \"title\": \"Keywords\", \"items\": { \"type\": \"string\" } } }, \"required\": [ \"name\" ] }" +} +``` diff --git a/extensions/vocabulary-index-sql/docs/schema.sql b/extensions/vocabulary-index-sql/docs/schema.sql index 773df12..e6bd609 100644 --- a/extensions/vocabulary-index-sql/docs/schema.sql +++ b/extensions/vocabulary-index-sql/docs/schema.sql @@ -1,10 +1,10 @@ -- table: edc_vocabulary CREATE TABLE IF NOT EXISTS edc_vocabulary ( - id VARCHAR NOT NULL, + id VARCHAR NOT NULL, created_at BIGINT NOT NULL, json_schema JSON DEFAULT '{}', - vocabulary_name VARCHAR NOT NULL, + name VARCHAR NOT NULL, PRIMARY KEY (id) ); diff --git a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/VocabularyStatements.java b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/VocabularyStatements.java index ff59798..fece7d0 100644 --- a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/VocabularyStatements.java +++ b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/VocabularyStatements.java @@ -71,7 +71,7 @@ default String getCreatedAtColumn() { /** * Generates a SQL query using sub-select statements out of the query spec. * - * @param query query to be executed + * @param query querySpec to be executed * @return A {@link SqlQueryStatement} that contains the SQL and statement parameters */ SqlQueryStatement createQuery(QuerySpec query); diff --git a/resources/sql/vocabulary-schema.sql b/resources/sql/vocabulary-schema.sql index 773df12..e6bd609 100644 --- a/resources/sql/vocabulary-schema.sql +++ b/resources/sql/vocabulary-schema.sql @@ -1,10 +1,10 @@ -- table: edc_vocabulary CREATE TABLE IF NOT EXISTS edc_vocabulary ( - id VARCHAR NOT NULL, + id VARCHAR NOT NULL, created_at BIGINT NOT NULL, json_schema JSON DEFAULT '{}', - vocabulary_name VARCHAR NOT NULL, + name VARCHAR NOT NULL, PRIMARY KEY (id) ); From 762031e8fd18d09ff7bcd17ed12158b2b6c75a57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Tue, 21 May 2024 12:46:44 +0200 Subject: [PATCH 054/174] vocabulary improvements --- .../vocabulary/controller/VocabularyApi.java | 4 ++- .../service/VocabularyServiceImpl.java | 23 +++++++++++++++ .../storage/InMemoryVocabularyIndex.java | 13 +++++++++ .../JsonObjectFromVocabularyTransformer.java | 7 ++++- .../JsonObjectToVocabularyTransformer.java | 8 ++++++ .../validator/VocabularyValidator.java | 2 ++ .../vocabulary-index-sql/docs/schema.sql | 2 ++ .../sql/index/SqlVocabularyIndex.java | 20 ++++++++++++- .../schema/BaseSqlDialectStatements.java | 4 +++ .../index/schema/VocabularyStatements.java | 8 ++++++ .../schema/postgres/VocabularyMapping.java | 2 ++ resources/sql/vocabulary-schema.sql | 2 ++ .../spi/vocabulary/VocabularyIndex.java | 8 ++++++ .../spi/vocabulary/domain/Vocabulary.java | 28 +++++++++++++++++++ 14 files changed, 128 insertions(+), 3 deletions(-) diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/controller/VocabularyApi.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/controller/VocabularyApi.java index acb5a62..32475d3 100644 --- a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/controller/VocabularyApi.java +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/controller/VocabularyApi.java @@ -125,7 +125,9 @@ record VocabularyOutputSchema( { "@id": "vocabularyId", "name": "vocabulary name", - "jsonSchema": "{ \\"title\\": \\"vocabulary\\", \\"type\\": \\"object\\", \\"properties\\": { \\"name\\": { \\"type\\": \\"string\\", \\"title\\": \\"Name\\" }, \\"dct:keyword\\": { \\"type\\": \\"array\\", \\"title\\": \\"Keywords\\", \\"items\\": { \\"type\\": \\"string\\" } } }, \\"required\\": [ \\"name\\" ], \\"@context\\": { \\"dct\\": \\"http:\\/\\/purl.org\\/dc\\/terms\\/\" } }" + "jsonSchema": "{ \\"title\\": \\"vocabulary\\", \\"type\\": \\"object\\", \\"properties\\": { \\"name\\": { \\"type\\": \\"string\\", \\"title\\": \\"Name\\" }, \\"dct:keyword\\": { \\"type\\": \\"array\\", \\"title\\": \\"Keywords\\", \\"items\\": { \\"type\\": \\"string\\" } } }, \\"required\\": [ \\"name\\" ], \\"@context\\": { \\"dct\\": \\"http:\\/\\/purl.org\\/dc\\/terms\\/\" } }", + "category": "dataset", + "defaultVocabulary": true } """; } diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/service/VocabularyServiceImpl.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/service/VocabularyServiceImpl.java index e72ed5c..e44f803 100644 --- a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/service/VocabularyServiceImpl.java +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/service/VocabularyServiceImpl.java @@ -43,8 +43,19 @@ public ServiceResult> search() { @Override public ServiceResult create(Vocabulary vocabulary) { return transactionContext.execute(() -> { + // Managing default vocabulary + var defaultVocabulary = index.getDefaultVocabulary(); + + // Create new vocabulary var createResult = index.create(vocabulary); + if (createResult.succeeded()) { + // There is only a default vocabulary + if (defaultVocabulary != null && vocabulary.isDefaultVocabulary()) { + defaultVocabulary.setDefaultVocabulary(false); + index.updateVocabulary(defaultVocabulary); + } + return ServiceResult.success(vocabulary); } return ServiceResult.fromFailure(createResult); @@ -62,7 +73,19 @@ public ServiceResult delete(String vocabularyId) { @Override public ServiceResult update(Vocabulary vocabulary) { return transactionContext.execute(() -> { + // Managing default vocabulary + var defaultVocabulary = index.getDefaultVocabulary(); + + // Update vocabulary var updatedVocabulary = index.updateVocabulary(vocabulary); + + // There is only a default vocabulary + if (defaultVocabulary != null && vocabulary.isDefaultVocabulary() && + ! defaultVocabulary.getId().equals(vocabulary.getId())) { + defaultVocabulary.setDefaultVocabulary(false); + index.updateVocabulary(defaultVocabulary); + } + return ServiceResult.from(updatedVocabulary); }); } diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/storage/InMemoryVocabularyIndex.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/storage/InMemoryVocabularyIndex.java index 1ce1afb..27e3f3c 100644 --- a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/storage/InMemoryVocabularyIndex.java +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/storage/InMemoryVocabularyIndex.java @@ -104,4 +104,17 @@ private Vocabulary delete(String vocabularyId) { return cache.remove(vocabularyId); } + @Override + public Vocabulary getDefaultVocabulary() { + lock.readLock().lock(); + try { + return cache.values().stream() + .filter(vocabulary -> vocabulary.isDefaultVocabulary()) + .findFirst() + .orElse(null); + } finally { + lock.readLock().unlock(); + } + } + } diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectFromVocabularyTransformer.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectFromVocabularyTransformer.java index 92599d5..7d44812 100644 --- a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectFromVocabularyTransformer.java +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectFromVocabularyTransformer.java @@ -16,6 +16,9 @@ import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.EDC_VOCABULARY_TYPE; import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_JSON_SCHEMA; import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_NAME; +import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_CATEGORY; +import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_DEFAULT_VOCABULARY; + /** * Creates a JsonObject from a {@link Vocabulary} @@ -39,7 +42,9 @@ public JsonObjectFromVocabularyTransformer(JsonBuilderFactory jsonFactory, Objec .add(ID, vocabulary.getId()) .add(TYPE, EDC_VOCABULARY_TYPE) .add(PROPERTY_NAME, vocabulary.getName()) - .add(PROPERTY_JSON_SCHEMA, vocabulary.getJsonSchema()); + .add(PROPERTY_JSON_SCHEMA, vocabulary.getJsonSchema()) + .add(PROPERTY_CATEGORY, vocabulary.getCategory()) + .add(PROPERTY_DEFAULT_VOCABULARY, vocabulary.isDefaultVocabulary()); return builder.build(); } diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectToVocabularyTransformer.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectToVocabularyTransformer.java index be99616..989b458 100644 --- a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectToVocabularyTransformer.java +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectToVocabularyTransformer.java @@ -9,6 +9,8 @@ import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_JSON_SCHEMA; import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_NAME; +import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_CATEGORY; +import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_DEFAULT_VOCABULARY; /** * Converts from an {@link Vocabulary} as a {@link JsonObject} in JSON-LD expanded form to an {@link Vocabulary}. @@ -30,9 +32,15 @@ public JsonObjectToVocabularyTransformer() { visitProperties(jsonObject, key -> switch (key) { case PROPERTY_NAME -> value -> builder.name(transformString(value, context)); case PROPERTY_JSON_SCHEMA -> value -> builder.jsonSchema(transformString(value, context)); + case PROPERTY_CATEGORY -> value -> builder.category(transformString(value, context)); + case PROPERTY_DEFAULT_VOCABULARY -> value -> builder.defaultVocabulary(transformBoolean(value, context)); default -> doNothing(); }); + if (!jsonObject.containsKey(PROPERTY_DEFAULT_VOCABULARY)) { + builder.defaultVocabulary(false); + } + return builderResult(builder::build, context); } diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/validator/VocabularyValidator.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/validator/VocabularyValidator.java index 1b1753c..e7d0723 100644 --- a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/validator/VocabularyValidator.java +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/validator/VocabularyValidator.java @@ -8,6 +8,7 @@ import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_JSON_SCHEMA; import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_NAME; +import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_CATEGORY; /** * Validator for Vocabulary @@ -21,6 +22,7 @@ public static Validator instance() { return JsonObjectValidator.newValidator() .verifyId(OptionalIdNotBlank::new) .verify(PROPERTY_NAME, MandatoryObject::new) + .verify(PROPERTY_CATEGORY, MandatoryObject::new) .verify(PROPERTY_JSON_SCHEMA, MandatoryJsonField::new) .build(); } diff --git a/extensions/vocabulary-index-sql/docs/schema.sql b/extensions/vocabulary-index-sql/docs/schema.sql index e6bd609..2208b90 100644 --- a/extensions/vocabulary-index-sql/docs/schema.sql +++ b/extensions/vocabulary-index-sql/docs/schema.sql @@ -5,6 +5,8 @@ CREATE TABLE IF NOT EXISTS edc_vocabulary created_at BIGINT NOT NULL, json_schema JSON DEFAULT '{}', name VARCHAR NOT NULL, + category VARCHAR NOT NULL, + default_vocabulary BOOLEAN DEFAULT FALSE, PRIMARY KEY (id) ); diff --git a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/SqlVocabularyIndex.java b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/SqlVocabularyIndex.java index e2207c4..db63787 100644 --- a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/SqlVocabularyIndex.java +++ b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/SqlVocabularyIndex.java @@ -74,7 +74,7 @@ public StoreResult create(Vocabulary vocabulary) { return transactionContext.execute(() -> { try (var connection = getConnection()) { if (existsById(vocabularyId, connection)) { - var msg = format(VocabularyIndex.VOCABULARY_NOT_FOUND_TEMPLATE, vocabularyId); + var msg = format(VocabularyIndex.VOCABULARY_EXISTS_TEMPLATE, vocabularyId); return StoreResult.alreadyExists(msg); } @@ -82,6 +82,8 @@ public StoreResult create(Vocabulary vocabulary) { vocabularyId, vocabulary.getCreatedAt(), vocabulary.getName(), + vocabulary.getCategory(), + vocabulary.isDefaultVocabulary(), toJson(vocabulary.getJsonSchema()) ); @@ -121,6 +123,8 @@ public StoreResult updateVocabulary(Vocabulary vocabulary) { queryExecutor.execute(connection, vocabularyStatements.getUpdateVocabularyTemplate(), vocabulary.getName(), toJson(vocabulary.getJsonSchema()), + vocabulary.getCategory(), + vocabulary.isDefaultVocabulary(), vocabularyId ); @@ -134,6 +138,18 @@ public StoreResult updateVocabulary(Vocabulary vocabulary) { }); } + @Override + public Vocabulary getDefaultVocabulary() { + try (var connection = getConnection()) { + var querySpec = QuerySpec.Builder.newInstance().filter(criterion("default_vocabulary", "=", true)).build(); + var statement = vocabularyStatements.createQuery(querySpec); + return queryExecutor.query(connection, true, this::mapVocabulary, statement.getQueryAsString(), statement.getParameters()) + .findFirst().orElse(null); + } catch (SQLException e) { + throw new EdcPersistenceException(e); + } + } + private int mapRowCount(ResultSet resultSet) throws SQLException { return resultSet.getInt(vocabularyStatements.getCountVariableName()); } @@ -150,6 +166,8 @@ private Vocabulary mapVocabulary(ResultSet resultSet) throws SQLException { .id(resultSet.getString(vocabularyStatements.getVocabularyIdColumn())) .createdAt(resultSet.getLong(vocabularyStatements.getCreatedAtColumn())) .name(resultSet.getString(vocabularyStatements.getNameColumn())) + .category(resultSet.getString(vocabularyStatements.getCategoryColumn())) + .defaultVocabulary(resultSet.getBoolean(vocabularyStatements.getDefaultVocabularyColumn())) .jsonSchema(resultSet.getString(vocabularyStatements.getJsonSchemaColumn())) .build(); } diff --git a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/BaseSqlDialectStatements.java b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/BaseSqlDialectStatements.java index 19e2fda..898b1e6 100644 --- a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/BaseSqlDialectStatements.java +++ b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/BaseSqlDialectStatements.java @@ -24,6 +24,8 @@ public String getInsertVocabularyTemplate() { .column(getVocabularyIdColumn()) .column(getCreatedAtColumn()) .column(getNameColumn()) + .column(getCategoryColumn()) + .column(getDefaultVocabularyColumn()) .jsonColumn(getJsonSchemaColumn()) .insertInto(getVocabularyTable()); } @@ -33,6 +35,8 @@ public String getUpdateVocabularyTemplate() { return executeStatement() .column(getNameColumn()) .jsonColumn(getJsonSchemaColumn()) + .column(getCategoryColumn()) + .column(getDefaultVocabularyColumn()) .update(getVocabularyTable(), getVocabularyIdColumn()); } diff --git a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/VocabularyStatements.java b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/VocabularyStatements.java index fece7d0..4bfb0c0 100644 --- a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/VocabularyStatements.java +++ b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/VocabularyStatements.java @@ -33,6 +33,14 @@ default String getJsonSchemaColumn() { return "json_schema"; } + default String getCategoryColumn() { + return "category"; + } + + default String getDefaultVocabularyColumn() { + return "default_vocabulary"; + } + default String getCreatedAtColumn() { return "created_at"; } diff --git a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/postgres/VocabularyMapping.java b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/postgres/VocabularyMapping.java index 9df69b4..5790fc8 100644 --- a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/postgres/VocabularyMapping.java +++ b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/postgres/VocabularyMapping.java @@ -14,6 +14,8 @@ public VocabularyMapping(VocabularyStatements statements) { add("id", statements.getVocabularyIdColumn()); add("createdAt", statements.getCreatedAtColumn()); add("name", statements.getNameColumn()); + add("category", statements.getCategoryColumn()); + add("default_vocabulary", statements.getDefaultVocabularyColumn()); add("json_schema", new JsonFieldTranslator(statements.getJsonSchemaColumn())); } diff --git a/resources/sql/vocabulary-schema.sql b/resources/sql/vocabulary-schema.sql index e6bd609..2208b90 100644 --- a/resources/sql/vocabulary-schema.sql +++ b/resources/sql/vocabulary-schema.sql @@ -5,6 +5,8 @@ CREATE TABLE IF NOT EXISTS edc_vocabulary created_at BIGINT NOT NULL, json_schema JSON DEFAULT '{}', name VARCHAR NOT NULL, + category VARCHAR NOT NULL, + default_vocabulary BOOLEAN DEFAULT FALSE, PRIMARY KEY (id) ); diff --git a/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/VocabularyIndex.java b/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/VocabularyIndex.java index 4bf45b4..dc3f408 100644 --- a/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/VocabularyIndex.java +++ b/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/VocabularyIndex.java @@ -58,4 +58,12 @@ public interface VocabularyIndex { */ StoreResult updateVocabulary(Vocabulary vocabulary); + /** + * Fetches the default {@link Vocabulary} from the metadata backend. + * + * @return The {@link Vocabulary} if one was found, or null otherwise. + * @throws NullPointerException If {@code vocabularyId} was null or empty. + */ + Vocabulary getDefaultVocabulary(); + } diff --git a/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/domain/Vocabulary.java b/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/domain/Vocabulary.java index b2f25ae..27c159a 100644 --- a/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/domain/Vocabulary.java +++ b/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/domain/Vocabulary.java @@ -20,11 +20,15 @@ public class Vocabulary extends Entity { public static final String PROPERTY_ID = EDC_NAMESPACE + "id"; public static final String PROPERTY_NAME = EDC_NAMESPACE + "name"; public static final String PROPERTY_JSON_SCHEMA = EDC_NAMESPACE + "jsonSchema"; + public static final String PROPERTY_CATEGORY = EDC_NAMESPACE + "category"; + public static final String PROPERTY_DEFAULT_VOCABULARY = EDC_NAMESPACE + "defaultVocabulary"; public static final String EDC_VOCABULARY_TYPE = EDC_NAMESPACE + "Vocabulary"; private String name; private String jsonSchema; + private String category; + private boolean defaultVocabulary; private Vocabulary() { } @@ -37,11 +41,25 @@ public String getJsonSchema() { return jsonSchema; } + public String getCategory() { + return category; + } + + public boolean isDefaultVocabulary() { + return defaultVocabulary; + } + + public void setDefaultVocabulary(boolean _defaultVocabulary) { + defaultVocabulary = _defaultVocabulary; + } + public Builder toBuilder() { return Vocabulary.Builder.newInstance() .id(id) .name(name) .jsonSchema(jsonSchema) + .category(category) + .defaultVocabulary(defaultVocabulary) .createdAt(createdAt); } @@ -74,6 +92,16 @@ public Builder jsonSchema(String jsonSchema) { return self(); } + public Builder category(String category) { + entity.category = category; + return self(); + } + + public Builder defaultVocabulary(boolean defaultVocabulary) { + entity.defaultVocabulary = defaultVocabulary; + return self(); + } + @Override public Builder createdAt(long value) { entity.createdAt = value; From 8bb6b9337a05ee739402cccfa5a85519e0f2e933 Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Wed, 22 May 2024 14:18:35 +0200 Subject: [PATCH 055/174] 431555 Se elimina campo defaultVocabulary --- .../vocabulary/VocabularyApiExtension.java | 22 ++----------------- .../vocabulary/controller/VocabularyApi.java | 4 ++-- .../controller/VocabularyApiController.java | 4 +--- .../service/VocabularyServiceImpl.java | 21 ------------------ .../storage/InMemoryVocabularyIndex.java | 16 +------------- .../JsonObjectFromVocabularyTransformer.java | 8 ++----- .../JsonObjectToVocabularyTransformer.java | 8 +------ .../validator/MandatoryJsonField.java | 2 +- .../validator/VocabularyValidator.java | 2 +- .../sql/index/SqlVocabularyIndex.java | 18 +-------------- .../schema/BaseSqlDialectStatements.java | 4 +--- .../index/schema/VocabularyStatements.java | 4 ---- .../schema/postgres/VocabularyMapping.java | 1 - resources/sql/vocabulary-schema.sql | 3 +-- .../spi/vocabulary/VocabularyIndex.java | 10 +-------- .../spi/vocabulary/domain/Vocabulary.java | 15 ------------- 16 files changed, 15 insertions(+), 127 deletions(-) diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/VocabularyApiExtension.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/VocabularyApiExtension.java index 09b3e91..febe52e 100644 --- a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/VocabularyApiExtension.java +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/VocabularyApiExtension.java @@ -23,11 +23,11 @@ import org.upm.inesdata.vocabulary.transformer.JsonObjectToVocabularyTransformer; import org.upm.inesdata.vocabulary.validator.VocabularyValidator; +import java.util.Map; + import static org.eclipse.edc.spi.constants.CoreConstants.JSON_LD; import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.EDC_VOCABULARY_TYPE; -import java.util.Map; - /** * Extension that provides an API for managing vocabularies */ @@ -74,14 +74,6 @@ public VocabularyService vocabularyService() { return new VocabularyServiceImpl(vocabularyIndex, transactionContext); } - /** - * Provides a default in memory vocabularyIndex - */ - @Provider(isDefault = true) - public VocabularyIndex defaultVocabularyIndex() { - return getVocabularyIndex(); - } - /** * Initializes the service */ @@ -107,14 +99,4 @@ public void initialize(ServiceExtensionContext context) { healthCheckService.addLivenessProvider(() -> successResult); } } - - /** - * Creates a InMemoryVocabularyIndex if not exists - */ - private InMemoryVocabularyIndex getVocabularyIndex() { - if (defaultVocabularyIndex == null) { - defaultVocabularyIndex = new InMemoryVocabularyIndex(); - } - return defaultVocabularyIndex; - } } diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/controller/VocabularyApi.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/controller/VocabularyApi.java index 32475d3..e96d394 100644 --- a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/controller/VocabularyApi.java +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/controller/VocabularyApi.java @@ -13,6 +13,7 @@ import jakarta.json.JsonObject; import org.eclipse.edc.api.model.ApiCoreSchema; import org.eclipse.edc.connector.controlplane.contract.spi.types.offer.ContractOffer; +import org.upm.inesdata.spi.vocabulary.domain.Vocabulary; import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.ID; import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.TYPE; @@ -126,8 +127,7 @@ record VocabularyOutputSchema( "@id": "vocabularyId", "name": "vocabulary name", "jsonSchema": "{ \\"title\\": \\"vocabulary\\", \\"type\\": \\"object\\", \\"properties\\": { \\"name\\": { \\"type\\": \\"string\\", \\"title\\": \\"Name\\" }, \\"dct:keyword\\": { \\"type\\": \\"array\\", \\"title\\": \\"Keywords\\", \\"items\\": { \\"type\\": \\"string\\" } } }, \\"required\\": [ \\"name\\" ], \\"@context\\": { \\"dct\\": \\"http:\\/\\/purl.org\\/dc\\/terms\\/\" } }", - "category": "dataset", - "defaultVocabulary": true + "category": "dataset" } """; } diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/controller/VocabularyApiController.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/controller/VocabularyApiController.java index 48bc1d6..523115c 100644 --- a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/controller/VocabularyApiController.java +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/controller/VocabularyApiController.java @@ -11,7 +11,7 @@ import jakarta.ws.rs.PathParam; import jakarta.ws.rs.Produces; import jakarta.ws.rs.core.MediaType; - +import org.eclipse.edc.api.model.IdResponse; import org.eclipse.edc.spi.EdcException; import org.eclipse.edc.spi.monitor.Monitor; import org.eclipse.edc.spi.result.Result; @@ -28,8 +28,6 @@ import static org.eclipse.edc.web.spi.exception.ServiceResultHandler.exceptionMapper; import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.EDC_VOCABULARY_TYPE; -import org.eclipse.edc.api.model.IdResponse; - /** * Implementation of the controller for {@link Vocabulary} managing. */ diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/service/VocabularyServiceImpl.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/service/VocabularyServiceImpl.java index e44f803..9ed1d0b 100644 --- a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/service/VocabularyServiceImpl.java +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/service/VocabularyServiceImpl.java @@ -2,7 +2,6 @@ import org.eclipse.edc.spi.result.ServiceResult; import org.eclipse.edc.transaction.spi.TransactionContext; - import org.upm.inesdata.spi.vocabulary.VocabularyIndex; import org.upm.inesdata.spi.vocabulary.VocabularyService; import org.upm.inesdata.spi.vocabulary.domain.Vocabulary; @@ -43,19 +42,10 @@ public ServiceResult> search() { @Override public ServiceResult create(Vocabulary vocabulary) { return transactionContext.execute(() -> { - // Managing default vocabulary - var defaultVocabulary = index.getDefaultVocabulary(); - // Create new vocabulary var createResult = index.create(vocabulary); if (createResult.succeeded()) { - // There is only a default vocabulary - if (defaultVocabulary != null && vocabulary.isDefaultVocabulary()) { - defaultVocabulary.setDefaultVocabulary(false); - index.updateVocabulary(defaultVocabulary); - } - return ServiceResult.success(vocabulary); } return ServiceResult.fromFailure(createResult); @@ -73,19 +63,8 @@ public ServiceResult delete(String vocabularyId) { @Override public ServiceResult update(Vocabulary vocabulary) { return transactionContext.execute(() -> { - // Managing default vocabulary - var defaultVocabulary = index.getDefaultVocabulary(); - // Update vocabulary var updatedVocabulary = index.updateVocabulary(vocabulary); - - // There is only a default vocabulary - if (defaultVocabulary != null && vocabulary.isDefaultVocabulary() && - ! defaultVocabulary.getId().equals(vocabulary.getId())) { - defaultVocabulary.setDefaultVocabulary(false); - index.updateVocabulary(defaultVocabulary); - } - return ServiceResult.from(updatedVocabulary); }); } diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/storage/InMemoryVocabularyIndex.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/storage/InMemoryVocabularyIndex.java index 27e3f3c..d961d79 100644 --- a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/storage/InMemoryVocabularyIndex.java +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/storage/InMemoryVocabularyIndex.java @@ -1,8 +1,8 @@ package org.upm.inesdata.vocabulary.storage; import org.eclipse.edc.spi.result.StoreResult; -import org.upm.inesdata.spi.vocabulary.domain.Vocabulary; import org.upm.inesdata.spi.vocabulary.VocabularyIndex; +import org.upm.inesdata.spi.vocabulary.domain.Vocabulary; import java.util.Map; import java.util.Objects; @@ -103,18 +103,4 @@ public StoreResult updateVocabulary(Vocabulary vocabulary) { private Vocabulary delete(String vocabularyId) { return cache.remove(vocabularyId); } - - @Override - public Vocabulary getDefaultVocabulary() { - lock.readLock().lock(); - try { - return cache.values().stream() - .filter(vocabulary -> vocabulary.isDefaultVocabulary()) - .findFirst() - .orElse(null); - } finally { - lock.readLock().unlock(); - } - } - } diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectFromVocabularyTransformer.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectFromVocabularyTransformer.java index 7d44812..811cec0 100644 --- a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectFromVocabularyTransformer.java +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectFromVocabularyTransformer.java @@ -3,7 +3,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import jakarta.json.JsonBuilderFactory; import jakarta.json.JsonObject; - import org.eclipse.edc.jsonld.spi.transformer.AbstractJsonLdTransformer; import org.eclipse.edc.transform.spi.TransformerContext; import org.jetbrains.annotations.NotNull; @@ -12,12 +11,10 @@ import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.ID; import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.TYPE; - import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.EDC_VOCABULARY_TYPE; +import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_CATEGORY; import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_JSON_SCHEMA; import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_NAME; -import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_CATEGORY; -import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_DEFAULT_VOCABULARY; /** @@ -43,8 +40,7 @@ public JsonObjectFromVocabularyTransformer(JsonBuilderFactory jsonFactory, Objec .add(TYPE, EDC_VOCABULARY_TYPE) .add(PROPERTY_NAME, vocabulary.getName()) .add(PROPERTY_JSON_SCHEMA, vocabulary.getJsonSchema()) - .add(PROPERTY_CATEGORY, vocabulary.getCategory()) - .add(PROPERTY_DEFAULT_VOCABULARY, vocabulary.isDefaultVocabulary()); + .add(PROPERTY_CATEGORY, vocabulary.getCategory()); return builder.build(); } diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectToVocabularyTransformer.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectToVocabularyTransformer.java index 989b458..b46d88e 100644 --- a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectToVocabularyTransformer.java +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectToVocabularyTransformer.java @@ -7,10 +7,9 @@ import org.jetbrains.annotations.Nullable; import org.upm.inesdata.spi.vocabulary.domain.Vocabulary; +import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_CATEGORY; import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_JSON_SCHEMA; import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_NAME; -import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_CATEGORY; -import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_DEFAULT_VOCABULARY; /** * Converts from an {@link Vocabulary} as a {@link JsonObject} in JSON-LD expanded form to an {@link Vocabulary}. @@ -33,14 +32,9 @@ public JsonObjectToVocabularyTransformer() { case PROPERTY_NAME -> value -> builder.name(transformString(value, context)); case PROPERTY_JSON_SCHEMA -> value -> builder.jsonSchema(transformString(value, context)); case PROPERTY_CATEGORY -> value -> builder.category(transformString(value, context)); - case PROPERTY_DEFAULT_VOCABULARY -> value -> builder.defaultVocabulary(transformBoolean(value, context)); default -> doNothing(); }); - if (!jsonObject.containsKey(PROPERTY_DEFAULT_VOCABULARY)) { - builder.defaultVocabulary(false); - } - return builderResult(builder::build, context); } diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/validator/MandatoryJsonField.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/validator/MandatoryJsonField.java index 02dfd40..d21e323 100644 --- a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/validator/MandatoryJsonField.java +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/validator/MandatoryJsonField.java @@ -4,11 +4,11 @@ import jakarta.json.JsonObject; import jakarta.json.JsonReader; import jakarta.json.stream.JsonParsingException; -import java.io.StringReader; import org.eclipse.edc.validator.jsonobject.JsonLdPath; import org.eclipse.edc.validator.spi.ValidationResult; import org.eclipse.edc.validator.spi.Validator; +import java.io.StringReader; import java.util.Optional; import static java.lang.String.format; diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/validator/VocabularyValidator.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/validator/VocabularyValidator.java index e7d0723..494c487 100644 --- a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/validator/VocabularyValidator.java +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/validator/VocabularyValidator.java @@ -6,9 +6,9 @@ import org.eclipse.edc.validator.jsonobject.validators.OptionalIdNotBlank; import org.eclipse.edc.validator.spi.Validator; +import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_CATEGORY; import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_JSON_SCHEMA; import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_NAME; -import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_CATEGORY; /** * Validator for Vocabulary diff --git a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/SqlVocabularyIndex.java b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/SqlVocabularyIndex.java index db63787..6cb0121 100644 --- a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/SqlVocabularyIndex.java +++ b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/SqlVocabularyIndex.java @@ -9,10 +9,9 @@ import org.eclipse.edc.transaction.datasource.spi.DataSourceRegistry; import org.eclipse.edc.transaction.spi.TransactionContext; import org.jetbrains.annotations.Nullable; - +import org.upm.inesdata.spi.vocabulary.VocabularyIndex; import org.upm.inesdata.spi.vocabulary.domain.Vocabulary; import org.upm.inesdata.vocabulary.sql.index.schema.VocabularyStatements; -import org.upm.inesdata.spi.vocabulary.VocabularyIndex; import java.sql.Connection; import java.sql.ResultSet; @@ -83,7 +82,6 @@ public StoreResult create(Vocabulary vocabulary) { vocabulary.getCreatedAt(), vocabulary.getName(), vocabulary.getCategory(), - vocabulary.isDefaultVocabulary(), toJson(vocabulary.getJsonSchema()) ); @@ -124,7 +122,6 @@ public StoreResult updateVocabulary(Vocabulary vocabulary) { vocabulary.getName(), toJson(vocabulary.getJsonSchema()), vocabulary.getCategory(), - vocabulary.isDefaultVocabulary(), vocabularyId ); @@ -138,18 +135,6 @@ public StoreResult updateVocabulary(Vocabulary vocabulary) { }); } - @Override - public Vocabulary getDefaultVocabulary() { - try (var connection = getConnection()) { - var querySpec = QuerySpec.Builder.newInstance().filter(criterion("default_vocabulary", "=", true)).build(); - var statement = vocabularyStatements.createQuery(querySpec); - return queryExecutor.query(connection, true, this::mapVocabulary, statement.getQueryAsString(), statement.getParameters()) - .findFirst().orElse(null); - } catch (SQLException e) { - throw new EdcPersistenceException(e); - } - } - private int mapRowCount(ResultSet resultSet) throws SQLException { return resultSet.getInt(vocabularyStatements.getCountVariableName()); } @@ -167,7 +152,6 @@ private Vocabulary mapVocabulary(ResultSet resultSet) throws SQLException { .createdAt(resultSet.getLong(vocabularyStatements.getCreatedAtColumn())) .name(resultSet.getString(vocabularyStatements.getNameColumn())) .category(resultSet.getString(vocabularyStatements.getCategoryColumn())) - .defaultVocabulary(resultSet.getBoolean(vocabularyStatements.getDefaultVocabularyColumn())) .jsonSchema(resultSet.getString(vocabularyStatements.getJsonSchemaColumn())) .build(); } diff --git a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/BaseSqlDialectStatements.java b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/BaseSqlDialectStatements.java index 898b1e6..0725adc 100644 --- a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/BaseSqlDialectStatements.java +++ b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/BaseSqlDialectStatements.java @@ -1,9 +1,9 @@ package org.upm.inesdata.vocabulary.sql.index.schema; -import org.upm.inesdata.vocabulary.sql.index.schema.postgres.VocabularyMapping; import org.eclipse.edc.spi.query.QuerySpec; import org.eclipse.edc.sql.translation.SqlOperatorTranslator; import org.eclipse.edc.sql.translation.SqlQueryStatement; +import org.upm.inesdata.vocabulary.sql.index.schema.postgres.VocabularyMapping; import static java.lang.String.format; @@ -25,7 +25,6 @@ public String getInsertVocabularyTemplate() { .column(getCreatedAtColumn()) .column(getNameColumn()) .column(getCategoryColumn()) - .column(getDefaultVocabularyColumn()) .jsonColumn(getJsonSchemaColumn()) .insertInto(getVocabularyTable()); } @@ -36,7 +35,6 @@ public String getUpdateVocabularyTemplate() { .column(getNameColumn()) .jsonColumn(getJsonSchemaColumn()) .column(getCategoryColumn()) - .column(getDefaultVocabularyColumn()) .update(getVocabularyTable(), getVocabularyIdColumn()); } diff --git a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/VocabularyStatements.java b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/VocabularyStatements.java index 4bfb0c0..0ccb9ff 100644 --- a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/VocabularyStatements.java +++ b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/VocabularyStatements.java @@ -37,10 +37,6 @@ default String getCategoryColumn() { return "category"; } - default String getDefaultVocabularyColumn() { - return "default_vocabulary"; - } - default String getCreatedAtColumn() { return "created_at"; } diff --git a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/postgres/VocabularyMapping.java b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/postgres/VocabularyMapping.java index 5790fc8..dfd627c 100644 --- a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/postgres/VocabularyMapping.java +++ b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/postgres/VocabularyMapping.java @@ -15,7 +15,6 @@ public VocabularyMapping(VocabularyStatements statements) { add("createdAt", statements.getCreatedAtColumn()); add("name", statements.getNameColumn()); add("category", statements.getCategoryColumn()); - add("default_vocabulary", statements.getDefaultVocabularyColumn()); add("json_schema", new JsonFieldTranslator(statements.getJsonSchemaColumn())); } diff --git a/resources/sql/vocabulary-schema.sql b/resources/sql/vocabulary-schema.sql index 2208b90..cb8d1a8 100644 --- a/resources/sql/vocabulary-schema.sql +++ b/resources/sql/vocabulary-schema.sql @@ -5,8 +5,7 @@ CREATE TABLE IF NOT EXISTS edc_vocabulary created_at BIGINT NOT NULL, json_schema JSON DEFAULT '{}', name VARCHAR NOT NULL, - category VARCHAR NOT NULL, - default_vocabulary BOOLEAN DEFAULT FALSE, + category VARCHAR NOT NULL PRIMARY KEY (id) ); diff --git a/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/VocabularyIndex.java b/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/VocabularyIndex.java index dc3f408..8a89aeb 100644 --- a/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/VocabularyIndex.java +++ b/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/VocabularyIndex.java @@ -1,9 +1,9 @@ package org.upm.inesdata.spi.vocabulary; -import org.upm.inesdata.spi.vocabulary.domain.Vocabulary; import org.eclipse.edc.runtime.metamodel.annotation.ExtensionPoint; import org.eclipse.edc.spi.persistence.EdcPersistenceException; import org.eclipse.edc.spi.result.StoreResult; +import org.upm.inesdata.spi.vocabulary.domain.Vocabulary; import java.util.stream.Stream; @@ -58,12 +58,4 @@ public interface VocabularyIndex { */ StoreResult updateVocabulary(Vocabulary vocabulary); - /** - * Fetches the default {@link Vocabulary} from the metadata backend. - * - * @return The {@link Vocabulary} if one was found, or null otherwise. - * @throws NullPointerException If {@code vocabularyId} was null or empty. - */ - Vocabulary getDefaultVocabulary(); - } diff --git a/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/domain/Vocabulary.java b/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/domain/Vocabulary.java index 27c159a..9fece22 100644 --- a/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/domain/Vocabulary.java +++ b/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/domain/Vocabulary.java @@ -21,14 +21,12 @@ public class Vocabulary extends Entity { public static final String PROPERTY_NAME = EDC_NAMESPACE + "name"; public static final String PROPERTY_JSON_SCHEMA = EDC_NAMESPACE + "jsonSchema"; public static final String PROPERTY_CATEGORY = EDC_NAMESPACE + "category"; - public static final String PROPERTY_DEFAULT_VOCABULARY = EDC_NAMESPACE + "defaultVocabulary"; public static final String EDC_VOCABULARY_TYPE = EDC_NAMESPACE + "Vocabulary"; private String name; private String jsonSchema; private String category; - private boolean defaultVocabulary; private Vocabulary() { } @@ -45,13 +43,6 @@ public String getCategory() { return category; } - public boolean isDefaultVocabulary() { - return defaultVocabulary; - } - - public void setDefaultVocabulary(boolean _defaultVocabulary) { - defaultVocabulary = _defaultVocabulary; - } public Builder toBuilder() { return Vocabulary.Builder.newInstance() @@ -59,7 +50,6 @@ public Builder toBuilder() { .name(name) .jsonSchema(jsonSchema) .category(category) - .defaultVocabulary(defaultVocabulary) .createdAt(createdAt); } @@ -97,11 +87,6 @@ public Builder category(String category) { return self(); } - public Builder defaultVocabulary(boolean defaultVocabulary) { - entity.defaultVocabulary = defaultVocabulary; - return self(); - } - @Override public Builder createdAt(long value) { entity.createdAt = value; From 16807607039e07f235205efc8ee7f31f9e564673 Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Thu, 23 May 2024 08:26:41 +0200 Subject: [PATCH 056/174] schema sql --- extensions/vocabulary-index-sql/docs/schema.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/extensions/vocabulary-index-sql/docs/schema.sql b/extensions/vocabulary-index-sql/docs/schema.sql index 2208b90..cb8d1a8 100644 --- a/extensions/vocabulary-index-sql/docs/schema.sql +++ b/extensions/vocabulary-index-sql/docs/schema.sql @@ -5,8 +5,7 @@ CREATE TABLE IF NOT EXISTS edc_vocabulary created_at BIGINT NOT NULL, json_schema JSON DEFAULT '{}', name VARCHAR NOT NULL, - category VARCHAR NOT NULL, - default_vocabulary BOOLEAN DEFAULT FALSE, + category VARCHAR NOT NULL PRIMARY KEY (id) ); From 28466280475baa4f450bf5a510231c2cf752a002 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Fri, 24 May 2024 13:34:43 +0200 Subject: [PATCH 057/174] added Sovity policy-time-interval and policy-always-true extensions --- extensions/policy-always-true/LICENSE | 201 ++++++++++++++++++ extensions/policy-always-true/README.md | 31 +++ .../policy-always-true/build.gradle.kts | 11 + .../policy/AlwaysTruePolicyConstants.java | 25 +++ .../policy/AlwaysTruePolicyExtension.java | 66 ++++++ .../AlwaysTruePolicyDefinitionService.java | 71 +++++++ .../services/AlwaysTruePolicyService.java | 52 +++++ ...rg.eclipse.edc.spi.system.ServiceExtension | 1 + extensions/policy-time-interval/LICENSE | 201 ++++++++++++++++++ extensions/policy-time-interval/README.md | 35 +++ .../policy-time-interval/build.gradle.kts | 10 + .../policy/PolicyEvaluationTimeExtension.java | 54 +++++ .../policy/PolicyEvaluationTimeFunction.java | 52 +++++ ...rg.eclipse.edc.spi.system.ServiceExtension | 1 + gradle/libs.versions.toml | 4 +- launchers/connector/build.gradle.kts | 4 + settings.gradle.kts | 4 +- 17 files changed, 821 insertions(+), 2 deletions(-) create mode 100644 extensions/policy-always-true/LICENSE create mode 100644 extensions/policy-always-true/README.md create mode 100644 extensions/policy-always-true/build.gradle.kts create mode 100644 extensions/policy-always-true/src/main/java/de/sovity/edc/extension/policy/AlwaysTruePolicyConstants.java create mode 100644 extensions/policy-always-true/src/main/java/de/sovity/edc/extension/policy/AlwaysTruePolicyExtension.java create mode 100644 extensions/policy-always-true/src/main/java/de/sovity/edc/extension/policy/services/AlwaysTruePolicyDefinitionService.java create mode 100644 extensions/policy-always-true/src/main/java/de/sovity/edc/extension/policy/services/AlwaysTruePolicyService.java create mode 100644 extensions/policy-always-true/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension create mode 100644 extensions/policy-time-interval/LICENSE create mode 100644 extensions/policy-time-interval/README.md create mode 100644 extensions/policy-time-interval/build.gradle.kts create mode 100644 extensions/policy-time-interval/src/main/java/de/sovity/edc/extension/policy/PolicyEvaluationTimeExtension.java create mode 100644 extensions/policy-time-interval/src/main/java/de/sovity/edc/extension/policy/PolicyEvaluationTimeFunction.java create mode 100644 extensions/policy-time-interval/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension diff --git a/extensions/policy-always-true/LICENSE b/extensions/policy-always-true/LICENSE new file mode 100644 index 0000000..e4dafcf --- /dev/null +++ b/extensions/policy-always-true/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2022 sovity.de + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/extensions/policy-always-true/README.md b/extensions/policy-always-true/README.md new file mode 100644 index 0000000..fe61a0f --- /dev/null +++ b/extensions/policy-always-true/README.md @@ -0,0 +1,31 @@ + +
+
+ + Logo + + +

EDC-Connector Extension:
Always True Policy

+ +

+ Report Bug + · + Request Feature +

+
+ +## About this Extension +This extension creates a Policy Definition `always-true` on EDC startup. + +## Why does this extension exist? + +While the default behavior for contract definitions with empty policies is not "default deny", +our UI will be ensuring non-empty access and contract policies. + +Therefore, it is of interest to have an `always-true` policy to explicitly enable full access in contract definitions. + +## License +Apache License 2.0 - see [LICENSE](../../LICENSE) + +## Contact +sovity GmbH - contact@sovity.de diff --git a/extensions/policy-always-true/build.gradle.kts b/extensions/policy-always-true/build.gradle.kts new file mode 100644 index 0000000..bc0d897 --- /dev/null +++ b/extensions/policy-always-true/build.gradle.kts @@ -0,0 +1,11 @@ +plugins { + `java-library` + id("com.gmv.inesdata.edc-application") +} + +dependencies { + api(libs.edc.spi.core) + api(libs.edc.policy.engine.spi) + api(libs.edc.control.plane.spi) + implementation(libs.edc.api.core) +} diff --git a/extensions/policy-always-true/src/main/java/de/sovity/edc/extension/policy/AlwaysTruePolicyConstants.java b/extensions/policy-always-true/src/main/java/de/sovity/edc/extension/policy/AlwaysTruePolicyConstants.java new file mode 100644 index 0000000..ec9115b --- /dev/null +++ b/extensions/policy-always-true/src/main/java/de/sovity/edc/extension/policy/AlwaysTruePolicyConstants.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2022 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.extension.policy; + +public class AlwaysTruePolicyConstants { + public static final String EXTENSION_NAME = "Policy Function: ALWAYS_TRUE"; + public static final String EXPRESSION_LEFT_VALUE = "ALWAYS_TRUE"; + public static final String EXPRESSION_RIGHT_VALUE = "true"; + public static final String POLICY_DEFINITION_ID = "always-true"; + + private AlwaysTruePolicyConstants() { + } +} diff --git a/extensions/policy-always-true/src/main/java/de/sovity/edc/extension/policy/AlwaysTruePolicyExtension.java b/extensions/policy-always-true/src/main/java/de/sovity/edc/extension/policy/AlwaysTruePolicyExtension.java new file mode 100644 index 0000000..9d6d7bc --- /dev/null +++ b/extensions/policy-always-true/src/main/java/de/sovity/edc/extension/policy/AlwaysTruePolicyExtension.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2022 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.extension.policy; + +import de.sovity.edc.extension.policy.services.AlwaysTruePolicyDefinitionService; +import de.sovity.edc.extension.policy.services.AlwaysTruePolicyService; +import org.eclipse.edc.connector.controlplane.services.spi.policydefinition.PolicyDefinitionService; +import org.eclipse.edc.policy.engine.spi.PolicyEngine; +import org.eclipse.edc.policy.engine.spi.RuleBindingRegistry; +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.spi.monitor.Monitor; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; + +import static de.sovity.edc.extension.policy.AlwaysTruePolicyConstants.EXTENSION_NAME; + +/** + * Extension: Policy Definition "Always True". + */ +public class AlwaysTruePolicyExtension implements ServiceExtension { + @Inject + private Monitor monitor; + + @Inject + private RuleBindingRegistry ruleBindingRegistry; + + @Inject + private PolicyDefinitionService policyDefinitionService; + + @Inject + private PolicyEngine policyEngine; + + @Override + public String name() { + return EXTENSION_NAME; + } + + @Override + public void initialize(ServiceExtensionContext context) { + var alwaysTruePolicyService = new AlwaysTruePolicyService(ruleBindingRegistry, policyEngine); + alwaysTruePolicyService.registerPolicy(); + } + + @Override + public void start() { + var alwaysTruePolicyDefinitionService = new AlwaysTruePolicyDefinitionService(policyDefinitionService); + if (!alwaysTruePolicyDefinitionService.exists()) { + monitor.info("Creating Always True Policy Definition."); + alwaysTruePolicyDefinitionService.create(); + } else { + monitor.debug("Skipping Always True Policy Definition creation, policy definition already exists."); + } + } +} diff --git a/extensions/policy-always-true/src/main/java/de/sovity/edc/extension/policy/services/AlwaysTruePolicyDefinitionService.java b/extensions/policy-always-true/src/main/java/de/sovity/edc/extension/policy/services/AlwaysTruePolicyDefinitionService.java new file mode 100644 index 0000000..cd89f7c --- /dev/null +++ b/extensions/policy-always-true/src/main/java/de/sovity/edc/extension/policy/services/AlwaysTruePolicyDefinitionService.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2022 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.extension.policy.services; + +import org.eclipse.edc.connector.controlplane.policy.spi.PolicyDefinition; +import org.eclipse.edc.connector.controlplane.services.spi.policydefinition.PolicyDefinitionService; +import org.eclipse.edc.policy.model.Action; +import org.eclipse.edc.policy.model.AtomicConstraint; +import org.eclipse.edc.policy.model.LiteralExpression; +import org.eclipse.edc.policy.model.Operator; +import org.eclipse.edc.policy.model.Permission; +import org.eclipse.edc.policy.model.Policy; + +import static de.sovity.edc.extension.policy.AlwaysTruePolicyConstants.EXPRESSION_LEFT_VALUE; +import static de.sovity.edc.extension.policy.AlwaysTruePolicyConstants.EXPRESSION_RIGHT_VALUE; +import static de.sovity.edc.extension.policy.AlwaysTruePolicyConstants.POLICY_DEFINITION_ID; + +/** + * Creates policy definition "always-true". + */ +public class AlwaysTruePolicyDefinitionService { + private final PolicyDefinitionService policyDefinitionService; + + public AlwaysTruePolicyDefinitionService(PolicyDefinitionService policyDefinitionService) { + this.policyDefinitionService = policyDefinitionService; + } + + /** + * Checks if policy definition "always-true" exists + * + * @return if exists + */ + public boolean exists() { + return policyDefinitionService.findById(POLICY_DEFINITION_ID) != null; + } + + /** + * Creates policy definition "always-true". + */ + public void create() { + var alwaysTrueConstraint = AtomicConstraint.Builder.newInstance() + .leftExpression(new LiteralExpression(EXPRESSION_LEFT_VALUE)) + .operator(Operator.EQ) + .rightExpression(new LiteralExpression(EXPRESSION_RIGHT_VALUE)) + .build(); + var alwaysTruePermission = Permission.Builder.newInstance() + .action(Action.Builder.newInstance().type("USE").build()) + .constraint(alwaysTrueConstraint) + .build(); + var policy = Policy.Builder.newInstance() + .permission(alwaysTruePermission) + .build(); + var policyDefinition = PolicyDefinition.Builder.newInstance() + .id(POLICY_DEFINITION_ID) + .policy(policy) + .build(); + policyDefinitionService.create(policyDefinition); + } +} diff --git a/extensions/policy-always-true/src/main/java/de/sovity/edc/extension/policy/services/AlwaysTruePolicyService.java b/extensions/policy-always-true/src/main/java/de/sovity/edc/extension/policy/services/AlwaysTruePolicyService.java new file mode 100644 index 0000000..7c1b82a --- /dev/null +++ b/extensions/policy-always-true/src/main/java/de/sovity/edc/extension/policy/services/AlwaysTruePolicyService.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2022 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.extension.policy.services; + +import de.sovity.edc.extension.policy.AlwaysTruePolicyConstants; +import org.eclipse.edc.policy.engine.spi.PolicyEngine; +import org.eclipse.edc.policy.engine.spi.RuleBindingRegistry; +import org.eclipse.edc.policy.model.Operator; +import org.eclipse.edc.policy.model.Permission; + +import static org.eclipse.edc.policy.engine.spi.PolicyEngine.ALL_SCOPES; + +/** + * Creates policy "Always True". + *

+ * To be exact, it resolves to true iff constraint is {@link AlwaysTruePolicyConstants#EXPRESSION_LEFT_VALUE} + * "EQ" {@link AlwaysTruePolicyConstants#EXPRESSION_RIGHT_VALUE}. + */ +public class AlwaysTruePolicyService { + private final RuleBindingRegistry ruleBindingRegistry; + private final PolicyEngine policyEngine; + + public AlwaysTruePolicyService(RuleBindingRegistry ruleBindingRegistry, PolicyEngine policyEngine) { + this.ruleBindingRegistry = ruleBindingRegistry; + this.policyEngine = policyEngine; + } + + public void registerPolicy() { + ruleBindingRegistry.bind("USE", ALL_SCOPES); + ruleBindingRegistry.bind(AlwaysTruePolicyConstants.EXPRESSION_LEFT_VALUE, ALL_SCOPES); + policyEngine.registerFunction( + ALL_SCOPES, + Permission.class, + AlwaysTruePolicyConstants.EXPRESSION_LEFT_VALUE, + (operator, rightValue, rule, context1) -> operator.equals(Operator.EQ) && + rightValue.toString().equals(AlwaysTruePolicyConstants.EXPRESSION_RIGHT_VALUE) + ); + } + +} diff --git a/extensions/policy-always-true/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/policy-always-true/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 0000000..1882fd0 --- /dev/null +++ b/extensions/policy-always-true/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1 @@ +de.sovity.edc.extension.policy.AlwaysTruePolicyExtension diff --git a/extensions/policy-time-interval/LICENSE b/extensions/policy-time-interval/LICENSE new file mode 100644 index 0000000..e4dafcf --- /dev/null +++ b/extensions/policy-time-interval/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2022 sovity.de + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/extensions/policy-time-interval/README.md b/extensions/policy-time-interval/README.md new file mode 100644 index 0000000..e6c543c --- /dev/null +++ b/extensions/policy-time-interval/README.md @@ -0,0 +1,35 @@ + +
+

+ + Logo + + +

EDC-Connector Extension:
Time Interval Restricted Policy

+ +

+ Report Bug + · + Request Feature +

+
+ +## About this Extension + +This extension adds a policy function that validates the time of data consumption. + +Adds permission function with left side expression `POLICY_EVALUATION_TIME` and supported +operators `EQ`, `NEQ`, `LT`, `LEQ`, `GT`, `GEQ`. The right side expression is expected to be in the following date +format `yyyy-MM-dd'T'HH:mm:ss.SSSXXX`. + +## Why does this extension exist? + +Limiting data offers to specific valid durations. + +## License + +Apache License 2.0 - see [LICENSE](../../LICENSE) + +## Contact + +sovity GmbH - contact@sovity.de diff --git a/extensions/policy-time-interval/build.gradle.kts b/extensions/policy-time-interval/build.gradle.kts new file mode 100644 index 0000000..7f6ecec --- /dev/null +++ b/extensions/policy-time-interval/build.gradle.kts @@ -0,0 +1,10 @@ +plugins { + `java-library` + id("com.gmv.inesdata.edc-application") +} + +dependencies { + api(libs.edc.auth.spi) + api(libs.edc.policy.engine.spi) + testImplementation(libs.edc.core.junit) +} diff --git a/extensions/policy-time-interval/src/main/java/de/sovity/edc/extension/policy/PolicyEvaluationTimeExtension.java b/extensions/policy-time-interval/src/main/java/de/sovity/edc/extension/policy/PolicyEvaluationTimeExtension.java new file mode 100644 index 0000000..feb80ae --- /dev/null +++ b/extensions/policy-time-interval/src/main/java/de/sovity/edc/extension/policy/PolicyEvaluationTimeExtension.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2022 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.extension.policy; + +import org.eclipse.edc.policy.engine.spi.PolicyEngine; +import org.eclipse.edc.policy.engine.spi.RuleBindingRegistry; +import org.eclipse.edc.policy.model.Permission; +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; + +import static org.eclipse.edc.policy.engine.spi.PolicyEngine.ALL_SCOPES; + +public class PolicyEvaluationTimeExtension implements ServiceExtension { + + private static final String KEY_POLICY_EVALUATION_TIME = "POLICY_EVALUATION_TIME"; + private static final String EXTENSION_NAME = "Policy Function: POLICY_EVALUATION_TIME"; + + @Inject + private RuleBindingRegistry ruleBindingRegistry; + + @Inject + private PolicyEngine policyEngine; + + @Override + public String name() { + return EXTENSION_NAME; + } + + @Override + public void initialize(ServiceExtensionContext context) { + var monitor = context.getMonitor(); + + ruleBindingRegistry.bind("USE", ALL_SCOPES); + ruleBindingRegistry.bind(KEY_POLICY_EVALUATION_TIME, ALL_SCOPES); + policyEngine.registerFunction( + ALL_SCOPES, + Permission.class, + KEY_POLICY_EVALUATION_TIME, + new PolicyEvaluationTimeFunction(monitor)); + } +} diff --git a/extensions/policy-time-interval/src/main/java/de/sovity/edc/extension/policy/PolicyEvaluationTimeFunction.java b/extensions/policy-time-interval/src/main/java/de/sovity/edc/extension/policy/PolicyEvaluationTimeFunction.java new file mode 100644 index 0000000..0a46127 --- /dev/null +++ b/extensions/policy-time-interval/src/main/java/de/sovity/edc/extension/policy/PolicyEvaluationTimeFunction.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2022 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.extension.policy; + +import org.eclipse.edc.policy.engine.spi.AtomicConstraintFunction; +import org.eclipse.edc.policy.engine.spi.PolicyContext; +import org.eclipse.edc.policy.model.Operator; +import org.eclipse.edc.policy.model.Permission; +import org.eclipse.edc.spi.monitor.Monitor; + +import java.time.OffsetDateTime; +import java.time.format.DateTimeParseException; + +public class PolicyEvaluationTimeFunction implements AtomicConstraintFunction { + private final Monitor monitor; + + public PolicyEvaluationTimeFunction(Monitor monitor) { + this.monitor = monitor; + } + + @Override + public boolean evaluate(Operator operator, Object rightValue, Permission rule, PolicyContext context) { + try { + var policyDate = OffsetDateTime.parse((String) rightValue); + var nowDate = OffsetDateTime.now(); + return switch (operator) { + case LT -> nowDate.isBefore(policyDate); + case LEQ -> nowDate.isBefore(policyDate) || nowDate.equals(policyDate); + case GT -> nowDate.isAfter(policyDate); + case GEQ -> nowDate.isAfter(policyDate) || nowDate.equals(policyDate); + case EQ -> nowDate.equals(policyDate); + case NEQ -> !nowDate.equals(policyDate); + default -> false; + }; + } catch (DateTimeParseException e) { + monitor.severe("Failed to parse right value of constraint to date."); + return false; + } + } +} diff --git a/extensions/policy-time-interval/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/policy-time-interval/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 0000000..4deab3d --- /dev/null +++ b/extensions/policy-time-interval/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1 @@ +de.sovity.edc.extension.policy.PolicyEvaluationTimeExtension diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 9ec91d0..7c9ef3f 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -15,7 +15,7 @@ swagger-annotations-jakarta = "2.2.21" [libraries] edc-api-core = { module = "org.eclipse.edc:api-core", version.ref = "edc" } edc-api-management-config = { module = "org.eclipse.edc:management-api-configuration", version.ref = "edc" } -edc-auth-spi = { module = "org.eclipse.edc:api-core", version.ref = "edc" } +edc-auth-spi = { module = "org.eclipse.edc:auth-spi", version.ref = "edc" } edc-boot = { module = "org.eclipse.edc:boot", version.ref = "edc" } edc-build-plugin = { module = "org.eclipse.edc.edc-build:org.eclipse.edc.edc-build.gradle.plugin", version.ref = "edc" } edc-configuration-filesystem = { module = "org.eclipse.edc:configuration-filesystem", version.ref = "edc" } @@ -23,6 +23,7 @@ edc-connector-core = { module = "org.eclipse.edc:connector-core", version.ref = edc-control-plane-api = { module = "org.eclipse.edc:control-plane-api", version.ref = "edc" } edc-control-plane-api-client = { module = "org.eclipse.edc:control-plane-api-client", version.ref = "edc" } edc-control-plane-core = { module = "org.eclipse.edc:control-plane-core", version.ref = "edc" } +edc-control-plane-spi = { module = "org.eclipse.edc:control-plane-spi", version.ref = "edc" } edc-data-plane-api = { module = "org.eclipse.edc:data-plane-api", version.ref = "edc" } edc-data-plane-control-api = { module = "org.eclipse.edc:data-plane-control-api", version.ref = "edc" } edc-data-plane-core = { module = "org.eclipse.edc:data-plane-core", version.ref = "edc" } @@ -38,6 +39,7 @@ edc-iam-oauth2-service = { module = "org.eclipse.edc:oauth2-service", version.re edc-json-ld-lib = { module = "org.eclipse.edc:json-ld-lib", version.ref = "edc" } edc-management-api = { module = "org.eclipse.edc:management-api", version.ref = "edc" } edc-micrometer-core = { module = "org.eclipse.edc:micrometer-core", version.ref = "edc" } +edc-policy-engine-spi = { module = "org.eclipse.edc:policy-engine-spi", version.ref = "edc" } edc-spi-core = { module = "org.eclipse.edc:core-spi", version.ref = "edc" } edc-spi-jsonld = { module = "org.eclipse.edc:json-ld-spi", version.ref = "edc" } edc-spi-transform = { module = "org.eclipse.edc:transform-spi", version.ref = "edc" } diff --git a/launchers/connector/build.gradle.kts b/launchers/connector/build.gradle.kts index 726afe4..9c2ab18 100644 --- a/launchers/connector/build.gradle.kts +++ b/launchers/connector/build.gradle.kts @@ -39,6 +39,10 @@ dependencies { // Vocabularios implementation(project(":extensions:vocabulary-api")) + // Policies + implementation(project(":extensions:policy-always-true")) + implementation(project(":extensions:policy-time-interval")) + // Persistencia comun implementation(libs.edc.sql.core) implementation(libs.edc.sql.edr) diff --git a/settings.gradle.kts b/settings.gradle.kts index 2375bba..e5c3707 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -9,8 +9,10 @@ rootProject.name = "inesdata-connector" include(":spi:vocabulary-spi") // Extensions -include(":extensions:participants-from-configuration") include(":extensions:auth-oauth2-jwt") +include(":extensions:participants-from-configuration") +include(":extensions:policy-always-true") +include(":extensions:policy-time-interval") include(":extensions:vocabulary-api") include(":extensions:vocabulary-index-sql") From 9d87a28c62242eb38a13d782e31c172e9baf40ea Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Wed, 29 May 2024 14:13:46 +0200 Subject: [PATCH 058/174] Endpoint nuevo storageasset --- extensions/store-asset-api/README.md | 19 +++ extensions/store-asset-api/build.gradle.kts | 33 +++++ .../StorageAssetApiExtension.java | 99 ++++++++++++++ .../storageasset/config/AppConfig.java | 18 +++ .../controller/StorageAssetApi.java | 50 +++++++ .../controller/StorageAssetApiController.java | 125 ++++++++++++++++++ .../storageasset/service/S3Service.java | 102 ++++++++++++++ ...rg.eclipse.edc.spi.system.ServiceExtension | 1 + gradle/libs.versions.toml | 10 +- launchers/connector/build.gradle.kts | 3 + settings.gradle.kts | 1 + 11 files changed, 460 insertions(+), 1 deletion(-) create mode 100644 extensions/store-asset-api/README.md create mode 100644 extensions/store-asset-api/build.gradle.kts create mode 100644 extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/StorageAssetApiExtension.java create mode 100644 extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/config/AppConfig.java create mode 100644 extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/controller/StorageAssetApi.java create mode 100644 extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/controller/StorageAssetApiController.java create mode 100644 extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/service/S3Service.java create mode 100644 extensions/store-asset-api/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension diff --git a/extensions/store-asset-api/README.md b/extensions/store-asset-api/README.md new file mode 100644 index 0000000..6fbc14d --- /dev/null +++ b/extensions/store-asset-api/README.md @@ -0,0 +1,19 @@ +# Vocabulary API + +Provides a management API for work with vocabularies. This API expands the functionality of the control-plane management API to be able to handle Vocabulary entities. + + +## Vocabulary entity + +An example of a Vocabulary entity is shown below. + +``` +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@id": "vocabularyId", + "name": "Vocabulary name", + "jsonSchema": "{ \"title\": \"vocabulary\", \"type\": \"object\", \"properties\": { \"name\": { \"type\": \"string\", \"title\": \"Name\" }, \"keyword\": { \"type\": \"array\", \"title\": \"Keywords\", \"items\": { \"type\": \"string\" } } }, \"required\": [ \"name\" ] }" +} +``` diff --git a/extensions/store-asset-api/build.gradle.kts b/extensions/store-asset-api/build.gradle.kts new file mode 100644 index 0000000..0d222d9 --- /dev/null +++ b/extensions/store-asset-api/build.gradle.kts @@ -0,0 +1,33 @@ +plugins { + `java-library` + id("com.gmv.inesdata.edc-application") +} + +dependencies { + api(libs.edc.spi.asset) + api(libs.edc.spi.core) + implementation(libs.edc.spi.transform) + implementation(libs.edc.web.spi) + + implementation(libs.edc.connector.core) + implementation(libs.edc.api.core) + implementation(libs.edc.lib.util) + implementation(libs.edc.lib.transform) + implementation(libs.edc.dsp.api.configuration) + implementation(libs.edc.api.management.config) + implementation(libs.edc.transaction.spi) + implementation(libs.edc.lib.validator) + implementation(libs.edc.validator.spi) + implementation(libs.swagger.annotations.jakarta) + implementation(libs.jakarta.rsApi) + implementation(libs.jakarta.eeApi) + implementation(libs.parsson) + implementation(libs.jersey) + implementation(libs.edc.json.ld.lib) + implementation(libs.aws.s3) + implementation(libs.aws.s3.transfer) + implementation(libs.edc.api.asset) + implementation(libs.edc.control.plane.transform) + runtimeOnly(libs.edc.spi.jsonld) + runtimeOnly(libs.edc.json.ld.lib) +} diff --git a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/StorageAssetApiExtension.java b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/StorageAssetApiExtension.java new file mode 100644 index 0000000..c604b2f --- /dev/null +++ b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/StorageAssetApiExtension.java @@ -0,0 +1,99 @@ +package org.upm.inesdata.storageasset; + +import jakarta.json.Json; +import org.eclipse.edc.api.validation.DataAddressValidator; +import org.eclipse.edc.connector.api.management.configuration.ManagementApiConfiguration; +import org.eclipse.edc.connector.controlplane.api.management.asset.validation.AssetValidator; +import org.eclipse.edc.connector.controlplane.services.spi.asset.AssetService; +import org.eclipse.edc.connector.controlplane.transform.edc.from.JsonObjectFromAssetTransformer; +import org.eclipse.edc.connector.controlplane.transform.edc.to.JsonObjectToAssetTransformer; +import org.eclipse.edc.jsonld.spi.JsonLd; +import org.eclipse.edc.runtime.metamodel.annotation.Extension; +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; +import org.eclipse.edc.spi.types.TypeManager; +import org.eclipse.edc.transaction.spi.TransactionContext; +import org.eclipse.edc.transform.spi.TypeTransformerRegistry; +import org.eclipse.edc.validator.spi.JsonObjectValidatorRegistry; +import org.eclipse.edc.web.spi.WebService; +import org.upm.inesdata.storageasset.controller.StorageAssetApiController; +import org.upm.inesdata.storageasset.service.S3Service; +import software.amazon.awssdk.regions.Region; + +import java.util.Map; + +import static org.eclipse.edc.connector.controlplane.asset.spi.domain.Asset.EDC_ASSET_TYPE; +import static org.eclipse.edc.spi.constants.CoreConstants.JSON_LD; +import static org.eclipse.edc.spi.types.domain.DataAddress.EDC_DATA_ADDRESS_TYPE; +/** + * Extension that provides an API for managing vocabularies + */ +@Extension(value = StorageAssetApiExtension.NAME) +public class StorageAssetApiExtension implements ServiceExtension { + + public static final String NAME = "StorageAsset API Extension"; + + @Inject + private AssetService assetService; + + @Inject + private WebService webService; + + @Inject + private ManagementApiConfiguration config; + + @Inject + private TypeManager typeManager; + + @Inject + private TransactionContext transactionContext; + + @Inject + private TypeTransformerRegistry transformerRegistry; + + @Inject + private JsonObjectValidatorRegistry validator; + @Inject + private JsonLd jsonLd; + + + @Override + public String name() { + return NAME; + } + + /** + * Initializes the service + */ + @Override + public void initialize(ServiceExtensionContext context) { + var monitor = context.getMonitor(); + + var managementApiTransformerRegistry = transformerRegistry.forContext("management-api"); + + var factory = Json.createBuilderFactory(Map.of()); + var jsonLdMapper = typeManager.getMapper(JSON_LD); + managementApiTransformerRegistry.register(new JsonObjectFromAssetTransformer(factory, jsonLdMapper)); + managementApiTransformerRegistry.register(new JsonObjectToAssetTransformer()); + + validator.register(EDC_ASSET_TYPE, AssetValidator.instance()); + validator.register(EDC_DATA_ADDRESS_TYPE, DataAddressValidator.instance()); + + // Leer las variables de entorno + String accessKey = context.getSetting("edc.aws.access.key",""); + String secretKey = context.getSetting("edc.aws.secret.access.key",""); + String endpointOverride = context.getSetting("edc.aws.endpoint.override",""); + String regionName = context.getSetting("edc.aws.region",""); + String bucketName = context.getSetting("edc.aws.bucket.name",""); + + Region region = Region.of(regionName); + + // Crear una instancia de S3Service + S3Service s3Service = new S3Service(accessKey, secretKey, endpointOverride, region, bucketName); + + var storageAssetApiController = new StorageAssetApiController(assetService, managementApiTransformerRegistry, monitor, validator,s3Service, + jsonLd, context); + webService.registerResource(config.getContextAlias(), storageAssetApiController); + } +} diff --git a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/config/AppConfig.java b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/config/AppConfig.java new file mode 100644 index 0000000..d73e79e --- /dev/null +++ b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/config/AppConfig.java @@ -0,0 +1,18 @@ +package org.upm.inesdata.storageasset.config; + +import jakarta.ws.rs.ApplicationPath; +import jakarta.ws.rs.core.Application; +import org.upm.inesdata.storageasset.controller.StorageAssetApiController; + +import java.util.HashSet; +import java.util.Set; + +@ApplicationPath("/api") +public class AppConfig extends Application { + @Override + public Set> getClasses() { + Set> classes = new HashSet<>(); + classes.add(StorageAssetApiController.class); + return classes; + } +} \ No newline at end of file diff --git a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/controller/StorageAssetApi.java b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/controller/StorageAssetApi.java new file mode 100644 index 0000000..9392365 --- /dev/null +++ b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/controller/StorageAssetApi.java @@ -0,0 +1,50 @@ +package org.upm.inesdata.storageasset.controller; + +import io.swagger.v3.oas.annotations.OpenAPIDefinition; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.info.Info; +import io.swagger.v3.oas.annotations.media.ArraySchema; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.json.JsonObject; +import jakarta.ws.rs.core.MediaType; +import org.eclipse.edc.api.model.ApiCoreSchema; +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; +import org.glassfish.jersey.media.multipart.FormDataParam; + +import java.io.InputStream; + +@OpenAPIDefinition( + info = @Info(description = "Manages the connector s3 assets.", + title = "S3 Asset API", version = "1")) +@Tag(name = "S3Asset") +public interface StorageAssetApi { + + /** + * Creates a new storage asset + * + * @param fileInputStream the input stream of the file to be uploaded + * @param fileDetail the details of the file to be uploaded + * @param assetJson the input stream of the asset metadata in JSON format + * @return JsonObject with the created asset + */ + @Operation(description = "Creates a new S3 asset", + requestBody = @RequestBody(content = @Content(mediaType = MediaType.MULTIPART_FORM_DATA, schema = @Schema( + type = "object", requiredProperties = {"file", "json"} + ))), + responses = { + @ApiResponse(responseCode = "200", description = "S3 asset was created successfully", + content = @Content(schema = @Schema(implementation = ApiCoreSchema.IdResponseSchema.class))), + @ApiResponse(responseCode = "400", description = "Request body was malformed", + content = @Content(array = @ArraySchema(schema = @Schema(implementation = ApiCoreSchema.ApiErrorDetailSchema.class)))), + @ApiResponse(responseCode = "409", description = "Could not create asset, because an asset with that ID already exists", + content = @Content(array = @ArraySchema(schema = @Schema(implementation = ApiCoreSchema.ApiErrorDetailSchema.class)))) + } + ) + JsonObject createStorageAsset(@FormDataParam("file") InputStream fileInputStream, + @FormDataParam("file") FormDataContentDisposition fileDetail, + @FormDataParam("json") JsonObject assetJson); +} diff --git a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/controller/StorageAssetApiController.java b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/controller/StorageAssetApiController.java new file mode 100644 index 0000000..4cc64a8 --- /dev/null +++ b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/controller/StorageAssetApiController.java @@ -0,0 +1,125 @@ +package org.upm.inesdata.storageasset.controller; + +import jakarta.json.JsonObject; +import jakarta.servlet.annotation.MultipartConfig; +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import org.eclipse.edc.api.model.IdResponse; +import org.eclipse.edc.connector.controlplane.asset.spi.domain.Asset; +import org.eclipse.edc.connector.controlplane.services.spi.asset.AssetService; +import org.eclipse.edc.jsonld.spi.JsonLd; +import org.eclipse.edc.spi.EdcException; +import org.eclipse.edc.spi.monitor.Monitor; +import org.eclipse.edc.spi.system.ServiceExtensionContext; +import org.eclipse.edc.transform.spi.TypeTransformerRegistry; +import org.eclipse.edc.validator.spi.JsonObjectValidatorRegistry; +import org.eclipse.edc.web.spi.exception.InvalidRequestException; +import org.eclipse.edc.web.spi.exception.ValidationFailureException; +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; +import org.glassfish.jersey.media.multipart.FormDataParam; +import org.upm.inesdata.storageasset.service.S3Service; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; + +import static org.eclipse.edc.connector.controlplane.asset.spi.domain.Asset.EDC_ASSET_TYPE; +import static org.eclipse.edc.web.spi.exception.ServiceResultHandler.exceptionMapper; + +@MultipartConfig +@Consumes(MediaType.MULTIPART_FORM_DATA) +@Produces(MediaType.APPLICATION_JSON) +@Path("/s3assets") +public class StorageAssetApiController implements StorageAssetApi { + private final TypeTransformerRegistry transformerRegistry; + private final AssetService service; + private final Monitor monitor; + private final JsonObjectValidatorRegistry validator; + private final S3Service s3Service; + + private final JsonLd jsonLd; + + private final ServiceExtensionContext context; + + public StorageAssetApiController(AssetService service, TypeTransformerRegistry transformerRegistry, Monitor monitor, + JsonObjectValidatorRegistry validator, S3Service s3Service, JsonLd jsonLd, ServiceExtensionContext context) { + this.transformerRegistry = transformerRegistry; + this.service = service; + this.monitor = monitor; + this.validator = validator; + this.s3Service = s3Service; + this.jsonLd = jsonLd; + this.context = context; + } + + @POST + @Override + public JsonObject createStorageAsset(@FormDataParam("file") InputStream fileInputStream, + @FormDataParam("file") FormDataContentDisposition fileDetail, @FormDataParam("json") JsonObject assetJson) { + + String fileName = fileDetail.getFileName(); + + InputStream bufferedInputStream = new BufferedInputStream(fileInputStream); + + JsonObject expand = jsonLd.expand(assetJson).orElseThrow((f) -> new EdcException("Failed to expand request")); + // Validación + validator.validate(EDC_ASSET_TYPE, expand).orElseThrow(ValidationFailureException::new); + + // Transformación + var asset = transformerRegistry.transform(expand, Asset.class).orElseThrow(InvalidRequestException::new); + + // Guardar fichero en MinIO + // Calcular el tamaño del fichero manualmente + long contentLength = 0; + try { + contentLength = getFileSize(bufferedInputStream); + } catch (IOException e) { + throw new EdcException("Failed to process file size", e); + } + s3Service.uploadFile(fileName, bufferedInputStream, contentLength); + try { + setStorageProperties(asset, fileName); + + // Creación de asset + var idResponse = service.create(asset) + .map(a -> IdResponse.Builder.newInstance().id(a.getId()).createdAt(a.getCreatedAt()).build()) + .orElseThrow(exceptionMapper(Asset.class, asset.getId())); + + return transformerRegistry.transform(idResponse, JsonObject.class) + .orElseThrow(f -> new EdcException(f.getFailureDetail())); + } catch (Exception e) { + // Eliminar el archivo en caso de fallo + s3Service.deleteFile(fileName); + throw new EdcException("Failed to process multipart data", e); + } + } + + private long getFileSize(InputStream inputStream) throws IOException { + byte[] buffer = new byte[8192]; + int bytesRead; + long size = 0; + + inputStream.mark(Integer.MAX_VALUE); + + while ((bytesRead = inputStream.read(buffer)) != -1) { + size += bytesRead; + } + + inputStream.reset(); + + return size; + } + + private void setStorageProperties(Asset asset, String fileName) { + String regionName = context.getSetting("edc.aws.region", ""); + String bucketName = context.getSetting("edc.aws.bucket.name", ""); + asset.getPrivateProperties().put("storageAssetFile", fileName); + asset.getDataAddress().setKeyName(fileName); + asset.getDataAddress().setType("InesDataStore"); + asset.getDataAddress().getProperties().put("bucketName", bucketName); + asset.getDataAddress().getProperties().put("region", regionName); + } +} diff --git a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/service/S3Service.java b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/service/S3Service.java new file mode 100644 index 0000000..c6036cf --- /dev/null +++ b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/service/S3Service.java @@ -0,0 +1,102 @@ +package org.upm.inesdata.storageasset.service; + +import org.eclipse.edc.spi.EdcException; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3Configuration; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.transfer.s3.S3TransferManager; +import software.amazon.awssdk.transfer.s3.model.CompletedUpload; +import software.amazon.awssdk.transfer.s3.model.UploadRequest; +import software.amazon.awssdk.core.async.AsyncRequestBody; + +import java.io.InputStream; +import java.net.URI; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Servicio para manejar operaciones de almacenamiento en S3. + */ +public class S3Service { + private final S3AsyncClient s3AsyncClient; + private final S3TransferManager transferManager; + private final String bucketName; + private final ExecutorService executorService; + + public S3Service(String accessKey, String secretKey, String endpointOverride, Region region, String bucketName) { + this.s3AsyncClient = S3AsyncClient.builder() + .region(region) + .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKey, secretKey))) + .endpointOverride(URI.create(endpointOverride)) + .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()) + .build(); + this.transferManager = S3TransferManager.builder().s3Client(s3AsyncClient).build(); + this.bucketName = bucketName; + this.executorService = Executors.newFixedThreadPool(10); // Crear un pool de hilos fijo + } + + public String uploadFile(String key, InputStream inputStream, long contentLength) { + // Verificar si el archivo ya existe + boolean exists = doesObjectExist(bucketName, key).join(); + if (exists) { + throw new EdcException("File with key " + key + " already exists."); + } + + PutObjectRequest objectRequest = PutObjectRequest.builder() + .bucket(bucketName) + .key(key) + .build(); + + AsyncRequestBody requestBody = AsyncRequestBody.fromInputStream(inputStream, contentLength, executorService); + + UploadRequest uploadRequest = UploadRequest.builder() + .putObjectRequest(objectRequest) + .requestBody(requestBody) + .build(); + + CompletableFuture upload = transferManager.upload(uploadRequest).completionFuture(); + upload.join(); // Esperar a que la carga se complete + + return key; + } + + public void deleteFile(String key) { + DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder() + .bucket(bucketName) + .key(key) + .build(); + s3AsyncClient.deleteObject(deleteObjectRequest).join(); // Esperar a que se complete la eliminación + } + + public void close() { + transferManager.close(); + s3AsyncClient.close(); + executorService.shutdown(); + } + + public CompletableFuture doesObjectExist(String bucketName, String key) { + HeadObjectRequest headObjectRequest = HeadObjectRequest.builder() + .bucket(bucketName) + .key(key) + .build(); + + return s3AsyncClient.headObject(headObjectRequest) + .thenApply(response -> true) + .exceptionally(ex -> { + if (ex.getCause() instanceof NoSuchKeyException || (ex.getCause() instanceof S3Exception && ((S3Exception) ex.getCause()).statusCode() == 404)) { + return false; + } else { + throw new RuntimeException("Error checking if object exists", ex); + } + }); + } + +} diff --git a/extensions/store-asset-api/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/store-asset-api/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 0000000..6bdc8db --- /dev/null +++ b/extensions/store-asset-api/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1 @@ +org.upm.inesdata.storageasset.StorageAssetApiExtension \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 7c9ef3f..c9eb809 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -10,6 +10,9 @@ jupiter = "5.10.2" mockito = "5.2.0" postgres = "42.7.3" rsApi = "4.0.0" +rseeApi = "10.0.0" +parsson = "1.1.4" +jersey = "3.0.0" swagger-annotations-jakarta = "2.2.21" [libraries] @@ -24,6 +27,7 @@ edc-control-plane-api = { module = "org.eclipse.edc:control-plane-api", version. edc-control-plane-api-client = { module = "org.eclipse.edc:control-plane-api-client", version.ref = "edc" } edc-control-plane-core = { module = "org.eclipse.edc:control-plane-core", version.ref = "edc" } edc-control-plane-spi = { module = "org.eclipse.edc:control-plane-spi", version.ref = "edc" } +edc-control-plane-transform = { module = "org.eclipse.edc:control-plane-transform", version.ref = "edc" } edc-data-plane-api = { module = "org.eclipse.edc:data-plane-api", version.ref = "edc" } edc-data-plane-control-api = { module = "org.eclipse.edc:data-plane-control-api", version.ref = "edc" } edc-data-plane-core = { module = "org.eclipse.edc:data-plane-core", version.ref = "edc" } @@ -43,6 +47,8 @@ edc-policy-engine-spi = { module = "org.eclipse.edc:policy-engine-spi", version. edc-spi-core = { module = "org.eclipse.edc:core-spi", version.ref = "edc" } edc-spi-jsonld = { module = "org.eclipse.edc:json-ld-spi", version.ref = "edc" } edc-spi-transform = { module = "org.eclipse.edc:transform-spi", version.ref = "edc" } +edc-spi-asset = { module = "org.eclipse.edc:asset-spi", version.ref = "edc" } +edc-api-asset = { module = "org.eclipse.edc:asset-api", version.ref = "edc" } edc-transaction-local = { module = "org.eclipse.edc:transaction-local", version.ref = "edc" } edc-transaction-datasource-spi = { module = "org.eclipse.edc:transaction-datasource-spi", version.ref = "edc" } edc-transaction-spi = { module = "org.eclipse.edc:transaction-spi", version.ref = "edc" } @@ -52,7 +58,9 @@ edc-vault-filesystem = { module = "org.eclipse.edc:vault-filesystem", version.re edc-validator-spi = { module = "org.eclipse.edc:validator-spi", version.ref = "edc" } edc-web-spi = { module = "org.eclipse.edc:web-spi", version.ref = "edc" } jakarta-rsApi = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version.ref = "rsApi" } - +jakarta-eeApi = { module = "jakarta.platform:jakarta.jakartaee-api", version.ref = "rseeApi" } +parsson = { module = "org.eclipse.parsson:jakarta.json", version.ref = "parsson" } +jersey = { module = "org.glassfish.jersey.media:jersey-media-multipart", version.ref = "jersey" } edc-lib-transform = { module = "org.eclipse.edc:transform-lib", version.ref = "edc" } edc-lib-util = { module = "org.eclipse.edc:util-lib", version.ref = "edc" } edc-lib-validator = { module = "org.eclipse.edc:validator-lib", version.ref = "edc" } diff --git a/launchers/connector/build.gradle.kts b/launchers/connector/build.gradle.kts index 9c2ab18..5a131a6 100644 --- a/launchers/connector/build.gradle.kts +++ b/launchers/connector/build.gradle.kts @@ -74,6 +74,9 @@ dependencies { implementation(libs.edc.federated.catalog.spi) implementation(libs.edc.federated.catalog.core) implementation(libs.edc.federated.catalog.api) + + // Storage assets + implementation(project(":extensions:store-asset-api")) runtimeOnly(libs.edc.transaction.local) runtimeOnly(libs.postgres) diff --git a/settings.gradle.kts b/settings.gradle.kts index e5c3707..d127ea0 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -15,6 +15,7 @@ include(":extensions:policy-always-true") include(":extensions:policy-time-interval") include(":extensions:vocabulary-api") include(":extensions:vocabulary-index-sql") +include(":extensions:store-asset-api") // Connector include(":launchers:connector") From 06dd1579b4f2f745f1e504744fe525c2b463fa19 Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Wed, 29 May 2024 16:49:49 +0200 Subject: [PATCH 059/174] Correcciones --- extensions/store-asset-api/build.gradle.kts | 1 + .../StorageAssetApiExtension.java | 5 ++-- .../controller/StorageAssetApiController.java | 30 +++++++++---------- .../storageasset/service/S3Service.java | 8 +++-- 4 files changed, 24 insertions(+), 20 deletions(-) diff --git a/extensions/store-asset-api/build.gradle.kts b/extensions/store-asset-api/build.gradle.kts index 0d222d9..300da81 100644 --- a/extensions/store-asset-api/build.gradle.kts +++ b/extensions/store-asset-api/build.gradle.kts @@ -28,6 +28,7 @@ dependencies { implementation(libs.aws.s3.transfer) implementation(libs.edc.api.asset) implementation(libs.edc.control.plane.transform) + implementation(libs.edc.aws.s3.core) runtimeOnly(libs.edc.spi.jsonld) runtimeOnly(libs.edc.json.ld.lib) } diff --git a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/StorageAssetApiExtension.java b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/StorageAssetApiExtension.java index c604b2f..a6f5ef8 100644 --- a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/StorageAssetApiExtension.java +++ b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/StorageAssetApiExtension.java @@ -92,8 +92,9 @@ public void initialize(ServiceExtensionContext context) { // Crear una instancia de S3Service S3Service s3Service = new S3Service(accessKey, secretKey, endpointOverride, region, bucketName); - var storageAssetApiController = new StorageAssetApiController(assetService, managementApiTransformerRegistry, monitor, validator,s3Service, - jsonLd, context); + var storageAssetApiController = new StorageAssetApiController(assetService, managementApiTransformerRegistry, + validator,s3Service, + jsonLd, bucketName, regionName); webService.registerResource(config.getContextAlias(), storageAssetApiController); } } diff --git a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/controller/StorageAssetApiController.java b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/controller/StorageAssetApiController.java index 4cc64a8..8e208fa 100644 --- a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/controller/StorageAssetApiController.java +++ b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/controller/StorageAssetApiController.java @@ -12,9 +12,9 @@ import org.eclipse.edc.connector.controlplane.services.spi.asset.AssetService; import org.eclipse.edc.jsonld.spi.JsonLd; import org.eclipse.edc.spi.EdcException; -import org.eclipse.edc.spi.monitor.Monitor; -import org.eclipse.edc.spi.system.ServiceExtensionContext; +import org.eclipse.edc.spi.constants.CoreConstants; import org.eclipse.edc.transform.spi.TypeTransformerRegistry; +import org.eclipse.edc.util.string.StringUtils; import org.eclipse.edc.validator.spi.JsonObjectValidatorRegistry; import org.eclipse.edc.web.spi.exception.InvalidRequestException; import org.eclipse.edc.web.spi.exception.ValidationFailureException; @@ -36,23 +36,23 @@ public class StorageAssetApiController implements StorageAssetApi { private final TypeTransformerRegistry transformerRegistry; private final AssetService service; - private final Monitor monitor; private final JsonObjectValidatorRegistry validator; private final S3Service s3Service; private final JsonLd jsonLd; - private final ServiceExtensionContext context; + private final String bucketName; + private final String region; - public StorageAssetApiController(AssetService service, TypeTransformerRegistry transformerRegistry, Monitor monitor, - JsonObjectValidatorRegistry validator, S3Service s3Service, JsonLd jsonLd, ServiceExtensionContext context) { + public StorageAssetApiController(AssetService service, TypeTransformerRegistry transformerRegistry, + JsonObjectValidatorRegistry validator, S3Service s3Service, JsonLd jsonLd, String bucketName, String region) { this.transformerRegistry = transformerRegistry; this.service = service; - this.monitor = monitor; this.validator = validator; this.s3Service = s3Service; this.jsonLd = jsonLd; - this.context = context; + this.bucketName = bucketName; + this.region = region; } @POST @@ -79,9 +79,11 @@ public JsonObject createStorageAsset(@FormDataParam("file") InputStream fileInpu } catch (IOException e) { throw new EdcException("Failed to process file size", e); } - s3Service.uploadFile(fileName, bufferedInputStream, contentLength); + String folder = String.valueOf(asset.getDataAddress().getProperties().get(CoreConstants.EDC_NAMESPACE+"folder")); + String fullKey = StringUtils.isNullOrBlank(folder) || "null".equals(folder)?fileName:(folder.endsWith("/") ? folder + fileName : folder + "/" + fileName); + s3Service.uploadFile(fullKey,bufferedInputStream, contentLength); try { - setStorageProperties(asset, fileName); + setStorageProperties(asset, fullKey); // Creación de asset var idResponse = service.create(asset) @@ -92,7 +94,7 @@ public JsonObject createStorageAsset(@FormDataParam("file") InputStream fileInpu .orElseThrow(f -> new EdcException(f.getFailureDetail())); } catch (Exception e) { // Eliminar el archivo en caso de fallo - s3Service.deleteFile(fileName); + s3Service.deleteFile(fullKey); throw new EdcException("Failed to process multipart data", e); } } @@ -114,12 +116,10 @@ private long getFileSize(InputStream inputStream) throws IOException { } private void setStorageProperties(Asset asset, String fileName) { - String regionName = context.getSetting("edc.aws.region", ""); - String bucketName = context.getSetting("edc.aws.bucket.name", ""); asset.getPrivateProperties().put("storageAssetFile", fileName); asset.getDataAddress().setKeyName(fileName); asset.getDataAddress().setType("InesDataStore"); - asset.getDataAddress().getProperties().put("bucketName", bucketName); - asset.getDataAddress().getProperties().put("region", regionName); + asset.getDataAddress().getProperties().put(CoreConstants.EDC_NAMESPACE+ "bucketName", bucketName); + asset.getDataAddress().getProperties().put(CoreConstants.EDC_NAMESPACE+"region", region); } } diff --git a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/service/S3Service.java b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/service/S3Service.java index c6036cf..20a1527 100644 --- a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/service/S3Service.java +++ b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/service/S3Service.java @@ -1,6 +1,7 @@ package org.upm.inesdata.storageasset.service; import org.eclipse.edc.spi.EdcException; +import org.eclipse.edc.web.spi.exception.ObjectConflictException; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; @@ -44,10 +45,11 @@ public S3Service(String accessKey, String secretKey, String endpointOverride, Re } public String uploadFile(String key, InputStream inputStream, long contentLength) { + // Verificar si el archivo ya existe boolean exists = doesObjectExist(bucketName, key).join(); if (exists) { - throw new EdcException("File with key " + key + " already exists."); + throw new ObjectConflictException("File with key " + key + " already exists."); } PutObjectRequest objectRequest = PutObjectRequest.builder() @@ -69,9 +71,10 @@ public String uploadFile(String key, InputStream inputStream, long contentLength } public void deleteFile(String key) { + // Ajustar la clave para incluir la carpeta DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder() .bucket(bucketName) - .key(key) + .key(fullKey) .build(); s3AsyncClient.deleteObject(deleteObjectRequest).join(); // Esperar a que se complete la eliminación } @@ -98,5 +101,4 @@ public CompletableFuture doesObjectExist(String bucketName, String key) } }); } - } From f579747807fda7b3efd3dc3ea9eceacfbc72428a Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Wed, 29 May 2024 16:53:43 +0200 Subject: [PATCH 060/174] =?UTF-8?q?Error=20de=20compilaci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/org/upm/inesdata/storageasset/service/S3Service.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/service/S3Service.java b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/service/S3Service.java index 20a1527..204e79f 100644 --- a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/service/S3Service.java +++ b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/service/S3Service.java @@ -74,7 +74,7 @@ public void deleteFile(String key) { // Ajustar la clave para incluir la carpeta DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder() .bucket(bucketName) - .key(fullKey) + .key(key) .build(); s3AsyncClient.deleteObject(deleteObjectRequest).join(); // Esperar a que se complete la eliminación } From 1fd52161e492590d794ff598bf3929330136e18c Mon Sep 17 00:00:00 2001 From: Juan Calero Date: Fri, 31 May 2024 08:37:24 +0200 Subject: [PATCH 061/174] Adding SQL resources to the final image So the orchestration engine can be easily scripted to init the BD without the connector code --- .github/workflows/build-publish.yml | 5 ++--- docker/Dockerfile | 5 +++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-publish.yml b/.github/workflows/build-publish.yml index 91d43db..356caf5 100644 --- a/.github/workflows/build-publish.yml +++ b/.github/workflows/build-publish.yml @@ -6,7 +6,6 @@ on: branches: - master - develop - - feature/pipeline-* tags: - v* env: @@ -36,7 +35,7 @@ jobs: # - name: Push image run: | - IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME + IMAGE_ID=ghcr.io/$IMAGE_NAME # This changes all uppercase characters to lowercase. IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') @@ -45,7 +44,7 @@ jobs: # This strips the "v" prefix from the tag name. [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') # This uses the Docker `latest` tag convention. - [ "$VERSION" == "main" ] && VERSION=latest + [ "$VERSION" == "develop" ] && VERSION=latest echo IMAGE_ID=$IMAGE_ID echo VERSION=$VERSION docker tag $IMAGE_NAME $IMAGE_ID:$VERSION diff --git a/docker/Dockerfile b/docker/Dockerfile index 1c4a733..35eb42b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -19,6 +19,9 @@ RUN mkdir -p ${CONNECTOR_RESOURCES} RUN chown -R ${USER_NAME}:0 ${CONNECTOR_HOME} \ && chmod -R g=u ${CONNECTOR_HOME} +# Helpful utility while the connector is not able to create its own database +RUN apt-get update && apt-get install -y postgresql-client + # User by default USER ${USER_NAME} @@ -27,6 +30,8 @@ WORKDIR ${CONNECTOR_HOME} # Copy connector COPY $CONNECTOR_JAR connector.jar +COPY resources/* ${CONNECTOR_RESOURCES}/ + # Use "exec" for Kubernetes graceful termination (SIGINT) to reach JVM. ENTRYPOINT [ "sh", "-c", \ From ffc168194407de29fe26dfa8695d6898ffe970d5 Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Mon, 3 Jun 2024 17:23:36 +0200 Subject: [PATCH 062/174] Resueltos comentarios MR. --- extensions/store-asset-api/README.md | 20 ++----------------- .../storageasset/config/AppConfig.java | 18 ----------------- gradle/libs.versions.toml | 10 +++++----- 3 files changed, 7 insertions(+), 41 deletions(-) delete mode 100644 extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/config/AppConfig.java diff --git a/extensions/store-asset-api/README.md b/extensions/store-asset-api/README.md index 6fbc14d..022c913 100644 --- a/extensions/store-asset-api/README.md +++ b/extensions/store-asset-api/README.md @@ -1,19 +1,3 @@ -# Vocabulary API +# S3 Asset API -Provides a management API for work with vocabularies. This API expands the functionality of the control-plane management API to be able to handle Vocabulary entities. - - -## Vocabulary entity - -An example of a Vocabulary entity is shown below. - -``` -{ - "@context": { - "@vocab": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@id": "vocabularyId", - "name": "Vocabulary name", - "jsonSchema": "{ \"title\": \"vocabulary\", \"type\": \"object\", \"properties\": { \"name\": { \"type\": \"string\", \"title\": \"Name\" }, \"keyword\": { \"type\": \"array\", \"title\": \"Keywords\", \"items\": { \"type\": \"string\" } } }, \"required\": [ \"name\" ] }" -} -``` +Provides a management API for uploading files to S3 and creating assets. diff --git a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/config/AppConfig.java b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/config/AppConfig.java deleted file mode 100644 index d73e79e..0000000 --- a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/config/AppConfig.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.upm.inesdata.storageasset.config; - -import jakarta.ws.rs.ApplicationPath; -import jakarta.ws.rs.core.Application; -import org.upm.inesdata.storageasset.controller.StorageAssetApiController; - -import java.util.HashSet; -import java.util.Set; - -@ApplicationPath("/api") -public class AppConfig extends Application { - @Override - public Set> getClasses() { - Set> classes = new HashSet<>(); - classes.add(StorageAssetApiController.class); - return classes; - } -} \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index c9eb809..f13176c 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -16,6 +16,7 @@ jersey = "3.0.0" swagger-annotations-jakarta = "2.2.21" [libraries] +edc-api-asset = { module = "org.eclipse.edc:asset-api", version.ref = "edc" } edc-api-core = { module = "org.eclipse.edc:api-core", version.ref = "edc" } edc-api-management-config = { module = "org.eclipse.edc:management-api-configuration", version.ref = "edc" } edc-auth-spi = { module = "org.eclipse.edc:auth-spi", version.ref = "edc" } @@ -48,7 +49,6 @@ edc-spi-core = { module = "org.eclipse.edc:core-spi", version.ref = "edc" } edc-spi-jsonld = { module = "org.eclipse.edc:json-ld-spi", version.ref = "edc" } edc-spi-transform = { module = "org.eclipse.edc:transform-spi", version.ref = "edc" } edc-spi-asset = { module = "org.eclipse.edc:asset-spi", version.ref = "edc" } -edc-api-asset = { module = "org.eclipse.edc:asset-api", version.ref = "edc" } edc-transaction-local = { module = "org.eclipse.edc:transaction-local", version.ref = "edc" } edc-transaction-datasource-spi = { module = "org.eclipse.edc:transaction-datasource-spi", version.ref = "edc" } edc-transaction-spi = { module = "org.eclipse.edc:transaction-spi", version.ref = "edc" } @@ -57,10 +57,6 @@ edc-transfer-pull-http-receiver = { module = "org.eclipse.edc:transfer-pull-http edc-vault-filesystem = { module = "org.eclipse.edc:vault-filesystem", version.ref = "edc" } edc-validator-spi = { module = "org.eclipse.edc:validator-spi", version.ref = "edc" } edc-web-spi = { module = "org.eclipse.edc:web-spi", version.ref = "edc" } -jakarta-rsApi = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version.ref = "rsApi" } -jakarta-eeApi = { module = "jakarta.platform:jakarta.jakartaee-api", version.ref = "rseeApi" } -parsson = { module = "org.eclipse.parsson:jakarta.json", version.ref = "parsson" } -jersey = { module = "org.glassfish.jersey.media:jersey-media-multipart", version.ref = "jersey" } edc-lib-transform = { module = "org.eclipse.edc:transform-lib", version.ref = "edc" } edc-lib-util = { module = "org.eclipse.edc:util-lib", version.ref = "edc" } edc-lib-validator = { module = "org.eclipse.edc:validator-lib", version.ref = "edc" } @@ -95,6 +91,10 @@ edc-core-junit = { module = "org.eclipse.edc:junit", version.ref = "edc" } # No EDC references aws-s3 = { module = "software.amazon.awssdk:s3", version.ref = "aws" } aws-s3-transfer = { module = "software.amazon.awssdk:s3-transfer-manager", version.ref = "aws" } +jakarta-rsApi = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version.ref = "rsApi" } +jakarta-eeApi = { module = "jakarta.platform:jakarta.jakartaee-api", version.ref = "rseeApi" } +jersey = { module = "org.glassfish.jersey.media:jersey-media-multipart", version.ref = "jersey" } +parsson = { module = "org.eclipse.parsson:jakarta.json", version.ref = "parsson" } postgres = { module = "org.postgresql:postgresql", version.ref = "postgres" } swagger-annotations-jakarta = { module = "io.swagger.core.v3:swagger-annotations-jakarta", version.ref = "swagger-annotations-jakarta" } From 0aa7bf2572a0da9ab504d2748622f4e38cd400cb Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Wed, 5 Jun 2024 09:44:16 +0200 Subject: [PATCH 063/174] Error sintaxis vocabularios --- resources/sql/vocabulary-schema.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/sql/vocabulary-schema.sql b/resources/sql/vocabulary-schema.sql index cb8d1a8..c1883a4 100644 --- a/resources/sql/vocabulary-schema.sql +++ b/resources/sql/vocabulary-schema.sql @@ -5,7 +5,7 @@ CREATE TABLE IF NOT EXISTS edc_vocabulary created_at BIGINT NOT NULL, json_schema JSON DEFAULT '{}', name VARCHAR NOT NULL, - category VARCHAR NOT NULL + category VARCHAR NOT NULL, PRIMARY KEY (id) ); From 5c7e6d401387365b086e292db4065a4bfadc8fda Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Fri, 7 Jun 2024 10:50:31 +0200 Subject: [PATCH 064/174] fix vault keys --- .../inesdata/storageasset/StorageAssetApiExtension.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/StorageAssetApiExtension.java b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/StorageAssetApiExtension.java index a6f5ef8..f325388 100644 --- a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/StorageAssetApiExtension.java +++ b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/StorageAssetApiExtension.java @@ -10,6 +10,7 @@ import org.eclipse.edc.jsonld.spi.JsonLd; import org.eclipse.edc.runtime.metamodel.annotation.Extension; import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.spi.security.Vault; import org.eclipse.edc.spi.system.ServiceExtension; import org.eclipse.edc.spi.system.ServiceExtensionContext; import org.eclipse.edc.spi.types.TypeManager; @@ -57,6 +58,8 @@ public class StorageAssetApiExtension implements ServiceExtension { @Inject private JsonLd jsonLd; + @Inject + private Vault vault; @Override public String name() { @@ -81,8 +84,8 @@ public void initialize(ServiceExtensionContext context) { validator.register(EDC_DATA_ADDRESS_TYPE, DataAddressValidator.instance()); // Leer las variables de entorno - String accessKey = context.getSetting("edc.aws.access.key",""); - String secretKey = context.getSetting("edc.aws.secret.access.key",""); + var accessKey = vault.resolveSecret(context.getSetting("edc.aws.access.key", "")); + var secretKey = vault.resolveSecret(context.getSetting("edc.aws.secret.access.key", "")); String endpointOverride = context.getSetting("edc.aws.endpoint.override",""); String regionName = context.getSetting("edc.aws.region",""); String bucketName = context.getSetting("edc.aws.bucket.name",""); From 1b8db6bd3b277a32fa8d3001a69154eaa04b1bdd Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Fri, 7 Jun 2024 11:07:54 +0200 Subject: [PATCH 065/174] Improvement --- .../storageasset/StorageAssetApiExtension.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/StorageAssetApiExtension.java b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/StorageAssetApiExtension.java index f325388..319249e 100644 --- a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/StorageAssetApiExtension.java +++ b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/StorageAssetApiExtension.java @@ -34,7 +34,13 @@ public class StorageAssetApiExtension implements ServiceExtension { public static final String NAME = "StorageAsset API Extension"; - + public static final String DEFAULT_VALUE = ""; + public static final String AWS_ACCESS_KEY = "edc.aws.access.key"; + public static final String AWS_SECRET_ACCESS = "edc.aws.secret.access.key"; + public static final String AWS_ENDPOINT_OVERRIDE = "edc.aws.endpoint.override"; + public static final String AWS_REGION = "edc.aws.region"; + public static final String AWS_BUCKET_NAME = "edc.aws.bucket.name"; + @Inject private AssetService assetService; @@ -84,11 +90,11 @@ public void initialize(ServiceExtensionContext context) { validator.register(EDC_DATA_ADDRESS_TYPE, DataAddressValidator.instance()); // Leer las variables de entorno - var accessKey = vault.resolveSecret(context.getSetting("edc.aws.access.key", "")); - var secretKey = vault.resolveSecret(context.getSetting("edc.aws.secret.access.key", "")); - String endpointOverride = context.getSetting("edc.aws.endpoint.override",""); - String regionName = context.getSetting("edc.aws.region",""); - String bucketName = context.getSetting("edc.aws.bucket.name",""); + var accessKey = vault.resolveSecret(context.getSetting(AWS_ACCESS_KEY, DEFAULT_VALUE)); + var secretKey = vault.resolveSecret(context.getSetting(AWS_SECRET_ACCESS, DEFAULT_VALUE)); + var endpointOverride = context.getSetting(AWS_ENDPOINT_OVERRIDE, DEFAULT_VALUE); + var regionName = context.getSetting(AWS_REGION, DEFAULT_VALUE); + var bucketName = context.getSetting(AWS_BUCKET_NAME, DEFAULT_VALUE); Region region = Region.of(regionName); From 459b18d0bf1d4384bc02644f0bf35dfbd4de2bbc Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Thu, 13 Jun 2024 12:37:25 +0200 Subject: [PATCH 066/174] =?UTF-8?q?Persistencia=20cat=C3=A1logo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../federated-catalog-cache-sql/README.md | 30 +++ .../build.gradle.kts | 17 ++ .../federated-catalog-cache-sql/docs/er.puml | 9 + .../docs/schema.sql | 12 + .../SqlFederatedCacheServiceExtension.java | 69 +++++ .../sql/index/SqlFederatedCacheStore.java | 253 ++++++++++++++++++ .../schema/BaseSqlDialectStatements.java | 229 ++++++++++++++++ .../schema/SqlFederatedCatalogStatements.java | 151 +++++++++++ .../postgres/PostgresDialectStatements.java | 17 ++ .../postgres/SqlFederatedCatalogMapping.java | 19 ++ ...rg.eclipse.edc.spi.system.ServiceExtension | 1 + launchers/connector/build.gradle.kts | 1 + resources/sql/federated-catalog-schema.sql | 44 +++ settings.gradle.kts | 1 + 14 files changed, 853 insertions(+) create mode 100644 extensions/federated-catalog-cache-sql/README.md create mode 100644 extensions/federated-catalog-cache-sql/build.gradle.kts create mode 100644 extensions/federated-catalog-cache-sql/docs/er.puml create mode 100644 extensions/federated-catalog-cache-sql/docs/schema.sql create mode 100644 extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheServiceExtension.java create mode 100644 extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java create mode 100644 extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java create mode 100644 extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/SqlFederatedCatalogStatements.java create mode 100644 extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/postgres/PostgresDialectStatements.java create mode 100644 extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/postgres/SqlFederatedCatalogMapping.java create mode 100644 extensions/federated-catalog-cache-sql/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension create mode 100644 resources/sql/federated-catalog-schema.sql diff --git a/extensions/federated-catalog-cache-sql/README.md b/extensions/federated-catalog-cache-sql/README.md new file mode 100644 index 0000000..2e8e55c --- /dev/null +++ b/extensions/federated-catalog-cache-sql/README.md @@ -0,0 +1,30 @@ +# SQL Vocabulary + +Provides SQL persistence for vocabularies. + +## Prerequisites + +Please apply this [schema](docs/schema.sql) to your SQL database. + +## Entity Diagram + +![ER Diagram](//www.plantuml.com/plantuml/png/SoWkIImgAStDuKhDAyaigLH8JKcEByjFJamgpKaigbIevb9Gq5B8JB5IA2ufoinBLx2n2V2simEBvYNcfiB4mG9PnVbvmSaPgRc9ACB9HQc99QafZYLM2ZdvO35TNQvQBeVKl1IWnG00) + + +## Configuration + +| Key | Description | Mandatory | +|:---|:---|---| +| edc.datasource.vocabulary.name | Datasource used to store vocabularies | X | diff --git a/extensions/federated-catalog-cache-sql/build.gradle.kts b/extensions/federated-catalog-cache-sql/build.gradle.kts new file mode 100644 index 0000000..fbbfe23 --- /dev/null +++ b/extensions/federated-catalog-cache-sql/build.gradle.kts @@ -0,0 +1,17 @@ +plugins { + `java-library` + id("com.gmv.inesdata.edc-application") +} + +dependencies { + implementation(libs.edc.federated.catalog.api) + api(libs.edc.spi.core) + api(libs.edc.transaction.spi) + implementation(libs.edc.transaction.spi) + implementation(libs.edc.transaction.datasource.spi) + implementation(libs.edc.sql.core) + implementation(libs.edc.lib.util) + implementation(libs.edc.federated.catalog.core) +} + + diff --git a/extensions/federated-catalog-cache-sql/docs/er.puml b/extensions/federated-catalog-cache-sql/docs/er.puml new file mode 100644 index 0000000..f081ea7 --- /dev/null +++ b/extensions/federated-catalog-cache-sql/docs/er.puml @@ -0,0 +1,9 @@ +@startuml +entity edc_vocabulary { + * id: string <> + * name: string + * jsonSchema: string + * createdAt: long + -- +} +@enduml \ No newline at end of file diff --git a/extensions/federated-catalog-cache-sql/docs/schema.sql b/extensions/federated-catalog-cache-sql/docs/schema.sql new file mode 100644 index 0000000..cb8d1a8 --- /dev/null +++ b/extensions/federated-catalog-cache-sql/docs/schema.sql @@ -0,0 +1,12 @@ +-- table: edc_vocabulary +CREATE TABLE IF NOT EXISTS edc_vocabulary +( + id VARCHAR NOT NULL, + created_at BIGINT NOT NULL, + json_schema JSON DEFAULT '{}', + name VARCHAR NOT NULL, + category VARCHAR NOT NULL + PRIMARY KEY (id) +); + +COMMENT ON COLUMN edc_vocabulary.json_schema IS 'JSON Schema with the vocabulary'; diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheServiceExtension.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheServiceExtension.java new file mode 100644 index 0000000..3b5bc24 --- /dev/null +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheServiceExtension.java @@ -0,0 +1,69 @@ +package org.upm.inesdata.federated.sql.index; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.eclipse.edc.catalog.spi.FederatedCacheStore; +import org.eclipse.edc.runtime.metamodel.annotation.Extension; +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.runtime.metamodel.annotation.Provider; +import org.eclipse.edc.runtime.metamodel.annotation.Provides; +import org.eclipse.edc.runtime.metamodel.annotation.Setting; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; +import org.eclipse.edc.spi.types.TypeManager; +import org.eclipse.edc.sql.QueryExecutor; +import org.eclipse.edc.transaction.datasource.spi.DataSourceRegistry; +import org.eclipse.edc.transaction.spi.TransactionContext; +import org.upm.inesdata.federated.sql.index.schema.SqlFederatedCatalogStatements; +import org.upm.inesdata.federated.sql.index.schema.postgres.PostgresDialectStatements; + +/** + * Extension that stores vocabylaries in SQL databases + */ +@Provides({ FederatedCacheStore.class }) +@Extension(value = "SQL federatedCatalog index") +public class SqlFederatedCacheServiceExtension implements ServiceExtension { + + /** + * Name of the federatedCatalog datasource. + */ + @Setting(required = true) + public static final String DATASOURCE_SETTING_NAME = "edc.datasource.federatedCatalog.name"; + + @Inject + private DataSourceRegistry dataSourceRegistry; + + @Inject + private TransactionContext transactionContext; + + @Inject(required = false) + private SqlFederatedCatalogStatements dialect; + + @Inject + private TypeManager typeManager; + + @Inject + private QueryExecutor queryExecutor; + + + @Provider(isDefault = true) + public FederatedCacheStore defaultCacheStore() { + return new SqlFederatedCacheStore(dataSourceRegistry, DATASOURCE_SETTING_NAME,transactionContext,getObjectMapper(),dialect,queryExecutor); + } + + @Override + public void initialize(ServiceExtensionContext context) { + var dataSourceName = context.getConfig().getString(DATASOURCE_SETTING_NAME, DataSourceRegistry.DEFAULT_DATASOURCE); + + var sqlFederatedCacheStore = new SqlFederatedCacheStore(dataSourceRegistry, dataSourceName, transactionContext, typeManager.getMapper(), + getDialect(), queryExecutor); + + context.registerService(FederatedCacheStore.class, sqlFederatedCacheStore); + } + + private SqlFederatedCatalogStatements getDialect() { + return dialect != null ? dialect : new PostgresDialectStatements(); + } + private ObjectMapper getObjectMapper() { + return typeManager.getMapper(); + } +} diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java new file mode 100644 index 0000000..5bf5370 --- /dev/null +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java @@ -0,0 +1,253 @@ +package org.upm.inesdata.federated.sql.index; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.eclipse.edc.catalog.spi.FederatedCacheStore; +import org.eclipse.edc.connector.controlplane.catalog.spi.Catalog; +import org.eclipse.edc.connector.controlplane.catalog.spi.DataService; +import org.eclipse.edc.connector.controlplane.catalog.spi.Dataset; +import org.eclipse.edc.connector.controlplane.catalog.spi.Distribution; +import org.eclipse.edc.policy.model.Policy; +import org.eclipse.edc.spi.persistence.EdcPersistenceException; +import org.eclipse.edc.spi.query.Criterion; +import org.eclipse.edc.spi.query.QuerySpec; +import org.eclipse.edc.spi.result.StoreResult; +import org.eclipse.edc.sql.QueryExecutor; +import org.eclipse.edc.sql.store.AbstractSqlStore; +import org.eclipse.edc.sql.translation.SqlQueryStatement; +import org.eclipse.edc.transaction.datasource.spi.DataSourceRegistry; +import org.eclipse.edc.transaction.spi.TransactionContext; +import org.upm.inesdata.federated.sql.index.schema.SqlFederatedCatalogStatements; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +public class SqlFederatedCacheStore extends AbstractSqlStore implements FederatedCacheStore { + + private final SqlFederatedCatalogStatements databaseStatements; + + public SqlFederatedCacheStore(DataSourceRegistry dataSourceRegistry, + String dataSourceName, + TransactionContext transactionContext, + ObjectMapper objectMapper, + SqlFederatedCatalogStatements databaseStatements, + QueryExecutor queryExecutor) { + super(dataSourceRegistry, dataSourceName, transactionContext, objectMapper, queryExecutor); + this.databaseStatements = Objects.requireNonNull(databaseStatements); + } + @Override + public void save(Catalog catalog) { + Objects.requireNonNull(catalog); + + transactionContext.execute(() -> { + try (var connection = getConnection()) { + connection.setAutoCommit(false); + + if (catalog.getDataServices() != null) { + for (DataService dataService : catalog.getDataServices()) { + queryExecutor.execute(connection, databaseStatements.getInsertDataServiceTemplate(), + dataService.getId(), + dataService.getEndpointDescription(), + dataService.getEndpointUrl() + ); + queryExecutor.execute(connection, databaseStatements.getInsertCatalogDataServiceTemplate(), + catalog.getId(), + dataService.getId() + ); + } + } + + if (catalog.getDatasets() != null) { + for (Dataset dataset : catalog.getDatasets()) { + queryExecutor.execute(connection, databaseStatements.getInsertDatasetTemplate(), + dataset.getId(), + toJson(dataset.getOffers()), + toJson(dataset.getProperties()), + catalog.getId() + ); + + if (dataset.getDistributions() != null) { + for (Distribution distribution : dataset.getDistributions()) { + DataService dataService = distribution.getDataService(); + if (!dataServiceExists(dataService.getId())) { + queryExecutor.execute(connection, databaseStatements.getInsertDataServiceTemplate(), + dataService.getId(), + dataService.getEndpointDescription(), + dataService.getEndpointUrl() + ); + } + queryExecutor.execute(connection, databaseStatements.getInsertDistributionTemplate(), + distribution.getFormat(), + dataService.getId(), + dataset.getId() + ); + } + } + } + } + + queryExecutor.execute(connection, databaseStatements.getInsertCatalogTemplate(), + catalog.getId(), + catalog.getParticipantId(), + toJson(catalog.getProperties()), + false + ); + + connection.commit(); + return StoreResult.success(); + } catch (Exception e) { + try (var connection = getConnection()) { + connection.rollback(); + } catch (SQLException rollbackEx) { + throw new EdcPersistenceException("Failed to rollback transaction.", rollbackEx); + } + throw new EdcPersistenceException(e); + } + }); + } + + + @Override + public Collection query(List query) { + return transactionContext.execute(() -> { + try (var connection = getConnection()) { + QuerySpec querySpec = QuerySpec.Builder.newInstance().filter(query).build(); + SqlQueryStatement queryStatement = databaseStatements.createQuery(querySpec); + return queryExecutor.query(connection, true, this::mapResultSetToCatalog, queryStatement.getQueryAsString(), queryStatement.getParameters()).collect(Collectors.toList()); + } catch (SQLException e) { + throw new EdcPersistenceException(e); + } + }); + } + + + + + @Override + public void deleteExpired() { + transactionContext.execute(() -> { + try (var connection = getConnection()) { + queryExecutor.execute(connection, databaseStatements.getDeleteExpiredCatalogsTemplate(), true); + return null; + } catch (SQLException e) { + throw new EdcPersistenceException(e); + } + }); + } + + @Override + public void expireAll() { + transactionContext.execute(() -> { + try (var connection = getConnection()) { + queryExecutor.execute(connection, databaseStatements.getExpireAllCatalogsTemplate(), true, false); + return null; + } catch (SQLException e) { + throw new EdcPersistenceException(e); + } + }); + } + + private boolean dataServiceExists(String dataServiceId) throws SQLException { + return getDataServiceById(dataServiceId)!=null; + } + + private Catalog mapResultSetToCatalog(ResultSet resultSet) throws SQLException { + String id = resultSet.getString("id"); + String participantId = resultSet.getString("participant_id"); + Map properties = fromJson(resultSet.getString("properties"), Map.class); + List datasets = getDatasetsForCatalog(id); + List dataServices = getDataServicesForCatalog(id); + + return Catalog.Builder.newInstance() + .id(id) + .participantId(participantId) + .datasets(datasets) + .dataServices(dataServices) + .properties(properties) + .build(); + } + + private List getDatasetsForCatalog(String catalogId) { + try (var connection = getConnection()) { + String sql = databaseStatements.getSelectDatasetsForCatalogTemplate(); + return queryExecutor.query(connection, false, this::mapResultSetToDataset, sql, catalogId).collect(Collectors.toList()); + } catch (SQLException e) { + throw new EdcPersistenceException(e); + } + } + + private Dataset mapResultSetToDataset(ResultSet resultSet) throws SQLException { + String id = resultSet.getString("id"); + Map properties = fromJson(resultSet.getString("properties"), Map.class); + Map offers = fromJson(resultSet.getString("offers"), Map.class); + + List distributions = getDistributionsForDataset(id); + + Dataset.Builder datasetBuilder = Dataset.Builder.newInstance() + .id(id) + .properties(properties) + .distributions(distributions); + + if (offers != null) { + offers.forEach(datasetBuilder::offer); + } + + return datasetBuilder.build(); + } + + private List getDistributionsForDataset(String datasetId) { + try (var connection = getConnection()) { + String sql = databaseStatements.getSelectDistributionsForDatasetTemplate(); + return queryExecutor.query(connection, false, this::mapResultSetToDistribution, sql, datasetId).collect(Collectors.toList()); + } catch (SQLException e) { + throw new EdcPersistenceException(e); + } + } + + private Distribution mapResultSetToDistribution(ResultSet resultSet) throws SQLException { + String format = resultSet.getString("format"); + String dataServiceId = resultSet.getString("data_service_id"); + + DataService dataService = getDataServiceById(dataServiceId); + + return Distribution.Builder.newInstance() + .format(format) + .dataService(dataService) + .build(); + } + + private DataService getDataServiceById(String dataServiceId) { + try (var connection = getConnection()) { + String sql = databaseStatements.getSelectDataServicesForIdTemplate(); + return queryExecutor.query(connection, false, this::mapResultSetToDataService, sql, dataServiceId).findFirst().orElse(null); + } catch (SQLException e) { + throw new EdcPersistenceException(e); + } + } + + private List getDataServicesForCatalog(String catalogId) { + try (var connection = getConnection()) { + String sql = databaseStatements.getSelectDataServicesForCatalogTemplate(); + return queryExecutor.query(connection, false, this::mapResultSetToDataService, sql, catalogId).collect( + Collectors.toList()); + } catch (SQLException e) { + throw new EdcPersistenceException(e); + } + } + + private DataService mapResultSetToDataService(ResultSet resultSet) throws SQLException { + String id = resultSet.getString("id"); + String terms = resultSet.getString("terms"); + String endpointUrl = resultSet.getString("endpoint_url"); + + return DataService.Builder.newInstance() + .id(id) + .endpointDescription(terms) + .endpointUrl(endpointUrl) + .build(); + } +} diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java new file mode 100644 index 0000000..969cf73 --- /dev/null +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java @@ -0,0 +1,229 @@ +package org.upm.inesdata.federated.sql.index.schema; + +import org.eclipse.edc.spi.query.QuerySpec; +import org.eclipse.edc.sql.translation.SqlOperatorTranslator; +import org.eclipse.edc.sql.translation.SqlQueryStatement; +import org.upm.inesdata.federated.sql.index.schema.postgres.SqlFederatedCatalogMapping; + +import static java.lang.String.format; + +public class BaseSqlDialectStatements implements SqlFederatedCatalogStatements { + + protected final SqlOperatorTranslator operatorTranslator; + + public BaseSqlDialectStatements(SqlOperatorTranslator operatorTranslator) { + this.operatorTranslator = operatorTranslator; + } + + @Override + public String getInsertCatalogTemplate() { + return executeStatement() + .column(getCatalogIdColumn()) + .column(getCatalogParticipantIdColumn()) + .jsonColumn(getCatalogPropertiesColumn()) + .column(getCatalogExpiredColumn()) + .insertInto(getCatalogTable()); + } + + @Override + public String getUpdateCatalogTemplate() { + return executeStatement() + .column(getCatalogParticipantIdColumn()) + .jsonColumn(getCatalogPropertiesColumn()) + .column(getCatalogExpiredColumn()) + .update(getCatalogTable(), getCatalogIdColumn()); + } + + @Override + public String getCountCatalogByIdClause() { + return format("SELECT COUNT(*) AS %s FROM %s WHERE %s = ?", + getCountVariableName(), + getCatalogTable(), + getCatalogIdColumn()); + } + + @Override + public String getSelectCatalogTemplate() { + return format("SELECT * FROM %s AS a", getCatalogTable()); + } + + @Override + public String getDeleteCatalogByIdTemplate() { + return executeStatement() + .delete(getCatalogTable(), getCatalogIdColumn()); + } + + @Override + public String getInsertDataServiceTemplate() { + return executeStatement() + .column(getDataServiceIdColumn()) + .column(getDataServiceTermsColumn()) + .column(getDataServiceEndpointUrlColumn()) + .insertInto(getDataServiceTable()); + } + + @Override + public String getUpdateDataServiceTemplate() { + return executeStatement() + .column(getDataServiceTermsColumn()) + .column(getDataServiceEndpointUrlColumn()) + .update(getDataServiceTable(), getDataServiceIdColumn()); + } + + @Override + public String getCountDataServiceByIdClause() { + return format("SELECT COUNT(*) AS %s FROM %s WHERE %s = ?", + getCountVariableName(), + getDataServiceTable(), + getDataServiceIdColumn()); + } + + @Override + public String getSelectDataServiceTemplate() { + return format("SELECT * FROM %s AS a", getDataServiceTable()); + } + + @Override + public String getDeleteDataServiceByIdTemplate() { + return executeStatement() + .delete(getDataServiceTable(), getDataServiceIdColumn()); + } + + @Override + public String getInsertCatalogDataServiceTemplate() { + return executeStatement() + .column("catalog_id") + .column("dataset_id") + .insertInto(getCatalogDataServiceTable()); + } + + // Dataset CRUD methods + @Override + public String getInsertDatasetTemplate() { + return executeStatement() + .column(getDatasetIdColumn()) + .jsonColumn(getDatasetOffersColumn()) + .jsonColumn(getDatasetPropertiesColumn()) + .column(getDatasetCatalogIdColumn()) + .insertInto(getDatasetTable()); + } + + @Override + public String getUpdateDatasetTemplate() { + return executeStatement() + .jsonColumn(getDatasetOffersColumn()) + .jsonColumn(getDatasetPropertiesColumn()) + .column(getDatasetCatalogIdColumn()) + .update(getDatasetTable(), getDatasetIdColumn()); + } + + @Override + public String getCountDatasetByIdClause() { + return format("SELECT COUNT(*) AS %s FROM %s WHERE %s = ?", + getCountVariableName(), + getDatasetTable(), + getDatasetIdColumn()); + } + + @Override + public String getSelectDatasetTemplate() { + return format("SELECT * FROM %s AS a", getDatasetTable()); + } + + @Override + public String getDeleteDatasetByIdTemplate() { + return executeStatement() + .delete(getDatasetTable(), getDatasetIdColumn()); + } + + @Override + public String getInsertDistributionTemplate() { + return executeStatement() + .column(getDistributionFormatColumn()) + .column(getDistributionDataServiceIdColumn()) + .column(getDistributionDatasetIdColumn()) + .insertInto(getDistributionTable()); + } + + @Override + public String getUpdateDistributionTemplate() { + return executeStatement() + .column(getDistributionFormatColumn()) + .column(getDistributionDataServiceIdColumn()) + .update(getDistributionTable(), getDistributionDatasetIdColumn()); + } + + @Override + public String getCountDistributionByIdClause() { + return format("SELECT COUNT(*) AS %s FROM %s WHERE %s = ?", + getCountVariableName(), + getDistributionTable(), + getDistributionDatasetIdColumn()); + } + + @Override + public String getSelectDistributionTemplate() { + return format("SELECT * FROM %s AS a", getDistributionTable()); + } + + @Override + public String getDeleteDistributionByIdTemplate() { + return executeStatement() + .delete(getDistributionTable(), getDistributionDatasetIdColumn()); + } + + @Override + public String getInsertDatasetDistributionTemplate() { + return executeStatement() + .column("dataset_id") + .column("distribution_id") + .insertInto("edc_dataset_distribution"); + } + @Override + public String getCountVariableName() { + return "COUNT"; + } + + @Override + public SqlQueryStatement createQuery(QuerySpec querySpec) { + return new SqlQueryStatement(getSelectCatalogTemplate(), querySpec, new SqlFederatedCatalogMapping(this), operatorTranslator); + + } + @Override + public String getDeleteExpiredCatalogsTemplate() { + return executeStatement() + .delete(getCatalogTable(), getCatalogExpiredColumn()); + } + + @Override + public String getExpireAllCatalogsTemplate() { + return executeStatement() + .column(getCatalogExpiredColumn()) + .update(getCatalogTable(), getCatalogExpiredColumn()); + } + + @Override + public String getSelectDatasetsForCatalogTemplate() { + return format("SELECT * FROM %s AS a WHERE catalog_id = ?v", getDatasetTable()); + } + + @Override + public String getSelectDistributionsForDatasetTemplate() { + return format("SELECT * FROM %s AS a WHERE dataset_id = ?v", getDistributionTable()); + } + @Override + public String getSelectDataServicesForCatalogTemplate() { + return format( + "SELECT ds.* FROM %s AS ds " + + "JOIN %s AS cds ON ds.id = cds.data_service_id " + + "WHERE cds.catalog_id = ?", + getDataServiceTable(), + getCatalogDataServiceTable() + ); + } + + @Override + public String getSelectDataServicesForIdTemplate() { + return format("SELECT * FROM %s AS a WHERE id = ?v", getDataServiceTable()); + } +} diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/SqlFederatedCatalogStatements.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/SqlFederatedCatalogStatements.java new file mode 100644 index 0000000..48a335f --- /dev/null +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/SqlFederatedCatalogStatements.java @@ -0,0 +1,151 @@ +package org.upm.inesdata.federated.sql.index.schema; + +import org.eclipse.edc.runtime.metamodel.annotation.ExtensionPoint; +import org.eclipse.edc.spi.query.QuerySpec; +import org.eclipse.edc.sql.statement.SqlStatements; +import org.eclipse.edc.sql.translation.SqlQueryStatement; + +@ExtensionPoint +public interface SqlFederatedCatalogStatements extends SqlStatements { + + // Catalog CRUD methods + default String getCatalogTable() { + return "edc_catalog"; + } + + default String getCatalogIdColumn() { + return "id"; + } + + default String getCatalogParticipantIdColumn() { + return "participant_id"; + } + + default String getCatalogPropertiesColumn() { + return "properties"; + } + + default String getCatalogExpiredColumn() { + return "expired"; + } + + String getInsertCatalogTemplate(); + + String getUpdateCatalogTemplate(); + + String getSelectCatalogTemplate(); + + String getCountCatalogByIdClause(); + + String getDeleteCatalogByIdTemplate(); + + // DataService CRUD methods + default String getDataServiceTable() { + return "edc_data_service"; + } + + default String getDataServiceIdColumn() { + return "id"; + } + + default String getDataServiceTermsColumn() { + return "terms"; + } + + default String getDataServiceEndpointUrlColumn() { + return "endpoint_url"; + } + + String getInsertDataServiceTemplate(); + + String getUpdateDataServiceTemplate(); + + String getSelectDataServiceTemplate(); + + String getCountDataServiceByIdClause(); + + String getDeleteDataServiceByIdTemplate(); + + String getInsertCatalogDataServiceTemplate(); + + default String getCatalogDataServiceTable() { + return "edc_catalog_data_service"; + } + + + // Dataset CRUD methods + default String getDatasetTable() { + return "edc_dataset"; + } + + default String getDatasetIdColumn() { + return "id"; + } + + default String getDatasetOffersColumn() { + return "offers"; + } + + default String getDatasetPropertiesColumn() { + return "properties"; + } + + default String getDatasetCatalogIdColumn() { + return "catalog_id"; + } + + String getInsertDatasetTemplate(); + + String getUpdateDatasetTemplate(); + + String getSelectDatasetTemplate(); + + String getCountDatasetByIdClause(); + + String getDeleteDatasetByIdTemplate(); + + + // Distribution CRUD methods + default String getDistributionTable() { + return "edc_distribution"; + } + + + default String getDistributionFormatColumn() { + return "format"; + } + + default String getDistributionDataServiceIdColumn() { + return "data_service_id"; + } + + default String getDistributionDatasetIdColumn() { + return "dataset_id"; + } + String getInsertDistributionTemplate(); + + String getUpdateDistributionTemplate(); + + String getSelectDistributionTemplate(); + + String getCountDistributionByIdClause(); + + String getDeleteDistributionByIdTemplate(); + + // Methods for creating SQL query using sub-select statements + SqlQueryStatement createQuery(QuerySpec query); + + String getDeleteExpiredCatalogsTemplate(); + + String getExpireAllCatalogsTemplate(); + String getInsertDatasetDistributionTemplate(); + String getCountVariableName(); + + String getSelectDatasetsForCatalogTemplate(); + + String getSelectDistributionsForDatasetTemplate(); + + String getSelectDataServicesForCatalogTemplate(); + + String getSelectDataServicesForIdTemplate(); +} diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/postgres/PostgresDialectStatements.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/postgres/PostgresDialectStatements.java new file mode 100644 index 0000000..c8085e8 --- /dev/null +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/postgres/PostgresDialectStatements.java @@ -0,0 +1,17 @@ +package org.upm.inesdata.federated.sql.index.schema.postgres; + +import org.upm.inesdata.federated.sql.index.schema.BaseSqlDialectStatements; +import org.eclipse.edc.sql.dialect.PostgresDialect; +import org.eclipse.edc.sql.translation.PostgresqlOperatorTranslator; + +public class PostgresDialectStatements extends BaseSqlDialectStatements { + + public PostgresDialectStatements() { + super(new PostgresqlOperatorTranslator()); + } + + @Override + public String getFormatAsJsonOperator() { + return PostgresDialect.getJsonCastOperator(); + } +} diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/postgres/SqlFederatedCatalogMapping.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/postgres/SqlFederatedCatalogMapping.java new file mode 100644 index 0000000..5efd016 --- /dev/null +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/postgres/SqlFederatedCatalogMapping.java @@ -0,0 +1,19 @@ +package org.upm.inesdata.federated.sql.index.schema.postgres; + +import org.eclipse.edc.sql.translation.JsonFieldTranslator; +import org.eclipse.edc.sql.translation.TranslationMapping; +import org.upm.inesdata.federated.sql.index.schema.SqlFederatedCatalogStatements; + +/** + * Maps fields of a onto the + * corresponding SQL schema (= column names) enabling access through Postgres JSON operators where applicable + */ +public class SqlFederatedCatalogMapping extends TranslationMapping { + + public SqlFederatedCatalogMapping(SqlFederatedCatalogStatements statements) { + add("id", statements.getCatalogIdColumn()); + add("offers", new JsonFieldTranslator(statements.getDatasetOffersColumn())); + add("properties", new JsonFieldTranslator(statements.getCatalogPropertiesColumn())); + } + +} diff --git a/extensions/federated-catalog-cache-sql/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/federated-catalog-cache-sql/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 0000000..d2ea9aa --- /dev/null +++ b/extensions/federated-catalog-cache-sql/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1 @@ +org.upm.inesdata.federated.sql.index.SqlFederatedCacheServiceExtension \ No newline at end of file diff --git a/launchers/connector/build.gradle.kts b/launchers/connector/build.gradle.kts index 5a131a6..a38e3dc 100644 --- a/launchers/connector/build.gradle.kts +++ b/launchers/connector/build.gradle.kts @@ -74,6 +74,7 @@ dependencies { implementation(libs.edc.federated.catalog.spi) implementation(libs.edc.federated.catalog.core) implementation(libs.edc.federated.catalog.api) + implementation(project(":extensions:federated-catalog-cache-sql")) // Storage assets implementation(project(":extensions:store-asset-api")) diff --git a/resources/sql/federated-catalog-schema.sql b/resources/sql/federated-catalog-schema.sql new file mode 100644 index 0000000..348152b --- /dev/null +++ b/resources/sql/federated-catalog-schema.sql @@ -0,0 +1,44 @@ +-- Tabla para Catalog +CREATE TABLE IF NOT EXISTS edc_catalog ( + id VARCHAR NOT NULL, + participant_id VARCHAR(255), + properties JSON DEFAULT '{}', + expired BOOLEAN, + PRIMARY KEY (id) +); + +-- Tabla para Dataset +CREATE TABLE IF NOT EXISTS edc_dataset ( + id VARCHAR NOT NULL, + offers JSON DEFAULT '{}', + properties JSON DEFAULT '{}', + catalog_id VARCHAR, + FOREIGN KEY (catalog_id) REFERENCES edc_catalog(id), + PRIMARY KEY (id) +); + +-- Tabla para DataService +CREATE TABLE IF NOT EXISTS edc_data_service ( + id VARCHAR NOT NULL, + terms VARCHAR(255), + endpoint_url VARCHAR(255), + PRIMARY KEY (id) +); + +-- Tabla para Distribution +CREATE TABLE IF NOT EXISTS edc_distribution ( + format VARCHAR(255), + data_service_id VARCHAR, + dataset_id VARCHAR, + FOREIGN KEY (data_service_id) REFERENCES edc_data_service(id), + FOREIGN KEY (dataset_id) REFERENCES edc_dataset(id) +); + +-- Tabla de relación entre Catalog y DataService (para la lista de dataServices en Catalog) +CREATE TABLE IF NOT EXISTS edc_catalog_data_service ( + catalog_id VARCHAR, + data_service_id VARCHAR, + PRIMARY KEY (catalog_id, data_service_id), + FOREIGN KEY (catalog_id) REFERENCES edc_catalog(id), + FOREIGN KEY (data_service_id) REFERENCES edc_data_service(id) +); \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index d127ea0..7730723 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -16,6 +16,7 @@ include(":extensions:policy-time-interval") include(":extensions:vocabulary-api") include(":extensions:vocabulary-index-sql") include(":extensions:store-asset-api") +include(":extensions:federated-catalog-cache-sql") // Connector include(":launchers:connector") From b1bbc1a2181287abf97362b584cf8f429278d49a Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Thu, 13 Jun 2024 12:55:01 +0200 Subject: [PATCH 067/174] Correcciones --- .../inesdata/federated/sql/index/SqlFederatedCacheStore.java | 2 +- .../federated/sql/index/schema/BaseSqlDialectStatements.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java index 5bf5370..bbf8e99 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java @@ -45,7 +45,7 @@ public void save(Catalog catalog) { transactionContext.execute(() -> { try (var connection = getConnection()) { - connection.setAutoCommit(false); +// connection.setAutoCommit(false); if (catalog.getDataServices() != null) { for (DataService dataService : catalog.getDataServices()) { diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java index 969cf73..bdcb2c4 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java @@ -93,7 +93,7 @@ public String getDeleteDataServiceByIdTemplate() { public String getInsertCatalogDataServiceTemplate() { return executeStatement() .column("catalog_id") - .column("dataset_id") + .column("data_service_id") .insertInto(getCatalogDataServiceTable()); } From 93ee33cfb638add5a07bbb6c703491d0eb7f44e6 Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Thu, 13 Jun 2024 13:01:24 +0200 Subject: [PATCH 068/174] En primer lgar catalog --- .../sql/index/SqlFederatedCacheStore.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java index bbf8e99..3197954 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java @@ -47,6 +47,13 @@ public void save(Catalog catalog) { try (var connection = getConnection()) { // connection.setAutoCommit(false); + queryExecutor.execute(connection, databaseStatements.getInsertCatalogTemplate(), + catalog.getId(), + catalog.getParticipantId(), + toJson(catalog.getProperties()), + false + ); + if (catalog.getDataServices() != null) { for (DataService dataService : catalog.getDataServices()) { queryExecutor.execute(connection, databaseStatements.getInsertDataServiceTemplate(), @@ -90,12 +97,7 @@ public void save(Catalog catalog) { } } - queryExecutor.execute(connection, databaseStatements.getInsertCatalogTemplate(), - catalog.getId(), - catalog.getParticipantId(), - toJson(catalog.getProperties()), - false - ); + connection.commit(); return StoreResult.success(); From eb017710c811b64f01924a4ef5509e3d80d89d98 Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Thu, 13 Jun 2024 13:16:29 +0200 Subject: [PATCH 069/174] =?UTF-8?q?Transacci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../federated/sql/index/SqlFederatedCacheStore.java | 7 ------- .../sql/index/schema/BaseSqlDialectStatements.java | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java index 3197954..efe74ce 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java @@ -98,15 +98,8 @@ public void save(Catalog catalog) { } - - connection.commit(); return StoreResult.success(); } catch (Exception e) { - try (var connection = getConnection()) { - connection.rollback(); - } catch (SQLException rollbackEx) { - throw new EdcPersistenceException("Failed to rollback transaction.", rollbackEx); - } throw new EdcPersistenceException(e); } }); diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java index bdcb2c4..beedf02 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java @@ -224,6 +224,6 @@ public String getSelectDataServicesForCatalogTemplate() { @Override public String getSelectDataServicesForIdTemplate() { - return format("SELECT * FROM %s AS a WHERE id = ?v", getDataServiceTable()); + return format("SELECT * FROM %s AS a WHERE id = ?", getDataServiceTable()); } } From b6da41b3e6ec926e709e189db6ecd9f4cf19ad77 Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Thu, 13 Jun 2024 13:48:37 +0200 Subject: [PATCH 070/174] borrado antes de persistencia --- .../sql/index/SqlFederatedCacheStore.java | 27 +++++++++++++++++++ .../schema/BaseSqlDialectStatements.java | 23 ++++++++++++++++ .../schema/SqlFederatedCatalogStatements.java | 8 ++++++ 3 files changed, 58 insertions(+) diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java index efe74ce..bbf992d 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java @@ -18,6 +18,7 @@ import org.eclipse.edc.transaction.spi.TransactionContext; import org.upm.inesdata.federated.sql.index.schema.SqlFederatedCatalogStatements; +import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.util.Collection; @@ -47,6 +48,9 @@ public void save(Catalog catalog) { try (var connection = getConnection()) { // connection.setAutoCommit(false); + // Eliminar toda la información relacionada con el catálogo + deleteRelatedCatalogData(connection, catalog.getId()); + queryExecutor.execute(connection, databaseStatements.getInsertCatalogTemplate(), catalog.getId(), catalog.getParticipantId(), @@ -146,6 +150,29 @@ public void expireAll() { }); } + + private void deleteRelatedCatalogData(Connection connection, String catalogId) throws SQLException { + // Eliminar las distribuciones relacionadas con los datasets del catálogo + String deleteDistributionsSql = databaseStatements.getDeleteDistributionsForCatalogTemplate(); + queryExecutor.execute(connection, deleteDistributionsSql, catalogId); + + // Eliminar las relaciones entre el catálogo y los servicios de datos + String deleteCatalogDataServicesSql = databaseStatements.getDeleteCatalogDataServicesTemplate(); + queryExecutor.execute(connection, deleteCatalogDataServicesSql, catalogId); + + // Eliminar los servicios de datos que no están relacionados con ningún catálogo o distribución + String deleteOrphanDataServicesSql = databaseStatements.getDeleteOrphanDataServicesTemplate(); + queryExecutor.execute(connection, deleteOrphanDataServicesSql); + + // Eliminar los datasets del catálogo + String deleteDatasetsSql = databaseStatements.getDeleteDatasetsForCatalogTemplate(); + queryExecutor.execute(connection, deleteDatasetsSql, catalogId); + + // Eliminar el propio catálogo + String deleteCatalogSql = databaseStatements.getDeleteCatalogByIdTemplate(); + queryExecutor.execute(connection, deleteCatalogSql, catalogId); + } + private boolean dataServiceExists(String dataServiceId) throws SQLException { return getDataServiceById(dataServiceId)!=null; } diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java index beedf02..58ece87 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java @@ -226,4 +226,27 @@ public String getSelectDataServicesForCatalogTemplate() { public String getSelectDataServicesForIdTemplate() { return format("SELECT * FROM %s AS a WHERE id = ?", getDataServiceTable()); } + + + @Override + public String getDeleteDistributionsForCatalogTemplate() { + return format("DELETE FROM %s WHERE dataset_id IN (SELECT id FROM %s WHERE catalog_id = ?)", getDistributionTable(), getDatasetTable()); + } + + @Override + public String getDeleteCatalogDataServicesTemplate() { + + return format("DELETE FROM %s WHERE catalog_id = ?",getCatalogDataServiceTable()); + } + + @Override + public String getDeleteOrphanDataServicesTemplate() { + return format("DELETE FROM %s WHERE id NOT IN (SELECT data_service_id FROM %s)" + + " AND id NOT IN (SELECT data_service_id FROM %s)",getDataServiceTable(), getCatalogDataServiceTable(), getDistributionTable()); + } + + @Override + public String getDeleteDatasetsForCatalogTemplate() { + return format("DELETE FROM %s WHERE catalog_id = ?",getDatasetTable()); + } } diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/SqlFederatedCatalogStatements.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/SqlFederatedCatalogStatements.java index 48a335f..e58e270 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/SqlFederatedCatalogStatements.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/SqlFederatedCatalogStatements.java @@ -148,4 +148,12 @@ default String getDistributionDatasetIdColumn() { String getSelectDataServicesForCatalogTemplate(); String getSelectDataServicesForIdTemplate(); + + String getDeleteDistributionsForCatalogTemplate(); + + String getDeleteCatalogDataServicesTemplate(); + + String getDeleteOrphanDataServicesTemplate(); + + String getDeleteDatasetsForCatalogTemplate(); } From 244a2e1c5b899e37ca7913c44116fb2a59969be0 Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Thu, 13 Jun 2024 14:25:10 +0200 Subject: [PATCH 071/174] =?UTF-8?q?Borrado=20en=20transacci=C3=B3n=20disti?= =?UTF-8?q?nta?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../federated/sql/index/SqlFederatedCacheStore.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java index bbf992d..1f2f5aa 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java @@ -43,14 +43,22 @@ public SqlFederatedCacheStore(DataSourceRegistry dataSourceRegistry, @Override public void save(Catalog catalog) { Objects.requireNonNull(catalog); - transactionContext.execute(() -> { try (var connection = getConnection()) { -// connection.setAutoCommit(false); + // connection.setAutoCommit(false); // Eliminar toda la información relacionada con el catálogo deleteRelatedCatalogData(connection, catalog.getId()); + return StoreResult.success(); + } catch (Exception e) { + throw new EdcPersistenceException(e); + } + }); + transactionContext.execute(() -> { + try (var connection = getConnection()) { +// connection.setAutoCommit(false); + queryExecutor.execute(connection, databaseStatements.getInsertCatalogTemplate(), catalog.getId(), catalog.getParticipantId(), From 9a8fb3b92dbb04427324a97ab94bb3421c1d94f5 Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Thu, 13 Jun 2024 15:52:38 +0200 Subject: [PATCH 072/174] Delete por participantId --- .../sql/index/SqlFederatedCacheStore.java | 14 +++++++------- .../sql/index/schema/BaseSqlDialectStatements.java | 4 ++-- .../schema/SqlFederatedCatalogStatements.java | 2 +- resources/sql/federated-catalog-schema.sql | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java index 1f2f5aa..24f8f1c 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java @@ -48,7 +48,7 @@ public void save(Catalog catalog) { // connection.setAutoCommit(false); // Eliminar toda la información relacionada con el catálogo - deleteRelatedCatalogData(connection, catalog.getId()); + deleteRelatedCatalogData(connection, catalog); return StoreResult.success(); } catch (Exception e) { @@ -159,14 +159,14 @@ public void expireAll() { } - private void deleteRelatedCatalogData(Connection connection, String catalogId) throws SQLException { + private void deleteRelatedCatalogData(Connection connection, Catalog catalog){ // Eliminar las distribuciones relacionadas con los datasets del catálogo String deleteDistributionsSql = databaseStatements.getDeleteDistributionsForCatalogTemplate(); - queryExecutor.execute(connection, deleteDistributionsSql, catalogId); + queryExecutor.execute(connection, deleteDistributionsSql, catalog.getId()); // Eliminar las relaciones entre el catálogo y los servicios de datos String deleteCatalogDataServicesSql = databaseStatements.getDeleteCatalogDataServicesTemplate(); - queryExecutor.execute(connection, deleteCatalogDataServicesSql, catalogId); + queryExecutor.execute(connection, deleteCatalogDataServicesSql, catalog.getId()); // Eliminar los servicios de datos que no están relacionados con ningún catálogo o distribución String deleteOrphanDataServicesSql = databaseStatements.getDeleteOrphanDataServicesTemplate(); @@ -174,11 +174,11 @@ private void deleteRelatedCatalogData(Connection connection, String catalogId) t // Eliminar los datasets del catálogo String deleteDatasetsSql = databaseStatements.getDeleteDatasetsForCatalogTemplate(); - queryExecutor.execute(connection, deleteDatasetsSql, catalogId); + queryExecutor.execute(connection, deleteDatasetsSql, catalog.getId()); // Eliminar el propio catálogo - String deleteCatalogSql = databaseStatements.getDeleteCatalogByIdTemplate(); - queryExecutor.execute(connection, deleteCatalogSql, catalogId); + String deleteCatalogSql = databaseStatements.getDeleteCatalogByParticipantIdTemplate(); + queryExecutor.execute(connection, deleteCatalogSql, catalog.getParticipantId()); } private boolean dataServiceExists(String dataServiceId) throws SQLException { diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java index 58ece87..4dd57b1 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java @@ -48,9 +48,9 @@ public String getSelectCatalogTemplate() { } @Override - public String getDeleteCatalogByIdTemplate() { + public String getDeleteCatalogByParticipantIdTemplate() { return executeStatement() - .delete(getCatalogTable(), getCatalogIdColumn()); + .delete(getCatalogTable(), getCatalogParticipantIdColumn()); } @Override diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/SqlFederatedCatalogStatements.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/SqlFederatedCatalogStatements.java index e58e270..2ccc766 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/SqlFederatedCatalogStatements.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/SqlFederatedCatalogStatements.java @@ -37,7 +37,7 @@ default String getCatalogExpiredColumn() { String getCountCatalogByIdClause(); - String getDeleteCatalogByIdTemplate(); + String getDeleteCatalogByParticipantIdTemplate(); // DataService CRUD methods default String getDataServiceTable() { diff --git a/resources/sql/federated-catalog-schema.sql b/resources/sql/federated-catalog-schema.sql index 348152b..1a6724d 100644 --- a/resources/sql/federated-catalog-schema.sql +++ b/resources/sql/federated-catalog-schema.sql @@ -1,10 +1,10 @@ -- Tabla para Catalog CREATE TABLE IF NOT EXISTS edc_catalog ( id VARCHAR NOT NULL, - participant_id VARCHAR(255), + participant_id VARCHAR(255) NOT NULL, properties JSON DEFAULT '{}', expired BOOLEAN, - PRIMARY KEY (id) + PRIMARY KEY (participant_id) ); -- Tabla para Dataset From fe5b8fa1135b5b1665e890d889549b42ce168fc7 Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Thu, 13 Jun 2024 16:27:31 +0200 Subject: [PATCH 073/174] Unique participant_id --- .../sql/index/SqlFederatedCacheStore.java | 50 +++++++++++++------ .../schema/BaseSqlDialectStatements.java | 6 +++ .../schema/SqlFederatedCatalogStatements.java | 2 + resources/sql/federated-catalog-schema.sql | 3 +- 4 files changed, 46 insertions(+), 15 deletions(-) diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java index 24f8f1c..ec9623b 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java @@ -161,24 +161,29 @@ public void expireAll() { private void deleteRelatedCatalogData(Connection connection, Catalog catalog){ // Eliminar las distribuciones relacionadas con los datasets del catálogo - String deleteDistributionsSql = databaseStatements.getDeleteDistributionsForCatalogTemplate(); - queryExecutor.execute(connection, deleteDistributionsSql, catalog.getId()); + Catalog catalogByParticipantId = getCatalogByParticipantId(catalog.getParticipantId()); - // Eliminar las relaciones entre el catálogo y los servicios de datos - String deleteCatalogDataServicesSql = databaseStatements.getDeleteCatalogDataServicesTemplate(); - queryExecutor.execute(connection, deleteCatalogDataServicesSql, catalog.getId()); + if(catalogByParticipantId!=null && catalogByParticipantId.getId()!=null){ + String deleteDistributionsSql = databaseStatements.getDeleteDistributionsForCatalogTemplate(); + queryExecutor.execute(connection, deleteDistributionsSql, catalogByParticipantId.getId()); - // Eliminar los servicios de datos que no están relacionados con ningún catálogo o distribución - String deleteOrphanDataServicesSql = databaseStatements.getDeleteOrphanDataServicesTemplate(); - queryExecutor.execute(connection, deleteOrphanDataServicesSql); + // Eliminar las relaciones entre el catálogo y los servicios de datos + String deleteCatalogDataServicesSql = databaseStatements.getDeleteCatalogDataServicesTemplate(); + queryExecutor.execute(connection, deleteCatalogDataServicesSql, catalogByParticipantId.getId()); - // Eliminar los datasets del catálogo - String deleteDatasetsSql = databaseStatements.getDeleteDatasetsForCatalogTemplate(); - queryExecutor.execute(connection, deleteDatasetsSql, catalog.getId()); + // Eliminar los servicios de datos que no están relacionados con ningún catálogo o distribución + String deleteOrphanDataServicesSql = databaseStatements.getDeleteOrphanDataServicesTemplate(); + queryExecutor.execute(connection, deleteOrphanDataServicesSql); + + // Eliminar los datasets del catálogo + String deleteDatasetsSql = databaseStatements.getDeleteDatasetsForCatalogTemplate(); + queryExecutor.execute(connection, deleteDatasetsSql, catalogByParticipantId.getId()); + + // Eliminar el propio catálogo + String deleteCatalogSql = databaseStatements.getDeleteCatalogByParticipantIdTemplate(); + queryExecutor.execute(connection, deleteCatalogSql, catalog.getParticipantId()); + } - // Eliminar el propio catálogo - String deleteCatalogSql = databaseStatements.getDeleteCatalogByParticipantIdTemplate(); - queryExecutor.execute(connection, deleteCatalogSql, catalog.getParticipantId()); } private boolean dataServiceExists(String dataServiceId) throws SQLException { @@ -259,6 +264,23 @@ private DataService getDataServiceById(String dataServiceId) { } } + private Catalog getCatalogByParticipantId(String participantId) { + try (var connection = getConnection()) { + String selectCatalog = databaseStatements.getSelectCatalogForParticipantIdTemplate(); + return queryExecutor.query(connection, false, this::mapResultSetToCatalogSimple, selectCatalog, participantId).findFirst().orElse(null); + } catch (SQLException e) { + throw new EdcPersistenceException(e); + } + } + + private Catalog mapResultSetToCatalogSimple(ResultSet resultSet) throws SQLException { + String id = resultSet.getString("id"); + return Catalog.Builder.newInstance() + .id(id) + .build(); + } + + private List getDataServicesForCatalog(String catalogId) { try (var connection = getConnection()) { String sql = databaseStatements.getSelectDataServicesForCatalogTemplate(); diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java index 4dd57b1..a806857 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java @@ -249,4 +249,10 @@ public String getDeleteOrphanDataServicesTemplate() { public String getDeleteDatasetsForCatalogTemplate() { return format("DELETE FROM %s WHERE catalog_id = ?",getDatasetTable()); } + + + @Override + public String getSelectCatalogForParticipantIdTemplate() { + return format("SELECT * FROM %s AS a WHERE participant_id = ?", getCatalogParticipantIdColumn()); + } } diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/SqlFederatedCatalogStatements.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/SqlFederatedCatalogStatements.java index 2ccc766..08be995 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/SqlFederatedCatalogStatements.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/SqlFederatedCatalogStatements.java @@ -156,4 +156,6 @@ default String getDistributionDatasetIdColumn() { String getDeleteOrphanDataServicesTemplate(); String getDeleteDatasetsForCatalogTemplate(); + + String getSelectCatalogForParticipantIdTemplate(); } diff --git a/resources/sql/federated-catalog-schema.sql b/resources/sql/federated-catalog-schema.sql index 1a6724d..dfb7bcf 100644 --- a/resources/sql/federated-catalog-schema.sql +++ b/resources/sql/federated-catalog-schema.sql @@ -4,7 +4,8 @@ CREATE TABLE IF NOT EXISTS edc_catalog ( participant_id VARCHAR(255) NOT NULL, properties JSON DEFAULT '{}', expired BOOLEAN, - PRIMARY KEY (participant_id) + PRIMARY KEY (id), + UNIQUE (participant_id) ); -- Tabla para Dataset From c433b367dcb31224ca9d5fe48780a68835867918 Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Thu, 13 Jun 2024 17:15:29 +0200 Subject: [PATCH 074/174] Error query --- .../federated/sql/index/SqlFederatedCacheStore.java | 7 +++++-- .../sql/index/schema/BaseSqlDialectStatements.java | 6 +++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java index ec9623b..4107822 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java @@ -1,5 +1,6 @@ package org.upm.inesdata.federated.sql.index; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import org.eclipse.edc.catalog.spi.FederatedCacheStore; import org.eclipse.edc.connector.controlplane.catalog.spi.Catalog; @@ -218,7 +219,7 @@ private List getDatasetsForCatalog(String catalogId) { private Dataset mapResultSetToDataset(ResultSet resultSet) throws SQLException { String id = resultSet.getString("id"); Map properties = fromJson(resultSet.getString("properties"), Map.class); - Map offers = fromJson(resultSet.getString("offers"), Map.class); + Map offers = fromJson(resultSet.getString("offers"), new TypeReference>() {}); List distributions = getDistributionsForDataset(id); @@ -228,7 +229,9 @@ private Dataset mapResultSetToDataset(ResultSet resultSet) throws SQLException { .distributions(distributions); if (offers != null) { - offers.forEach(datasetBuilder::offer); + offers.forEach((key, value) -> + datasetBuilder.offer(key, value) + ); } return datasetBuilder.build(); diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java index a806857..0fa64d5 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java @@ -204,12 +204,12 @@ public String getExpireAllCatalogsTemplate() { @Override public String getSelectDatasetsForCatalogTemplate() { - return format("SELECT * FROM %s AS a WHERE catalog_id = ?v", getDatasetTable()); + return format("SELECT * FROM %s AS a WHERE catalog_id = ?", getDatasetTable()); } @Override public String getSelectDistributionsForDatasetTemplate() { - return format("SELECT * FROM %s AS a WHERE dataset_id = ?v", getDistributionTable()); + return format("SELECT * FROM %s AS a WHERE dataset_id = ?", getDistributionTable()); } @Override public String getSelectDataServicesForCatalogTemplate() { @@ -253,6 +253,6 @@ public String getDeleteDatasetsForCatalogTemplate() { @Override public String getSelectCatalogForParticipantIdTemplate() { - return format("SELECT * FROM %s AS a WHERE participant_id = ?", getCatalogParticipantIdColumn()); + return format("SELECT * FROM %s AS a WHERE participant_id = ?", getCatalogTable()); } } From 44ab7474efa555bb15b7503b4a3fb0731a752a64 Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Thu, 13 Jun 2024 17:24:27 +0200 Subject: [PATCH 075/174] =?UTF-8?q?=C3=BAltimos=20retoques?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../federated-catalog-cache-sql/README.md | 15 +- .../SqlFederatedCacheServiceExtension.java | 2 +- .../sql/index/SqlFederatedCacheStore.java | 452 ++++++++---------- .../schema/BaseSqlDialectStatements.java | 111 ----- .../schema/SqlFederatedCatalogStatements.java | 31 -- .../postgres/SqlFederatedCatalogMapping.java | 2 +- 6 files changed, 209 insertions(+), 404 deletions(-) diff --git a/extensions/federated-catalog-cache-sql/README.md b/extensions/federated-catalog-cache-sql/README.md index 2e8e55c..0f84c52 100644 --- a/extensions/federated-catalog-cache-sql/README.md +++ b/extensions/federated-catalog-cache-sql/README.md @@ -1,6 +1,6 @@ -# SQL Vocabulary +# SQL Federated Catalog -Provides SQL persistence for vocabularies. +Provides SQL persistence for federated catalog. ## Prerequisites @@ -12,11 +12,8 @@ Please apply this [schema](docs/schema.sql) to your SQL database. diff --git a/extensions/federated-catalog-cache-sql/docs/er.puml b/extensions/federated-catalog-cache-sql/docs/er.puml index 0ddf4d7..7c10bdd 100644 --- a/extensions/federated-catalog-cache-sql/docs/er.puml +++ b/extensions/federated-catalog-cache-sql/docs/er.puml @@ -1,6 +1,43 @@ @startuml entity edc_catalog { * id: string <> + * participantId: string + * properties: Map + * expired: boolean -- } -@enduml \ No newline at end of file + +entity edc_dataset { + * id: string <> + * offers: Map + * properties: Map + * catalogId: string <> + -- +} + +entity edc_data_service { + * id: string <> + * terms: string + * endpointUrl: string + -- +} + +entity edc_distribution { + * format: string + * dataServiceId: string <> + * datasetId: string <> + -- +} + +entity edc_catalog_data_service { + * catalogId: string <> + * dataServiceId: string <> + -- +} + +edc_catalog ||--o{ edc_dataset: contains +edc_catalog ||--o{ edc_catalog_data_service: contains +edc_data_service ||--o{ edc_distribution: provides +edc_dataset ||--o{ edc_distribution: contains +edc_data_service ||--o{ edc_catalog_data_service: contains +@enduml diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheServiceExtension.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheServiceExtension.java index 6898921..5cf3f50 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheServiceExtension.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheServiceExtension.java @@ -45,12 +45,17 @@ public class SqlFederatedCacheServiceExtension implements ServiceExtension { @Inject private QueryExecutor queryExecutor; - + /** + * Provides a defaultCacheStore implementation + */ @Provider(isDefault = true) public PaginatedFederatedCacheStoreIndex defaultCacheStore() { return new SqlFederatedCacheStore(dataSourceRegistry, DATASOURCE_SETTING_NAME,transactionContext,getObjectMapper(),dialect,queryExecutor); } + /** + * Initializes the service + */ @Override public void initialize(ServiceExtensionContext context) { var dataSourceName = context.getConfig().getString(DATASOURCE_SETTING_NAME, DataSourceRegistry.DEFAULT_DATASOURCE); diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java index 424abfd..3ccc864 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.json.JsonObject; import org.eclipse.edc.connector.controlplane.catalog.spi.Catalog; import org.eclipse.edc.connector.controlplane.catalog.spi.DataService; import org.eclipse.edc.connector.controlplane.catalog.spi.Dataset; @@ -33,11 +34,26 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +/** + * Implementation of the {@link PaginatedFederatedCacheStoreIndex} that uses SQL for storing and retrieving federated + * catalog data. This class extends {@link AbstractSqlStore} and provides methods for saving, querying, and managing + * catalogs in a federated cache. + */ public class SqlFederatedCacheStore extends AbstractSqlStore implements PaginatedFederatedCacheStoreIndex { public static final String INTERNAL_CATALOG_ID = "internal_catalog_id"; private final SqlFederatedCatalogStatements databaseStatements; + /** + * Constructs a SqlFederatedCacheStore with the specified dependencies. + * + * @param dataSourceRegistry the registry for data sources. + * @param dataSourceName the name of the data source. + * @param transactionContext the context for handling transactions. + * @param objectMapper the object mapper for JSON processing. + * @param databaseStatements the SQL statements specific to the federated catalog. + * @param queryExecutor the executor for running SQL queries. + */ public SqlFederatedCacheStore(DataSourceRegistry dataSourceRegistry, String dataSourceName, TransactionContext transactionContext, ObjectMapper objectMapper, SqlFederatedCatalogStatements databaseStatements, QueryExecutor queryExecutor) { @@ -45,6 +61,12 @@ public SqlFederatedCacheStore(DataSourceRegistry dataSourceRegistry, String data this.databaseStatements = Objects.requireNonNull(databaseStatements); } + /** + * Saves the provided catalog into the federated cache database. + * + * @param catalog the catalog to be saved. + * @throws NullPointerException if the catalog is null. + */ @Override public void save(Catalog catalog) { Objects.requireNonNull(catalog); @@ -68,6 +90,13 @@ public void save(Catalog catalog) { }); } + /** + * Queries the federated cache based on the provided criteria. + * + * @param query the list of criteria to filter the query. + * @return a collection of catalogs that match the query criteria. + * @throws EdcPersistenceException if a SQL error occurs during the query. + */ @Override public Collection query(List query) { return transactionContext.execute(() -> { @@ -82,6 +111,11 @@ public Collection query(List query) { }); } + /** + * Deletes expired catalogs from the federated cache database. + * + * @throws EdcPersistenceException if a SQL error occurs during the deletion. + */ @Override public void deleteExpired() { transactionContext.execute(() -> { @@ -94,6 +128,11 @@ public void deleteExpired() { }); } + /** + * Expires all catalogs in the federated cache database. + * + * @throws EdcPersistenceException if a SQL error occurs during the operation. + */ @Override public void expireAll() { transactionContext.execute(() -> { @@ -106,6 +145,11 @@ public void expireAll() { }); } + /** + * (non-javadoc) + * + * @see PaginatedFederatedCacheStoreIndex#queryPagination(QuerySpec) + */ @Override public Collection queryPagination(QuerySpec querySpec) { return transactionContext.execute(() -> { diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java index e9973d1..ef51d38 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java @@ -8,14 +8,28 @@ import static java.lang.String.format; +/** + * Base implementation of {@link SqlFederatedCatalogStatements} providing SQL statements + * for managing federated catalog entities in a SQL database. + */ public class BaseSqlDialectStatements implements SqlFederatedCatalogStatements { protected final SqlOperatorTranslator operatorTranslator; + /** + * Constructs a {@code BaseSqlDialectStatements} instance with the specified SQL operator translator. + * + * @param operatorTranslator the SQL operator translator used for translating SQL operators. + */ public BaseSqlDialectStatements(SqlOperatorTranslator operatorTranslator) { this.operatorTranslator = operatorTranslator; } + /** + * {@inheritDoc} + * + * @see SqlFederatedCatalogStatements#getInsertCatalogTemplate() + */ @Override public String getInsertCatalogTemplate() { return executeStatement() @@ -26,17 +40,32 @@ public String getInsertCatalogTemplate() { .insertInto(getCatalogTable()); } + /** + * {@inheritDoc} + * + * @see SqlFederatedCatalogStatements#getSelectCatalogTemplate() + */ @Override public String getSelectCatalogTemplate() { return format("SELECT * FROM %s AS a", getCatalogTable()); } + /** + * {@inheritDoc} + * + * @see SqlFederatedCatalogStatements#getDeleteCatalogByParticipantIdTemplate() + */ @Override public String getDeleteCatalogByParticipantIdTemplate() { return executeStatement() .delete(getCatalogTable(), getCatalogParticipantIdColumn()); } + /** + * {@inheritDoc} + * + * @see SqlFederatedCatalogStatements#getInsertDataServiceTemplate() + */ @Override public String getInsertDataServiceTemplate() { return executeStatement() @@ -46,6 +75,11 @@ public String getInsertDataServiceTemplate() { .insertInto(getDataServiceTable()); } + /** + * {@inheritDoc} + * + * @see SqlFederatedCatalogStatements#getInsertCatalogDataServiceTemplate() + */ @Override public String getInsertCatalogDataServiceTemplate() { return executeStatement() @@ -54,7 +88,11 @@ public String getInsertCatalogDataServiceTemplate() { .insertInto(getCatalogDataServiceTable()); } - // Dataset CRUD methods + /** + * {@inheritDoc} + * + * @see SqlFederatedCatalogStatements#getInsertDatasetTemplate() + */ @Override public String getInsertDatasetTemplate() { return executeStatement() @@ -65,6 +103,11 @@ public String getInsertDatasetTemplate() { .insertInto(getDatasetTable()); } + /** + * {@inheritDoc} + * + * @see SqlFederatedCatalogStatements#getInsertDistributionTemplate() + */ @Override public String getInsertDistributionTemplate() { return executeStatement() @@ -74,22 +117,42 @@ public String getInsertDistributionTemplate() { .insertInto(getDistributionTable()); } + /** + * {@inheritDoc} + * + * @see SqlFederatedCatalogStatements#createQuery(QuerySpec) + */ @Override public SqlQueryStatement createQuery(QuerySpec querySpec) { return new SqlQueryStatement(getSelectCatalogTemplate(), querySpec, new SqlFederatedCatalogMapping(this), operatorTranslator); } + /** + * {@inheritDoc} + * + * @see SqlFederatedCatalogStatements#createDatasetQuery(QuerySpec) + */ @Override public SqlQueryStatement createDatasetQuery(QuerySpec querySpec) { return new SqlQueryStatement(getSelectDatasetTemplate(), querySpec, new SqlDatasetMapping(this), operatorTranslator); } + /** + * {@inheritDoc} + * + * @see SqlFederatedCatalogStatements#getDeleteExpiredCatalogsTemplate() + */ @Override public String getDeleteExpiredCatalogsTemplate() { return executeStatement() .delete(getCatalogTable(), getCatalogExpiredColumn()); } + /** + * {@inheritDoc} + * + * @see SqlFederatedCatalogStatements#getExpireAllCatalogsTemplate() + */ @Override public String getExpireAllCatalogsTemplate() { return executeStatement() @@ -97,15 +160,31 @@ public String getExpireAllCatalogsTemplate() { .update(getCatalogTable(), getCatalogExpiredColumn()); } + /** + * {@inheritDoc} + * + * @see SqlFederatedCatalogStatements#getSelectDatasetsForCatalogTemplate() + */ @Override public String getSelectDatasetsForCatalogTemplate() { return format("SELECT * FROM %s AS a WHERE catalog_id = ?", getDatasetTable()); } + /** + * {@inheritDoc} + * + * @see SqlFederatedCatalogStatements#getSelectDistributionsForDatasetTemplate() + */ @Override public String getSelectDistributionsForDatasetTemplate() { return format("SELECT * FROM %s AS a WHERE dataset_id = ?", getDistributionTable()); } + + /** + * {@inheritDoc} + * + * @see SqlFederatedCatalogStatements#getSelectDataServicesForCatalogTemplate() + */ @Override public String getSelectDataServicesForCatalogTemplate() { return format( @@ -117,42 +196,72 @@ public String getSelectDataServicesForCatalogTemplate() { ); } + /** + * {@inheritDoc} + * + * @see SqlFederatedCatalogStatements#getSelectDataServicesForIdTemplate() + */ @Override public String getSelectDataServicesForIdTemplate() { return format("SELECT * FROM %s AS a WHERE id = ?", getDataServiceTable()); } - + /** + * {@inheritDoc} + * + * @see SqlFederatedCatalogStatements#getDeleteDistributionsForCatalogTemplate() + */ @Override public String getDeleteDistributionsForCatalogTemplate() { return format("DELETE FROM %s WHERE dataset_id IN (SELECT id FROM %s WHERE catalog_id = ?)", getDistributionTable(), getDatasetTable()); } + /** + * {@inheritDoc} + * + * @see SqlFederatedCatalogStatements#getDeleteCatalogDataServicesTemplate() + */ @Override public String getDeleteCatalogDataServicesTemplate() { - - return format("DELETE FROM %s WHERE catalog_id = ?",getCatalogDataServiceTable()); + return format("DELETE FROM %s WHERE catalog_id = ?", getCatalogDataServiceTable()); } + /** + * {@inheritDoc} + * + * @see SqlFederatedCatalogStatements#getDeleteOrphanDataServicesTemplate() + */ @Override public String getDeleteOrphanDataServicesTemplate() { return format("DELETE FROM %s WHERE id NOT IN (SELECT data_service_id FROM %s)" + - " AND id NOT IN (SELECT data_service_id FROM %s)",getDataServiceTable(), getCatalogDataServiceTable(), getDistributionTable()); + " AND id NOT IN (SELECT data_service_id FROM %s)", getDataServiceTable(), getCatalogDataServiceTable(), getDistributionTable()); } + /** + * {@inheritDoc} + * + * @see SqlFederatedCatalogStatements#getDeleteDatasetsForCatalogTemplate() + */ @Override public String getDeleteDatasetsForCatalogTemplate() { - return format("DELETE FROM %s WHERE catalog_id = ?",getDatasetTable()); + return format("DELETE FROM %s WHERE catalog_id = ?", getDatasetTable()); } - + /** + * {@inheritDoc} + * + * @see SqlFederatedCatalogStatements#getSelectCatalogForParticipantIdTemplate() + */ @Override public String getSelectCatalogForParticipantIdTemplate() { return format("SELECT * FROM %s AS a WHERE participant_id = ?", getCatalogTable()); } - - + /** + * {@inheritDoc} + * + * @see SqlFederatedCatalogStatements#getSelectDatasetTemplate() + */ @Override public String getSelectDatasetTemplate() { return format("SELECT * FROM %s AS a", getDatasetTable()); diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/SqlFederatedCatalogStatements.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/SqlFederatedCatalogStatements.java index e2e1266..def47e4 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/SqlFederatedCatalogStatements.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/SqlFederatedCatalogStatements.java @@ -5,130 +5,334 @@ import org.eclipse.edc.sql.statement.SqlStatements; import org.eclipse.edc.sql.translation.SqlQueryStatement; +/** + * SQL statements interface for managing federated catalog data. Extends {@link SqlStatements} and provides methods for + * retrieving SQL queries and templates related to CRUD operations on catalogs, data services, datasets, distributions, + * and other related entities. + * + * This interface includes default methods for retrieving table names and column names specific to each entity, as well + * as methods for generating SQL templates and statements for various operations. + */ @ExtensionPoint public interface SqlFederatedCatalogStatements extends SqlStatements { - // Catalog CRUD methods - default String getCatalogTable() { - return "edc_catalog"; - } - - default String getCatalogIdColumn() { - return "id"; - } - - default String getCatalogParticipantIdColumn() { - return "participant_id"; - } - - default String getCatalogPropertiesColumn() { - return "properties"; - } - - default String getCatalogExpiredColumn() { - return "expired"; - } - - String getInsertCatalogTemplate(); - - String getSelectCatalogTemplate(); - - String getDeleteCatalogByParticipantIdTemplate(); - - // DataService CRUD methods - default String getDataServiceTable() { - return "edc_data_service"; - } - - default String getDataServiceIdColumn() { - return "id"; - } - - default String getDataServiceTermsColumn() { - return "terms"; - } - - default String getDataServiceEndpointUrlColumn() { - return "endpoint_url"; - } - - String getInsertDataServiceTemplate(); - - String getInsertCatalogDataServiceTemplate(); - - default String getCatalogDataServiceTable() { - return "edc_catalog_data_service"; - } - - - // Dataset CRUD methods - default String getDatasetTable() { - return "edc_dataset"; - } - - default String getDatasetIdColumn() { - return "id"; - } - - default String getDatasetOffersColumn() { - return "offers"; - } - - default String getDatasetPropertiesColumn() { - return "properties"; - } - - default String getDatasetCatalogIdColumn() { - return "catalog_id"; - } - - String getInsertDatasetTemplate(); - - - // Distribution CRUD methods - default String getDistributionTable() { - return "edc_distribution"; - } - - - default String getDistributionFormatColumn() { - return "format"; - } - - default String getDistributionDataServiceIdColumn() { - return "data_service_id"; - } - - default String getDistributionDatasetIdColumn() { - return "dataset_id"; - } - String getInsertDistributionTemplate(); - // Methods for creating SQL query using sub-select statements - SqlQueryStatement createQuery(QuerySpec query); - - SqlQueryStatement createDatasetQuery(QuerySpec querySpec); - - String getDeleteExpiredCatalogsTemplate(); - - String getExpireAllCatalogsTemplate(); - - String getSelectDatasetsForCatalogTemplate(); - - String getSelectDistributionsForDatasetTemplate(); - - String getSelectDataServicesForCatalogTemplate(); - - String getSelectDataServicesForIdTemplate(); - - String getDeleteDistributionsForCatalogTemplate(); - - String getDeleteCatalogDataServicesTemplate(); - - String getDeleteOrphanDataServicesTemplate(); - - String getDeleteDatasetsForCatalogTemplate(); - - String getSelectCatalogForParticipantIdTemplate(); - - String getSelectDatasetTemplate(); + /** + * Retrieves the name of the catalog table. + * + * @return the name of the catalog table. + */ + default String getCatalogTable() { + return "edc_catalog"; + } + + /** + * Retrieves the name of the column storing catalog IDs. + * + * @return the name of the catalog ID column. + */ + default String getCatalogIdColumn() { + return "id"; + } + + /** + * Retrieves the name of the column storing participant IDs associated with catalogs. + * + * @return the name of the participant ID column. + */ + default String getCatalogParticipantIdColumn() { + return "participant_id"; + } + + /** + * Retrieves the name of the column storing properties of catalogs as JSON. + * + * @return the name of the properties column. + */ + default String getCatalogPropertiesColumn() { + return "properties"; + } + + /** + * Retrieves the name of the column indicating if a catalog has expired. + * + * @return the name of the expired column. + */ + default String getCatalogExpiredColumn() { + return "expired"; + } + + /** + * Retrieves the SQL template for inserting a new catalog. + * + * @return the SQL template for inserting a catalog. + */ + String getInsertCatalogTemplate(); + + /** + * Retrieves the SQL template for selecting catalogs. + * + * @return the SQL template for selecting catalogs. + */ + String getSelectCatalogTemplate(); + + /** + * Retrieves the SQL template for deleting catalogs by participant ID. + * + * @return the SQL template for deleting catalogs by participant ID. + */ + String getDeleteCatalogByParticipantIdTemplate(); + + /** + * Retrieves the name of the data service table. + * + * @return the name of the data service table. + */ + default String getDataServiceTable() { + return "edc_data_service"; + } + + /** + * Retrieves the name of the column storing data service IDs. + * + * @return the name of the data service ID column. + */ + default String getDataServiceIdColumn() { + return "id"; + } + + /** + * Retrieves the name of the column storing terms associated with data services. + * + * @return the name of the terms column. + */ + default String getDataServiceTermsColumn() { + return "terms"; + } + + /** + * Retrieves the name of the column storing endpoint URLs of data services. + * + * @return the name of the endpoint URL column. + */ + default String getDataServiceEndpointUrlColumn() { + return "endpoint_url"; + } + + /** + * Retrieves the SQL template for inserting a new data service. + * + * @return the SQL template for inserting a data service. + */ + String getInsertDataServiceTemplate(); + + /** + * Retrieves the SQL template for inserting a mapping between a catalog and a data service. + * + * @return the SQL template for inserting a catalog-data service mapping. + */ + String getInsertCatalogDataServiceTemplate(); + + /** + * Retrieves the name of the table storing mappings between catalogs and data services. + * + * @return the name of the catalog-data service mapping table. + */ + default String getCatalogDataServiceTable() { + return "edc_catalog_data_service"; + } + + /** + * Retrieves the name of the dataset table. + * + * @return the name of the dataset table. + */ + default String getDatasetTable() { + return "edc_dataset"; + } + + /** + * Retrieves the name of the column storing dataset IDs. + * + * @return the name of the dataset ID column. + */ + default String getDatasetIdColumn() { + return "id"; + } + + /** + * Retrieves the name of the column storing offers associated with datasets. + * + * @return the name of the offers column. + */ + default String getDatasetOffersColumn() { + return "offers"; + } + + /** + * Retrieves the name of the column storing properties of datasets as JSON. + * + * @return the name of the properties column. + */ + default String getDatasetPropertiesColumn() { + return "properties"; + } + + /** + * Retrieves the name of the column storing the catalog ID associated with datasets. + * + * @return the name of the catalog ID column. + */ + default String getDatasetCatalogIdColumn() { + return "catalog_id"; + } + + /** + * Retrieves the SQL template for inserting a new dataset. + * + * @return the SQL template for inserting a dataset. + */ + String getInsertDatasetTemplate(); + + /** + * Retrieves the name of the distribution table. + * + * @return the name of the distribution table. + */ + default String getDistributionTable() { + return "edc_distribution"; + } + + /** + * Retrieves the name of the column storing distribution formats. + * + * @return the name of the format column. + */ + default String getDistributionFormatColumn() { + return "format"; + } + + /** + * Retrieves the name of the column storing data service IDs associated with distributions. + * + * @return the name of the data service ID column. + */ + default String getDistributionDataServiceIdColumn() { + return "data_service_id"; + } + + /** + * Retrieves the name of the column storing dataset IDs associated with distributions. + * + * @return the name of the dataset ID column. + */ + default String getDistributionDatasetIdColumn() { + return "dataset_id"; + } + + /** + * Retrieves the SQL template for inserting a new distribution. + * + * @return the SQL template for inserting a distribution. + */ + String getInsertDistributionTemplate(); + + /** + * Creates an SQL query statement based on the provided query specification. + * + * @param query the query specification defining filters, sorting, and pagination. + * @return an SQL query statement. + */ + SqlQueryStatement createQuery(QuerySpec query); + + /** + * Creates an SQL query statement specifically for datasets based on the provided query specification. + * + * @param querySpec the query specification defining filters, sorting, and pagination for datasets. + * @return an SQL query statement for datasets. + */ + SqlQueryStatement createDatasetQuery(QuerySpec querySpec); + + /** + * Retrieves the SQL template for deleting expired catalogs. + * + * @return the SQL template for deleting expired catalogs. + */ + String getDeleteExpiredCatalogsTemplate(); + + /** + * Retrieves the SQL template for expiring all catalogs. + * + * @return the SQL template for expiring all catalogs. + */ + String getExpireAllCatalogsTemplate(); + + /** + * Retrieves the SQL template for selecting datasets associated with a catalog. + * + * @return the SQL template for selecting datasets for a catalog. + */ + String getSelectDatasetsForCatalogTemplate(); + + /** + * Retrieves the SQL template for selecting distributions associated with a dataset. + * + * @return the SQL template for selecting distributions for a dataset. + */ + String getSelectDistributionsForDatasetTemplate(); + + /** + * Retrieves the SQL template for selecting data services associated with a catalog. + * + * @return the SQL template for selecting data services for a catalog. + */ + String getSelectDataServicesForCatalogTemplate(); + + /** + * Retrieves the SQL template for selecting data services by ID. + * + * @return the SQL template for selecting data services by ID. + */ + String getSelectDataServicesForIdTemplate(); + + /** + * Retrieves the SQL template for deleting distributions associated with a catalog. + * + * @return the SQL template for deleting distributions for a catalog. + */ + String getDeleteDistributionsForCatalogTemplate(); + + /** + * Retrieves the SQL template for deleting mappings between catalogs and data services. + * + * @return the SQL template for deleting catalog-data service mappings. + */ + String getDeleteCatalogDataServicesTemplate(); + + /** + * Retrieves the SQL template for deleting orphaned data services. + * + * @return the SQL template for deleting orphaned data services. + */ + String getDeleteOrphanDataServicesTemplate(); + + /** + * Retrieves the SQL template for deleting datasets associated with a catalog. + * + * @return the SQL template for deleting datasets for a catalog. + */ + String getDeleteDatasetsForCatalogTemplate(); + + /** + * Retrieves the SQL template for selecting a catalog by participant ID. + * + * @return the SQL template for selecting a catalog by participant ID. + */ + String getSelectCatalogForParticipantIdTemplate(); + + /** + * Retrieves the SQL template for selecting a dataset. + * + * @return the SQL template for selecting a dataset. + */ + String getSelectDatasetTemplate(); } diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/postgres/PostgresDialectStatements.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/postgres/PostgresDialectStatements.java index c8085e8..87c6006 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/postgres/PostgresDialectStatements.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/postgres/PostgresDialectStatements.java @@ -4,14 +4,25 @@ import org.eclipse.edc.sql.dialect.PostgresDialect; import org.eclipse.edc.sql.translation.PostgresqlOperatorTranslator; +/** + * PostgreSQL-specific SQL dialect statements used for JSON operations. Extends {@link BaseSqlDialectStatements} and + * provides PostgreSQL-specific implementations for JSON-related operations. + */ public class PostgresDialectStatements extends BaseSqlDialectStatements { + /** + * Constructs a PostgresDialectStatements object using a PostgreSQL operator translator. + */ + public PostgresDialectStatements() { + super(new PostgresqlOperatorTranslator()); + } - public PostgresDialectStatements() { - super(new PostgresqlOperatorTranslator()); - } - - @Override - public String getFormatAsJsonOperator() { - return PostgresDialect.getJsonCastOperator(); - } + /** + * Retrieves the PostgreSQL operator for formatting as JSON. + * + * @return the PostgreSQL JSON cast operator. + */ + @Override + public String getFormatAsJsonOperator() { + return PostgresDialect.getJsonCastOperator(); + } } diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/postgres/SqlDatasetMapping.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/postgres/SqlDatasetMapping.java index 5bd053e..af8b028 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/postgres/SqlDatasetMapping.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/postgres/SqlDatasetMapping.java @@ -5,11 +5,16 @@ import org.upm.inesdata.federated.sql.index.schema.SqlFederatedCatalogStatements; /** - * Maps fields of a federated catalog onto the + * Maps fields of a dataset of federated catalog onto the * corresponding SQL schema (= column names) enabling access through Postgres JSON operators where applicable */ public class SqlDatasetMapping extends TranslationMapping { - + /** + * Constructs a mapping for SQL dataset columns using the provided SQL federated catalog statements. + * This mapping specifies how dataset fields correspond to database columns. + * + * @param statements the SQL statements specific to the federated catalog schema. + */ public SqlDatasetMapping(SqlFederatedCatalogStatements statements) { add("id", statements.getDatasetIdColumn()); add("offers", new JsonFieldTranslator(statements.getDatasetOffersColumn())); diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/postgres/SqlFederatedCatalogMapping.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/postgres/SqlFederatedCatalogMapping.java index 5f0a8ac..4599f1f 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/postgres/SqlFederatedCatalogMapping.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/postgres/SqlFederatedCatalogMapping.java @@ -5,15 +5,20 @@ import org.upm.inesdata.federated.sql.index.schema.SqlFederatedCatalogStatements; /** - * Maps fields of a federated catalog onto the - * corresponding SQL schema (= column names) enabling access through Postgres JSON operators where applicable + * Maps fields of a federated catalog onto the corresponding SQL schema (= column names) enabling access through + * Postgres JSON operators where applicable */ public class SqlFederatedCatalogMapping extends TranslationMapping { - - public SqlFederatedCatalogMapping(SqlFederatedCatalogStatements statements) { - add("id", statements.getCatalogIdColumn()); - add("offers", new JsonFieldTranslator(statements.getDatasetOffersColumn())); - add("properties", new JsonFieldTranslator(statements.getCatalogPropertiesColumn())); - } + /** + * Constructs a mapping for SQL federated catalog columns using the provided SQL federated catalog statements. This + * mapping specifies how federated catalog fields correspond to database columns. + * + * @param statements the SQL statements specific to the federated catalog schema. + */ + public SqlFederatedCatalogMapping(SqlFederatedCatalogStatements statements) { + add("id", statements.getCatalogIdColumn()); + add("offers", new JsonFieldTranslator(statements.getDatasetOffersColumn())); + add("properties", new JsonFieldTranslator(statements.getCatalogPropertiesColumn())); + } } diff --git a/spi/federated-catalog-cache-spi/README.md b/spi/federated-catalog-cache-spi/README.md index a17be64..b50cb96 100644 --- a/spi/federated-catalog-cache-spi/README.md +++ b/spi/federated-catalog-cache-spi/README.md @@ -1 +1 @@ -This module contains extension points and interfaces specifically for the Inesdata Vocabulary feature. \ No newline at end of file +This module contains extension points and interfaces specifically for the Inesdata federated catalog feature. \ No newline at end of file diff --git a/spi/federated-catalog-cache-spi/src/main/java/org/upm/inesdata/spi/federated/FederatedCatalogCacheService.java b/spi/federated-catalog-cache-spi/src/main/java/org/upm/inesdata/spi/federated/FederatedCatalogCacheService.java index 6578bb3..676b2bf 100644 --- a/spi/federated-catalog-cache-spi/src/main/java/org/upm/inesdata/spi/federated/FederatedCatalogCacheService.java +++ b/spi/federated-catalog-cache-spi/src/main/java/org/upm/inesdata/spi/federated/FederatedCatalogCacheService.java @@ -13,6 +13,12 @@ */ public interface FederatedCatalogCacheService { + /** + * Searches for catalogs based on the given query specification and returns a paginated result. + * + * @param querySpec the specification of the query which includes filters, sorting, and pagination details. + * @return a ServiceResult containing a collection of catalogs that match the query criteria. + */ ServiceResult> searchPagination(QuerySpec querySpec); } \ No newline at end of file diff --git a/spi/federated-catalog-cache-spi/src/main/java/org/upm/inesdata/spi/federated/domain/Vocabulary.java b/spi/federated-catalog-cache-spi/src/main/java/org/upm/inesdata/spi/federated/domain/Vocabulary.java deleted file mode 100644 index 9fece22..0000000 --- a/spi/federated-catalog-cache-spi/src/main/java/org/upm/inesdata/spi/federated/domain/Vocabulary.java +++ /dev/null @@ -1,113 +0,0 @@ -package org.upm.inesdata.spi.vocabulary.domain; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; -import org.eclipse.edc.spi.entity.Entity; - -import java.util.Objects; -import java.util.UUID; - -import static org.eclipse.edc.spi.constants.CoreConstants.EDC_NAMESPACE; - - -/** - * The {@link Vocabulary} contains the vocabylary information. - */ -@JsonDeserialize(builder = Vocabulary.Builder.class) -public class Vocabulary extends Entity { - - public static final String PROPERTY_ID = EDC_NAMESPACE + "id"; - public static final String PROPERTY_NAME = EDC_NAMESPACE + "name"; - public static final String PROPERTY_JSON_SCHEMA = EDC_NAMESPACE + "jsonSchema"; - public static final String PROPERTY_CATEGORY = EDC_NAMESPACE + "category"; - public static final String EDC_VOCABULARY_TYPE = EDC_NAMESPACE + "Vocabulary"; - - - private String name; - private String jsonSchema; - private String category; - - private Vocabulary() { - } - - public String getName() { - return name; - } - - public String getJsonSchema() { - return jsonSchema; - } - - public String getCategory() { - return category; - } - - - public Builder toBuilder() { - return Vocabulary.Builder.newInstance() - .id(id) - .name(name) - .jsonSchema(jsonSchema) - .category(category) - .createdAt(createdAt); - } - - @JsonPOJOBuilder(withPrefix = "") - public static class Builder extends Entity.Builder { - - protected Builder(Vocabulary vocabulary) { - super(vocabulary); - } - - @JsonCreator - public static Builder newInstance() { - return new Builder(new Vocabulary()); - } - - @Override - public Builder id(String id) { - entity.id = id; - return self(); - } - - public Builder name(String name) { - entity.name = name; - return self(); - } - - public Builder jsonSchema(String jsonSchema) { - Objects.requireNonNull(jsonSchema); - entity.jsonSchema = jsonSchema; - return self(); - } - - public Builder category(String category) { - entity.category = category; - return self(); - } - - @Override - public Builder createdAt(long value) { - entity.createdAt = value; - return self(); - } - - @Override - public Builder self() { - return this; - } - - @Override - public Vocabulary build() { - super.build(); - - if (entity.getId() == null) { - id(UUID.randomUUID().toString()); - } - - return entity; - } - } - -} diff --git a/spi/federated-catalog-cache-spi/src/main/java/org/upm/inesdata/spi/federated/index/PaginatedFederatedCacheStoreIndex.java b/spi/federated-catalog-cache-spi/src/main/java/org/upm/inesdata/spi/federated/index/PaginatedFederatedCacheStoreIndex.java index 819c4a5..7cc8192 100644 --- a/spi/federated-catalog-cache-spi/src/main/java/org/upm/inesdata/spi/federated/index/PaginatedFederatedCacheStoreIndex.java +++ b/spi/federated-catalog-cache-spi/src/main/java/org/upm/inesdata/spi/federated/index/PaginatedFederatedCacheStoreIndex.java @@ -14,7 +14,7 @@ public interface PaginatedFederatedCacheStoreIndex extends FederatedCacheStore { /** - * Queries the store for {@code ContractOffer}s + * Queries the store for {@code Catalog}s * * @param querySpec A list of criteria and pagination the dataset must fulfill * @return A collection of catalog that are already in the store and that satisfy a given list of criteria and pagination. diff --git a/spi/federated-catalog-cache-spi/src/main/java/org/upm/inesdata/spi/federated/package-info.java b/spi/federated-catalog-cache-spi/src/main/java/org/upm/inesdata/spi/federated/package-info.java deleted file mode 100644 index 69fc60f..0000000 --- a/spi/federated-catalog-cache-spi/src/main/java/org/upm/inesdata/spi/federated/package-info.java +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright (c) 2020 - 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -@Spi("Vocabulary services") -package org.upm.inesdata.spi.vocabulary; - -import org.eclipse.edc.runtime.metamodel.annotation.Spi; From edc8b673f0a48f56f6f67e8e472dd8ca6b94bc63 Mon Sep 17 00:00:00 2001 From: ppel Date: Wed, 19 Jun 2024 13:05:19 +0200 Subject: [PATCH 094/174] #434249 - Updated README --- extensions/federated-catalog-cache-api/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/federated-catalog-cache-api/README.md b/extensions/federated-catalog-cache-api/README.md index 022c913..4fe8e86 100644 --- a/extensions/federated-catalog-cache-api/README.md +++ b/extensions/federated-catalog-cache-api/README.md @@ -1,3 +1,3 @@ -# S3 Asset API +Federated Catalog API -Provides a management API for uploading files to S3 and creating assets. +Provides a management API for getting the federated catalog paginated. From 285623532ef695611cf2ea9745a385526c4fc102 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Wed, 19 Jun 2024 15:49:26 +0200 Subject: [PATCH 095/174] add hashicorp vault extension --- .gitignore | 3 + gradle/libs.versions.toml | 1 + launchers/connector/build.gradle.kts | 7 +- .../upm/inesdata/custom/StartupExtension.java | 97 ------------------- .../monitor/ConnectorMonitorExtension.java | 13 +++ ...rg.eclipse.edc.spi.system.MonitorExtension | 1 + ...rg.eclipse.edc.spi.system.ServiceExtension | 1 - .../spi/vocabulary/VocabularyService.java | 5 +- .../inesdata/spi/vocabulary/package-info.java | 14 --- 9 files changed, 25 insertions(+), 117 deletions(-) delete mode 100644 launchers/connector/src/main/java/org/upm/inesdata/custom/StartupExtension.java create mode 100644 launchers/connector/src/main/java/org/upm/inesdata/monitor/ConnectorMonitorExtension.java create mode 100644 launchers/connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.MonitorExtension delete mode 100644 launchers/connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension diff --git a/.gitignore b/.gitignore index 3958a43..ee1f192 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,6 @@ bin/ .vscode .env + +# Ignore dev resources +resources-dev/ \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 8293fd3..a5a776f 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -60,6 +60,7 @@ edc-transfer-data-plane-signaling = { module = "org.eclipse.edc:transfer-data-pl edc-transfer-pull-http-receiver = { module = "org.eclipse.edc:transfer-pull-http-dynamic-receiver", version.ref = "edc" } edc-validator-data-address-http-data = { module = "org.eclipse.edc:validator-data-address-http-data", version.ref = "edc" } edc-validator-spi = { module = "org.eclipse.edc:validator-spi", version.ref = "edc" } +edc-vault-hashicorp = { module = "org.eclipse.edc:vault-hashicorp", version.ref = "edc" } edc-web-spi = { module = "org.eclipse.edc:web-spi", version.ref = "edc" } # SQL Extensions diff --git a/launchers/connector/build.gradle.kts b/launchers/connector/build.gradle.kts index 40c154c..a58950b 100644 --- a/launchers/connector/build.gradle.kts +++ b/launchers/connector/build.gradle.kts @@ -29,6 +29,9 @@ dependencies { implementation(libs.edc.data.plane.selector.api) implementation(libs.edc.data.plane.selector.core) + // Secretos + implementation(libs.edc.vault.hashicorp) + // Transferencia implementation(libs.edc.transfer.data.plane.signaling) implementation(libs.edc.transfer.pull.http.receiver) @@ -88,11 +91,7 @@ application { mainClass.set("org.eclipse.edc.boot.system.runtime.BaseRuntime") } -var distTar = tasks.getByName("distTar") -var distZip = tasks.getByName("distZip") - tasks.withType { mergeServiceFiles() archiveFileName.set("connector-app.jar") - dependsOn(distTar, distZip) } diff --git a/launchers/connector/src/main/java/org/upm/inesdata/custom/StartupExtension.java b/launchers/connector/src/main/java/org/upm/inesdata/custom/StartupExtension.java deleted file mode 100644 index 2ad84d2..0000000 --- a/launchers/connector/src/main/java/org/upm/inesdata/custom/StartupExtension.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation - * - */ - -package org.upm.inesdata.custom; - -import org.upm.inesdata.monitor.Slf4jMonitor; -import org.eclipse.edc.runtime.metamodel.annotation.Inject; -import org.eclipse.edc.spi.security.Vault; -import org.eclipse.edc.spi.system.ServiceExtension; -import org.eclipse.edc.spi.system.ServiceExtensionContext; -import org.eclipse.edc.spi.monitor.Monitor; -import org.eclipse.edc.spi.system.MonitorExtension; - -public class StartupExtension implements ServiceExtension, MonitorExtension { - - @Override - public Monitor getMonitor() { - - return new Slf4jMonitor(); - } - - @Inject - private Vault vault; - - private static final String PUBLIC_KEY = """ - -----BEGIN CERTIFICATE----- - MIIDETCCAfkCFBEOuzF+mRgrzrxVv70zNDUEv+nUMA0GCSqGSIb3DQEBCwUAMEUx - CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl - cm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjQwMTMxMTIyNTUyWhcNMjUwMTMwMTIy - NTUyWjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE - CgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC - AQ8AMIIBCgKCAQEAxHbsW755sB/5YwjPyk0xPJwyQkxkGV2SS4sQQCCz8KV/QoEi - lk/PzUqUsE2e7LqavWL5+FPSic79ZKEljeeNB1z5xoE2YnKEs55MLi/PgfLJHJ9M - bA42lJBYI37MeGv0kGkIQDIrtpDjBfSgGSINXeTjs0T/l5sXXPTfSRm4URWX3I+Q - O9ACYLb7Cz//G8HdYHp8MHNa38x4BeBku7cT7xgEUPfhu4LCnBW1u0pMbeT8A2M9 - zz/TIOJKwUeupacXjJ/tqHGLYMgBLCSRNK4zFJiGzhDbEZA6+NMUp660t7Su6M5i - x3WIffKlIcxi0bnk9slhjK5NynJ3H/uvbfYnCwIDAQABMA0GCSqGSIb3DQEBCwUA - A4IBAQBAttZGxh1k87S8OHA6L8fmu/PEtjdkJV3Cu7J3xTGfmSBVMTStl2NI4D4i - FJUFdTJjvEibEaQEETz2zJakwdp7JGG7ip7e/dKsP0wVMC63MZ8eeDzST7IzBLk6 - mjb+Co0+2p/9rHPBz1AFpHRcJ9jaORh8FlctfcxcBsr6dMptGtCZMzam26WwePxa - pNd+wBNFHuAXHyTzHl3NIq2Bd+56Roen5muZKv4xqI7iVBbVrejvVyXNDhrqh9QB - VushjclGdc2GSdCVpUhiv1srX59/yX5HBZG8HRQSgCXpcSiGIhiMmd1gaU/qrXa8 - kwmd6tdKUcAiLU7qXEJX4iBziobm - -----END CERTIFICATE----- - """; - - private static final String PRIVATE_KEY = """ - -----BEGIN PRIVATE KEY----- - MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDEduxbvnmwH/lj - CM/KTTE8nDJCTGQZXZJLixBAILPwpX9CgSKWT8/NSpSwTZ7supq9Yvn4U9KJzv1k - oSWN540HXPnGgTZicoSznkwuL8+B8skcn0xsDjaUkFgjfsx4a/SQaQhAMiu2kOMF - 9KAZIg1d5OOzRP+Xmxdc9N9JGbhRFZfcj5A70AJgtvsLP/8bwd1genwwc1rfzHgF - 4GS7txPvGARQ9+G7gsKcFbW7Skxt5PwDYz3PP9Mg4krBR66lpxeMn+2ocYtgyAEs - JJE0rjMUmIbOENsRkDr40xSnrrS3tK7ozmLHdYh98qUhzGLRueT2yWGMrk3Kcncf - +69t9icLAgMBAAECggEAMrh/IdplgS5oDITQXGqcXQj4QryFFTVemQkwa1XmDqkz - VyCOpN67B8WK6I9JidVNMLq2TUGat8BME+g0kVOnybBbvb9LIpRtJRAnm771JBfh - ivWnXbg3qBgx4QwRzr8UNxoeqrApL+ts6PM0R/jOGaEHlhcoo2PDXtjLMoyk1K26 - JdXibZ8TjqS59MG9BB8j2/NiyaQxC46k8FPz5UXnLAVr8z8kzi0P/LfuZdFO3BjB - 1Pgsgy7zme32nUCTJSQbCUKngY7KAwaeuPVji5QbkJ0Tk1tWop9IKg43RzqVp+Me - QZWDgIyeWMRkmKb6GXrl6HjuOFEymMQkPlRH7QPDcQKBgQDjOaYaPFQIgkkKAoMa - DGT0bv1RdPWithYHDTKpa6xMqfZ5/H+09i/+wVH1P8j2osylZOunsCVoIeeeTuv4 - VYC+VTOboOFdbzKD9gOyvZ4nUox+nT/1awK/ZBRcX1GrItZLs7QbrWHswT1VWTcO - ccj7wVIKs15yuBuV0w97qA1h0QKBgQDdWA2v1Yj0jgOn8leCSGj2LlzYObuEWEn4 - lCNn0b5ZABTqFpG6viNpalyTUHDXGYt3mZpEBpW2O+x7Bv9EO8+G9TtkW13NL+eD - 4+4wftRx4LTuOlG5wAicNG0dvaobMZ3Ba/v+KCs2pQaC6pC0E12j5euPoTl6pUxL - 0caANS72GwKBgA6Yu3Tf/z30sKB1/NoFhc61+ovrCYSEUfHuBR0hD5Z1LKI1eDa1 - weuJm6w3hr9hQeg2DXVeOWjxFKH0y+/N/lE7w+7xbMjFxeb4PNCUob5A7UmM7Hjl - gO2ujihI70QhHscKKj3E64978mazLng9Ev4pMw4A68ZO0WbUgiAdcY6hAoGBAJA6 - YDDDI4q3PwqgjwUFyd8gNr9uXxHNteWUIb30nePAeeBO9IaKfuihtdEICX8f1hEI - tcy9vbARO/auiIcdsgxun1S1dFnPbTwzuXFAM45AraTNu5Md3n5qau+GNuGhYvo2 - Mu1Zn07sGWIVFlVAiTPTDbt5gsq4Cw8ckgvYm9EhAoGAJEi0oDOOPS8skP71HsEm - fwMhp414LL7xFf4LZj8RL4+vDfzyNnbQ3/zpGKCzwO2MWAQx4GB9hNrX8meGLJXP - w5zMub2RLs0x4pZPuIQVIfcf6Ryuwx/kM3uZAj6lUm5LpfsWJNFZylC2ixuLTLgn - B1fcrLsMChYYsJvCfzElLSs= - -----END PRIVATE KEY----- - """; - - - @Override - public void initialize(ServiceExtensionContext context) { - vault.storeSecret("public-key", PUBLIC_KEY); - vault.storeSecret("private-key", PRIVATE_KEY); - vault.storeSecret("access-key", "DrJ6PmnJ3QtlxYowRSLi"); - vault.storeSecret("secret-key", "gjaVnJstoeV4bWLrJkmswKBMIZxY10PjZOJhp2qk"); - } -} diff --git a/launchers/connector/src/main/java/org/upm/inesdata/monitor/ConnectorMonitorExtension.java b/launchers/connector/src/main/java/org/upm/inesdata/monitor/ConnectorMonitorExtension.java new file mode 100644 index 0000000..f18cab4 --- /dev/null +++ b/launchers/connector/src/main/java/org/upm/inesdata/monitor/ConnectorMonitorExtension.java @@ -0,0 +1,13 @@ +package org.upm.inesdata.monitor; + +import org.eclipse.edc.spi.monitor.Monitor; +import org.eclipse.edc.spi.system.MonitorExtension; + +public class ConnectorMonitorExtension implements MonitorExtension { + + @Override + public Monitor getMonitor() { + + return new Slf4jMonitor(); + } +} \ No newline at end of file diff --git a/launchers/connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.MonitorExtension b/launchers/connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.MonitorExtension new file mode 100644 index 0000000..48bca39 --- /dev/null +++ b/launchers/connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.MonitorExtension @@ -0,0 +1 @@ +org.upm.inesdata.monitor.ConnectorMonitorExtension diff --git a/launchers/connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/launchers/connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension deleted file mode 100644 index 269011a..0000000 --- a/launchers/connector/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension +++ /dev/null @@ -1 +0,0 @@ -org.upm.inesdata.custom.StartupExtension diff --git a/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/VocabularyService.java b/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/VocabularyService.java index fccf217..9f89194 100644 --- a/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/VocabularyService.java +++ b/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/VocabularyService.java @@ -1,7 +1,10 @@ +/* + * INESData - UPM + */ package org.upm.inesdata.spi.vocabulary; -import org.upm.inesdata.spi.vocabulary.domain.Vocabulary; import org.eclipse.edc.spi.result.ServiceResult; +import org.upm.inesdata.spi.vocabulary.domain.Vocabulary; import java.util.List; diff --git a/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/package-info.java b/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/package-info.java index 69fc60f..239702c 100644 --- a/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/package-info.java +++ b/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/package-info.java @@ -1,17 +1,3 @@ -/* - * Copyright (c) 2020 - 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - @Spi("Vocabulary services") package org.upm.inesdata.spi.vocabulary; From f3dbf5e06d9edc941eaa7923ccde38023853f966 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Mon, 24 Jun 2024 09:27:43 +0200 Subject: [PATCH 096/174] remove false positive of vault-hashicorp-0.7.0.jar --- suppressions.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/suppressions.xml b/suppressions.xml index 97081c8..7e1a11b 100644 --- a/suppressions.xml +++ b/suppressions.xml @@ -23,4 +23,11 @@ cpe:/a:jetty:jetty cpe:/a:eclipse:jetty + + + ^pkg:maven/org\.eclipse\.edc/vault\-hashicorp@.*$ + cpe:/a:hashicorp:vault + From 70e172925aeba2e0f4512022ca7ab8392d9e391e Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Tue, 25 Jun 2024 11:07:23 +0200 Subject: [PATCH 097/174] Upgrade --- gradle/libs.versions.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index a5a776f..52eeda9 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -3,14 +3,14 @@ format.version = "1.1" [versions] assertj = "3.26.0" -aws = "2.25.66" -edc = "0.7.0" +aws = "2.26.7" +edc = "0.7.1" jakarta-json = "2.0.1" -jersey = "3.0.0" +jersey = "3.1.7" junit-pioneer = "2.2.0" jupiter = "5.10.2" mockito = "5.2.0" -parsson = "1.1.4" +parsson = "1.1.6" postgres = "42.7.3" rsApi = "4.0.0" rseeApi = "10.0.0" From 79f6635b4030ce2e68df31d69e6f66647496c492 Mon Sep 17 00:00:00 2001 From: ppel Date: Tue, 25 Jun 2024 12:47:56 +0200 Subject: [PATCH 098/174] #434678 - Added participantId to properties in dataset --- extensions/federated-catalog-cache-sql/docs/schema.sql | 5 +++-- .../inesdata/federated/sql/index/SqlFederatedCacheStore.java | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/extensions/federated-catalog-cache-sql/docs/schema.sql b/extensions/federated-catalog-cache-sql/docs/schema.sql index dfb7bcf..1a88970 100644 --- a/extensions/federated-catalog-cache-sql/docs/schema.sql +++ b/extensions/federated-catalog-cache-sql/docs/schema.sql @@ -15,7 +15,7 @@ CREATE TABLE IF NOT EXISTS edc_dataset ( properties JSON DEFAULT '{}', catalog_id VARCHAR, FOREIGN KEY (catalog_id) REFERENCES edc_catalog(id), - PRIMARY KEY (id) + PRIMARY KEY (id, catalog_id) ); -- Tabla para DataService @@ -31,8 +31,9 @@ CREATE TABLE IF NOT EXISTS edc_distribution ( format VARCHAR(255), data_service_id VARCHAR, dataset_id VARCHAR, + catalog_id VARCHAR, FOREIGN KEY (data_service_id) REFERENCES edc_data_service(id), - FOREIGN KEY (dataset_id) REFERENCES edc_dataset(id) + FOREIGN KEY (dataset_id, catalog_id) REFERENCES edc_dataset(id, catalog_id) ); -- Tabla de relación entre Catalog y DataService (para la lista de dataServices en Catalog) diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java index 3ccc864..b8dae1e 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java @@ -8,6 +8,7 @@ import org.eclipse.edc.connector.controlplane.catalog.spi.Dataset; import org.eclipse.edc.connector.controlplane.catalog.spi.Distribution; import org.eclipse.edc.policy.model.Policy; +import org.eclipse.edc.spi.constants.CoreConstants; import org.eclipse.edc.spi.persistence.EdcPersistenceException; import org.eclipse.edc.spi.query.Criterion; import org.eclipse.edc.spi.query.QuerySpec; @@ -318,6 +319,7 @@ private void insertDataServices(Catalog catalog, Connection connection) { private void insertDatasets(Catalog catalog, Connection connection) throws SQLException { if (catalog.getDatasets() != null) { for (Dataset dataset : catalog.getDatasets()) { + dataset.getProperties().put(CoreConstants.EDC_NAMESPACE + "participantId", catalog.getParticipantId()); queryExecutor.execute(connection, databaseStatements.getInsertDatasetTemplate(), dataset.getId(), toJson(dataset.getOffers()), toJson(dataset.getProperties()), catalog.getId()); From 411ef92b4dc9a80da8ed1d9d1b995540a2e337d3 Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Tue, 25 Jun 2024 13:21:10 +0200 Subject: [PATCH 099/174] Upgrade 0.7.1 --- .../inesdata/countelements/CountElementsApiExtension.java | 8 +++----- .../federated/FederatedCatalogCacheApiExtension.java | 8 +++----- .../sql/index/SqlFederatedCacheServiceExtension.java | 6 +++--- .../federated/sql/index/SqlFederatedCacheStore.java | 5 ++--- .../inesdata/storageasset/StorageAssetApiExtension.java | 7 ++----- .../upm/inesdata/vocabulary/VocabularyApiExtension.java | 7 ++----- gradle/libs.versions.toml | 5 +++-- .../index/PaginatedFederatedCacheStoreIndex.java | 4 ++-- 8 files changed, 20 insertions(+), 30 deletions(-) diff --git a/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/CountElementsApiExtension.java b/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/CountElementsApiExtension.java index f11392f..a36615c 100644 --- a/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/CountElementsApiExtension.java +++ b/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/CountElementsApiExtension.java @@ -1,7 +1,6 @@ package org.upm.inesdata.countelements; import jakarta.json.Json; -import org.eclipse.edc.connector.api.management.configuration.ManagementApiConfiguration; import org.eclipse.edc.runtime.metamodel.annotation.Extension; import org.eclipse.edc.runtime.metamodel.annotation.Inject; import org.eclipse.edc.runtime.metamodel.annotation.Provider; @@ -12,6 +11,8 @@ import org.eclipse.edc.transaction.spi.TransactionContext; import org.eclipse.edc.transform.spi.TypeTransformerRegistry; import org.eclipse.edc.web.spi.WebService; +import org.eclipse.edc.web.spi.configuration.ApiContext; +import org.eclipse.edc.web.spi.configuration.context.ManagementApiUrl; import org.upm.inesdata.countelements.controller.CountElementsApiController; import org.upm.inesdata.countelements.service.CountElementsServiceImpl; import org.upm.inesdata.countelements.transformer.JsonObjectFromCountElementTransformer; @@ -39,9 +40,6 @@ public class CountElementsApiExtension implements ServiceExtension { @Inject(required = false) private HealthCheckService healthCheckService; - @Inject - private ManagementApiConfiguration config; - @Inject private TypeManager typeManager; @@ -75,6 +73,6 @@ public void initialize(ServiceExtensionContext context) { managementApiTransformerRegistry.register(new JsonObjectFromCountElementTransformer(factory, jsonLdMapper)); var countElementsApiController = new CountElementsApiController(countElementsService(), managementApiTransformerRegistry); - webService.registerResource(config.getContextAlias(), countElementsApiController); + webService.registerResource(ApiContext.MANAGEMENT, countElementsApiController); } } diff --git a/extensions/federated-catalog-cache-api/src/main/java/org/upm/inesdata/federated/FederatedCatalogCacheApiExtension.java b/extensions/federated-catalog-cache-api/src/main/java/org/upm/inesdata/federated/FederatedCatalogCacheApiExtension.java index 7931ea4..620fd80 100644 --- a/extensions/federated-catalog-cache-api/src/main/java/org/upm/inesdata/federated/FederatedCatalogCacheApiExtension.java +++ b/extensions/federated-catalog-cache-api/src/main/java/org/upm/inesdata/federated/FederatedCatalogCacheApiExtension.java @@ -1,7 +1,7 @@ package org.upm.inesdata.federated; import jakarta.json.Json; -import org.eclipse.edc.connector.api.management.configuration.ManagementApiConfiguration; +import org.eclipse.edc.connector.api.management.configuration.ManagementApiConfigurationExtension; import org.eclipse.edc.connector.controlplane.transform.edc.from.JsonObjectFromAssetTransformer; import org.eclipse.edc.connector.controlplane.transform.edc.to.JsonObjectToAssetTransformer; import org.eclipse.edc.jsonld.spi.JsonLd; @@ -16,6 +16,7 @@ import org.eclipse.edc.transform.spi.TypeTransformerRegistry; import org.eclipse.edc.validator.spi.JsonObjectValidatorRegistry; import org.eclipse.edc.web.spi.WebService; +import org.eclipse.edc.web.spi.configuration.ApiContext; import org.upm.inesdata.federated.controller.FederatedCatalogCacheApiController; import org.upm.inesdata.federated.service.FederatedCatalogCacheServiceImpl; import org.upm.inesdata.spi.federated.FederatedCatalogCacheService; @@ -34,9 +35,6 @@ public class FederatedCatalogCacheApiExtension implements ServiceExtension { @Inject private WebService webService; - @Inject - private ManagementApiConfiguration config; - @Inject private TypeManager typeManager; @@ -85,6 +83,6 @@ public void initialize(ServiceExtensionContext context) { var federatedCatalogCacheApiController = new FederatedCatalogCacheApiController(this.federatedCatalogCacheService(), managementApiTransformerRegistry, validator,monitor); - webService.registerResource(config.getContextAlias(), federatedCatalogCacheApiController); + webService.registerResource(ApiContext.MANAGEMENT, federatedCatalogCacheApiController); } } diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheServiceExtension.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheServiceExtension.java index 5cf3f50..9e6570b 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheServiceExtension.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheServiceExtension.java @@ -1,7 +1,7 @@ package org.upm.inesdata.federated.sql.index; import com.fasterxml.jackson.databind.ObjectMapper; -import org.eclipse.edc.catalog.spi.FederatedCacheStore; +import org.eclipse.edc.catalog.spi.FederatedCatalogCache; import org.eclipse.edc.runtime.metamodel.annotation.Extension; import org.eclipse.edc.runtime.metamodel.annotation.Inject; import org.eclipse.edc.runtime.metamodel.annotation.Provider; @@ -20,7 +20,7 @@ /** * Extension that stores federatedCatalogs in SQL databases */ -@Provides({ FederatedCacheStore.class, PaginatedFederatedCacheStoreIndex.class }) +@Provides({ FederatedCatalogCache.class, PaginatedFederatedCacheStoreIndex.class }) @Extension(value = "SQL federatedCatalog index") public class SqlFederatedCacheServiceExtension implements ServiceExtension { @@ -64,7 +64,7 @@ public void initialize(ServiceExtensionContext context) { getDialect(), queryExecutor); context.registerService(PaginatedFederatedCacheStoreIndex.class, sqlFederatedCacheStore); - context.registerService(FederatedCacheStore.class, sqlFederatedCacheStore); + context.registerService(FederatedCatalogCache.class, sqlFederatedCacheStore); } private SqlFederatedCatalogStatements getDialect() { diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java index 3ccc864..0f37e3b 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/SqlFederatedCacheStore.java @@ -98,11 +98,10 @@ public void save(Catalog catalog) { * @throws EdcPersistenceException if a SQL error occurs during the query. */ @Override - public Collection query(List query) { + public Collection query(QuerySpec query) { return transactionContext.execute(() -> { try (var connection = getConnection()) { - QuerySpec querySpec = QuerySpec.Builder.newInstance().filter(query).build(); - SqlQueryStatement queryStatement = databaseStatements.createQuery(querySpec); + SqlQueryStatement queryStatement = databaseStatements.createQuery(query); return queryExecutor.query(connection, true, this::mapResultSetToCatalog, queryStatement.getQueryAsString(), queryStatement.getParameters()).collect(Collectors.toList()); } catch (SQLException e) { diff --git a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/StorageAssetApiExtension.java b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/StorageAssetApiExtension.java index 319249e..3b248f8 100644 --- a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/StorageAssetApiExtension.java +++ b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/StorageAssetApiExtension.java @@ -2,7 +2,6 @@ import jakarta.json.Json; import org.eclipse.edc.api.validation.DataAddressValidator; -import org.eclipse.edc.connector.api.management.configuration.ManagementApiConfiguration; import org.eclipse.edc.connector.controlplane.api.management.asset.validation.AssetValidator; import org.eclipse.edc.connector.controlplane.services.spi.asset.AssetService; import org.eclipse.edc.connector.controlplane.transform.edc.from.JsonObjectFromAssetTransformer; @@ -18,6 +17,7 @@ import org.eclipse.edc.transform.spi.TypeTransformerRegistry; import org.eclipse.edc.validator.spi.JsonObjectValidatorRegistry; import org.eclipse.edc.web.spi.WebService; +import org.eclipse.edc.web.spi.configuration.ApiContext; import org.upm.inesdata.storageasset.controller.StorageAssetApiController; import org.upm.inesdata.storageasset.service.S3Service; import software.amazon.awssdk.regions.Region; @@ -47,9 +47,6 @@ public class StorageAssetApiExtension implements ServiceExtension { @Inject private WebService webService; - @Inject - private ManagementApiConfiguration config; - @Inject private TypeManager typeManager; @@ -104,6 +101,6 @@ public void initialize(ServiceExtensionContext context) { var storageAssetApiController = new StorageAssetApiController(assetService, managementApiTransformerRegistry, validator,s3Service, jsonLd, bucketName, regionName); - webService.registerResource(config.getContextAlias(), storageAssetApiController); + webService.registerResource(ApiContext.MANAGEMENT, storageAssetApiController); } } diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/VocabularyApiExtension.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/VocabularyApiExtension.java index febe52e..c2d774a 100644 --- a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/VocabularyApiExtension.java +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/VocabularyApiExtension.java @@ -1,7 +1,6 @@ package org.upm.inesdata.vocabulary; import jakarta.json.Json; -import org.eclipse.edc.connector.api.management.configuration.ManagementApiConfiguration; import org.eclipse.edc.runtime.metamodel.annotation.Extension; import org.eclipse.edc.runtime.metamodel.annotation.Inject; import org.eclipse.edc.runtime.metamodel.annotation.Provider; @@ -14,6 +13,7 @@ import org.eclipse.edc.transform.spi.TypeTransformerRegistry; import org.eclipse.edc.validator.spi.JsonObjectValidatorRegistry; import org.eclipse.edc.web.spi.WebService; +import org.eclipse.edc.web.spi.configuration.ApiContext; import org.upm.inesdata.spi.vocabulary.VocabularyIndex; import org.upm.inesdata.spi.vocabulary.VocabularyService; import org.upm.inesdata.vocabulary.controller.VocabularyApiController; @@ -46,9 +46,6 @@ public class VocabularyApiExtension implements ServiceExtension { @Inject(required = false) private HealthCheckService healthCheckService; - @Inject - private ManagementApiConfiguration config; - @Inject private TypeManager typeManager; @@ -90,7 +87,7 @@ public void initialize(ServiceExtensionContext context) { validator.register(EDC_VOCABULARY_TYPE, VocabularyValidator.instance()); var vocabularyApiController = new VocabularyApiController(this.vocabularyService(), managementApiTransformerRegistry, monitor, validator); - webService.registerResource(config.getContextAlias(), vocabularyApiController); + webService.registerResource(ApiContext.MANAGEMENT, vocabularyApiController); // contribute to the liveness probe if (healthCheckService != null) { diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 52eeda9..b8252b2 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -5,6 +5,7 @@ format.version = "1.1" assertj = "3.26.0" aws = "2.26.7" edc = "0.7.1" +edc-aws = "0.7.1-SNAPSHOT" jakarta-json = "2.0.1" jersey = "3.1.7" junit-pioneer = "2.2.0" @@ -76,8 +77,8 @@ edc-sql-transfer-process-store = { module = "org.eclipse.edc:transfer-process-st edc-sql-data-plane-store = { module = "org.eclipse.edc:data-plane-store-sql", version.ref = "edc" } # EDC aws s3 stuff -edc-aws-s3-core = { module = "org.eclipse.edc:aws-s3-core", version.ref = "edc" } -edc-data-plane-aws-s3 = { module = "org.eclipse.edc:data-plane-aws-s3", version.ref = "edc" } +edc-aws-s3-core = { module = "org.eclipse.edc:aws-s3-core", version.ref = "edc-aws" } +edc-data-plane-aws-s3 = { module = "org.eclipse.edc:data-plane-aws-s3", version.ref = "edc-aws" } # EDC Observability edc-observability-api = { module = "org.eclipse.edc:api-observability", version.ref = "edc" } diff --git a/spi/federated-catalog-cache-spi/src/main/java/org/upm/inesdata/spi/federated/index/PaginatedFederatedCacheStoreIndex.java b/spi/federated-catalog-cache-spi/src/main/java/org/upm/inesdata/spi/federated/index/PaginatedFederatedCacheStoreIndex.java index 7cc8192..499ae68 100644 --- a/spi/federated-catalog-cache-spi/src/main/java/org/upm/inesdata/spi/federated/index/PaginatedFederatedCacheStoreIndex.java +++ b/spi/federated-catalog-cache-spi/src/main/java/org/upm/inesdata/spi/federated/index/PaginatedFederatedCacheStoreIndex.java @@ -1,6 +1,6 @@ package org.upm.inesdata.spi.federated.index; -import org.eclipse.edc.catalog.spi.FederatedCacheStore; +import org.eclipse.edc.catalog.spi.FederatedCatalogCache; import org.eclipse.edc.connector.controlplane.catalog.spi.Catalog; import org.eclipse.edc.runtime.metamodel.annotation.ExtensionPoint; import org.eclipse.edc.spi.query.QuerySpec; @@ -11,7 +11,7 @@ * Datastore interface for objects. */ @ExtensionPoint -public interface PaginatedFederatedCacheStoreIndex extends FederatedCacheStore { +public interface PaginatedFederatedCacheStoreIndex extends FederatedCatalogCache { /** * Queries the store for {@code Catalog}s From 344f0cdf0c77da5b48e32d33c586b919cecbab73 Mon Sep 17 00:00:00 2001 From: ppel Date: Wed, 26 Jun 2024 13:19:38 +0200 Subject: [PATCH 100/174] Added dependencies --- gradle/libs.versions.toml | 3 +++ launchers/connector/build.gradle.kts | 1 + 2 files changed, 4 insertions(+) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b8252b2..2875eef 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -21,6 +21,7 @@ swagger-annotations-jakarta = "2.2.21" edc-api-asset = { module = "org.eclipse.edc:asset-api", version.ref = "edc" } edc-api-core = { module = "org.eclipse.edc:api-core", version.ref = "edc" } edc-api-management-config = { module = "org.eclipse.edc:management-api-configuration", version.ref = "edc" } +edc-api-control-config = { module = "org.eclipse.edc:control-api-configuration", version.ref = "edc" } edc-auth-spi = { module = "org.eclipse.edc:auth-spi", version.ref = "edc" } edc-build-plugin = { module = "org.eclipse.edc.edc-build:org.eclipse.edc.edc-build.gradle.plugin", version.ref = "edc" } edc-configuration-filesystem = { module = "org.eclipse.edc:configuration-filesystem", version.ref = "edc" } @@ -36,6 +37,8 @@ edc-data-plane-core = { module = "org.eclipse.edc:data-plane-core", version.ref edc-data-plane-http = { module = "org.eclipse.edc:data-plane-http", version.ref = "edc" } edc-data-plane-selector-api = { module = "org.eclipse.edc:data-plane-selector-api", version.ref = "edc" } edc-data-plane-selector-core = { module = "org.eclipse.edc:data-plane-selector-core", version.ref = "edc" } +edc-data-plane-selector-control-api = { module = "org.eclipse.edc:data-plane-selector-control-api", version.ref = "edc" } +edc-data-plane-selector-client = { module = "org.eclipse.edc:data-plane-selector-client", version.ref = "edc" } edc-data-plane-self-registration = { module = "org.eclipse.edc:data-plane-self-registration", version.ref = "edc" } edc-dsp = { module = "org.eclipse.edc:dsp", version.ref = "edc" } edc-dsp-api-configuration = { module = "org.eclipse.edc:dsp-http-api-configuration", version.ref = "edc" } diff --git a/launchers/connector/build.gradle.kts b/launchers/connector/build.gradle.kts index 7773377..fefcab5 100644 --- a/launchers/connector/build.gradle.kts +++ b/launchers/connector/build.gradle.kts @@ -9,6 +9,7 @@ dependencies { // Common libs implementation(libs.edc.dsp) implementation(libs.edc.management.api) + implementation(libs.edc.api.control.config) // Temporary libs implementation(libs.edc.configuration.filesystem) From 1673f6343e2deafd00715c2545016aca3120798c Mon Sep 17 00:00:00 2001 From: ppel Date: Wed, 26 Jun 2024 14:33:10 +0200 Subject: [PATCH 101/174] #434678 - Changed SQL schema in resources --- resources/sql/federated-catalog-schema.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/resources/sql/federated-catalog-schema.sql b/resources/sql/federated-catalog-schema.sql index dfb7bcf..1a88970 100644 --- a/resources/sql/federated-catalog-schema.sql +++ b/resources/sql/federated-catalog-schema.sql @@ -15,7 +15,7 @@ CREATE TABLE IF NOT EXISTS edc_dataset ( properties JSON DEFAULT '{}', catalog_id VARCHAR, FOREIGN KEY (catalog_id) REFERENCES edc_catalog(id), - PRIMARY KEY (id) + PRIMARY KEY (id, catalog_id) ); -- Tabla para DataService @@ -31,8 +31,9 @@ CREATE TABLE IF NOT EXISTS edc_distribution ( format VARCHAR(255), data_service_id VARCHAR, dataset_id VARCHAR, + catalog_id VARCHAR, FOREIGN KEY (data_service_id) REFERENCES edc_data_service(id), - FOREIGN KEY (dataset_id) REFERENCES edc_dataset(id) + FOREIGN KEY (dataset_id, catalog_id) REFERENCES edc_dataset(id, catalog_id) ); -- Tabla de relación entre Catalog y DataService (para la lista de dataServices en Catalog) From ffd607a4c081ca39b12c835be0a3c73eee76b467 Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Thu, 27 Jun 2024 14:00:27 +0200 Subject: [PATCH 102/174] =?UTF-8?q?extensi=C3=B3n=20para=20a=C3=B1adir=20g?= =?UTF-8?q?enerator=20function=20de=20AmazonS3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../extended-data-plane-public-api/README.md | 3 + .../build.gradle.kts | 9 ++ .../ExtendedDataPlanePublicApiExtension.java | 115 ++++++++++++++++++ ...rg.eclipse.edc.spi.system.ServiceExtension | 1 + launchers/connector/build.gradle.kts | 3 + settings.gradle.kts | 1 + 6 files changed, 132 insertions(+) create mode 100644 extensions/extended-data-plane-public-api/README.md create mode 100644 extensions/extended-data-plane-public-api/build.gradle.kts create mode 100644 extensions/extended-data-plane-public-api/src/main/java/org/upm/inesdata/extendeddataplanepublic/ExtendedDataPlanePublicApiExtension.java create mode 100644 extensions/extended-data-plane-public-api/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension diff --git a/extensions/extended-data-plane-public-api/README.md b/extensions/extended-data-plane-public-api/README.md new file mode 100644 index 0000000..dcf14cf --- /dev/null +++ b/extensions/extended-data-plane-public-api/README.md @@ -0,0 +1,3 @@ +# Extended data plane public API + +Provides an extension of 'DataPlanePublicApiV2Extension'. diff --git a/extensions/extended-data-plane-public-api/build.gradle.kts b/extensions/extended-data-plane-public-api/build.gradle.kts new file mode 100644 index 0000000..37a6aca --- /dev/null +++ b/extensions/extended-data-plane-public-api/build.gradle.kts @@ -0,0 +1,9 @@ +plugins { + `java-library` + id("com.gmv.inesdata.edc-application") +} + +dependencies { + api(libs.edc.spi.core) + implementation(libs.edc.data.plane.public.api) +} diff --git a/extensions/extended-data-plane-public-api/src/main/java/org/upm/inesdata/extendeddataplanepublic/ExtendedDataPlanePublicApiExtension.java b/extensions/extended-data-plane-public-api/src/main/java/org/upm/inesdata/extendeddataplanepublic/ExtendedDataPlanePublicApiExtension.java new file mode 100644 index 0000000..5b77525 --- /dev/null +++ b/extensions/extended-data-plane-public-api/src/main/java/org/upm/inesdata/extendeddataplanepublic/ExtendedDataPlanePublicApiExtension.java @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation + * + */ + +package org.upm.inesdata.extendeddataplanepublic; + +import org.eclipse.edc.connector.dataplane.api.controller.DataPlanePublicApiV2Controller; +import org.eclipse.edc.connector.dataplane.spi.Endpoint; +import org.eclipse.edc.connector.dataplane.spi.iam.DataPlaneAuthorizationService; +import org.eclipse.edc.connector.dataplane.spi.iam.PublicEndpointGeneratorService; +import org.eclipse.edc.connector.dataplane.spi.pipeline.PipelineService; +import org.eclipse.edc.runtime.metamodel.annotation.Extension; +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.runtime.metamodel.annotation.Setting; +import org.eclipse.edc.runtime.metamodel.annotation.SettingContext; +import org.eclipse.edc.spi.system.ExecutorInstrumentation; +import org.eclipse.edc.spi.system.Hostname; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; +import org.eclipse.edc.web.spi.WebServer; +import org.eclipse.edc.web.spi.WebService; +import org.eclipse.edc.web.spi.configuration.ApiContext; +import org.eclipse.edc.web.spi.configuration.WebServiceConfigurer; +import org.eclipse.edc.web.spi.configuration.WebServiceSettings; + +import java.util.concurrent.Executors; + +/** + * This extension provides generic endpoints which are open to public participants of the Dataspace to execute + * requests on the actual data source. + */ +@Extension(value = ExtendedDataPlanePublicApiExtension.NAME) +public class ExtendedDataPlanePublicApiExtension implements ServiceExtension { + public static final String NAME = "Data Plane Public API"; + + private static final int DEFAULT_PUBLIC_PORT = 8185; + private static final String PUBLIC_CONTEXT_PATH = "/api/v2/public"; + + @SettingContext("Public API context setting key") + private static final String PUBLIC_CONFIG_KEY = "web.http." + ApiContext.PUBLIC; + + @Setting(value = "Base url of the public API endpoint without the trailing slash. This should correspond to the values configured " + + "in '" + DEFAULT_PUBLIC_PORT + "' and '" + PUBLIC_CONTEXT_PATH + "'.", defaultValue = "http://:" + DEFAULT_PUBLIC_PORT + PUBLIC_CONTEXT_PATH) + private static final String PUBLIC_ENDPOINT = "edc.dataplane.api.public.baseurl"; + + private static final int DEFAULT_THREAD_POOL = 10; + private static final WebServiceSettings PUBLIC_SETTINGS = WebServiceSettings.Builder.newInstance() + .apiConfigKey(PUBLIC_CONFIG_KEY) + .contextAlias(ApiContext.PUBLIC) + .defaultPath(PUBLIC_CONTEXT_PATH) + .defaultPort(DEFAULT_PUBLIC_PORT) + .name(NAME) + .build(); + + @Inject + private WebServer webServer; + + @Inject + private WebServiceConfigurer webServiceConfigurer; + + @Inject + private PipelineService pipelineService; + + @Inject + private WebService webService; + + @Inject + private ExecutorInstrumentation executorInstrumentation; + + @Inject + private DataPlaneAuthorizationService authorizationService; + + @Inject + private PublicEndpointGeneratorService generatorService; + + @Inject + private Hostname hostname; + + @Override + public String name() { + return NAME; + } + + @Override + public void initialize(ServiceExtensionContext context) { + var config = context.getConfig(PUBLIC_CONFIG_KEY); + var configuration = webServiceConfigurer.configure(config, webServer, PUBLIC_SETTINGS); + var executorService = executorInstrumentation.instrument( + Executors.newFixedThreadPool(DEFAULT_THREAD_POOL), + "Data plane proxy transfers" + ); + + var publicEndpoint = context.getSetting(PUBLIC_ENDPOINT, null); + if (publicEndpoint == null) { + publicEndpoint = "http://%s:%d%s".formatted(hostname.get(), configuration.getPort(), configuration.getPath()); + context.getMonitor().warning("Config property '%s' was not specified, the default '%s' will be used.".formatted(PUBLIC_ENDPOINT, publicEndpoint)); + } + var endpoint = Endpoint.url(publicEndpoint); + generatorService.addGeneratorFunction("HttpData", dataAddress -> endpoint); + generatorService.addGeneratorFunction("AmazonS3", dataAddress -> endpoint); + + var publicApiController = new DataPlanePublicApiV2Controller(pipelineService, executorService, authorizationService); + webService.registerResource(ApiContext.PUBLIC, publicApiController); + } +} diff --git a/extensions/extended-data-plane-public-api/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/extended-data-plane-public-api/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 0000000..d33192c --- /dev/null +++ b/extensions/extended-data-plane-public-api/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1 @@ +org.upm.inesdata.extendeddataplanepublic.ExtendedDataPlanePublicApiExtension \ No newline at end of file diff --git a/launchers/connector/build.gradle.kts b/launchers/connector/build.gradle.kts index fefcab5..7d485ea 100644 --- a/launchers/connector/build.gradle.kts +++ b/launchers/connector/build.gradle.kts @@ -90,6 +90,9 @@ dependencies { // Count elements implementation(project(":extensions:count-elements-api")) + //Data plane public api + implementation(project(":extensions:extended-data-plane-public-api")) + runtimeOnly(libs.edc.transaction.local) runtimeOnly(libs.postgres) } diff --git a/settings.gradle.kts b/settings.gradle.kts index 9943ffb..7803ccb 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -22,6 +22,7 @@ include(":extensions:federated-catalog-cache-sql") include(":extensions:federated-catalog-cache-api") include(":extensions:count-elements-api") include(":extensions:count-elements-sql") +include(":extensions:extended-data-plane-public-api") // Connector include(":launchers:connector") From 0186aeeb3f939782646d93fb51efb70dc5e40ce0 Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Thu, 27 Jun 2024 16:01:24 +0200 Subject: [PATCH 103/174] Inesdata store transferencia solucionada --- .../storageasset/controller/StorageAssetApiController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/controller/StorageAssetApiController.java b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/controller/StorageAssetApiController.java index 8e208fa..04af4df 100644 --- a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/controller/StorageAssetApiController.java +++ b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/controller/StorageAssetApiController.java @@ -118,7 +118,7 @@ private long getFileSize(InputStream inputStream) throws IOException { private void setStorageProperties(Asset asset, String fileName) { asset.getPrivateProperties().put("storageAssetFile", fileName); asset.getDataAddress().setKeyName(fileName); - asset.getDataAddress().setType("InesDataStore"); + asset.getDataAddress().setType("AmazonS3"); asset.getDataAddress().getProperties().put(CoreConstants.EDC_NAMESPACE+ "bucketName", bucketName); asset.getDataAddress().getProperties().put(CoreConstants.EDC_NAMESPACE+"region", region); } From 9934e9dd96437d6cd5aa6fa4a9efe8a7cc7482d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Fri, 28 Jun 2024 08:31:07 +0200 Subject: [PATCH 104/174] suppress false positives --- suppressions.xml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/suppressions.xml b/suppressions.xml index 7e1a11b..6cb7b22 100644 --- a/suppressions.xml +++ b/suppressions.xml @@ -30,4 +30,32 @@ ^pkg:maven/org\.eclipse\.edc/vault\-hashicorp@.*$ cpe:/a:hashicorp:vault + + + ^pkg:maven/org\.eclipse\.edc/control\-plane\-spi@.*$ + cpe:2.3:a:plane:plane + + + + ^pkg:maven/org\.eclipse\.edc/control\-plane\-transform@.*$ + cpe:2.3:a:plane:plane + + + + ^pkg:maven/org\.eclipse\.edc/data\-plane\-selector\-spi@.*$ + cpe:2.3:a:plane:plane + + + + ^pkg:maven/org\.eclipse\.edc/data\-plane\-spi@.*$ + cpe:2.3:a:plane:plane + From 413ce26fefb52ba9f6f11dc6ede0be84437c1b71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n?= Date: Fri, 28 Jun 2024 08:45:09 +0200 Subject: [PATCH 105/174] suppress plane:plane false positives --- suppressions.xml | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/suppressions.xml b/suppressions.xml index 6cb7b22..31f715d 100644 --- a/suppressions.xml +++ b/suppressions.xml @@ -33,29 +33,11 @@ - ^pkg:maven/org\.eclipse\.edc/control\-plane\-spi@.*$ - cpe:2.3:a:plane:plane - - - - ^pkg:maven/org\.eclipse\.edc/control\-plane\-transform@.*$ - cpe:2.3:a:plane:plane - - - - ^pkg:maven/org\.eclipse\.edc/data\-plane\-selector\-spi@.*$ - cpe:2.3:a:plane:plane - - - - ^pkg:maven/org\.eclipse\.edc/data\-plane\-spi@.*$ - cpe:2.3:a:plane:plane + ^pkg:maven/org\.eclipse\.edc/.*@.*$ + cpe:/a:plane:plane From bd26b6e89f7dc6c13b4a1b61ad04f34bf5b8d7a3 Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Fri, 28 Jun 2024 09:38:19 +0200 Subject: [PATCH 106/174] README --- extensions/extended-data-plane-public-api/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/extended-data-plane-public-api/README.md b/extensions/extended-data-plane-public-api/README.md index dcf14cf..377ec9a 100644 --- a/extensions/extended-data-plane-public-api/README.md +++ b/extensions/extended-data-plane-public-api/README.md @@ -1,3 +1,3 @@ # Extended data plane public API -Provides an extension of 'DataPlanePublicApiV2Extension'. +Provides an extension of 'DataPlanePublicApiV2Extension' that allows performing HTTP-PULL from AmazonS3 destinations. \ No newline at end of file From 906fb52f8e0ca7e8e5e9d0f09554a99d416a08c0 Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Thu, 4 Jul 2024 08:47:02 +0200 Subject: [PATCH 107/174] =?UTF-8?q?extensiones=20de=20auditor=C3=ADa?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- extensions/audit-configuration/README.md | 18 ++++++ .../audit-configuration/build.gradle.kts | 17 ++++++ .../upm/inesdata/audit/AuditExtension.java | 34 +++++++++++ .../audit/HttpRequestInterceptor.java | 56 +++++++++++++++++++ ...rg.eclipse.edc.spi.system.ServiceExtension | 1 + .../audit-event-configuration/README.md | 18 ++++++ .../build.gradle.kts | 17 ++++++ .../auditevent/AuditEventSubscriber.java | 20 +++++++ .../AuditEventSubscriptionExtension.java | 21 +++++++ ...rg.eclipse.edc.spi.system.ServiceExtension | 1 + launchers/connector/build.gradle.kts | 4 ++ settings.gradle.kts | 2 + 12 files changed, 209 insertions(+) create mode 100644 extensions/audit-configuration/README.md create mode 100644 extensions/audit-configuration/build.gradle.kts create mode 100644 extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/AuditExtension.java create mode 100644 extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/HttpRequestInterceptor.java create mode 100644 extensions/audit-configuration/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension create mode 100644 extensions/audit-event-configuration/README.md create mode 100644 extensions/audit-event-configuration/build.gradle.kts create mode 100644 extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriber.java create mode 100644 extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriptionExtension.java create mode 100644 extensions/audit-event-configuration/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension diff --git a/extensions/audit-configuration/README.md b/extensions/audit-configuration/README.md new file mode 100644 index 0000000..51bb140 --- /dev/null +++ b/extensions/audit-configuration/README.md @@ -0,0 +1,18 @@ +# Oauth2 JWT Token Authentication Service + +This extension provides the capability to authorizate the request to the connector management API. The extension will access the Bearer token provided in the Authorization header and validate that it is a valid JWT-encoded bearer token. It is necessary to have the `org.eclipse.edc:oauth2-core` extension correctly configured. + +To authorize a user, the roles of the provided JWT token must contain: +- a valid role from those configured in `allowedRoles` +- a role with the `connector name` + +## Configuration + +Example configuration: + +```properties +edc.api.auth.oauth2.allowedRoles.1.role=connector-admin +edc.api.auth.oauth2.allowedRoles.2.role=connector-management +``` + +The `edc.api.auth.oauth2.allowedRoles` will be used by the federated catalog to retrieve the list of allowed roles that can perform requests on the managemente API connector. \ No newline at end of file diff --git a/extensions/audit-configuration/build.gradle.kts b/extensions/audit-configuration/build.gradle.kts new file mode 100644 index 0000000..9261f24 --- /dev/null +++ b/extensions/audit-configuration/build.gradle.kts @@ -0,0 +1,17 @@ +plugins { + `java-library` + id("com.gmv.inesdata.edc-application") +} + +dependencies { + api(libs.edc.auth.spi) + + implementation(libs.edc.iam.oauth2.core) + implementation(libs.jakarta.rsApi) + + testImplementation(libs.edc.core.junit) + testImplementation(libs.assertj) + testImplementation(libs.mockito.core) +} + + diff --git a/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/AuditExtension.java b/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/AuditExtension.java new file mode 100644 index 0000000..3583152 --- /dev/null +++ b/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/AuditExtension.java @@ -0,0 +1,34 @@ +package org.upm.inesdata.audit; + +import jakarta.servlet.Filter; +import org.eclipse.edc.runtime.metamodel.annotation.Extension; +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.runtime.metamodel.annotation.Provides; +import org.eclipse.edc.spi.monitor.Monitor; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; +import org.eclipse.edc.web.spi.WebService; + +@Provides(HttpRequestInterceptor.class) +@Extension(value = AuditExtension.NAME) +public class AuditExtension implements ServiceExtension { + + public static final String NAME = "Audit configuration"; + + + @Inject + private WebService webService; + + @Override + public String name() { + return NAME; + } + + @Override + public void initialize(ServiceExtensionContext context) { + // Registrar el interceptor HTTP + context.registerService(Filter.class, new HttpRequestInterceptor(context.getMonitor())); + webService.registerResource(new HttpRequestInterceptor(context.getMonitor())); + context.getMonitor().info("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA - Registered Filter"); + } +} diff --git a/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/HttpRequestInterceptor.java b/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/HttpRequestInterceptor.java new file mode 100644 index 0000000..b16c213 --- /dev/null +++ b/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/HttpRequestInterceptor.java @@ -0,0 +1,56 @@ +package org.upm.inesdata.audit; + +import jakarta.servlet.Filter; +import jakarta.servlet.FilterChain; +import jakarta.servlet.FilterConfig; +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletRequest; +import jakarta.servlet.ServletResponse; +import jakarta.servlet.annotation.WebFilter; +import jakarta.servlet.http.HttpServletRequest; +import org.eclipse.edc.spi.monitor.Monitor; + +import java.io.IOException; +import java.util.Enumeration; + +@WebFilter("/*") +public class HttpRequestInterceptor implements Filter { + + private final Monitor monitor; + + public HttpRequestInterceptor(Monitor monitor) { + this.monitor = monitor; + } + + @Override + public void init(FilterConfig filterConfig) throws ServletException { + // Inicialización si es necesaria + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws IOException, ServletException { + + HttpServletRequest httpRequest = (HttpServletRequest) request; + + // Auditar la petición + monitor.info("Intercepted Request: "); + monitor.info("Method: " + httpRequest.getMethod()); + monitor.info("URL: " + httpRequest.getRequestURL().toString()); + + // Imprimir cabeceras + Enumeration headerNames = httpRequest.getHeaderNames(); + while (headerNames.hasMoreElements()) { + String headerName = headerNames.nextElement(); + monitor.info(headerName + ": " + httpRequest.getHeader(headerName)); + } + + // Continuar con el siguiente filtro o recurso + chain.doFilter(request, response); + } + + @Override + public void destroy() { + // Limpieza si es necesaria + } +} \ No newline at end of file diff --git a/extensions/audit-configuration/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/audit-configuration/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 0000000..933cff4 --- /dev/null +++ b/extensions/audit-configuration/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1 @@ +org.upm.inesdata.audit.AuditExtension diff --git a/extensions/audit-event-configuration/README.md b/extensions/audit-event-configuration/README.md new file mode 100644 index 0000000..51bb140 --- /dev/null +++ b/extensions/audit-event-configuration/README.md @@ -0,0 +1,18 @@ +# Oauth2 JWT Token Authentication Service + +This extension provides the capability to authorizate the request to the connector management API. The extension will access the Bearer token provided in the Authorization header and validate that it is a valid JWT-encoded bearer token. It is necessary to have the `org.eclipse.edc:oauth2-core` extension correctly configured. + +To authorize a user, the roles of the provided JWT token must contain: +- a valid role from those configured in `allowedRoles` +- a role with the `connector name` + +## Configuration + +Example configuration: + +```properties +edc.api.auth.oauth2.allowedRoles.1.role=connector-admin +edc.api.auth.oauth2.allowedRoles.2.role=connector-management +``` + +The `edc.api.auth.oauth2.allowedRoles` will be used by the federated catalog to retrieve the list of allowed roles that can perform requests on the managemente API connector. \ No newline at end of file diff --git a/extensions/audit-event-configuration/build.gradle.kts b/extensions/audit-event-configuration/build.gradle.kts new file mode 100644 index 0000000..9261f24 --- /dev/null +++ b/extensions/audit-event-configuration/build.gradle.kts @@ -0,0 +1,17 @@ +plugins { + `java-library` + id("com.gmv.inesdata.edc-application") +} + +dependencies { + api(libs.edc.auth.spi) + + implementation(libs.edc.iam.oauth2.core) + implementation(libs.jakarta.rsApi) + + testImplementation(libs.edc.core.junit) + testImplementation(libs.assertj) + testImplementation(libs.mockito.core) +} + + diff --git a/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriber.java b/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriber.java new file mode 100644 index 0000000..3a1fcff --- /dev/null +++ b/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriber.java @@ -0,0 +1,20 @@ +package org.upm.inesdata.auditevent; + +import org.eclipse.edc.spi.event.Event; +import org.eclipse.edc.spi.event.EventEnvelope; +import org.eclipse.edc.spi.event.EventSubscriber; +import org.eclipse.edc.spi.monitor.Monitor; + +public class AuditEventSubscriber implements EventSubscriber { + + private final Monitor monitor; + + public AuditEventSubscriber(Monitor monitor) { + this.monitor = monitor; + } + + @Override + public void on(EventEnvelope event) { + monitor.info("el evento"+event.getPayload().toString()); + } +} \ No newline at end of file diff --git a/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriptionExtension.java b/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriptionExtension.java new file mode 100644 index 0000000..f9fe934 --- /dev/null +++ b/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriptionExtension.java @@ -0,0 +1,21 @@ +package org.upm.inesdata.auditevent; + +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.spi.event.Event; +import org.eclipse.edc.spi.event.EventRouter; +import org.eclipse.edc.spi.monitor.Monitor; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; + +public class AuditEventSubscriptionExtension implements ServiceExtension { + @Inject + private EventRouter eventRouter; + @Inject + private Monitor monitor; + + @Override + public void initialize(ServiceExtensionContext context) { + eventRouter.register(Event.class, new AuditEventSubscriber(monitor)); + eventRouter.registerSync(Event.class, new AuditEventSubscriber(monitor)); + } +} \ No newline at end of file diff --git a/extensions/audit-event-configuration/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/audit-event-configuration/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 0000000..10363ec --- /dev/null +++ b/extensions/audit-event-configuration/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1 @@ +org.upm.inesdata.auditevent.AuditEventSubscriptionExtension diff --git a/launchers/connector/build.gradle.kts b/launchers/connector/build.gradle.kts index 7d485ea..abc1da3 100644 --- a/launchers/connector/build.gradle.kts +++ b/launchers/connector/build.gradle.kts @@ -93,6 +93,10 @@ dependencies { //Data plane public api implementation(project(":extensions:extended-data-plane-public-api")) + // Storage assets + implementation(project(":extensions:audit-configuration")) + implementation(project(":extensions:audit-event-configuration")) + runtimeOnly(libs.edc.transaction.local) runtimeOnly(libs.postgres) } diff --git a/settings.gradle.kts b/settings.gradle.kts index 7803ccb..49a1eef 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -23,6 +23,8 @@ include(":extensions:federated-catalog-cache-api") include(":extensions:count-elements-api") include(":extensions:count-elements-sql") include(":extensions:extended-data-plane-public-api") +include(":extensions:audit-configuration") +include(":extensions:audit-event-configuration") // Connector include(":launchers:connector") From ebba017bbfbbaf383c1ff21a8d5a8bc8784f753d Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Thu, 4 Jul 2024 09:31:07 +0200 Subject: [PATCH 108/174] Control --- .../src/main/java/org/upm/inesdata/audit/AuditExtension.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/AuditExtension.java b/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/AuditExtension.java index 3583152..fd9072d 100644 --- a/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/AuditExtension.java +++ b/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/AuditExtension.java @@ -8,6 +8,7 @@ import org.eclipse.edc.spi.system.ServiceExtension; import org.eclipse.edc.spi.system.ServiceExtensionContext; import org.eclipse.edc.web.spi.WebService; +import org.eclipse.edc.web.spi.configuration.ApiContext; @Provides(HttpRequestInterceptor.class) @Extension(value = AuditExtension.NAME) @@ -28,7 +29,7 @@ public String name() { public void initialize(ServiceExtensionContext context) { // Registrar el interceptor HTTP context.registerService(Filter.class, new HttpRequestInterceptor(context.getMonitor())); - webService.registerResource(new HttpRequestInterceptor(context.getMonitor())); + webService.registerResource(ApiContext.CONTROL,new HttpRequestInterceptor(context.getMonitor())); context.getMonitor().info("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA - Registered Filter"); } } From d0cabaa2d6bfdaca123e991c09ae00c0640059a3 Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Thu, 4 Jul 2024 11:09:22 +0200 Subject: [PATCH 109/174] Cambios en la auditoria --- .../upm/inesdata/audit/AuditExtension.java | 3 +- .../audit/HttpRequestInterceptor.java | 46 +++++-------------- 2 files changed, 12 insertions(+), 37 deletions(-) diff --git a/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/AuditExtension.java b/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/AuditExtension.java index fd9072d..7069697 100644 --- a/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/AuditExtension.java +++ b/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/AuditExtension.java @@ -28,8 +28,7 @@ public String name() { @Override public void initialize(ServiceExtensionContext context) { // Registrar el interceptor HTTP - context.registerService(Filter.class, new HttpRequestInterceptor(context.getMonitor())); - webService.registerResource(ApiContext.CONTROL,new HttpRequestInterceptor(context.getMonitor())); + webService.registerResource(ApiContext.MANAGEMENT,new HttpRequestInterceptor(context.getMonitor())); context.getMonitor().info("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA - Registered Filter"); } } diff --git a/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/HttpRequestInterceptor.java b/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/HttpRequestInterceptor.java index b16c213..85e0d69 100644 --- a/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/HttpRequestInterceptor.java +++ b/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/HttpRequestInterceptor.java @@ -1,20 +1,15 @@ package org.upm.inesdata.audit; -import jakarta.servlet.Filter; -import jakarta.servlet.FilterChain; -import jakarta.servlet.FilterConfig; -import jakarta.servlet.ServletException; -import jakarta.servlet.ServletRequest; -import jakarta.servlet.ServletResponse; -import jakarta.servlet.annotation.WebFilter; -import jakarta.servlet.http.HttpServletRequest; import org.eclipse.edc.spi.monitor.Monitor; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.container.ContainerRequestFilter; +import jakarta.ws.rs.ext.Provider; import java.io.IOException; import java.util.Enumeration; -@WebFilter("/*") -public class HttpRequestInterceptor implements Filter { +@Provider +public class HttpRequestInterceptor implements ContainerRequestFilter { private final Monitor monitor; @@ -23,34 +18,15 @@ public HttpRequestInterceptor(Monitor monitor) { } @Override - public void init(FilterConfig filterConfig) throws ServletException { - // Inicialización si es necesaria - } - - @Override - public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) - throws IOException, ServletException { - - HttpServletRequest httpRequest = (HttpServletRequest) request; - + public void filter(ContainerRequestContext requestContext) throws IOException { // Auditar la petición monitor.info("Intercepted Request: "); - monitor.info("Method: " + httpRequest.getMethod()); - monitor.info("URL: " + httpRequest.getRequestURL().toString()); + monitor.info("Method: " + requestContext.getMethod()); + monitor.info("URL: " + requestContext.getUriInfo().getRequestUri().toString()); // Imprimir cabeceras - Enumeration headerNames = httpRequest.getHeaderNames(); - while (headerNames.hasMoreElements()) { - String headerName = headerNames.nextElement(); - monitor.info(headerName + ": " + httpRequest.getHeader(headerName)); + for (String headerName : requestContext.getHeaders().keySet()) { + monitor.info(headerName + ": " + String.join(", ", requestContext.getHeaders().get(headerName))); } - - // Continuar con el siguiente filtro o recurso - chain.doFilter(request, response); - } - - @Override - public void destroy() { - // Limpieza si es necesaria } -} \ No newline at end of file +} From c5a0536c3d8e73187b7eb5826f43ad5eebb557a4 Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Thu, 4 Jul 2024 13:58:40 +0200 Subject: [PATCH 110/174] Logs --- .../audit/HttpRequestInterceptor.java | 23 ++++++++----------- .../build.gradle.kts | 2 ++ .../auditevent/AuditEventSubscriber.java | 15 +++++++++++- gradle/libs.versions.toml | 2 ++ 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/HttpRequestInterceptor.java b/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/HttpRequestInterceptor.java index 85e0d69..8908a37 100644 --- a/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/HttpRequestInterceptor.java +++ b/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/HttpRequestInterceptor.java @@ -1,12 +1,11 @@ package org.upm.inesdata.audit; -import org.eclipse.edc.spi.monitor.Monitor; - import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.container.ContainerRequestFilter; import jakarta.ws.rs.ext.Provider; -import java.io.IOException; -import java.util.Enumeration; +import org.eclipse.edc.spi.monitor.Monitor; + +import java.text.MessageFormat; @Provider public class HttpRequestInterceptor implements ContainerRequestFilter { @@ -18,15 +17,11 @@ public HttpRequestInterceptor(Monitor monitor) { } @Override - public void filter(ContainerRequestContext requestContext) throws IOException { - // Auditar la petición - monitor.info("Intercepted Request: "); - monitor.info("Method: " + requestContext.getMethod()); - monitor.info("URL: " + requestContext.getUriInfo().getRequestUri().toString()); - - // Imprimir cabeceras - for (String headerName : requestContext.getHeaders().keySet()) { - monitor.info(headerName + ": " + String.join(", ", requestContext.getHeaders().get(headerName))); - } + public void filter(ContainerRequestContext requestContext){ + // Formatear y registrar el mensaje de auditoría + String auditLog = MessageFormat.format("[AUDIT][MANAGEMENT] User ''{0}'' calls ''{1}''", + "USER", + requestContext.getUriInfo().getRequestUri().toString()); + monitor.info(auditLog); } } diff --git a/extensions/audit-event-configuration/build.gradle.kts b/extensions/audit-event-configuration/build.gradle.kts index 9261f24..191d231 100644 --- a/extensions/audit-event-configuration/build.gradle.kts +++ b/extensions/audit-event-configuration/build.gradle.kts @@ -8,6 +8,8 @@ dependencies { implementation(libs.edc.iam.oauth2.core) implementation(libs.jakarta.rsApi) + implementation(libs.edc.contract.spi) + implementation(libs.edc.transfer.spi) testImplementation(libs.edc.core.junit) testImplementation(libs.assertj) diff --git a/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriber.java b/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriber.java index 3a1fcff..520f4f2 100644 --- a/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriber.java +++ b/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriber.java @@ -1,10 +1,14 @@ package org.upm.inesdata.auditevent; +import org.eclipse.edc.connector.controlplane.contract.spi.event.contractnegotiation.ContractNegotiationEvent; +import org.eclipse.edc.connector.controlplane.transfer.spi.event.TransferProcessEvent; import org.eclipse.edc.spi.event.Event; import org.eclipse.edc.spi.event.EventEnvelope; import org.eclipse.edc.spi.event.EventSubscriber; import org.eclipse.edc.spi.monitor.Monitor; +import java.text.MessageFormat; + public class AuditEventSubscriber implements EventSubscriber { private final Monitor monitor; @@ -15,6 +19,15 @@ public AuditEventSubscriber(Monitor monitor) { @Override public void on(EventEnvelope event) { - monitor.info("el evento"+event.getPayload().toString()); + if(event.getPayload() instanceof ContractNegotiationEvent){ + String simpleName = event.getPayload().getClass().getSimpleName(); + ContractNegotiationEvent payload = (ContractNegotiationEvent) event.getPayload(); + monitor.info(MessageFormat.format("[AUDIT][DSP] {0} from counterPartyId '{1}' with contractNegotiationId '{2}'",simpleName, payload.getCounterPartyId(),payload.getContractNegotiationId())); + }else if (event.getPayload() instanceof TransferProcessEvent){ + String simpleName = event.getPayload().getClass().getSimpleName(); + TransferProcessEvent payload = (TransferProcessEvent) event.getPayload(); + monitor.info(MessageFormat.format("[AUDIT][DSP] {0} from contractId '{1}' with assetId '{2}' for type '{3}'",simpleName, payload.getContractId(),payload.getAssetId(),payload.getType())); + } + } } \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 2875eef..deb5e1f 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -26,6 +26,7 @@ edc-auth-spi = { module = "org.eclipse.edc:auth-spi", version.ref = "edc" } edc-build-plugin = { module = "org.eclipse.edc.edc-build:org.eclipse.edc.edc-build.gradle.plugin", version.ref = "edc" } edc-configuration-filesystem = { module = "org.eclipse.edc:configuration-filesystem", version.ref = "edc" } edc-connector-core = { module = "org.eclipse.edc:connector-core", version.ref = "edc" } +edc-contract-spi = { module = "org.eclipse.edc:contract-spi", version.ref = "edc" } edc-control-plane-api = { module = "org.eclipse.edc:control-plane-api", version.ref = "edc" } edc-control-plane-api-client = { module = "org.eclipse.edc:control-plane-api-client", version.ref = "edc" } edc-control-plane-core = { module = "org.eclipse.edc:control-plane-core", version.ref = "edc" } @@ -62,6 +63,7 @@ edc-transaction-local = { module = "org.eclipse.edc:transaction-local", version. edc-transaction-spi = { module = "org.eclipse.edc:transaction-spi", version.ref = "edc" } edc-transfer-data-plane-signaling = { module = "org.eclipse.edc:transfer-data-plane-signaling", version.ref = "edc" } edc-transfer-pull-http-receiver = { module = "org.eclipse.edc:transfer-pull-http-dynamic-receiver", version.ref = "edc" } +edc-transfer-spi = { module = "org.eclipse.edc:transfer-spi", version.ref = "edc" } edc-validator-data-address-http-data = { module = "org.eclipse.edc:validator-data-address-http-data", version.ref = "edc" } edc-validator-spi = { module = "org.eclipse.edc:validator-spi", version.ref = "edc" } edc-vault-hashicorp = { module = "org.eclipse.edc:vault-hashicorp", version.ref = "edc" } From 368fdc61a2525a89e13809cb02a7455b1c53107b Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Thu, 4 Jul 2024 14:07:08 +0200 Subject: [PATCH 111/174] Log mensaje --- .../org/upm/inesdata/auditevent/AuditEventSubscriber.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriber.java b/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriber.java index 520f4f2..316a414 100644 --- a/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriber.java +++ b/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriber.java @@ -22,11 +22,11 @@ public void on(EventEnvelope event) { if(event.getPayload() instanceof ContractNegotiationEvent){ String simpleName = event.getPayload().getClass().getSimpleName(); ContractNegotiationEvent payload = (ContractNegotiationEvent) event.getPayload(); - monitor.info(MessageFormat.format("[AUDIT][DSP] {0} from counterPartyId '{1}' with contractNegotiationId '{2}'",simpleName, payload.getCounterPartyId(),payload.getContractNegotiationId())); + monitor.info(MessageFormat.format("[AUDIT][DSP] '{0}' from counterPartyId ''{1}'' with contractNegotiationId ''{2}''",simpleName, payload.getCounterPartyId(),payload.getContractNegotiationId())); }else if (event.getPayload() instanceof TransferProcessEvent){ String simpleName = event.getPayload().getClass().getSimpleName(); TransferProcessEvent payload = (TransferProcessEvent) event.getPayload(); - monitor.info(MessageFormat.format("[AUDIT][DSP] {0} from contractId '{1}' with assetId '{2}' for type '{3}'",simpleName, payload.getContractId(),payload.getAssetId(),payload.getType())); + monitor.info(MessageFormat.format("[AUDIT][DSP] '{0}' from contractId ''{1}'' with assetId ''{2}'' for type ''{3}''",simpleName, payload.getContractId(),payload.getAssetId(),payload.getType())); } } From d713feb5150ed98d04a9ab9bf801fe7509e938f1 Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Thu, 4 Jul 2024 14:17:43 +0200 Subject: [PATCH 112/174] Logs --- .../org/upm/inesdata/auditevent/AuditEventSubscriber.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriber.java b/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriber.java index 316a414..4482173 100644 --- a/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriber.java +++ b/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriber.java @@ -22,11 +22,11 @@ public void on(EventEnvelope event) { if(event.getPayload() instanceof ContractNegotiationEvent){ String simpleName = event.getPayload().getClass().getSimpleName(); ContractNegotiationEvent payload = (ContractNegotiationEvent) event.getPayload(); - monitor.info(MessageFormat.format("[AUDIT][DSP] '{0}' from counterPartyId ''{1}'' with contractNegotiationId ''{2}''",simpleName, payload.getCounterPartyId(),payload.getContractNegotiationId())); + monitor.info(MessageFormat.format("[AUDIT][DSP] ''{0}'' from counterPartyId ''{1}'' with contractNegotiationId ''{2}''",simpleName, payload.getCounterPartyId(),payload.getContractNegotiationId())); }else if (event.getPayload() instanceof TransferProcessEvent){ String simpleName = event.getPayload().getClass().getSimpleName(); TransferProcessEvent payload = (TransferProcessEvent) event.getPayload(); - monitor.info(MessageFormat.format("[AUDIT][DSP] '{0}' from contractId ''{1}'' with assetId ''{2}'' for type ''{3}''",simpleName, payload.getContractId(),payload.getAssetId(),payload.getType())); + monitor.info(MessageFormat.format("[AUDIT][DSP] ''{0}'' from contractId ''{1}'' with assetId ''{2}'' for type ''{3}''",simpleName, payload.getContractId(),payload.getAssetId(),payload.getType())); } } From 732d18e1558670d470ef943174aeedaceddec047 Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Fri, 5 Jul 2024 09:33:57 +0200 Subject: [PATCH 113/174] Cambios token y participant --- .../upm/inesdata/audit/AuditExtension.java | 10 ++--- .../audit/HttpRequestInterceptor.java | 44 +++++++++++++++++-- .../auditevent/AuditEventSubscriber.java | 8 ++-- .../AuditEventSubscriptionExtension.java | 4 +- 4 files changed, 52 insertions(+), 14 deletions(-) diff --git a/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/AuditExtension.java b/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/AuditExtension.java index 7069697..ccbe050 100644 --- a/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/AuditExtension.java +++ b/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/AuditExtension.java @@ -1,10 +1,9 @@ package org.upm.inesdata.audit; -import jakarta.servlet.Filter; import org.eclipse.edc.runtime.metamodel.annotation.Extension; import org.eclipse.edc.runtime.metamodel.annotation.Inject; import org.eclipse.edc.runtime.metamodel.annotation.Provides; -import org.eclipse.edc.spi.monitor.Monitor; +import org.eclipse.edc.spi.iam.IdentityService; import org.eclipse.edc.spi.system.ServiceExtension; import org.eclipse.edc.spi.system.ServiceExtensionContext; import org.eclipse.edc.web.spi.WebService; @@ -20,6 +19,9 @@ public class AuditExtension implements ServiceExtension { @Inject private WebService webService; + @Inject + private IdentityService identityService; + @Override public String name() { return NAME; @@ -27,8 +29,6 @@ public String name() { @Override public void initialize(ServiceExtensionContext context) { - // Registrar el interceptor HTTP - webService.registerResource(ApiContext.MANAGEMENT,new HttpRequestInterceptor(context.getMonitor())); - context.getMonitor().info("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA - Registered Filter"); + webService.registerResource(ApiContext.MANAGEMENT,new HttpRequestInterceptor(context.getMonitor(), identityService, context.getParticipantId())); } } diff --git a/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/HttpRequestInterceptor.java b/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/HttpRequestInterceptor.java index 8908a37..022d9e4 100644 --- a/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/HttpRequestInterceptor.java +++ b/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/HttpRequestInterceptor.java @@ -3,25 +3,61 @@ import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.container.ContainerRequestFilter; import jakarta.ws.rs.ext.Provider; +import org.eclipse.edc.spi.iam.IdentityService; +import org.eclipse.edc.spi.iam.TokenRepresentation; import org.eclipse.edc.spi.monitor.Monitor; import java.text.MessageFormat; +import java.util.List; +import java.util.Map; @Provider public class HttpRequestInterceptor implements ContainerRequestFilter { + private static final String TEMPLATE_AUDIT_LOG = "[AUDIT][''{0}''][MANAGEMENT] User ''{1}'' calls ''{2}''"; + + private static final String BEARER_PREFIX = "Bearer "; + private static final String REALM_ACCESS_CLAIM_NAME = "realm_access"; + private static final String ROLES_NAME = "roles"; + public static final String TOKEN_PROPERTY_PREFERRED_USERNAME = "preferred_username"; + private final Monitor monitor; + private final IdentityService identityService; - public HttpRequestInterceptor(Monitor monitor) { + private final String participantId; + + public HttpRequestInterceptor(Monitor monitor, IdentityService identityService, String participantId) { this.monitor = monitor; + this.identityService = identityService; + this.participantId = participantId; } @Override public void filter(ContainerRequestContext requestContext){ - // Formatear y registrar el mensaje de auditoría - String auditLog = MessageFormat.format("[AUDIT][MANAGEMENT] User ''{0}'' calls ''{1}''", - "USER", + String user = getUserFromRequest(requestContext); + String auditLog = MessageFormat.format(TEMPLATE_AUDIT_LOG, participantId, + user, requestContext.getUriInfo().getRequestUri().toString()); monitor.info(auditLog); } + + private String getUserFromRequest(ContainerRequestContext requestContext) { + List authorizationHeaders = requestContext.getHeaders().get("Authorization"); + if (authorizationHeaders != null && !authorizationHeaders.isEmpty()) { + String authHeader = authorizationHeaders.get(0); + if (authHeader.startsWith(BEARER_PREFIX)) { + String token = authHeader.replace(BEARER_PREFIX, ""); + return decodeJWT(token); + } + } + return "NotAuthenticatedUser"; + } + + private String decodeJWT(String token) { + var tokenRepresentation = TokenRepresentation.Builder.newInstance().token(token).build(); + var tokenValidation = identityService.verifyJwtToken(tokenRepresentation, null); + var claimToken = tokenValidation.getContent(); + var realmAccess = (Map) claimToken.getClaim(REALM_ACCESS_CLAIM_NAME); + return (String) realmAccess.get(TOKEN_PROPERTY_PREFERRED_USERNAME); + } } diff --git a/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriber.java b/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriber.java index 4482173..0a63a37 100644 --- a/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriber.java +++ b/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriber.java @@ -12,9 +12,11 @@ public class AuditEventSubscriber implements EventSubscriber { private final Monitor monitor; + private final String participantId; - public AuditEventSubscriber(Monitor monitor) { + public AuditEventSubscriber(Monitor monitor, String participantId) { this.monitor = monitor; + this.participantId = participantId; } @Override @@ -22,11 +24,11 @@ public void on(EventEnvelope event) { if(event.getPayload() instanceof ContractNegotiationEvent){ String simpleName = event.getPayload().getClass().getSimpleName(); ContractNegotiationEvent payload = (ContractNegotiationEvent) event.getPayload(); - monitor.info(MessageFormat.format("[AUDIT][DSP] ''{0}'' from counterPartyId ''{1}'' with contractNegotiationId ''{2}''",simpleName, payload.getCounterPartyId(),payload.getContractNegotiationId())); + monitor.info(MessageFormat.format("[AUDIT][''{0}''][DSP] ''{1}'' from counterPartyId ''{2}'' with contractNegotiationId ''{3}''",participantId,simpleName, payload.getCounterPartyId(),payload.getContractNegotiationId())); }else if (event.getPayload() instanceof TransferProcessEvent){ String simpleName = event.getPayload().getClass().getSimpleName(); TransferProcessEvent payload = (TransferProcessEvent) event.getPayload(); - monitor.info(MessageFormat.format("[AUDIT][DSP] ''{0}'' from contractId ''{1}'' with assetId ''{2}'' for type ''{3}''",simpleName, payload.getContractId(),payload.getAssetId(),payload.getType())); + monitor.info(MessageFormat.format("[AUDIT][''{0}''][DSP] ''{1}'' from contractId ''{2}'' with assetId ''{3}'' for type ''{4}''",participantId,simpleName, payload.getContractId(),payload.getAssetId(),payload.getType())); } } diff --git a/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriptionExtension.java b/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriptionExtension.java index f9fe934..d4bc9bb 100644 --- a/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriptionExtension.java +++ b/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriptionExtension.java @@ -15,7 +15,7 @@ public class AuditEventSubscriptionExtension implements ServiceExtension { @Override public void initialize(ServiceExtensionContext context) { - eventRouter.register(Event.class, new AuditEventSubscriber(monitor)); - eventRouter.registerSync(Event.class, new AuditEventSubscriber(monitor)); + eventRouter.register(Event.class, new AuditEventSubscriber(monitor,context.getParticipantId())); + eventRouter.registerSync(Event.class, new AuditEventSubscriber(monitor, context.getParticipantId())); } } \ No newline at end of file From 0160b7d0cfe28c389eb17cc68e148e4824d40d6c Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Fri, 5 Jul 2024 09:54:09 +0200 Subject: [PATCH 114/174] Error claims --- .../java/org/upm/inesdata/audit/HttpRequestInterceptor.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/HttpRequestInterceptor.java b/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/HttpRequestInterceptor.java index 022d9e4..acaf1d9 100644 --- a/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/HttpRequestInterceptor.java +++ b/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/HttpRequestInterceptor.java @@ -56,8 +56,6 @@ private String getUserFromRequest(ContainerRequestContext requestContext) { private String decodeJWT(String token) { var tokenRepresentation = TokenRepresentation.Builder.newInstance().token(token).build(); var tokenValidation = identityService.verifyJwtToken(tokenRepresentation, null); - var claimToken = tokenValidation.getContent(); - var realmAccess = (Map) claimToken.getClaim(REALM_ACCESS_CLAIM_NAME); - return (String) realmAccess.get(TOKEN_PROPERTY_PREFERRED_USERNAME); + return (String) tokenValidation.getContent().getClaims().get(TOKEN_PROPERTY_PREFERRED_USERNAME); } } From a21fabb1f5c5df0bffdd31db0aa041468849d32d Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Fri, 5 Jul 2024 10:52:08 +0200 Subject: [PATCH 115/174] =?UTF-8?q?=C3=9Altimos=20retoques?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- extensions/audit-configuration/README.md | 21 +++----- .../upm/inesdata/audit/AuditExtension.java | 17 ++++++- .../audit/HttpRequestInterceptor.java | 50 ++++++++++++++----- .../audit-event-configuration/README.md | 18 +++---- .../build.gradle.kts | 3 +- .../auditevent/AuditEventSubscriber.java | 29 ++++++++--- .../AuditEventSubscriptionExtension.java | 13 ++++- 7 files changed, 103 insertions(+), 48 deletions(-) diff --git a/extensions/audit-configuration/README.md b/extensions/audit-configuration/README.md index 51bb140..cfc1bee 100644 --- a/extensions/audit-configuration/README.md +++ b/extensions/audit-configuration/README.md @@ -1,18 +1,13 @@ -# Oauth2 JWT Token Authentication Service +# Audit Extension -This extension provides the capability to authorizate the request to the connector management API. The extension will access the Bearer token provided in the Authorization header and validate that it is a valid JWT-encoded bearer token. It is necessary to have the `org.eclipse.edc:oauth2-core` extension correctly configured. +This extension provides the capability to log audit events for HTTP requests made to the connector management API. The `AuditExtension` registers an `HttpRequestInterceptor` that logs details about the user making the request and the request URI. -To authorize a user, the roles of the provided JWT token must contain: -- a valid role from those configured in `allowedRoles` -- a role with the `connector name` +## Features -## Configuration - -Example configuration: +- Logs audit details for incoming HTTP requests. +- Extracts and verifies JWT tokens to log the username. +- Configurable participant ID for audit logging. -```properties -edc.api.auth.oauth2.allowedRoles.1.role=connector-admin -edc.api.auth.oauth2.allowedRoles.2.role=connector-management -``` +## Configuration -The `edc.api.auth.oauth2.allowedRoles` will be used by the federated catalog to retrieve the list of allowed roles that can perform requests on the managemente API connector. \ No newline at end of file +To configure the audit logging, you need to ensure that the `HttpRequestInterceptor` is registered with the web service. This is done within the `AuditExtension` class. diff --git a/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/AuditExtension.java b/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/AuditExtension.java index ccbe050..249a585 100644 --- a/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/AuditExtension.java +++ b/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/AuditExtension.java @@ -9,26 +9,39 @@ import org.eclipse.edc.web.spi.WebService; import org.eclipse.edc.web.spi.configuration.ApiContext; +/** + * Service extension for configuring audit logging. + * Registers the {@link HttpRequestInterceptor} as a resource with the web service. + */ @Provides(HttpRequestInterceptor.class) @Extension(value = AuditExtension.NAME) public class AuditExtension implements ServiceExtension { public static final String NAME = "Audit configuration"; - @Inject private WebService webService; @Inject private IdentityService identityService; + /** + * Returns the name of the extension. + * + * @return the name of the extension + */ @Override public String name() { return NAME; } + /** + * Initializes the service extension by registering the {@link HttpRequestInterceptor} with the web service. + * + * @param context the service extension context providing configuration and services + */ @Override public void initialize(ServiceExtensionContext context) { - webService.registerResource(ApiContext.MANAGEMENT,new HttpRequestInterceptor(context.getMonitor(), identityService, context.getParticipantId())); + webService.registerResource(ApiContext.MANAGEMENT, new HttpRequestInterceptor(context.getMonitor(), identityService, context.getParticipantId())); } } diff --git a/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/HttpRequestInterceptor.java b/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/HttpRequestInterceptor.java index acaf1d9..67b7543 100644 --- a/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/HttpRequestInterceptor.java +++ b/extensions/audit-configuration/src/main/java/org/upm/inesdata/audit/HttpRequestInterceptor.java @@ -9,38 +9,55 @@ import java.text.MessageFormat; import java.util.List; -import java.util.Map; +/** + * Intercepts HTTP requests to audit user actions. + * Logs details about the user and the request URI. + */ @Provider public class HttpRequestInterceptor implements ContainerRequestFilter { - private static final String TEMPLATE_AUDIT_LOG = "[AUDIT][''{0}''][MANAGEMENT] User ''{1}'' calls ''{2}''"; - + private static final String TEMPLATE_AUDIT_LOG = "[AUDIT][{0}][MANAGEMENT] User ''{1}'' calls ''{2}''"; private static final String BEARER_PREFIX = "Bearer "; - private static final String REALM_ACCESS_CLAIM_NAME = "realm_access"; - private static final String ROLES_NAME = "roles"; - public static final String TOKEN_PROPERTY_PREFERRED_USERNAME = "preferred_username"; + private static final String TOKEN_PROPERTY_PREFERRED_USERNAME = "preferred_username"; + private static final String TOKEN_USER_NOT_VALID_USER = "NotValidUser"; + private static final String TOKEN_USER_NOT_AUTHENTICATED_USER = "NotAuthenticatedUser"; private final Monitor monitor; private final IdentityService identityService; - private final String participantId; + /** + * Constructor for HttpRequestInterceptor. + * + * @param monitor the monitor interface used for logging + * @param identityService the identity service for token verification + * @param participantId the participant ID for audit log entries + */ public HttpRequestInterceptor(Monitor monitor, IdentityService identityService, String participantId) { this.monitor = monitor; this.identityService = identityService; this.participantId = participantId; } + /** + * Filters the HTTP request context to log audit details. + * + * @param requestContext the container request context + */ @Override - public void filter(ContainerRequestContext requestContext){ + public void filter(ContainerRequestContext requestContext) { String user = getUserFromRequest(requestContext); - String auditLog = MessageFormat.format(TEMPLATE_AUDIT_LOG, participantId, - user, - requestContext.getUriInfo().getRequestUri().toString()); + String auditLog = MessageFormat.format(TEMPLATE_AUDIT_LOG, participantId, user, requestContext.getUriInfo().getRequestUri().toString()); monitor.info(auditLog); } + /** + * Extracts the user from the HTTP request context by decoding the JWT token. + * + * @param requestContext the container request context + * @return the username extracted from the token, or a default value if not authenticated or token is invalid + */ private String getUserFromRequest(ContainerRequestContext requestContext) { List authorizationHeaders = requestContext.getHeaders().get("Authorization"); if (authorizationHeaders != null && !authorizationHeaders.isEmpty()) { @@ -50,12 +67,21 @@ private String getUserFromRequest(ContainerRequestContext requestContext) { return decodeJWT(token); } } - return "NotAuthenticatedUser"; + return TOKEN_USER_NOT_AUTHENTICATED_USER; } + /** + * Decodes the JWT token to extract the username. + * + * @param token the JWT token + * @return the username if token is valid, otherwise a default value indicating the token is not valid + */ private String decodeJWT(String token) { var tokenRepresentation = TokenRepresentation.Builder.newInstance().token(token).build(); var tokenValidation = identityService.verifyJwtToken(tokenRepresentation, null); + if (tokenValidation.failed()) { + return TOKEN_USER_NOT_VALID_USER; + } return (String) tokenValidation.getContent().getClaims().get(TOKEN_PROPERTY_PREFERRED_USERNAME); } } diff --git a/extensions/audit-event-configuration/README.md b/extensions/audit-event-configuration/README.md index 51bb140..6f343e6 100644 --- a/extensions/audit-event-configuration/README.md +++ b/extensions/audit-event-configuration/README.md @@ -1,18 +1,14 @@ -# Oauth2 JWT Token Authentication Service +# AuditEventSubscriptionExtension -This extension provides the capability to authorizate the request to the connector management API. The extension will access the Bearer token provided in the Authorization header and validate that it is a valid JWT-encoded bearer token. It is necessary to have the `org.eclipse.edc:oauth2-core` extension correctly configured. +This extension provides the capability to subscribe to audit events related to contract negotiations and transfer processes. The `AuditEventSubscriber` logs details about these events using the provided monitoring interface. -To authorize a user, the roles of the provided JWT token must contain: -- a valid role from those configured in `allowedRoles` -- a role with the `connector name` +## Features + +- Logs audit details for contract negotiation and transfer process events. +- Registers the `AuditEventSubscriber` with the event router for both asynchronous and synchronous event handling. ## Configuration -Example configuration: +To configure the `AuditEventSubscriptionExtension`, ensure that the `AuditEventSubscriber` is registered with the event router. This is done within the `AuditEventSubscriptionExtension` class. -```properties -edc.api.auth.oauth2.allowedRoles.1.role=connector-admin -edc.api.auth.oauth2.allowedRoles.2.role=connector-management -``` -The `edc.api.auth.oauth2.allowedRoles` will be used by the federated catalog to retrieve the list of allowed roles that can perform requests on the managemente API connector. \ No newline at end of file diff --git a/extensions/audit-event-configuration/build.gradle.kts b/extensions/audit-event-configuration/build.gradle.kts index 191d231..abfed22 100644 --- a/extensions/audit-event-configuration/build.gradle.kts +++ b/extensions/audit-event-configuration/build.gradle.kts @@ -5,8 +5,7 @@ plugins { dependencies { api(libs.edc.auth.spi) - - implementation(libs.edc.iam.oauth2.core) + implementation(libs.jakarta.rsApi) implementation(libs.edc.contract.spi) implementation(libs.edc.transfer.spi) diff --git a/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriber.java b/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriber.java index 0a63a37..a9649b1 100644 --- a/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriber.java +++ b/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriber.java @@ -9,27 +9,44 @@ import java.text.MessageFormat; +/** + * Subscriber for auditing contract negotiation and transfer process events. + * Logs event details using a specified monitoring interface. + */ public class AuditEventSubscriber implements EventSubscriber { + private static final String NEGOTIATION_TEMPLATE_AUDIT_LOG = "[AUDIT][{0}][DSP] ''{1}'' from counterPartyId ''{2}'' with contractNegotiationId ''{3}''"; + private static final String TRANSFER_TEMPLATE_AUDIT_LOG = "[AUDIT][{0}][DSP] ''{1}'' from contractId ''{2}'' with assetId ''{3}'' for type ''{4}''"; private final Monitor monitor; private final String participantId; + /** + * Constructor for AuditEventSubscriber. + * + * @param monitor the monitor interface used for logging + * @param participantId the participant ID for audit log entries + */ public AuditEventSubscriber(Monitor monitor, String participantId) { this.monitor = monitor; this.participantId = participantId; } + /** + * Processes the received event envelope and logs relevant details based on event type. + * + * @param event the event envelope containing the event to be processed + * @param the type of the event + */ @Override public void on(EventEnvelope event) { - if(event.getPayload() instanceof ContractNegotiationEvent){ + if (event.getPayload() instanceof ContractNegotiationEvent) { String simpleName = event.getPayload().getClass().getSimpleName(); ContractNegotiationEvent payload = (ContractNegotiationEvent) event.getPayload(); - monitor.info(MessageFormat.format("[AUDIT][''{0}''][DSP] ''{1}'' from counterPartyId ''{2}'' with contractNegotiationId ''{3}''",participantId,simpleName, payload.getCounterPartyId(),payload.getContractNegotiationId())); - }else if (event.getPayload() instanceof TransferProcessEvent){ + monitor.info(MessageFormat.format(NEGOTIATION_TEMPLATE_AUDIT_LOG, participantId, simpleName, payload.getCounterPartyId(), payload.getContractNegotiationId())); + } else if (event.getPayload() instanceof TransferProcessEvent) { String simpleName = event.getPayload().getClass().getSimpleName(); TransferProcessEvent payload = (TransferProcessEvent) event.getPayload(); - monitor.info(MessageFormat.format("[AUDIT][''{0}''][DSP] ''{1}'' from contractId ''{2}'' with assetId ''{3}'' for type ''{4}''",participantId,simpleName, payload.getContractId(),payload.getAssetId(),payload.getType())); + monitor.info(MessageFormat.format(TRANSFER_TEMPLATE_AUDIT_LOG, participantId, simpleName, payload.getContractId(), payload.getAssetId(), payload.getType())); } - } -} \ No newline at end of file +} diff --git a/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriptionExtension.java b/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriptionExtension.java index d4bc9bb..36b3e49 100644 --- a/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriptionExtension.java +++ b/extensions/audit-event-configuration/src/main/java/org/upm/inesdata/auditevent/AuditEventSubscriptionExtension.java @@ -7,15 +7,24 @@ import org.eclipse.edc.spi.system.ServiceExtension; import org.eclipse.edc.spi.system.ServiceExtensionContext; +/** + * Service extension for subscribing to audit events. + * Registers the {@link AuditEventSubscriber} with the event router for both asynchronous and synchronous event handling. + */ public class AuditEventSubscriptionExtension implements ServiceExtension { @Inject private EventRouter eventRouter; @Inject private Monitor monitor; + /** + * Initializes the service extension by registering the {@link AuditEventSubscriber} with the event router. + * + * @param context the service extension context providing configuration and services + */ @Override public void initialize(ServiceExtensionContext context) { - eventRouter.register(Event.class, new AuditEventSubscriber(monitor,context.getParticipantId())); + eventRouter.register(Event.class, new AuditEventSubscriber(monitor, context.getParticipantId())); eventRouter.registerSync(Event.class, new AuditEventSubscriber(monitor, context.getParticipantId())); } -} \ No newline at end of file +} From 4df735f52259793ffe09080162b646d832734e6e Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Fri, 5 Jul 2024 11:14:29 +0200 Subject: [PATCH 116/174] audit comment --- launchers/connector/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launchers/connector/build.gradle.kts b/launchers/connector/build.gradle.kts index abc1da3..232b27c 100644 --- a/launchers/connector/build.gradle.kts +++ b/launchers/connector/build.gradle.kts @@ -93,7 +93,7 @@ dependencies { //Data plane public api implementation(project(":extensions:extended-data-plane-public-api")) - // Storage assets + // Audit implementation(project(":extensions:audit-configuration")) implementation(project(":extensions:audit-event-configuration")) From 668a428e24971adfd167105564f84af747a4244d Mon Sep 17 00:00:00 2001 From: ppel Date: Thu, 11 Jul 2024 14:29:33 +0200 Subject: [PATCH 117/174] #436475 - Added new extension for searching datasets. Modified FederatedCatalog extension and CountElementsExtension to use this new search functionality --- .../CountElementsApiExtension.java | 6 +- .../controller/CountElementsApi.java | 3 +- .../CountElementsApiController.java | 30 +++- .../service/CountElementsServiceImpl.java | 5 +- .../count-elements-sql/build.gradle.kts | 1 + .../sql/index/SqlCountElementsIndex.java | 26 ++- .../schema/BaseSqlDialectStatements.java | 47 ++--- .../index/schema/CountElementsStatements.java | 47 +++++ .../schema/postgres/SqlDatasetMapping.java | 25 +++ .../build.gradle.kts | 1 + .../schema/BaseSqlDialectStatements.java | 5 +- .../schema/SqlFederatedCatalogStatements.java | 3 +- .../inesdata-search-extension/README.md | 18 ++ .../build.gradle.kts | 10 ++ .../CriterionToWhereClauseConverterImpl.java | 167 ++++++++++++++++++ .../extension/InesdataSearchExtension.java | 29 +++ .../extension/InesdataSqlQueryStatement.java | 51 ++++++ ...rg.eclipse.edc.spi.system.ServiceExtension | 1 + settings.gradle.kts | 1 + .../index/CountElementsIndex.java | 4 +- .../service/CountElementsService.java | 4 +- 21 files changed, 442 insertions(+), 42 deletions(-) create mode 100644 extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/schema/postgres/SqlDatasetMapping.java create mode 100644 extensions/inesdata-search-extension/README.md create mode 100644 extensions/inesdata-search-extension/build.gradle.kts create mode 100644 extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/CriterionToWhereClauseConverterImpl.java create mode 100644 extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/InesdataSearchExtension.java create mode 100644 extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/InesdataSqlQueryStatement.java create mode 100644 extensions/inesdata-search-extension/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension diff --git a/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/CountElementsApiExtension.java b/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/CountElementsApiExtension.java index a36615c..3eeb2e3 100644 --- a/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/CountElementsApiExtension.java +++ b/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/CountElementsApiExtension.java @@ -10,6 +10,7 @@ import org.eclipse.edc.spi.types.TypeManager; import org.eclipse.edc.transaction.spi.TransactionContext; import org.eclipse.edc.transform.spi.TypeTransformerRegistry; +import org.eclipse.edc.validator.spi.JsonObjectValidatorRegistry; import org.eclipse.edc.web.spi.WebService; import org.eclipse.edc.web.spi.configuration.ApiContext; import org.eclipse.edc.web.spi.configuration.context.ManagementApiUrl; @@ -49,6 +50,9 @@ public class CountElementsApiExtension implements ServiceExtension { @Inject private TransactionContext transactionContext; + @Inject + private JsonObjectValidatorRegistry validator; + @Override public String name() { return NAME; @@ -72,7 +76,7 @@ public void initialize(ServiceExtensionContext context) { var managementApiTransformerRegistry = transformerRegistry.forContext("management-api"); managementApiTransformerRegistry.register(new JsonObjectFromCountElementTransformer(factory, jsonLdMapper)); - var countElementsApiController = new CountElementsApiController(countElementsService(), managementApiTransformerRegistry); + var countElementsApiController = new CountElementsApiController(countElementsService(), managementApiTransformerRegistry, validator); webService.registerResource(ApiContext.MANAGEMENT, countElementsApiController); } } diff --git a/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/controller/CountElementsApi.java b/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/controller/CountElementsApi.java index bb40db4..16534de 100644 --- a/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/controller/CountElementsApi.java +++ b/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/controller/CountElementsApi.java @@ -8,6 +8,7 @@ import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.json.JsonObject; import org.eclipse.edc.api.model.ApiCoreSchema; @OpenAPIDefinition( @@ -32,5 +33,5 @@ public interface CountElementsApi { content = @Content(array = @ArraySchema(schema = @Schema(implementation = ApiCoreSchema.ApiErrorDetailSchema.class)))) } ) - long countElements(String entityType); + long countElements(String entityType, JsonObject querySpecJson); } diff --git a/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/controller/CountElementsApiController.java b/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/controller/CountElementsApiController.java index 5c3e4b6..fc59545 100644 --- a/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/controller/CountElementsApiController.java +++ b/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/controller/CountElementsApiController.java @@ -1,32 +1,44 @@ package org.upm.inesdata.countelements.controller; +import jakarta.json.JsonObject; import jakarta.ws.rs.BadRequestException; +import jakarta.ws.rs.Consumes; import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; import jakarta.ws.rs.Path; import jakarta.ws.rs.Produces; import jakarta.ws.rs.QueryParam; import jakarta.ws.rs.core.MediaType; +import org.eclipse.edc.spi.query.QuerySpec; import org.eclipse.edc.transform.spi.TypeTransformerRegistry; +import org.eclipse.edc.validator.spi.JsonObjectValidatorRegistry; +import org.eclipse.edc.web.spi.exception.InvalidRequestException; +import org.eclipse.edc.web.spi.exception.ValidationFailureException; import org.upm.inesdata.spi.countelements.service.CountElementsService; import java.util.Objects; +import static org.eclipse.edc.spi.query.QuerySpec.EDC_QUERY_SPEC_TYPE; + @Produces({MediaType.APPLICATION_JSON}) +@Consumes({ MediaType.APPLICATION_JSON }) @Path("/pagination") public class CountElementsApiController implements CountElementsApi { private final CountElementsService service; + private final JsonObjectValidatorRegistry validator; private final TypeTransformerRegistry transformerRegistry; - public CountElementsApiController(CountElementsService service, TypeTransformerRegistry transformerRegistry) { + public CountElementsApiController(CountElementsService service, TypeTransformerRegistry transformerRegistry, JsonObjectValidatorRegistry validator) { this.service = service; this.transformerRegistry = transformerRegistry; + this.validator = validator; } - @GET + @POST @Path("/count") @Override - public long countElements(@QueryParam("type") String entityType) { + public long countElements(@QueryParam("type") String entityType, JsonObject querySpecJson) { if (!Objects.equals(entityType, "asset") && !Objects.equals(entityType, "policyDefinition") && !Objects.equals(entityType, "contractDefinition") && !Objects.equals(entityType, "contractAgreement") @@ -35,7 +47,17 @@ public long countElements(@QueryParam("type") String entityType) { throw new BadRequestException("Entity type provided is not valid"); } - var count = service.countElements(entityType); + QuerySpec querySpec; + if (querySpecJson == null) { + querySpec = QuerySpec.Builder.newInstance().build(); + } else { + validator.validate(EDC_QUERY_SPEC_TYPE, querySpecJson).orElseThrow(ValidationFailureException::new); + + querySpec = transformerRegistry.transform(querySpecJson, QuerySpec.class) + .orElseThrow(InvalidRequestException::new); + } + + var count = service.countElements(entityType, querySpec); // JsonObject result = transformerRegistry.transform(count, JsonObject.class) // .orElseThrow(f -> new EdcException(f.getFailureDetail())); diff --git a/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/service/CountElementsServiceImpl.java b/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/service/CountElementsServiceImpl.java index 91ec4cc..6332128 100644 --- a/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/service/CountElementsServiceImpl.java +++ b/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/service/CountElementsServiceImpl.java @@ -1,5 +1,6 @@ package org.upm.inesdata.countelements.service; +import org.eclipse.edc.spi.query.QuerySpec; import org.eclipse.edc.transaction.spi.TransactionContext; import org.upm.inesdata.spi.countelements.domain.CountElement; import org.upm.inesdata.spi.countelements.index.CountElementsIndex; @@ -15,7 +16,7 @@ public CountElementsServiceImpl(CountElementsIndex countElementsIndex, Transacti } @Override - public CountElement countElements(String entityType) { - return transactionContext.execute(() -> countElementsIndex.countElements(entityType)); + public CountElement countElements(String entityType, QuerySpec querySpec) { + return transactionContext.execute(() -> countElementsIndex.countElements(entityType, querySpec)); } } diff --git a/extensions/count-elements-sql/build.gradle.kts b/extensions/count-elements-sql/build.gradle.kts index 91b8c6a..08dd665 100644 --- a/extensions/count-elements-sql/build.gradle.kts +++ b/extensions/count-elements-sql/build.gradle.kts @@ -6,6 +6,7 @@ plugins { dependencies { api(project(":spi:count-elements-spi")) implementation(project(":extensions:count-elements-api")) + api(project(":extensions:inesdata-search-extension")) api(libs.edc.spi.core) api(libs.edc.transaction.spi) implementation(libs.edc.transaction.spi) diff --git a/extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/SqlCountElementsIndex.java b/extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/SqlCountElementsIndex.java index 9f314c6..0930f73 100644 --- a/extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/SqlCountElementsIndex.java +++ b/extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/SqlCountElementsIndex.java @@ -2,8 +2,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.eclipse.edc.spi.persistence.EdcPersistenceException; +import org.eclipse.edc.spi.query.QuerySpec; import org.eclipse.edc.sql.QueryExecutor; import org.eclipse.edc.sql.store.AbstractSqlStore; +import org.eclipse.edc.sql.translation.SqlQueryStatement; import org.eclipse.edc.transaction.datasource.spi.DataSourceRegistry; import org.eclipse.edc.transaction.spi.TransactionContext; import org.upm.inesdata.countelements.sql.index.schema.CountElementsStatements; @@ -21,20 +23,28 @@ public class SqlCountElementsIndex extends AbstractSqlStore implements CountElem private final CountElementsStatements countElementsStatements; public SqlCountElementsIndex(DataSourceRegistry dataSourceRegistry, - String dataSourceName, - TransactionContext transactionContext, - ObjectMapper objectMapper, - CountElementsStatements countElementsStatements, - QueryExecutor queryExecutor) { + String dataSourceName, + TransactionContext transactionContext, + ObjectMapper objectMapper, + CountElementsStatements countElementsStatements, + QueryExecutor queryExecutor) { super(dataSourceRegistry, dataSourceName, transactionContext, objectMapper, queryExecutor); this.countElementsStatements = Objects.requireNonNull(countElementsStatements); } @Override - public CountElement countElements(String entityType) { + public CountElement countElements(String entityType, QuerySpec querySpec) { try (var connection = getConnection()) { - var sql = countElementsStatements.getCount(entityType); - long count = queryExecutor.single(connection, true, r -> r.getLong(1), sql); + long count; + if ("federatedCatalog".equals(entityType)) { + SqlQueryStatement dataSetQueryStatement = countElementsStatements.createCountDatasetQuery(entityType, querySpec); + count = queryExecutor.single(connection, true, r -> r.getLong(1), + dataSetQueryStatement.getQueryAsString(), dataSetQueryStatement.getParameters()); + } else { + var sql = countElementsStatements.getCount(entityType); + count = queryExecutor.single(connection, true, r -> r.getLong(1), sql); + } + return CountElement.Builder.newInstance().count(count).build(); } catch (SQLException e) { throw new EdcPersistenceException(e); diff --git a/extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/schema/BaseSqlDialectStatements.java b/extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/schema/BaseSqlDialectStatements.java index cedb579..18367f0 100644 --- a/extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/schema/BaseSqlDialectStatements.java +++ b/extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/schema/BaseSqlDialectStatements.java @@ -1,6 +1,9 @@ package org.upm.inesdata.countelements.sql.index.schema; +import org.eclipse.edc.spi.query.QuerySpec; import org.eclipse.edc.sql.translation.SqlOperatorTranslator; +import org.upm.inesdata.countelements.sql.index.schema.postgres.SqlDatasetMapping; +import org.upm.inesdata.search.extension.InesdataSqlQueryStatement; import static java.lang.String.format; @@ -15,31 +18,33 @@ public BaseSqlDialectStatements(SqlOperatorTranslator operatorTranslator) { this.operatorTranslator = operatorTranslator; } + /** + * {@inheritDoc} + * + * @see CountElementsStatements#getCount(String) + */ @Override public String getCount(String entityType) { - String tableName = null; - switch (entityType) { - case "asset": - tableName = getAssetTable(); - break; - case "policyDefinition": - tableName = getPolicyDefinitionTable(); - break; - case "contractDefinition": - tableName = getContractDefinitionTable(); - break; - case "contractAgreement": - tableName = getContractAgreementTable(); - break; - case "transferProcess": - tableName = getTransferProcessTable(); - break; - case "federatedCatalog": - tableName = getDatasetTable(); - break; - } + String tableName = switch (entityType) { + case "asset" -> getAssetTable(); + case "policyDefinition" -> getPolicyDefinitionTable(); + case "contractDefinition" -> getContractDefinitionTable(); + case "contractAgreement" -> getContractAgreementTable(); + case "transferProcess" -> getTransferProcessTable(); + case "federatedCatalog" -> getDatasetTable(); + default -> null; + }; return format("SELECT COUNT(*) FROM %s", tableName); } + /** + * {@inheritDoc} + * + * @see CountElementsStatements#createDatasetQuery(QuerySpec) + */ + @Override + public InesdataSqlQueryStatement createCountDatasetQuery(String entityType, QuerySpec querySpec) { + return new InesdataSqlQueryStatement(getCount(entityType), querySpec, new SqlDatasetMapping(this), operatorTranslator); + } } diff --git a/extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/schema/CountElementsStatements.java b/extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/schema/CountElementsStatements.java index 3bec855..93d29e4 100644 --- a/extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/schema/CountElementsStatements.java +++ b/extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/schema/CountElementsStatements.java @@ -1,7 +1,9 @@ package org.upm.inesdata.countelements.sql.index.schema; import org.eclipse.edc.runtime.metamodel.annotation.ExtensionPoint; +import org.eclipse.edc.spi.query.QuerySpec; import org.eclipse.edc.sql.statement.SqlStatements; +import org.upm.inesdata.search.extension.InesdataSqlQueryStatement; /** * Defines queries used by the SqlCountElementsIndexServiceExtension. @@ -51,6 +53,51 @@ default String getDatasetTable() { return "edc_dataset"; } + /** + * Retrieves the name of the column storing dataset IDs. + * + * @return the name of the dataset ID column. + */ + default String getDatasetIdColumn() { + return "id"; + } + + /** + * Retrieves the name of the column storing offers associated with datasets. + * + * @return the name of the offers column. + */ + default String getDatasetOffersColumn() { + return "offers"; + } + + /** + * Retrieves the name of the column storing properties of datasets as JSON. + * + * @return the name of the properties column. + */ + default String getDatasetPropertiesColumn() { + return "properties"; + } + + /** + * Retrieves the name of the column storing the catalog ID associated with datasets. + * + * @return the name of the catalog ID column. + */ + default String getDatasetCatalogIdColumn() { + return "catalog_id"; + } + + /** + * Creates an SQL query statement specifically for datasets based on the provided query specification. + * + * @param querySpec the query specification defining filters, sorting, and pagination for datasets. + * @param entityType the entity type (federatedCatalog) + * @return an SQL query statement for datasets. + */ + InesdataSqlQueryStatement createCountDatasetQuery(String entityType, QuerySpec querySpec); + /** * SELECT COUNT clause. */ diff --git a/extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/schema/postgres/SqlDatasetMapping.java b/extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/schema/postgres/SqlDatasetMapping.java new file mode 100644 index 0000000..ce71b5c --- /dev/null +++ b/extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/schema/postgres/SqlDatasetMapping.java @@ -0,0 +1,25 @@ +package org.upm.inesdata.countelements.sql.index.schema.postgres; + +import org.eclipse.edc.sql.translation.JsonFieldTranslator; +import org.eclipse.edc.sql.translation.TranslationMapping; +import org.upm.inesdata.countelements.sql.index.schema.CountElementsStatements; + +/** + * Maps fields of a dataset of federated catalog onto the + * corresponding SQL schema (= column names) enabling access through Postgres JSON operators where applicable + */ +public class SqlDatasetMapping extends TranslationMapping { + /** + * Constructs a mapping for SQL dataset columns using the provided SQL federated catalog statements. + * This mapping specifies how dataset fields correspond to database columns. + * + * @param statements the SQL statements specific to the federated catalog schema. + */ + public SqlDatasetMapping(CountElementsStatements statements) { + add("id", statements.getDatasetIdColumn()); + add("offers", new JsonFieldTranslator(statements.getDatasetOffersColumn())); + add("properties", new JsonFieldTranslator(statements.getDatasetPropertiesColumn())); + add("catalog_id", statements.getDatasetCatalogIdColumn()); + } + +} diff --git a/extensions/federated-catalog-cache-sql/build.gradle.kts b/extensions/federated-catalog-cache-sql/build.gradle.kts index bd0dcb4..48f92fd 100644 --- a/extensions/federated-catalog-cache-sql/build.gradle.kts +++ b/extensions/federated-catalog-cache-sql/build.gradle.kts @@ -8,6 +8,7 @@ dependencies { implementation(libs.edc.federated.catalog.api) api(libs.edc.spi.core) api(libs.edc.transaction.spi) + api(project(":extensions:inesdata-search-extension")) implementation(libs.edc.transaction.spi) implementation(libs.edc.transaction.datasource.spi) implementation(libs.edc.sql.core) diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java index ef51d38..317fee0 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java @@ -5,6 +5,7 @@ import org.eclipse.edc.sql.translation.SqlQueryStatement; import org.upm.inesdata.federated.sql.index.schema.postgres.SqlDatasetMapping; import org.upm.inesdata.federated.sql.index.schema.postgres.SqlFederatedCatalogMapping; +import org.upm.inesdata.search.extension.InesdataSqlQueryStatement; import static java.lang.String.format; @@ -133,8 +134,8 @@ public SqlQueryStatement createQuery(QuerySpec querySpec) { * @see SqlFederatedCatalogStatements#createDatasetQuery(QuerySpec) */ @Override - public SqlQueryStatement createDatasetQuery(QuerySpec querySpec) { - return new SqlQueryStatement(getSelectDatasetTemplate(), querySpec, new SqlDatasetMapping(this), operatorTranslator); + public InesdataSqlQueryStatement createDatasetQuery(QuerySpec querySpec) { + return new InesdataSqlQueryStatement(getSelectDatasetTemplate(), querySpec, new SqlDatasetMapping(this), operatorTranslator); } /** diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/SqlFederatedCatalogStatements.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/SqlFederatedCatalogStatements.java index def47e4..eb981c0 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/SqlFederatedCatalogStatements.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/SqlFederatedCatalogStatements.java @@ -4,6 +4,7 @@ import org.eclipse.edc.spi.query.QuerySpec; import org.eclipse.edc.sql.statement.SqlStatements; import org.eclipse.edc.sql.translation.SqlQueryStatement; +import org.upm.inesdata.search.extension.InesdataSqlQueryStatement; /** * SQL statements interface for managing federated catalog data. Extends {@link SqlStatements} and provides methods for @@ -250,7 +251,7 @@ default String getDistributionDatasetIdColumn() { * @param querySpec the query specification defining filters, sorting, and pagination for datasets. * @return an SQL query statement for datasets. */ - SqlQueryStatement createDatasetQuery(QuerySpec querySpec); + InesdataSqlQueryStatement createDatasetQuery(QuerySpec querySpec); /** * Retrieves the SQL template for deleting expired catalogs. diff --git a/extensions/inesdata-search-extension/README.md b/extensions/inesdata-search-extension/README.md new file mode 100644 index 0000000..51bb140 --- /dev/null +++ b/extensions/inesdata-search-extension/README.md @@ -0,0 +1,18 @@ +# Oauth2 JWT Token Authentication Service + +This extension provides the capability to authorizate the request to the connector management API. The extension will access the Bearer token provided in the Authorization header and validate that it is a valid JWT-encoded bearer token. It is necessary to have the `org.eclipse.edc:oauth2-core` extension correctly configured. + +To authorize a user, the roles of the provided JWT token must contain: +- a valid role from those configured in `allowedRoles` +- a role with the `connector name` + +## Configuration + +Example configuration: + +```properties +edc.api.auth.oauth2.allowedRoles.1.role=connector-admin +edc.api.auth.oauth2.allowedRoles.2.role=connector-management +``` + +The `edc.api.auth.oauth2.allowedRoles` will be used by the federated catalog to retrieve the list of allowed roles that can perform requests on the managemente API connector. \ No newline at end of file diff --git a/extensions/inesdata-search-extension/build.gradle.kts b/extensions/inesdata-search-extension/build.gradle.kts new file mode 100644 index 0000000..cfbb2cf --- /dev/null +++ b/extensions/inesdata-search-extension/build.gradle.kts @@ -0,0 +1,10 @@ +plugins { + `java-library` + id("com.gmv.inesdata.edc-application") +} + +dependencies { + implementation(libs.edc.sql.core) +} + + diff --git a/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/CriterionToWhereClauseConverterImpl.java b/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/CriterionToWhereClauseConverterImpl.java new file mode 100644 index 0000000..8c70404 --- /dev/null +++ b/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/CriterionToWhereClauseConverterImpl.java @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation + * + */ + +package org.upm.inesdata.search.extension; + +import org.eclipse.edc.spi.EdcException; +import org.eclipse.edc.spi.query.Criterion; +import org.eclipse.edc.sql.translation.CriterionToWhereClauseConverter; +import org.eclipse.edc.sql.translation.SqlOperatorTranslator; +import org.eclipse.edc.sql.translation.TranslationMapping; +import org.eclipse.edc.sql.translation.WhereClause; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static java.util.Collections.unmodifiableCollection; + +public class CriterionToWhereClauseConverterImpl implements CriterionToWhereClauseConverter { + + private static final String GENERIC_SEARCH = "genericSearch"; + private static final String ASSET_DATA_PROPERTY = "'https://w3id.org/edc/v0.0.1/ns/assetData'"; + private static final String [] COMMON_PROPERTIES = { + "https://w3id.org/edc/v0.0.1/ns/id", + "https://w3id.org/edc/v0.0.1/ns/name", + "https://w3id.org/edc/v0.0.1/ns/version", + "https://w3id.org/edc/v0.0.1/ns/contenttype", + "https://w3id.org/edc/v0.0.1/ns/contenttype", + "http://purl.org/dc/terms/format", + "http://www.w3.org/ns/dcat#keyword", + "http://www.w3.org/ns/dcat#byteSize", + "https://w3id.org/edc/v0.0.1/ns/shortDescription", + "https://w3id.org/edc/v0.0.1/ns/assetType", + "http://purl.org/dc/terms/description" + }; + private final TranslationMapping translationMapping; + private final SqlOperatorTranslator operatorTranslator; + + public CriterionToWhereClauseConverterImpl(TranslationMapping translationMapping, SqlOperatorTranslator operatorTranslator) { + this.translationMapping = translationMapping; + this.operatorTranslator = operatorTranslator; + } + + @Override + public WhereClause convert(Criterion criterion) { + var operator = operatorTranslator.translate(criterion.getOperator().toLowerCase()); + if (operator == null) { + throw new IllegalArgumentException("The operator '%s' is not supported".formatted(criterion.getOperator())); + } + + if (!operator.rightOperandClass().isAssignableFrom(criterion.getOperandRight().getClass())) { + throw new IllegalArgumentException("The operator '%s' requires the right-hand operand to be of type %s" + .formatted(criterion.getOperator(), operator.rightOperandClass().getSimpleName())); + } + + if (criterion.getOperandLeft().toString().startsWith(ASSET_DATA_PROPERTY)) { + return generateVocabularyWhereClause(criterion); + } else if (GENERIC_SEARCH.equals(criterion.getOperandLeft().toString())) { + return generateGenericPropertiesWhereClause(criterion); + } + + var whereClause = translationMapping.getWhereClause(criterion, operator); + if (whereClause == null) { + return new WhereClause("0 = ?", 1); + } + + return whereClause; + } + + private WhereClause generateGenericPropertiesWhereClause(Criterion criterion) { + String operator = criterion.getOperator(); + String rightValue = criterion.getOperandRight().toString(); + List values = new ArrayList<>(Collections.nCopies(COMMON_PROPERTIES.length, rightValue)); + + StringBuilder sqlWhereBuilder = new StringBuilder("("); + for (int i = 0; i < COMMON_PROPERTIES.length; i++) { + sqlWhereBuilder.append("properties ->> '") + .append(COMMON_PROPERTIES[i]) + .append("' ") + .append(operator) + .append(" ?"); + if (i < COMMON_PROPERTIES.length - 1) { + sqlWhereBuilder.append(" OR "); + } + } + sqlWhereBuilder.append(")"); + + return new WhereClause(sqlWhereBuilder.toString(), unmodifiableCollection(values)); + } + + private WhereClause generateVocabularyWhereClause(Criterion criterion) { + String[] propertiesList = splitByDotOutsideQuotes(criterion.getOperandLeft().toString()); + StringBuilder sqlWhereBuilder = new StringBuilder(); + + switch (propertiesList.length) { + case 3 -> + generateNonObjectPropertySQL(sqlWhereBuilder, propertiesList, criterion.getOperandRight().toString()); + case 4 -> + generateObjectPropertySQL(sqlWhereBuilder, propertiesList, criterion.getOperandRight().toString()); + default -> throw new IllegalArgumentException("Invalid vocabulary argument in the operandLeft: %s" + .formatted(criterion.getOperandLeft().toString())); + } + + return new WhereClause(sqlWhereBuilder.toString(), unmodifiableCollection(new ArrayList<>())); + } + + private void generateNonObjectPropertySQL(StringBuilder sqlWhereBuilder, String[] propertiesList, String operandRight) { + sqlWhereBuilder.append("(properties::jsonb -> ") + .append(propertiesList[0]) + .append(" -> ") + .append(propertiesList[1]) + .append(")::jsonb @> '[{") + .append(propertiesList[2].replaceAll("'", "\"")) + .append(": [{\"@value\": \"") + .append(operandRight) + .append("\"}]}]'::jsonb"); + } + + private void generateObjectPropertySQL(StringBuilder sqlWhereBuilder, String[] propertiesList, String operandRight) { + sqlWhereBuilder.append("EXISTS (SELECT 1 FROM jsonb_array_elements((properties::jsonb -> ") + .append(propertiesList[0]) + .append(" -> ") + .append(propertiesList[1]) + .append(")::jsonb) AS vocab WHERE vocab -> ") + .append(propertiesList[2]) + .append(" @> '[{") + .append(propertiesList[3].replaceAll("'", "\"")) + .append(": [{\"@value\": \"") + .append(operandRight) + .append("\"}]}]')"); + } + + private String[] splitByDotOutsideQuotes(String input) { + List parts = new ArrayList<>(); + + Pattern pattern = Pattern.compile("\\.(?=(?:[^']*'[^']*')*[^']*$)"); + + Matcher matcher = pattern.matcher(input); + int start = 0; + + while (matcher.find()) { + String part = input.substring(start, matcher.start()).trim(); + parts.add(part); + start = matcher.end(); + } + + if (start < input.length()) { + String lastPart = input.substring(start).trim(); + parts.add(lastPart); + } + + return parts.toArray(new String[0]); + } +} diff --git a/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/InesdataSearchExtension.java b/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/InesdataSearchExtension.java new file mode 100644 index 0000000..21b24a3 --- /dev/null +++ b/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/InesdataSearchExtension.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation + * + */ + +package org.upm.inesdata.search.extension; + +import org.eclipse.edc.runtime.metamodel.annotation.Extension; +import org.eclipse.edc.spi.system.ServiceExtension; + +@Extension(value = InesdataSearchExtension.NAME) +public class InesdataSearchExtension implements ServiceExtension { + + public static final String NAME = "Inesdata Search Extension"; + + @Override + public String name() { + return NAME; + } +} diff --git a/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/InesdataSqlQueryStatement.java b/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/InesdataSqlQueryStatement.java new file mode 100644 index 0000000..a464c2a --- /dev/null +++ b/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/InesdataSqlQueryStatement.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial API and implementation + * + */ + +package org.upm.inesdata.search.extension; + +import org.eclipse.edc.spi.query.QuerySpec; +import org.eclipse.edc.spi.query.SortOrder; +import org.eclipse.edc.sql.translation.CriterionToWhereClauseConverter; +import org.eclipse.edc.sql.translation.SortFieldConverterImpl; +import org.eclipse.edc.sql.translation.SqlOperatorTranslator; +import org.eclipse.edc.sql.translation.SqlQueryStatement; +import org.eclipse.edc.sql.translation.TranslationMapping; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static java.lang.String.format; +import static java.util.stream.Collectors.joining; + +/** + * Maps a {@link QuerySpec} to a single SQL {@code SELECT ... FROM ... WHERE ...} statement. The {@code SELECT ...} part + * is passed in through the constructor, and the rest of the query is assembled dynamically, based on the + * {@link QuerySpec} and the {@link TranslationMapping}. + */ +public class InesdataSqlQueryStatement extends SqlQueryStatement{ + + /** + * Initializes this SQL Query Statement. + * + * @param selectStatement The SELECT clause, e.g. {@code SELECT * FROM your_table} + * @param query a {@link QuerySpec} that contains a query in the canonical format + * @param rootModel A {@link TranslationMapping} that enables mapping from canonical to the SQL-specific + * model/format + * @param operatorTranslator the {@link SqlOperatorTranslator} instance. + */ + public InesdataSqlQueryStatement(String selectStatement, QuerySpec query, TranslationMapping rootModel, SqlOperatorTranslator operatorTranslator) { + super(selectStatement, query, rootModel, new CriterionToWhereClauseConverterImpl(rootModel, operatorTranslator)); + } +} diff --git a/extensions/inesdata-search-extension/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/inesdata-search-extension/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 0000000..67fef91 --- /dev/null +++ b/extensions/inesdata-search-extension/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1 @@ +org.upm.inesdata.search.extension.InesdataSearchExtension diff --git a/settings.gradle.kts b/settings.gradle.kts index 49a1eef..c140da8 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -25,6 +25,7 @@ include(":extensions:count-elements-sql") include(":extensions:extended-data-plane-public-api") include(":extensions:audit-configuration") include(":extensions:audit-event-configuration") +include(":extensions:inesdata-search-extension") // Connector include(":launchers:connector") diff --git a/spi/count-elements-spi/src/main/java/org/upm/inesdata/spi/countelements/index/CountElementsIndex.java b/spi/count-elements-spi/src/main/java/org/upm/inesdata/spi/countelements/index/CountElementsIndex.java index 031a8ea..d8a14a2 100644 --- a/spi/count-elements-spi/src/main/java/org/upm/inesdata/spi/countelements/index/CountElementsIndex.java +++ b/spi/count-elements-spi/src/main/java/org/upm/inesdata/spi/countelements/index/CountElementsIndex.java @@ -1,6 +1,7 @@ package org.upm.inesdata.spi.countelements.index; import org.eclipse.edc.runtime.metamodel.annotation.ExtensionPoint; +import org.eclipse.edc.spi.query.QuerySpec; import org.upm.inesdata.spi.countelements.domain.CountElement; /** @@ -13,7 +14,8 @@ public interface CountElementsIndex { * Counts all contract agreements * * @param entityType entity type + * @param querySpec filters * @return the number of contract agreements */ - CountElement countElements(String entityType); + CountElement countElements(String entityType, QuerySpec querySpec); } diff --git a/spi/count-elements-spi/src/main/java/org/upm/inesdata/spi/countelements/service/CountElementsService.java b/spi/count-elements-spi/src/main/java/org/upm/inesdata/spi/countelements/service/CountElementsService.java index 10d3802..be0bf5a 100644 --- a/spi/count-elements-spi/src/main/java/org/upm/inesdata/spi/countelements/service/CountElementsService.java +++ b/spi/count-elements-spi/src/main/java/org/upm/inesdata/spi/countelements/service/CountElementsService.java @@ -1,5 +1,6 @@ package org.upm.inesdata.spi.countelements.service; +import org.eclipse.edc.spi.query.QuerySpec; import org.upm.inesdata.spi.countelements.domain.CountElement; /** @@ -11,7 +12,8 @@ public interface CountElementsService { * Gets the total number of elements of an entity. * * @param entityType entity type + * @param querySpec filters * @return the total number of elements */ - CountElement countElements(String entityType); + CountElement countElements(String entityType, QuerySpec querySpec); } From 81534904eb8581b7cf31207b6d1fa7f5c0c6e485 Mon Sep 17 00:00:00 2001 From: ppel Date: Fri, 12 Jul 2024 10:46:35 +0200 Subject: [PATCH 118/174] #436475 - Throw InvalidRequestExcpetion when invalid length of vocabulary properties --- .../countelements/CountElementsApiExtension.java | 1 - .../controller/CountElementsApiController.java | 3 +-- extensions/inesdata-search-extension/build.gradle.kts | 3 ++- .../extension/CriterionToWhereClauseConverterImpl.java | 4 ++-- .../search/extension/InesdataSqlQueryStatement.java | 10 ---------- 5 files changed, 5 insertions(+), 16 deletions(-) diff --git a/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/CountElementsApiExtension.java b/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/CountElementsApiExtension.java index 3eeb2e3..edbf05d 100644 --- a/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/CountElementsApiExtension.java +++ b/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/CountElementsApiExtension.java @@ -13,7 +13,6 @@ import org.eclipse.edc.validator.spi.JsonObjectValidatorRegistry; import org.eclipse.edc.web.spi.WebService; import org.eclipse.edc.web.spi.configuration.ApiContext; -import org.eclipse.edc.web.spi.configuration.context.ManagementApiUrl; import org.upm.inesdata.countelements.controller.CountElementsApiController; import org.upm.inesdata.countelements.service.CountElementsServiceImpl; import org.upm.inesdata.countelements.transformer.JsonObjectFromCountElementTransformer; diff --git a/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/controller/CountElementsApiController.java b/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/controller/CountElementsApiController.java index fc59545..fafde2a 100644 --- a/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/controller/CountElementsApiController.java +++ b/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/controller/CountElementsApiController.java @@ -3,7 +3,6 @@ import jakarta.json.JsonObject; import jakarta.ws.rs.BadRequestException; import jakarta.ws.rs.Consumes; -import jakarta.ws.rs.GET; import jakarta.ws.rs.POST; import jakarta.ws.rs.Path; import jakarta.ws.rs.Produces; @@ -44,7 +43,7 @@ public long countElements(@QueryParam("type") String entityType, JsonObject quer && !Objects.equals(entityType, "contractAgreement") && !Objects.equals(entityType, "transferProcess") && !Objects.equals(entityType, "federatedCatalog")) { - throw new BadRequestException("Entity type provided is not valid"); + throw new InvalidRequestException("Entity type provided is not valid: %s".formatted(entityType)); } QuerySpec querySpec; diff --git a/extensions/inesdata-search-extension/build.gradle.kts b/extensions/inesdata-search-extension/build.gradle.kts index cfbb2cf..5baaa3b 100644 --- a/extensions/inesdata-search-extension/build.gradle.kts +++ b/extensions/inesdata-search-extension/build.gradle.kts @@ -4,7 +4,8 @@ plugins { } dependencies { - implementation(libs.edc.sql.core) + api(libs.edc.sql.core) + implementation(libs.edc.web.spi) } diff --git a/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/CriterionToWhereClauseConverterImpl.java b/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/CriterionToWhereClauseConverterImpl.java index 8c70404..ef1b007 100644 --- a/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/CriterionToWhereClauseConverterImpl.java +++ b/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/CriterionToWhereClauseConverterImpl.java @@ -14,12 +14,12 @@ package org.upm.inesdata.search.extension; -import org.eclipse.edc.spi.EdcException; import org.eclipse.edc.spi.query.Criterion; import org.eclipse.edc.sql.translation.CriterionToWhereClauseConverter; import org.eclipse.edc.sql.translation.SqlOperatorTranslator; import org.eclipse.edc.sql.translation.TranslationMapping; import org.eclipse.edc.sql.translation.WhereClause; +import org.eclipse.edc.web.spi.exception.InvalidRequestException; import java.util.ArrayList; import java.util.Collections; @@ -110,7 +110,7 @@ private WhereClause generateVocabularyWhereClause(Criterion criterion) { generateNonObjectPropertySQL(sqlWhereBuilder, propertiesList, criterion.getOperandRight().toString()); case 4 -> generateObjectPropertySQL(sqlWhereBuilder, propertiesList, criterion.getOperandRight().toString()); - default -> throw new IllegalArgumentException("Invalid vocabulary argument in the operandLeft: %s" + default -> throw new InvalidRequestException("Invalid vocabulary argument in the operandLeft: %s" .formatted(criterion.getOperandLeft().toString())); } diff --git a/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/InesdataSqlQueryStatement.java b/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/InesdataSqlQueryStatement.java index a464c2a..8764f59 100644 --- a/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/InesdataSqlQueryStatement.java +++ b/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/InesdataSqlQueryStatement.java @@ -15,20 +15,10 @@ package org.upm.inesdata.search.extension; import org.eclipse.edc.spi.query.QuerySpec; -import org.eclipse.edc.spi.query.SortOrder; -import org.eclipse.edc.sql.translation.CriterionToWhereClauseConverter; -import org.eclipse.edc.sql.translation.SortFieldConverterImpl; import org.eclipse.edc.sql.translation.SqlOperatorTranslator; import org.eclipse.edc.sql.translation.SqlQueryStatement; import org.eclipse.edc.sql.translation.TranslationMapping; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import static java.lang.String.format; -import static java.util.stream.Collectors.joining; - /** * Maps a {@link QuerySpec} to a single SQL {@code SELECT ... FROM ... WHERE ...} statement. The {@code SELECT ...} part * is passed in through the constructor, and the rest of the query is assembled dynamically, based on the From 6c574dee32263bfd5408321cde1e417125f585f9 Mon Sep 17 00:00:00 2001 From: ppel Date: Mon, 15 Jul 2024 09:37:15 +0200 Subject: [PATCH 119/174] #436475 - Updated readme --- .../inesdata-search-extension/README.md | 47 +++++++++++++------ 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/extensions/inesdata-search-extension/README.md b/extensions/inesdata-search-extension/README.md index 51bb140..3288d8d 100644 --- a/extensions/inesdata-search-extension/README.md +++ b/extensions/inesdata-search-extension/README.md @@ -1,18 +1,37 @@ -# Oauth2 JWT Token Authentication Service +# INESData search extension -This extension provides the capability to authorizate the request to the connector management API. The extension will access the Bearer token provided in the Authorization header and validate that it is a valid JWT-encoded bearer token. It is necessary to have the `org.eclipse.edc:oauth2-core` extension correctly configured. +This extension provides the capability to search inside the properties of an asset. +The functionality of this new search works as follows: +- To perform a search among the generic properties of the asset it is necessary to indicate 'genericSearch' as the value of the operandLeft +- To perform a search among the properties of a vocabulary, it is necessary to indicate 'https://w3id.org/edc/v0.0.1/ns/assetData' followed by the name of the vocabulary and the property to search for. An example is given in the following section. -To authorize a user, the roles of the provided JWT token must contain: -- a valid role from those configured in `allowedRoles` -- a role with the `connector name` +## Example -## Configuration - -Example configuration: - -```properties -edc.api.auth.oauth2.allowedRoles.1.role=connector-admin -edc.api.auth.oauth2.allowedRoles.2.role=connector-management +```json +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "offset": 0, + "limit": 5, + "sortOrder": "ASC", + "sortField": "id", + "filterExpression": [ + { + "operandLeft": "genericSearch", + "operator": "LIKE", + "operandRight": "%test%" + }, + { + "operandLeft": "'https://w3id.org/edc/v0.0.1/ns/assetData'.'https://w3id.org/edc/v0.0.1/ns/dcat-vocabulary'.'http://purl.org/dc/terms/language'", + "operator": "=", + "operandRight": "spanish" + }, + { + "operandLeft": "'https://w3id.org/edc/v0.0.1/ns/assetData'.'https://w3id.org/edc/v0.0.1/ns/dcat-vocabulary'.'http://purl.org/dc/terms/publisher'.'http://www.w3.org/2004/02/skos/core#notation'", + "operator": "=", + "operandRight": "notation-publisher" + } + ] +} ``` - -The `edc.api.auth.oauth2.allowedRoles` will be used by the federated catalog to retrieve the list of allowed roles that can perform requests on the managemente API connector. \ No newline at end of file From a7185da569a12aa4bfe905963e54688bae3d3f5a Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Tue, 16 Jul 2024 12:09:15 +0200 Subject: [PATCH 120/174] Registration service cambios --- .../README.md | 18 +++ .../build.gradle.kts | 17 +++ ...istrationServiceParticipantsExtension.java | 42 +++++++ .../catalog/ParticipantConfiguration.java | 117 ++++++++++++++++++ ...rg.eclipse.edc.spi.system.ServiceExtension | 1 + launchers/connector/build.gradle.kts | 3 +- settings.gradle.kts | 3 +- 7 files changed, 199 insertions(+), 2 deletions(-) create mode 100644 extensions/participants-from-registration-service/README.md create mode 100644 extensions/participants-from-registration-service/build.gradle.kts create mode 100644 extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/FromRegistrationServiceParticipantsExtension.java create mode 100644 extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/ParticipantConfiguration.java create mode 100644 extensions/participants-from-registration-service/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension diff --git a/extensions/participants-from-registration-service/README.md b/extensions/participants-from-registration-service/README.md new file mode 100644 index 0000000..f4e21ab --- /dev/null +++ b/extensions/participants-from-registration-service/README.md @@ -0,0 +1,18 @@ +# InesData Connector + +This extension provides the capability to obtain the list of participants of a data space in order to allow the federated catalog to obtain their data offerings. + +## Configuration + +Example configuration: + +```properties +edc.catalog.configuration.participant.1.name = connector-c1 +edc.catalog.configuration.participant.1.id = connector-c1 +edc.catalog.configuration.participant.1.targetUrl = http://localhost:19194/protocol +edc.catalog.configuration.participant.2.name = connector-c2 +edc.catalog.configuration.participant.2.id = connector-c2 +edc.catalog.configuration.participant.2.targetUrl = http://localhost:29194/protocol +``` + +The `edc.catalog.configuration.participant` will be used by the federated catalog to retrieve the list of participants whose catalog will be obtained for federating it. \ No newline at end of file diff --git a/extensions/participants-from-registration-service/build.gradle.kts b/extensions/participants-from-registration-service/build.gradle.kts new file mode 100644 index 0000000..115a78a --- /dev/null +++ b/extensions/participants-from-registration-service/build.gradle.kts @@ -0,0 +1,17 @@ +plugins { + `java-library` + id("com.gmv.inesdata.edc-application") +} + +dependencies { + implementation(libs.edc.federated.catalog.spi) + implementation(libs.edc.federated.catalog.core) + implementation(libs.ok.http3) + implementation(libs.edc.http.spi) + implementation(libs.edc.identity.did.spi) + + testImplementation(libs.edc.core.junit) + testImplementation(libs.junit.jupiter.api) + testImplementation(libs.assertj) + testImplementation(libs.mockito.core) +} diff --git a/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/FromRegistrationServiceParticipantsExtension.java b/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/FromRegistrationServiceParticipantsExtension.java new file mode 100644 index 0000000..3cd99b5 --- /dev/null +++ b/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/FromRegistrationServiceParticipantsExtension.java @@ -0,0 +1,42 @@ +package org.upm.inesdata.catalog; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.eclipse.edc.catalog.directory.InMemoryNodeDirectory; +import org.eclipse.edc.crawler.spi.TargetNodeDirectory; +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.runtime.metamodel.annotation.Provider; +import org.eclipse.edc.spi.iam.IdentityService; +import org.eclipse.edc.spi.iam.TokenParameters; +import org.eclipse.edc.spi.iam.TokenRepresentation; +import org.eclipse.edc.spi.result.Result; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; + + +/** + * Extension to set up federated target node directory using a configuration variable. + */ +public class FromRegistrationServiceParticipantsExtension implements ServiceExtension { + + @Inject + private IdentityService identityService; + + @Provider + public TargetNodeDirectory federatedCacheNodeDirectory(ServiceExtensionContext context) { + + var monitor = context.getMonitor(); + var participantConfig = new ParticipantConfiguration(monitor, new ObjectMapper()); + + var dir = new InMemoryNodeDirectory(); + + Result tokenRepresentationResult = identityService.obtainClientCredentials( + TokenParameters.Builder.newInstance().build()); + for (var target : participantConfig.getTargetNodes(context.getConfig(), tokenRepresentationResult)) { + // skipping null target nodes + if (target != null){ + dir.insert(target); + } + } + return dir; + } +} diff --git a/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/ParticipantConfiguration.java b/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/ParticipantConfiguration.java new file mode 100644 index 0000000..417d299 --- /dev/null +++ b/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/ParticipantConfiguration.java @@ -0,0 +1,117 @@ +package org.upm.inesdata.catalog; + +import java.util.ArrayList; +import java.util.List; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.client.ClientBuilder; +import jakarta.ws.rs.client.WebTarget; +import jakarta.ws.rs.core.MediaType; +import org.eclipse.edc.spi.iam.TokenRepresentation; +import org.eclipse.edc.spi.monitor.Monitor; + +import org.eclipse.edc.crawler.spi.TargetNode; +import org.eclipse.edc.runtime.metamodel.annotation.Setting; +import org.eclipse.edc.spi.result.Result; +import org.eclipse.edc.spi.system.configuration.Config; + +import static java.lang.String.format; + +/** + * Handles participants in configuration and transforms them into TargetNodes + */ +public class ParticipantConfiguration { + + public static final List SUPPORTED_PROTOCOLS = List.of("dataspace-protocol-http"); + + @Setting + public static final String EDC_CATALOG_REGISTRATION_SERVICE_HOST = "edc.catalog.registration.service.host"; + public static final String RESOURCE_URL = "/public/participants"; + + private final Monitor monitor; + private final Client client = ClientBuilder.newClient(); + private final ObjectMapper objectMapper; + + + /** + * Constructor + * + * @param monitor monitor + * @param objectMapper + */ + public ParticipantConfiguration(Monitor monitor, ObjectMapper objectMapper) { + this.monitor = monitor; + this.objectMapper = objectMapper; + } + + + + + /** + * Makes an HTTP GET request to the specified URL and returns the response as a string. + * + * @param url the URL to make the GET request to + * @param tokenRepresentationResult + * @return the response from the GET request + */ + public String makeHttpGetRequest(String url, Result tokenRepresentationResult) { + String token = tokenRepresentationResult.getContent().getToken(); + WebTarget target = client.target(url); + return target.request(MediaType.APPLICATION_JSON) + .header("Authorization", "Bearer " + token).get(String.class); + } + + /** + * Retrieve TargetNodes from configuration + * + * @param baseConfig EDC Configuration + * @param tokenRepresentationResult + * @return list of TargetNodes from configuration + */ + public List getTargetNodes(Config baseConfig, Result tokenRepresentationResult) { + var participantsConfig = baseConfig.getConfig(EDC_CATALOG_REGISTRATION_SERVICE_HOST); + + if (participantsConfig.getEntries().isEmpty()) { + monitor.severe("Error processing url registration service."); + return new ArrayList<>(); + } else { + var url = participantsConfig.getEntries().get(EDC_CATALOG_REGISTRATION_SERVICE_HOST) + RESOURCE_URL; + + try { + String response = makeHttpGetRequest(url, tokenRepresentationResult); + if(response==null){ + return new ArrayList<>(); + } + // Process the response and convert it to TargetNodes + // Assuming a method processResponseToTargetNodes(response) + return processResponseToTargetNodes(response); + } catch (Exception e) { + monitor.severe("Exception occurred while making HTTP GET request: " + e.getMessage()); + return new ArrayList<>(); + } + } + } + + private List processResponseToTargetNodes(String response) { + List targetNodes = new ArrayList<>(); + + try { + List nodes = objectMapper.readValue(response, new TypeReference<>() {}); + + for (JsonNode node : nodes) { + String participantId = node.get("participantId").asText(); + String url = node.get("url").asText(); + TargetNode targetNode = new TargetNode(participantId, participantId, url, SUPPORTED_PROTOCOLS); + targetNodes.add(targetNode); + } + } catch (Exception e) { + monitor.severe("Failed to deserialize the registration service response"); + } + + return targetNodes; + } + +} diff --git a/extensions/participants-from-registration-service/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/participants-from-registration-service/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 0000000..0c968bf --- /dev/null +++ b/extensions/participants-from-registration-service/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1 @@ +org.upm.inesdata.catalog.FromRegistrationServiceParticipantsExtension diff --git a/launchers/connector/build.gradle.kts b/launchers/connector/build.gradle.kts index 232b27c..c8a879d 100644 --- a/launchers/connector/build.gradle.kts +++ b/launchers/connector/build.gradle.kts @@ -77,7 +77,8 @@ dependencies { implementation(libs.edc.observability.api) // Federated Catalog - implementation(project(":extensions:participants-from-configuration")) + //implementation(project(":extensions:participants-from-configuration")) + implementation(project(":extensions:participants-from-registration-service")) implementation(libs.edc.federated.catalog.spi) implementation(libs.edc.federated.catalog.core) implementation(libs.edc.federated.catalog.api) diff --git a/settings.gradle.kts b/settings.gradle.kts index 49a1eef..6a96857 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -12,7 +12,8 @@ include(":spi:count-elements-spi") // Extensions include(":extensions:auth-oauth2-jwt") -include(":extensions:participants-from-configuration") +//include(":extensions:participants-from-configuration") +include(":extensions:participants-from-registration-service") include(":extensions:policy-always-true") include(":extensions:policy-time-interval") include(":extensions:vocabulary-api") From df338d67f59ea46c48b1a75a896a95ce1596f236 Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Tue, 16 Jul 2024 13:51:30 +0200 Subject: [PATCH 121/174] periodico --- .../build.gradle.kts | 3 -- ...istrationServiceParticipantsExtension.java | 46 +++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/extensions/participants-from-registration-service/build.gradle.kts b/extensions/participants-from-registration-service/build.gradle.kts index 115a78a..49480f1 100644 --- a/extensions/participants-from-registration-service/build.gradle.kts +++ b/extensions/participants-from-registration-service/build.gradle.kts @@ -6,9 +6,6 @@ plugins { dependencies { implementation(libs.edc.federated.catalog.spi) implementation(libs.edc.federated.catalog.core) - implementation(libs.ok.http3) - implementation(libs.edc.http.spi) - implementation(libs.edc.identity.did.spi) testImplementation(libs.edc.core.junit) testImplementation(libs.junit.jupiter.api) diff --git a/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/FromRegistrationServiceParticipantsExtension.java b/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/FromRegistrationServiceParticipantsExtension.java index 3cd99b5..b994bc3 100644 --- a/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/FromRegistrationServiceParticipantsExtension.java +++ b/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/FromRegistrationServiceParticipantsExtension.java @@ -3,8 +3,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.eclipse.edc.catalog.directory.InMemoryNodeDirectory; import org.eclipse.edc.crawler.spi.TargetNodeDirectory; +import org.eclipse.edc.crawler.spi.model.ExecutionPlan; +import org.eclipse.edc.crawler.spi.model.RecurringExecutionPlan; import org.eclipse.edc.runtime.metamodel.annotation.Inject; import org.eclipse.edc.runtime.metamodel.annotation.Provider; +import org.eclipse.edc.runtime.metamodel.annotation.Setting; import org.eclipse.edc.spi.iam.IdentityService; import org.eclipse.edc.spi.iam.TokenParameters; import org.eclipse.edc.spi.iam.TokenRepresentation; @@ -12,12 +15,26 @@ import org.eclipse.edc.spi.system.ServiceExtension; import org.eclipse.edc.spi.system.ServiceExtensionContext; +import java.time.Duration; +import java.util.Random; + +import static java.lang.String.format; +import static org.eclipse.edc.catalog.spi.CacheSettings.DEFAULT_EXECUTION_PERIOD_SECONDS; +import static org.eclipse.edc.catalog.spi.CacheSettings.DEFAULT_NUMBER_OF_CRAWLERS; +import static org.eclipse.edc.catalog.spi.CacheSettings.LOW_EXECUTION_PERIOD_SECONDS_THRESHOLD; /** * Extension to set up federated target node directory using a configuration variable. */ public class FromRegistrationServiceParticipantsExtension implements ServiceExtension { + @Setting("The time to elapse between two crawl runs") + public static final String EXECUTION_PLAN_PERIOD_SECONDS = "edc.participants.cache.execution.period.seconds"; + @Setting("The number of crawlers (execution threads) that should be used. The engine will re-use crawlers when necessary.") + public static final String NUM_CRAWLER_SETTING = "edc.participants.cache.partition.num.crawlers"; + @Setting("The initial delay for the cache crawler engine") + public static final String EXECUTION_PLAN_DELAY_SECONDS = "edc.participants.cache.execution.delay.seconds"; + @Inject private IdentityService identityService; @@ -39,4 +56,33 @@ public TargetNodeDirectory federatedCacheNodeDirectory(ServiceExtensionContext c } return dir; } + + + @Provider + public ExecutionPlan createRecurringExecutionPlan(ServiceExtensionContext context) { + var periodSeconds = context.getSetting(EXECUTION_PLAN_PERIOD_SECONDS, DEFAULT_EXECUTION_PERIOD_SECONDS); + var setting = context.getSetting(EXECUTION_PLAN_DELAY_SECONDS, null); + int initialDelaySeconds; + if ("random".equals(setting) || setting == null) { + initialDelaySeconds = randomSeconds(); + } else { + try { + initialDelaySeconds = Integer.parseInt(setting); + } catch (NumberFormatException ex) { + initialDelaySeconds = 0; + } + } + var monitor = context.getMonitor(); + if (periodSeconds < LOW_EXECUTION_PERIOD_SECONDS_THRESHOLD) { + var crawlers = context.getSetting(NUM_CRAWLER_SETTING, DEFAULT_NUMBER_OF_CRAWLERS); + monitor.warning(format("An execution period of %d seconds is very low (threshold = %d). This might result in the work queue to be ever growing." + + " A longer execution period or more crawler threads (currently using %d) should be considered.", periodSeconds, LOW_EXECUTION_PERIOD_SECONDS_THRESHOLD, crawlers)); + } + return new RecurringExecutionPlan(Duration.ofSeconds(periodSeconds), Duration.ofSeconds(initialDelaySeconds), monitor); + } + + private int randomSeconds() { + var rnd = new Random(); + return 10 + rnd.nextInt(90); + } } From ed3f92600de78696f23beecdcbd01b7a6e746061 Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Wed, 17 Jul 2024 08:31:17 +0200 Subject: [PATCH 122/174] =?UTF-8?q?configuraci=C3=B3n=20scheduler?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...istrationServiceParticipantsExtension.java | 81 +++++++++---------- 1 file changed, 37 insertions(+), 44 deletions(-) diff --git a/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/FromRegistrationServiceParticipantsExtension.java b/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/FromRegistrationServiceParticipantsExtension.java index b994bc3..abb3e35 100644 --- a/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/FromRegistrationServiceParticipantsExtension.java +++ b/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/FromRegistrationServiceParticipantsExtension.java @@ -3,10 +3,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.eclipse.edc.catalog.directory.InMemoryNodeDirectory; import org.eclipse.edc.crawler.spi.TargetNodeDirectory; -import org.eclipse.edc.crawler.spi.model.ExecutionPlan; -import org.eclipse.edc.crawler.spi.model.RecurringExecutionPlan; -import org.eclipse.edc.runtime.metamodel.annotation.Inject; -import org.eclipse.edc.runtime.metamodel.annotation.Provider; import org.eclipse.edc.runtime.metamodel.annotation.Setting; import org.eclipse.edc.spi.iam.IdentityService; import org.eclipse.edc.spi.iam.TokenParameters; @@ -14,75 +10,72 @@ import org.eclipse.edc.spi.result.Result; import org.eclipse.edc.spi.system.ServiceExtension; import org.eclipse.edc.spi.system.ServiceExtensionContext; +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.runtime.metamodel.annotation.Provider; -import java.time.Duration; -import java.util.Random; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; -import static java.lang.String.format; import static org.eclipse.edc.catalog.spi.CacheSettings.DEFAULT_EXECUTION_PERIOD_SECONDS; -import static org.eclipse.edc.catalog.spi.CacheSettings.DEFAULT_NUMBER_OF_CRAWLERS; -import static org.eclipse.edc.catalog.spi.CacheSettings.LOW_EXECUTION_PERIOD_SECONDS_THRESHOLD; -/** - * Extension to set up federated target node directory using a configuration variable. - */ public class FromRegistrationServiceParticipantsExtension implements ServiceExtension { @Setting("The time to elapse between two crawl runs") public static final String EXECUTION_PLAN_PERIOD_SECONDS = "edc.participants.cache.execution.period.seconds"; - @Setting("The number of crawlers (execution threads) that should be used. The engine will re-use crawlers when necessary.") - public static final String NUM_CRAWLER_SETTING = "edc.participants.cache.partition.num.crawlers"; - @Setting("The initial delay for the cache crawler engine") - public static final String EXECUTION_PLAN_DELAY_SECONDS = "edc.participants.cache.execution.delay.seconds"; @Inject private IdentityService identityService; - @Provider - public TargetNodeDirectory federatedCacheNodeDirectory(ServiceExtensionContext context) { + private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); + + private InMemoryNodeDirectory nodeDirectory; + @Override + public void initialize(ServiceExtensionContext context) { + var periodSeconds = context.getSetting(EXECUTION_PLAN_PERIOD_SECONDS, DEFAULT_EXECUTION_PERIOD_SECONDS); var monitor = context.getMonitor(); var participantConfig = new ParticipantConfiguration(monitor, new ObjectMapper()); - var dir = new InMemoryNodeDirectory(); + nodeDirectory = new InMemoryNodeDirectory(); // Initialize the directory + + scheduler.scheduleAtFixedRate(() -> { + try { + updateTargetNodeDirectory(context, participantConfig); + } catch (Exception e) { + monitor.severe("Error updating TargetNodeDirectory", e); + } + }, 0, periodSeconds, TimeUnit.SECONDS); + } + + private void updateTargetNodeDirectory(ServiceExtensionContext context, ParticipantConfiguration participantConfig) { + var newDir = new InMemoryNodeDirectory(); Result tokenRepresentationResult = identityService.obtainClientCredentials( TokenParameters.Builder.newInstance().build()); + for (var target : participantConfig.getTargetNodes(context.getConfig(), tokenRepresentationResult)) { // skipping null target nodes if (target != null){ - dir.insert(target); + newDir.insert(target); } } - return dir; + + updateDirectoryInContext(newDir); } + private synchronized void updateDirectoryInContext(InMemoryNodeDirectory newDir) { + this.nodeDirectory = newDir; + } @Provider - public ExecutionPlan createRecurringExecutionPlan(ServiceExtensionContext context) { - var periodSeconds = context.getSetting(EXECUTION_PLAN_PERIOD_SECONDS, DEFAULT_EXECUTION_PERIOD_SECONDS); - var setting = context.getSetting(EXECUTION_PLAN_DELAY_SECONDS, null); - int initialDelaySeconds; - if ("random".equals(setting) || setting == null) { - initialDelaySeconds = randomSeconds(); - } else { - try { - initialDelaySeconds = Integer.parseInt(setting); - } catch (NumberFormatException ex) { - initialDelaySeconds = 0; - } - } - var monitor = context.getMonitor(); - if (periodSeconds < LOW_EXECUTION_PERIOD_SECONDS_THRESHOLD) { - var crawlers = context.getSetting(NUM_CRAWLER_SETTING, DEFAULT_NUMBER_OF_CRAWLERS); - monitor.warning(format("An execution period of %d seconds is very low (threshold = %d). This might result in the work queue to be ever growing." + - " A longer execution period or more crawler threads (currently using %d) should be considered.", periodSeconds, LOW_EXECUTION_PERIOD_SECONDS_THRESHOLD, crawlers)); - } - return new RecurringExecutionPlan(Duration.ofSeconds(periodSeconds), Duration.ofSeconds(initialDelaySeconds), monitor); + public synchronized TargetNodeDirectory federatedCacheNodeDirectory(ServiceExtensionContext context) { + // Return the updated directory + return nodeDirectory; } - private int randomSeconds() { - var rnd = new Random(); - return 10 + rnd.nextInt(90); + @Override + public void shutdown() { + scheduler.shutdown(); } } From fc8f4677102b5630ec202fa84589255a343ae2a7 Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Wed, 17 Jul 2024 09:28:12 +0200 Subject: [PATCH 123/174] Correcciones --- ...istrationServiceParticipantsExtension.java | 17 ++++++------ .../inesdata/catalog/SharedNodeDirectory.java | 27 +++++++++++++++++++ 2 files changed, 36 insertions(+), 8 deletions(-) create mode 100644 extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/SharedNodeDirectory.java diff --git a/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/FromRegistrationServiceParticipantsExtension.java b/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/FromRegistrationServiceParticipantsExtension.java index abb3e35..ef00192 100644 --- a/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/FromRegistrationServiceParticipantsExtension.java +++ b/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/FromRegistrationServiceParticipantsExtension.java @@ -29,7 +29,7 @@ public class FromRegistrationServiceParticipantsExtension implements ServiceExte private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); - private InMemoryNodeDirectory nodeDirectory; + private SharedNodeDirectory sharedNodeDirectory = new SharedNodeDirectory(); @Override public void initialize(ServiceExtensionContext context) { @@ -37,15 +37,17 @@ public void initialize(ServiceExtensionContext context) { var monitor = context.getMonitor(); var participantConfig = new ParticipantConfiguration(monitor, new ObjectMapper()); - nodeDirectory = new InMemoryNodeDirectory(); // Initialize the directory + // Initial update + updateTargetNodeDirectory(context, participantConfig); + // Schedule periodic updates scheduler.scheduleAtFixedRate(() -> { try { updateTargetNodeDirectory(context, participantConfig); } catch (Exception e) { monitor.severe("Error updating TargetNodeDirectory", e); } - }, 0, periodSeconds, TimeUnit.SECONDS); + }, periodSeconds, periodSeconds, TimeUnit.SECONDS); } private void updateTargetNodeDirectory(ServiceExtensionContext context, ParticipantConfiguration participantConfig) { @@ -64,14 +66,13 @@ private void updateTargetNodeDirectory(ServiceExtensionContext context, Particip updateDirectoryInContext(newDir); } - private synchronized void updateDirectoryInContext(InMemoryNodeDirectory newDir) { - this.nodeDirectory = newDir; + private void updateDirectoryInContext(InMemoryNodeDirectory newDir) { + sharedNodeDirectory.update(newDir); } @Provider - public synchronized TargetNodeDirectory federatedCacheNodeDirectory(ServiceExtensionContext context) { - // Return the updated directory - return nodeDirectory; + public TargetNodeDirectory federatedCacheNodeDirectory(ServiceExtensionContext context) { + return sharedNodeDirectory; } @Override diff --git a/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/SharedNodeDirectory.java b/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/SharedNodeDirectory.java new file mode 100644 index 0000000..312d781 --- /dev/null +++ b/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/SharedNodeDirectory.java @@ -0,0 +1,27 @@ +package org.upm.inesdata.catalog; + +import org.eclipse.edc.catalog.directory.InMemoryNodeDirectory; +import org.eclipse.edc.crawler.spi.TargetNode; +import org.eclipse.edc.crawler.spi.TargetNodeDirectory; + +import java.util.List; + +public class SharedNodeDirectory implements TargetNodeDirectory { + + private volatile InMemoryNodeDirectory nodeDirectory = new InMemoryNodeDirectory(); + + public synchronized void update(InMemoryNodeDirectory newDirectory) { + this.nodeDirectory = newDirectory; + } + + @Override + public List getAll() { + return nodeDirectory.getAll(); + } + + @Override + public void insert(TargetNode node) { + nodeDirectory.insert(node); + } +} + From 893d1652c6bcda54a7ee642a00f5c524409586df Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Wed, 17 Jul 2024 12:42:44 +0200 Subject: [PATCH 124/174] README --- .../README.md | 37 ++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/extensions/participants-from-registration-service/README.md b/extensions/participants-from-registration-service/README.md index f4e21ab..0ba7dc7 100644 --- a/extensions/participants-from-registration-service/README.md +++ b/extensions/participants-from-registration-service/README.md @@ -1,18 +1,29 @@ -# InesData Connector +### Catalog Service Participants Extension -This extension provides the capability to obtain the list of participants of a data space in order to allow the federated catalog to obtain their data offerings. +This extension has the functionality for getting participant data within a service catalog. -## Configuration +#### Overview -Example configuration: +The `FromRegistrationServiceParticipantsExtension` class implements `ServiceExtension` to facilitate periodic updates of participant data from a registration service. It utilizes the Eclipse EDC framework for managing and processing metadata. -```properties -edc.catalog.configuration.participant.1.name = connector-c1 -edc.catalog.configuration.participant.1.id = connector-c1 -edc.catalog.configuration.participant.1.targetUrl = http://localhost:19194/protocol -edc.catalog.configuration.participant.2.name = connector-c2 -edc.catalog.configuration.participant.2.id = connector-c2 -edc.catalog.configuration.participant.2.targetUrl = http://localhost:29194/protocol -``` +#### Features -The `edc.catalog.configuration.participant` will be used by the federated catalog to retrieve the list of participants whose catalog will be obtained for federating it. \ No newline at end of file +- **Periodic Updates**: Periodically retrieves participant data from a configurable registration service. +- **Integration**: Integrates with Eclipse EDC for metadata management and service integration. +- **Concurrency**: Uses scheduled tasks for efficient and timely data updates. + +#### Setup + +1. **Dependencies**: Ensure dependencies like Eclipse EDC are included in the project. + +2. **Configuration**: Adjust settings such as `edc.participants.cache.execution.period.seconds` based on operational requirements. + +#### Usage + +The extension initializes by retrieving participant configurations and scheduling periodic updates using an in-memory directory (`InMemoryNodeDirectory`). Participant data is obtained via HTTP GET requests and transformed into `TargetNode` objects for ingestion into the directory. + +#### Components + +- **ParticipantConfiguration**: Manages HTTP requests to the registration service and transforms responses into `TargetNode` objects. + +- **SharedNodeDirectory**: Implements `TargetNodeDirectory` to maintain and update participant nodes across the application. From 36f32ea6e1de557ea695a21921597028a934086c Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Wed, 17 Jul 2024 14:23:57 +0200 Subject: [PATCH 125/174] Comentarios MR --- .../participants-from-registration-service/build.gradle.kts | 4 ---- .../org/upm/inesdata/catalog/ParticipantConfiguration.java | 6 +++--- launchers/connector/build.gradle.kts | 1 - settings.gradle.kts | 1 - 4 files changed, 3 insertions(+), 9 deletions(-) diff --git a/extensions/participants-from-registration-service/build.gradle.kts b/extensions/participants-from-registration-service/build.gradle.kts index 49480f1..5f567ef 100644 --- a/extensions/participants-from-registration-service/build.gradle.kts +++ b/extensions/participants-from-registration-service/build.gradle.kts @@ -7,8 +7,4 @@ dependencies { implementation(libs.edc.federated.catalog.spi) implementation(libs.edc.federated.catalog.core) - testImplementation(libs.edc.core.junit) - testImplementation(libs.junit.jupiter.api) - testImplementation(libs.assertj) - testImplementation(libs.mockito.core) } diff --git a/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/ParticipantConfiguration.java b/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/ParticipantConfiguration.java index 417d299..2975ed7 100644 --- a/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/ParticipantConfiguration.java +++ b/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/ParticipantConfiguration.java @@ -40,7 +40,7 @@ public class ParticipantConfiguration { * Constructor * * @param monitor monitor - * @param objectMapper + * @param objectMapper mapper */ public ParticipantConfiguration(Monitor monitor, ObjectMapper objectMapper) { this.monitor = monitor; @@ -54,7 +54,7 @@ public ParticipantConfiguration(Monitor monitor, ObjectMapper objectMapper) { * Makes an HTTP GET request to the specified URL and returns the response as a string. * * @param url the URL to make the GET request to - * @param tokenRepresentationResult + * @param tokenRepresentationResult token * @return the response from the GET request */ public String makeHttpGetRequest(String url, Result tokenRepresentationResult) { @@ -68,7 +68,7 @@ public String makeHttpGetRequest(String url, Result tokenRe * Retrieve TargetNodes from configuration * * @param baseConfig EDC Configuration - * @param tokenRepresentationResult + * @param tokenRepresentationResult token * @return list of TargetNodes from configuration */ public List getTargetNodes(Config baseConfig, Result tokenRepresentationResult) { diff --git a/launchers/connector/build.gradle.kts b/launchers/connector/build.gradle.kts index c8a879d..d45fe81 100644 --- a/launchers/connector/build.gradle.kts +++ b/launchers/connector/build.gradle.kts @@ -77,7 +77,6 @@ dependencies { implementation(libs.edc.observability.api) // Federated Catalog - //implementation(project(":extensions:participants-from-configuration")) implementation(project(":extensions:participants-from-registration-service")) implementation(libs.edc.federated.catalog.spi) implementation(libs.edc.federated.catalog.core) diff --git a/settings.gradle.kts b/settings.gradle.kts index 01ff52e..c265a56 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -12,7 +12,6 @@ include(":spi:count-elements-spi") // Extensions include(":extensions:auth-oauth2-jwt") -//include(":extensions:participants-from-configuration") include(":extensions:participants-from-registration-service") include(":extensions:policy-always-true") include(":extensions:policy-time-interval") From bf385e99c9cf81d3958756e2c12f1c13ce46671b Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Thu, 18 Jul 2024 10:43:49 +0200 Subject: [PATCH 126/174] Comentario --- .../FromRegistrationServiceParticipantsExtension.java | 10 +++++----- ...ration.java => ParticipantRegistrationService.java} | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) rename extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/{ParticipantConfiguration.java => ParticipantRegistrationService.java} (96%) diff --git a/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/FromRegistrationServiceParticipantsExtension.java b/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/FromRegistrationServiceParticipantsExtension.java index ef00192..e66d7dc 100644 --- a/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/FromRegistrationServiceParticipantsExtension.java +++ b/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/FromRegistrationServiceParticipantsExtension.java @@ -35,28 +35,28 @@ public class FromRegistrationServiceParticipantsExtension implements ServiceExte public void initialize(ServiceExtensionContext context) { var periodSeconds = context.getSetting(EXECUTION_PLAN_PERIOD_SECONDS, DEFAULT_EXECUTION_PERIOD_SECONDS); var monitor = context.getMonitor(); - var participantConfig = new ParticipantConfiguration(monitor, new ObjectMapper()); + var participantRegistrationService = new ParticipantRegistrationService(monitor, new ObjectMapper()); // Initial update - updateTargetNodeDirectory(context, participantConfig); + updateTargetNodeDirectory(context, participantRegistrationService); // Schedule periodic updates scheduler.scheduleAtFixedRate(() -> { try { - updateTargetNodeDirectory(context, participantConfig); + updateTargetNodeDirectory(context, participantRegistrationService); } catch (Exception e) { monitor.severe("Error updating TargetNodeDirectory", e); } }, periodSeconds, periodSeconds, TimeUnit.SECONDS); } - private void updateTargetNodeDirectory(ServiceExtensionContext context, ParticipantConfiguration participantConfig) { + private void updateTargetNodeDirectory(ServiceExtensionContext context, ParticipantRegistrationService participantRegistrationService) { var newDir = new InMemoryNodeDirectory(); Result tokenRepresentationResult = identityService.obtainClientCredentials( TokenParameters.Builder.newInstance().build()); - for (var target : participantConfig.getTargetNodes(context.getConfig(), tokenRepresentationResult)) { + for (var target : participantRegistrationService.getTargetNodes(context.getConfig(), tokenRepresentationResult)) { // skipping null target nodes if (target != null){ newDir.insert(target); diff --git a/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/ParticipantConfiguration.java b/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/ParticipantRegistrationService.java similarity index 96% rename from extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/ParticipantConfiguration.java rename to extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/ParticipantRegistrationService.java index 2975ed7..cb9e026 100644 --- a/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/ParticipantConfiguration.java +++ b/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/ParticipantRegistrationService.java @@ -23,7 +23,7 @@ /** * Handles participants in configuration and transforms them into TargetNodes */ -public class ParticipantConfiguration { +public class ParticipantRegistrationService { public static final List SUPPORTED_PROTOCOLS = List.of("dataspace-protocol-http"); @@ -42,7 +42,7 @@ public class ParticipantConfiguration { * @param monitor monitor * @param objectMapper mapper */ - public ParticipantConfiguration(Monitor monitor, ObjectMapper objectMapper) { + public ParticipantRegistrationService(Monitor monitor, ObjectMapper objectMapper) { this.monitor = monitor; this.objectMapper = objectMapper; } From 6fb9a5cb2a22e0c88f80a7c380964233742770f6 Mon Sep 17 00:00:00 2001 From: ppel Date: Mon, 22 Jul 2024 09:25:23 +0200 Subject: [PATCH 127/174] #436844 - Created new Shared context and new endpoint for getting vocabularies of a connector --- .../Oauth2JwtAuthenticationExtension.java | 5 +- extensions/shared-api-configuration/README.md | 0 .../shared-api-configuration/build.gradle.kts | 12 ++ .../SharedApiConfigurationExtension.java | 111 ++++++++++++++++++ .../shared/configuration/SharedApiUrl.java | 17 +++ ...haredOauth2JwtAuthenticationExtension.java | 39 ++++++ .../SharedOauth2JwtAuthenticationService.java | 69 +++++++++++ ...rg.eclipse.edc.spi.system.ServiceExtension | 2 + .../vocabulary/controller/VocabularyApi.java | 1 + .../service/VocabularyServiceImpl.java | 4 +- .../storage/InMemoryVocabularyIndex.java | 5 + .../JsonObjectFromVocabularyTransformer.java | 2 + .../JsonObjectToVocabularyTransformer.java | 2 + extensions/vocabulary-index-sql/docs/er.puml | 1 + .../vocabulary-index-sql/docs/schema.sql | 1 + .../sql/index/SqlVocabularyIndex.java | 15 +++ .../schema/BaseSqlDialectStatements.java | 1 + .../index/schema/VocabularyStatements.java | 4 + .../schema/postgres/VocabularyMapping.java | 1 + extensions/vocabulary-shared-api/README.md | 0 .../vocabulary-shared-api/build.gradle.kts | 26 ++++ .../api/VocabularySharedApiExtension.java | 89 ++++++++++++++ .../api/controller/VocabularySharedApi.java | 60 ++++++++++ .../VocabularySharedApiController.java | 55 +++++++++ .../service/VocabularySharedServiceImpl.java | 33 ++++++ ...bjectToConnectorVocabularyTransformer.java | 35 ++++++ ...rg.eclipse.edc.spi.system.ServiceExtension | 1 + .../vocabulary-shared-retrieval/README.md | 0 .../build.gradle.kts | 12 ++ .../VocabularySharedRetrievalExtension.java | 86 ++++++++++++++ ...rg.eclipse.edc.spi.system.ServiceExtension | 1 + gradle/libs.versions.toml | 6 +- launchers/connector/build.gradle.kts | 4 + settings.gradle.kts | 9 ++ .../spi/vocabulary/VocabularyIndex.java | 8 ++ .../vocabulary/VocabularySharedService.java | 18 +++ .../domain/ConnectorVocabulary.java | 55 +++++++++ .../spi/vocabulary/domain/Vocabulary.java | 11 ++ 38 files changed, 796 insertions(+), 5 deletions(-) create mode 100644 extensions/shared-api-configuration/README.md create mode 100644 extensions/shared-api-configuration/build.gradle.kts create mode 100644 extensions/shared-api-configuration/src/main/java/org/upm/inesdata/api/shared/configuration/SharedApiConfigurationExtension.java create mode 100644 extensions/shared-api-configuration/src/main/java/org/upm/inesdata/api/shared/configuration/SharedApiUrl.java create mode 100644 extensions/shared-api-configuration/src/main/java/org/upm/inesdata/api/shared/configuration/auth/SharedOauth2JwtAuthenticationExtension.java create mode 100644 extensions/shared-api-configuration/src/main/java/org/upm/inesdata/api/shared/configuration/auth/service/SharedOauth2JwtAuthenticationService.java create mode 100644 extensions/shared-api-configuration/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension create mode 100644 extensions/vocabulary-shared-api/README.md create mode 100644 extensions/vocabulary-shared-api/build.gradle.kts create mode 100644 extensions/vocabulary-shared-api/src/main/java/org/upm/inesdata/vocabulary/shared/api/VocabularySharedApiExtension.java create mode 100644 extensions/vocabulary-shared-api/src/main/java/org/upm/inesdata/vocabulary/shared/api/controller/VocabularySharedApi.java create mode 100644 extensions/vocabulary-shared-api/src/main/java/org/upm/inesdata/vocabulary/shared/api/controller/VocabularySharedApiController.java create mode 100644 extensions/vocabulary-shared-api/src/main/java/org/upm/inesdata/vocabulary/shared/api/service/VocabularySharedServiceImpl.java create mode 100644 extensions/vocabulary-shared-api/src/main/java/org/upm/inesdata/vocabulary/shared/api/transformer/JsonObjectToConnectorVocabularyTransformer.java create mode 100644 extensions/vocabulary-shared-api/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension create mode 100644 extensions/vocabulary-shared-retrieval/README.md create mode 100644 extensions/vocabulary-shared-retrieval/build.gradle.kts create mode 100644 extensions/vocabulary-shared-retrieval/src/main/java/org/upm/inesdata/vocabulary/shared/retrieval/VocabularySharedRetrievalExtension.java create mode 100644 extensions/vocabulary-shared-retrieval/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension create mode 100644 spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/VocabularySharedService.java create mode 100644 spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/domain/ConnectorVocabulary.java diff --git a/extensions/auth-oauth2-jwt/src/main/java/org/upm/inesdata/auth/Oauth2JwtAuthenticationExtension.java b/extensions/auth-oauth2-jwt/src/main/java/org/upm/inesdata/auth/Oauth2JwtAuthenticationExtension.java index d0ec2a3..bf8a085 100644 --- a/extensions/auth-oauth2-jwt/src/main/java/org/upm/inesdata/auth/Oauth2JwtAuthenticationExtension.java +++ b/extensions/auth-oauth2-jwt/src/main/java/org/upm/inesdata/auth/Oauth2JwtAuthenticationExtension.java @@ -1,6 +1,7 @@ package org.upm.inesdata.auth; import org.eclipse.edc.api.auth.spi.AuthenticationService; +import org.eclipse.edc.api.auth.spi.registry.ApiAuthenticationRegistry; import org.eclipse.edc.runtime.metamodel.annotation.Extension; import org.eclipse.edc.runtime.metamodel.annotation.Inject; import org.eclipse.edc.runtime.metamodel.annotation.Provides; @@ -29,6 +30,8 @@ public class Oauth2JwtAuthenticationExtension implements ServiceExtension { @Inject private IdentityService identityService; @Inject + private ApiAuthenticationRegistry authenticationRegistry; + @Inject private Vault vault; @Override @@ -45,6 +48,6 @@ public void initialize(ServiceExtensionContext context) { var rolesConfig = context.getConfig(AUTH_SETTING_ALLOWEDROLES); List allowedRoles = rolesConfig.partition().map(conf -> conf.getString(ROLE_PROPERTY_NAME)).collect(Collectors.toList()); - context.registerService(AuthenticationService.class, new Oauth2JwtAuthenticationService(identityService, participantId, allowedRoles)); + authenticationRegistry.register("management-api", new Oauth2JwtAuthenticationService(identityService, participantId, allowedRoles)); } } diff --git a/extensions/shared-api-configuration/README.md b/extensions/shared-api-configuration/README.md new file mode 100644 index 0000000..e69de29 diff --git a/extensions/shared-api-configuration/build.gradle.kts b/extensions/shared-api-configuration/build.gradle.kts new file mode 100644 index 0000000..a7f8129 --- /dev/null +++ b/extensions/shared-api-configuration/build.gradle.kts @@ -0,0 +1,12 @@ +plugins { + `java-library` + id("com.gmv.inesdata.edc-application") +} + +dependencies { + api(libs.edc.web.spi) + api(libs.edc.auth.spi) + api(libs.edc.iam.oauth2.service) + api(libs.edc.spi.jsonld) + api(libs.edc.jersey.providers.lib) +} \ No newline at end of file diff --git a/extensions/shared-api-configuration/src/main/java/org/upm/inesdata/api/shared/configuration/SharedApiConfigurationExtension.java b/extensions/shared-api-configuration/src/main/java/org/upm/inesdata/api/shared/configuration/SharedApiConfigurationExtension.java new file mode 100644 index 0000000..294decc --- /dev/null +++ b/extensions/shared-api-configuration/src/main/java/org/upm/inesdata/api/shared/configuration/SharedApiConfigurationExtension.java @@ -0,0 +1,111 @@ +package org.upm.inesdata.api.shared.configuration; + +import org.eclipse.edc.api.auth.spi.AuthenticationRequestFilter; +import org.eclipse.edc.api.auth.spi.registry.ApiAuthenticationRegistry; +import org.eclipse.edc.jsonld.spi.JsonLd; +import org.eclipse.edc.runtime.metamodel.annotation.Extension; +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.runtime.metamodel.annotation.Provides; +import org.eclipse.edc.runtime.metamodel.annotation.Setting; +import org.eclipse.edc.runtime.metamodel.annotation.SettingContext; +import org.eclipse.edc.spi.EdcException; +import org.eclipse.edc.spi.iam.IdentityService; +import org.eclipse.edc.spi.system.ExecutorInstrumentation; +import org.eclipse.edc.spi.system.Hostname; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; +import org.eclipse.edc.spi.types.TypeManager; +import org.eclipse.edc.web.jersey.providers.jsonld.JerseyJsonLdInterceptor; +import org.eclipse.edc.web.jersey.providers.jsonld.ObjectMapperProvider; +import org.eclipse.edc.web.spi.WebServer; +import org.eclipse.edc.web.spi.WebService; +import org.eclipse.edc.web.spi.configuration.WebServiceConfiguration; +import org.eclipse.edc.web.spi.configuration.WebServiceConfigurer; +import org.eclipse.edc.web.spi.configuration.WebServiceSettings; + +import java.net.URI; + +import static java.lang.String.format; +import static org.eclipse.edc.policy.model.OdrlNamespace.ODRL_PREFIX; +import static org.eclipse.edc.policy.model.OdrlNamespace.ODRL_SCHEMA; +import static org.eclipse.edc.spi.constants.CoreConstants.JSON_LD; + +/** + * This extension provides generic endpoints which are open to all connectors. + */ +@Provides(SharedApiUrl.class) +@Extension(value = SharedApiConfigurationExtension.NAME) +public class SharedApiConfigurationExtension implements ServiceExtension { + public static final String NAME = "Shared Public API"; + + private static final int DEFAULT_SHARED_PORT = 8186; + private static final String SHARED_CONTEXT_PATH = "/api/v1/shared"; + + @SettingContext("Shared API context setting key") + private static final String SHARED_CONFIG_KEY = "web.http.shared"; + + @Setting(value = "Base url of the shared API endpoint without the trailing slash. This should correspond to the values configured " + + "in '" + DEFAULT_SHARED_PORT + "' and '" + SHARED_CONTEXT_PATH + "'.", defaultValue = "http://:" + DEFAULT_SHARED_PORT + SHARED_CONTEXT_PATH) + private static final String SHARED_ENDPOINT = "edc.shared.endpoint"; + + private static final WebServiceSettings SHARED_SETTINGS = WebServiceSettings.Builder.newInstance() + .apiConfigKey(SHARED_CONFIG_KEY) + .contextAlias("shared") + .defaultPath(SHARED_CONTEXT_PATH) + .defaultPort(DEFAULT_SHARED_PORT) + .name(NAME) + .build(); + + private static final String SHARED_SCOPE = "SHARED_API"; + + @Inject + private WebServer webServer; + @Inject + private ApiAuthenticationRegistry authenticationRegistry; + @Inject + private WebServiceConfigurer webServiceConfigurer; + @Inject + private WebService webService; + @Inject + private ExecutorInstrumentation executorInstrumentation; + @Inject + private Hostname hostname; + @Inject + private IdentityService identityService; + @Inject + private JsonLd jsonLd; + @Inject + private TypeManager typeManager; + + @Override + public String name() { + return NAME; + } + + @Override + public void initialize(ServiceExtensionContext context) { + var config = context.getConfig(SHARED_CONFIG_KEY); + var configuration = webServiceConfigurer.configure(config, webServer, SHARED_SETTINGS); + + context.registerService(SharedApiUrl.class, sharedApiUrl(context, configuration)); + + var authenticationFilter = new AuthenticationRequestFilter(authenticationRegistry, "shared-api"); + webService.registerResource("shared", authenticationFilter); + + jsonLd.registerNamespace(ODRL_PREFIX, ODRL_SCHEMA, SHARED_SCOPE); + var jsonLdMapper = typeManager.getMapper(JSON_LD); + webService.registerResource("shared", new ObjectMapperProvider(jsonLdMapper)); + webService.registerResource("shared", new JerseyJsonLdInterceptor(jsonLd, jsonLdMapper, SHARED_SCOPE)); + } + + private SharedApiUrl sharedApiUrl(ServiceExtensionContext context, WebServiceConfiguration config) { + var callbackAddress = context.getSetting(SHARED_ENDPOINT, format("http://%s:%s%s", hostname.get(), config.getPort(), config.getPath())); + try { + var url = URI.create(callbackAddress); + return () -> url; + } catch (IllegalArgumentException e) { + context.getMonitor().severe("Error creating shared endpoint url", e); + throw new EdcException(e); + } + } +} diff --git a/extensions/shared-api-configuration/src/main/java/org/upm/inesdata/api/shared/configuration/SharedApiUrl.java b/extensions/shared-api-configuration/src/main/java/org/upm/inesdata/api/shared/configuration/SharedApiUrl.java new file mode 100644 index 0000000..7d44526 --- /dev/null +++ b/extensions/shared-api-configuration/src/main/java/org/upm/inesdata/api/shared/configuration/SharedApiUrl.java @@ -0,0 +1,17 @@ +package org.upm.inesdata.api.shared.configuration; + +import java.net.URI; + +/** + * Provides the Shared Api URL exposed, useful for setting callbacks. + */ +@FunctionalInterface +public interface SharedApiUrl { + + /** + * URI on which the Shared API is exposed + * + * @return Shared API URI. + */ + URI get(); +} diff --git a/extensions/shared-api-configuration/src/main/java/org/upm/inesdata/api/shared/configuration/auth/SharedOauth2JwtAuthenticationExtension.java b/extensions/shared-api-configuration/src/main/java/org/upm/inesdata/api/shared/configuration/auth/SharedOauth2JwtAuthenticationExtension.java new file mode 100644 index 0000000..1a2bed8 --- /dev/null +++ b/extensions/shared-api-configuration/src/main/java/org/upm/inesdata/api/shared/configuration/auth/SharedOauth2JwtAuthenticationExtension.java @@ -0,0 +1,39 @@ +package org.upm.inesdata.api.shared.configuration.auth; + +import org.eclipse.edc.api.auth.spi.AuthenticationService; +import org.eclipse.edc.api.auth.spi.registry.ApiAuthenticationRegistry; +import org.eclipse.edc.runtime.metamodel.annotation.Extension; +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.runtime.metamodel.annotation.Provides; +import org.eclipse.edc.spi.iam.IdentityService; +import org.eclipse.edc.spi.security.Vault; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; +import org.upm.inesdata.api.shared.configuration.auth.service.SharedOauth2JwtAuthenticationService; + +/** + * Extension that registers an SharedAuthenticationService that uses Oauth2 JWT Tokens + */ +@Provides(AuthenticationService.class) +@Extension(value = SharedOauth2JwtAuthenticationExtension.NAME) +public class SharedOauth2JwtAuthenticationExtension implements ServiceExtension { + + public static final String NAME = "Shared Oauth2 JWT Authentication"; + + @Inject + private IdentityService identityService; + @Inject + private ApiAuthenticationRegistry authenticationRegistry; + @Inject + private Vault vault; + + @Override + public String name() { + return NAME; + } + + @Override + public void initialize(ServiceExtensionContext context) { + authenticationRegistry.register("shared-api",new SharedOauth2JwtAuthenticationService(identityService)); + } +} diff --git a/extensions/shared-api-configuration/src/main/java/org/upm/inesdata/api/shared/configuration/auth/service/SharedOauth2JwtAuthenticationService.java b/extensions/shared-api-configuration/src/main/java/org/upm/inesdata/api/shared/configuration/auth/service/SharedOauth2JwtAuthenticationService.java new file mode 100644 index 0000000..6d2917b --- /dev/null +++ b/extensions/shared-api-configuration/src/main/java/org/upm/inesdata/api/shared/configuration/auth/service/SharedOauth2JwtAuthenticationService.java @@ -0,0 +1,69 @@ +package org.upm.inesdata.api.shared.configuration.auth.service; + +import jakarta.ws.rs.core.HttpHeaders; +import org.eclipse.edc.api.auth.spi.AuthenticationService; +import org.eclipse.edc.spi.iam.IdentityService; +import org.eclipse.edc.spi.iam.TokenRepresentation; +import org.eclipse.edc.web.spi.exception.AuthenticationFailedException; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * SharedAuthenticationService for access control based on JWT tokens + */ +public class SharedOauth2JwtAuthenticationService implements AuthenticationService { + private static final String BEARER_PREFIX = "Bearer "; + + private static final String AUTHORIZATION_HEADER_ERROR = HttpHeaders.AUTHORIZATION + " header not found"; + private static final String BEARER_PREFIX_ERROR = "Bearer token not found"; + + + private final IdentityService identityService; + + public SharedOauth2JwtAuthenticationService(IdentityService identityService) { + this.identityService = identityService; + } + + /** + * Checks whether a particular request is authorized based on the "Authorization" header. + * The header must contains a valid JWT token + * + * @param headers the headers + * @throws IllegalArgumentException The map of headers did not contain the "Authorization" header + * @return whether it is a valid JWT token or not + */ + @Override + public boolean isAuthenticated(Map> headers) { + + Objects.requireNonNull(headers, "headers"); + + // Get the Authorization header + var apiKey = headers.keySet().stream() + .filter(k -> k.equalsIgnoreCase(HttpHeaders.AUTHORIZATION)) + .map(headers::get) + .findFirst(); + + return apiKey.map(this::checkTokenValid).orElseThrow(() -> new AuthenticationFailedException(AUTHORIZATION_HEADER_ERROR)); + } + + private boolean checkTokenValid(List tokenKeys) { + return tokenKeys.size() == 1 && tokenKeys.stream().allMatch(this::isJwtTokenValid); + } + + @SuppressWarnings("unchecked") + private boolean isJwtTokenValid(String jwtToken) { + boolean valid = false; + + if (!jwtToken.startsWith(BEARER_PREFIX)) { + throw new AuthenticationFailedException(BEARER_PREFIX_ERROR); + } + + var tokenRepresentation = TokenRepresentation.Builder.newInstance().token(jwtToken.replace(BEARER_PREFIX, "")).build(); + var tokenValidation = identityService.verifyJwtToken(tokenRepresentation, null); + valid = tokenValidation.succeeded(); + + return valid; + } +} diff --git a/extensions/shared-api-configuration/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/shared-api-configuration/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 0000000..faa9ce2 --- /dev/null +++ b/extensions/shared-api-configuration/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1,2 @@ +org.upm.inesdata.api.shared.configuration.SharedApiConfigurationExtension +org.upm.inesdata.api.shared.configuration.auth.SharedOauth2JwtAuthenticationExtension \ No newline at end of file diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/controller/VocabularyApi.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/controller/VocabularyApi.java index e96d394..8d23914 100644 --- a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/controller/VocabularyApi.java +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/controller/VocabularyApi.java @@ -126,6 +126,7 @@ record VocabularyOutputSchema( { "@id": "vocabularyId", "name": "vocabulary name", + "connectorId": "connector-c1", "jsonSchema": "{ \\"title\\": \\"vocabulary\\", \\"type\\": \\"object\\", \\"properties\\": { \\"name\\": { \\"type\\": \\"string\\", \\"title\\": \\"Name\\" }, \\"dct:keyword\\": { \\"type\\": \\"array\\", \\"title\\": \\"Keywords\\", \\"items\\": { \\"type\\": \\"string\\" } } }, \\"required\\": [ \\"name\\" ], \\"@context\\": { \\"dct\\": \\"http:\\/\\/purl.org\\/dc\\/terms\\/\" } }", "category": "dataset" } diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/service/VocabularyServiceImpl.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/service/VocabularyServiceImpl.java index 9ed1d0b..3e50196 100644 --- a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/service/VocabularyServiceImpl.java +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/service/VocabularyServiceImpl.java @@ -13,8 +13,8 @@ */ public class VocabularyServiceImpl implements VocabularyService { - private final VocabularyIndex index; - private final TransactionContext transactionContext; + protected final VocabularyIndex index; + protected final TransactionContext transactionContext; /** * Constructor diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/storage/InMemoryVocabularyIndex.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/storage/InMemoryVocabularyIndex.java index d961d79..4ecf051 100644 --- a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/storage/InMemoryVocabularyIndex.java +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/storage/InMemoryVocabularyIndex.java @@ -97,6 +97,11 @@ public StoreResult updateVocabulary(Vocabulary vocabulary) { } } + @Override + public Stream searchVocabulariesByConnector(String connectorId) { + return null; + } + /** * Remove a vocabulary from cache based on its id */ diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectFromVocabularyTransformer.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectFromVocabularyTransformer.java index 811cec0..0feafec 100644 --- a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectFromVocabularyTransformer.java +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectFromVocabularyTransformer.java @@ -13,6 +13,7 @@ import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.TYPE; import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.EDC_VOCABULARY_TYPE; import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_CATEGORY; +import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_CONNECTOR_ID; import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_JSON_SCHEMA; import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_NAME; @@ -39,6 +40,7 @@ public JsonObjectFromVocabularyTransformer(JsonBuilderFactory jsonFactory, Objec .add(ID, vocabulary.getId()) .add(TYPE, EDC_VOCABULARY_TYPE) .add(PROPERTY_NAME, vocabulary.getName()) + .add(PROPERTY_CONNECTOR_ID, vocabulary.getConnectorId()) .add(PROPERTY_JSON_SCHEMA, vocabulary.getJsonSchema()) .add(PROPERTY_CATEGORY, vocabulary.getCategory()); diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectToVocabularyTransformer.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectToVocabularyTransformer.java index b46d88e..8fb0c6a 100644 --- a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectToVocabularyTransformer.java +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/transformer/JsonObjectToVocabularyTransformer.java @@ -8,6 +8,7 @@ import org.upm.inesdata.spi.vocabulary.domain.Vocabulary; import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_CATEGORY; +import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_CONNECTOR_ID; import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_JSON_SCHEMA; import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.PROPERTY_NAME; @@ -30,6 +31,7 @@ public JsonObjectToVocabularyTransformer() { visitProperties(jsonObject, key -> switch (key) { case PROPERTY_NAME -> value -> builder.name(transformString(value, context)); + case PROPERTY_CONNECTOR_ID -> value -> builder.connectorId(transformString(value, context)); case PROPERTY_JSON_SCHEMA -> value -> builder.jsonSchema(transformString(value, context)); case PROPERTY_CATEGORY -> value -> builder.category(transformString(value, context)); default -> doNothing(); diff --git a/extensions/vocabulary-index-sql/docs/er.puml b/extensions/vocabulary-index-sql/docs/er.puml index f081ea7..f667d09 100644 --- a/extensions/vocabulary-index-sql/docs/er.puml +++ b/extensions/vocabulary-index-sql/docs/er.puml @@ -2,6 +2,7 @@ entity edc_vocabulary { * id: string <> * name: string + * connectorId: string * jsonSchema: string * createdAt: long -- diff --git a/extensions/vocabulary-index-sql/docs/schema.sql b/extensions/vocabulary-index-sql/docs/schema.sql index cb8d1a8..891b291 100644 --- a/extensions/vocabulary-index-sql/docs/schema.sql +++ b/extensions/vocabulary-index-sql/docs/schema.sql @@ -5,6 +5,7 @@ CREATE TABLE IF NOT EXISTS edc_vocabulary created_at BIGINT NOT NULL, json_schema JSON DEFAULT '{}', name VARCHAR NOT NULL, + connector_id VARCHAR NOT NULL, category VARCHAR NOT NULL PRIMARY KEY (id) ); diff --git a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/SqlVocabularyIndex.java b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/SqlVocabularyIndex.java index 6cb0121..5e00c44 100644 --- a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/SqlVocabularyIndex.java +++ b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/SqlVocabularyIndex.java @@ -81,6 +81,7 @@ public StoreResult create(Vocabulary vocabulary) { vocabularyId, vocabulary.getCreatedAt(), vocabulary.getName(), + vocabulary.getConnectorId(), vocabulary.getCategory(), toJson(vocabulary.getJsonSchema()) ); @@ -135,6 +136,19 @@ public StoreResult updateVocabulary(Vocabulary vocabulary) { }); } + @Override + public Stream searchVocabulariesByConnector(String connectorId) { + return transactionContext.execute(() -> { + try { + var querySpec = QuerySpec.Builder.newInstance().filter(criterion("connectorId", "=", connectorId)).build(); + var statement = vocabularyStatements.createQuery(querySpec); + return queryExecutor.query(getConnection(), true, this::mapVocabulary, statement.getQueryAsString(), statement.getParameters()); + } catch (SQLException e) { + throw new EdcPersistenceException(e); + } + }); + } + private int mapRowCount(ResultSet resultSet) throws SQLException { return resultSet.getInt(vocabularyStatements.getCountVariableName()); } @@ -151,6 +165,7 @@ private Vocabulary mapVocabulary(ResultSet resultSet) throws SQLException { .id(resultSet.getString(vocabularyStatements.getVocabularyIdColumn())) .createdAt(resultSet.getLong(vocabularyStatements.getCreatedAtColumn())) .name(resultSet.getString(vocabularyStatements.getNameColumn())) + .connectorId(resultSet.getString(vocabularyStatements.getConnectorIdColumn())) .category(resultSet.getString(vocabularyStatements.getCategoryColumn())) .jsonSchema(resultSet.getString(vocabularyStatements.getJsonSchemaColumn())) .build(); diff --git a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/BaseSqlDialectStatements.java b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/BaseSqlDialectStatements.java index 0725adc..da1db31 100644 --- a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/BaseSqlDialectStatements.java +++ b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/BaseSqlDialectStatements.java @@ -24,6 +24,7 @@ public String getInsertVocabularyTemplate() { .column(getVocabularyIdColumn()) .column(getCreatedAtColumn()) .column(getNameColumn()) + .column(getConnectorIdColumn()) .column(getCategoryColumn()) .jsonColumn(getJsonSchemaColumn()) .insertInto(getVocabularyTable()); diff --git a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/VocabularyStatements.java b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/VocabularyStatements.java index 0ccb9ff..64ca1c9 100644 --- a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/VocabularyStatements.java +++ b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/VocabularyStatements.java @@ -29,6 +29,10 @@ default String getNameColumn() { return "name"; } + default String getConnectorIdColumn() { + return "connector_id"; + } + default String getJsonSchemaColumn() { return "json_schema"; } diff --git a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/postgres/VocabularyMapping.java b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/postgres/VocabularyMapping.java index dfd627c..f2bf61a 100644 --- a/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/postgres/VocabularyMapping.java +++ b/extensions/vocabulary-index-sql/src/main/java/org/upm/inesdata/vocabulary/sql/index/schema/postgres/VocabularyMapping.java @@ -14,6 +14,7 @@ public VocabularyMapping(VocabularyStatements statements) { add("id", statements.getVocabularyIdColumn()); add("createdAt", statements.getCreatedAtColumn()); add("name", statements.getNameColumn()); + add("connectorId", statements.getConnectorIdColumn()); add("category", statements.getCategoryColumn()); add("json_schema", new JsonFieldTranslator(statements.getJsonSchemaColumn())); } diff --git a/extensions/vocabulary-shared-api/README.md b/extensions/vocabulary-shared-api/README.md new file mode 100644 index 0000000..e69de29 diff --git a/extensions/vocabulary-shared-api/build.gradle.kts b/extensions/vocabulary-shared-api/build.gradle.kts new file mode 100644 index 0000000..faa5f5f --- /dev/null +++ b/extensions/vocabulary-shared-api/build.gradle.kts @@ -0,0 +1,26 @@ +plugins { + `java-library` + id("com.gmv.inesdata.edc-application") +} + +dependencies { + api(project(":spi:vocabulary-spi")) + api(project(":extensions:vocabulary-api")) + api(libs.edc.spi.core) + implementation(libs.edc.spi.transform) + implementation(libs.edc.web.spi) + + implementation(libs.edc.connector.core) + implementation(libs.edc.api.core) + implementation(libs.edc.lib.util) + implementation(libs.edc.lib.transform) + implementation(libs.edc.dsp.api.configuration) + implementation(libs.edc.api.management.config) + implementation(libs.swagger.annotations.jakarta) + implementation(libs.edc.transaction.spi) + implementation(libs.edc.lib.validator) + implementation(libs.edc.validator.spi) + implementation(libs.swagger.annotations.jakarta) + runtimeOnly(libs.edc.spi.jsonld) + runtimeOnly(libs.edc.json.ld.lib) +} \ No newline at end of file diff --git a/extensions/vocabulary-shared-api/src/main/java/org/upm/inesdata/vocabulary/shared/api/VocabularySharedApiExtension.java b/extensions/vocabulary-shared-api/src/main/java/org/upm/inesdata/vocabulary/shared/api/VocabularySharedApiExtension.java new file mode 100644 index 0000000..933db04 --- /dev/null +++ b/extensions/vocabulary-shared-api/src/main/java/org/upm/inesdata/vocabulary/shared/api/VocabularySharedApiExtension.java @@ -0,0 +1,89 @@ +package org.upm.inesdata.vocabulary.shared.api; + +import jakarta.json.Json; +import org.eclipse.edc.runtime.metamodel.annotation.Extension; +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.runtime.metamodel.annotation.Provider; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; +import org.eclipse.edc.spi.system.health.HealthCheckService; +import org.eclipse.edc.spi.types.TypeManager; +import org.eclipse.edc.transaction.spi.TransactionContext; +import org.eclipse.edc.transform.spi.TypeTransformerRegistry; +import org.eclipse.edc.validator.spi.JsonObjectValidatorRegistry; +import org.eclipse.edc.web.spi.WebService; +import org.upm.inesdata.spi.vocabulary.VocabularyIndex; +import org.upm.inesdata.spi.vocabulary.VocabularyService; +import org.upm.inesdata.spi.vocabulary.VocabularySharedService; +import org.upm.inesdata.vocabulary.service.VocabularyServiceImpl; +import org.upm.inesdata.vocabulary.shared.api.controller.VocabularySharedApi; +import org.upm.inesdata.vocabulary.shared.api.controller.VocabularySharedApiController; +import org.upm.inesdata.vocabulary.shared.api.service.VocabularySharedServiceImpl; +import org.upm.inesdata.vocabulary.shared.api.transformer.JsonObjectToConnectorVocabularyTransformer; +import org.upm.inesdata.vocabulary.transformer.JsonObjectFromVocabularyTransformer; +import org.upm.inesdata.vocabulary.transformer.JsonObjectToVocabularyTransformer; + +import java.util.Map; + +import static org.eclipse.edc.spi.constants.CoreConstants.JSON_LD; + +/** + * Extension that provides a shared API for getting vocabularies + */ +@Extension(value = VocabularySharedApiExtension.NAME) +public class VocabularySharedApiExtension implements ServiceExtension { + public static final String NAME = "Vocabulary Shared API Extension"; + + @Inject + private WebService webService; + + @Inject + private VocabularyIndex vocabularyIndex; + + @Inject(required = false) + private HealthCheckService healthCheckService; + + @Inject + private TypeManager typeManager; + + @Inject + private TransactionContext transactionContext; + + @Inject + private TypeTransformerRegistry transformerRegistry; + + @Inject + private JsonObjectValidatorRegistry validator; + + @Override + public String name() { + return NAME; + } + + /** + * Provides a default vocabularyService implementation + */ + @Provider(isDefault = true) + public VocabularySharedService vocabularySharedService() { + return new VocabularySharedServiceImpl(vocabularyIndex, transactionContext); + } + + /** + * Initializes the service + */ + @Override + public void initialize(ServiceExtensionContext context) { + var monitor = context.getMonitor(); + + var sharedApiTransformerRegistry = transformerRegistry.forContext("shared-api"); + + sharedApiTransformerRegistry.register(new JsonObjectToConnectorVocabularyTransformer()); + var factory = Json.createBuilderFactory(Map.of()); + var jsonLdMapper = typeManager.getMapper(JSON_LD); + sharedApiTransformerRegistry.register(new JsonObjectFromVocabularyTransformer(factory, jsonLdMapper)); + + var vocabularySharedApiController = new VocabularySharedApiController(sharedApiTransformerRegistry, this.vocabularySharedService(), monitor); + webService.registerResource("shared", vocabularySharedApiController); + + } +} diff --git a/extensions/vocabulary-shared-api/src/main/java/org/upm/inesdata/vocabulary/shared/api/controller/VocabularySharedApi.java b/extensions/vocabulary-shared-api/src/main/java/org/upm/inesdata/vocabulary/shared/api/controller/VocabularySharedApi.java new file mode 100644 index 0000000..2c87bdb --- /dev/null +++ b/extensions/vocabulary-shared-api/src/main/java/org/upm/inesdata/vocabulary/shared/api/controller/VocabularySharedApi.java @@ -0,0 +1,60 @@ +package org.upm.inesdata.vocabulary.shared.api.controller; + +import io.swagger.v3.oas.annotations.OpenAPIDefinition; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.info.Info; +import io.swagger.v3.oas.annotations.media.ArraySchema; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.json.JsonArray; +import jakarta.json.JsonObject; +import org.upm.inesdata.spi.vocabulary.domain.Vocabulary; +import org.upm.inesdata.vocabulary.controller.VocabularyApi; + +import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.ID; +import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.TYPE; +import static org.upm.inesdata.spi.vocabulary.domain.Vocabulary.EDC_VOCABULARY_TYPE; + +/** + * Controller for getting {@link Vocabulary} objects from connectors. + */ +@OpenAPIDefinition( + info = @Info(description = "Gets the connectors vocabularies.", + title = "Vocabulary Shared API", version = "1")) +@Tag(name = "Shared Vocabulary") +public interface VocabularySharedApi { + + /** + * Get all the vocabularies of a connector. + * + * @param connectorVocabularyJson the + * @return list of vocabularies of a connector + */ + @Operation(description = "Obtains all vocabularies from a connector", + requestBody = @RequestBody(content = @Content(schema = @Schema(implementation = ConnectorVocabularyInputSchema.class))), + responses = { + @ApiResponse(responseCode = "200", description = "List of vocabularies of a connector", + content = @Content(array = @ArraySchema(schema = @Schema(implementation = Vocabulary.class)))) + } + ) + JsonArray getVocabulariesFromConnector(JsonObject connectorVocabularyJson); + + /** + * Vocabulary output + */ + @ArraySchema() + @Schema(name = "ConnectorVocabularyInput", example = ConnectorVocabularyInputSchema.CONNECTOR_VOCABULARY_INPUT_EXAMPLE) + record ConnectorVocabularyInputSchema( + String connectorId + ) { + public static final String CONNECTOR_VOCABULARY_INPUT_EXAMPLE = """ + { + "connectorId": "connector-c1" + } + """; + } + +} diff --git a/extensions/vocabulary-shared-api/src/main/java/org/upm/inesdata/vocabulary/shared/api/controller/VocabularySharedApiController.java b/extensions/vocabulary-shared-api/src/main/java/org/upm/inesdata/vocabulary/shared/api/controller/VocabularySharedApiController.java new file mode 100644 index 0000000..a9b3591 --- /dev/null +++ b/extensions/vocabulary-shared-api/src/main/java/org/upm/inesdata/vocabulary/shared/api/controller/VocabularySharedApiController.java @@ -0,0 +1,55 @@ +package org.upm.inesdata.vocabulary.shared.api.controller; + +import jakarta.json.JsonArray; +import jakarta.json.JsonObject; +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import org.eclipse.edc.spi.monitor.Monitor; +import org.eclipse.edc.spi.result.Result; +import org.eclipse.edc.transform.spi.TypeTransformerRegistry; +import org.eclipse.edc.web.spi.exception.InvalidRequestException; +import org.upm.inesdata.spi.vocabulary.VocabularySharedService; +import org.upm.inesdata.spi.vocabulary.domain.ConnectorVocabulary; +import org.upm.inesdata.spi.vocabulary.domain.Vocabulary; + +import static jakarta.json.stream.JsonCollectors.toJsonArray; + +/** + * Implementation of the controller for getting {@link Vocabulary} list of a connector. + */ +@Consumes({MediaType.APPLICATION_JSON}) +@Produces({MediaType.APPLICATION_JSON}) +@Path("/connector-vocabularies") +public class VocabularySharedApiController implements VocabularySharedApi { + + private final TypeTransformerRegistry transformerRegistry; + private final VocabularySharedService vocabularySharedService; + private final Monitor monitor; + + /** + * Constructor + */ + public VocabularySharedApiController(TypeTransformerRegistry transformerRegistry, VocabularySharedService vocabularySharedService, Monitor monitor) { + this.transformerRegistry = transformerRegistry; + this.vocabularySharedService = vocabularySharedService; + this.monitor = monitor; + } + + @POST + @Path("/request") + @Override + public JsonArray getVocabulariesFromConnector(JsonObject connectorVocabularyJson) { + var connectorVocabulary = transformerRegistry.transform(connectorVocabularyJson, ConnectorVocabulary.class) + .orElseThrow(InvalidRequestException::new); + + return vocabularySharedService.searchVocabulariesByConnector(connectorVocabulary).getContent().stream() + .map(it -> transformerRegistry.transform(it, JsonObject.class)) + .peek(r -> r.onFailure(f -> monitor.warning(f.getFailureDetail()))) + .filter(Result::succeeded) + .map(Result::getContent) + .collect(toJsonArray()); + } +} diff --git a/extensions/vocabulary-shared-api/src/main/java/org/upm/inesdata/vocabulary/shared/api/service/VocabularySharedServiceImpl.java b/extensions/vocabulary-shared-api/src/main/java/org/upm/inesdata/vocabulary/shared/api/service/VocabularySharedServiceImpl.java new file mode 100644 index 0000000..c33b959 --- /dev/null +++ b/extensions/vocabulary-shared-api/src/main/java/org/upm/inesdata/vocabulary/shared/api/service/VocabularySharedServiceImpl.java @@ -0,0 +1,33 @@ +package org.upm.inesdata.vocabulary.shared.api.service; + +import org.eclipse.edc.spi.result.ServiceResult; +import org.eclipse.edc.transaction.spi.TransactionContext; +import org.upm.inesdata.spi.vocabulary.VocabularyIndex; +import org.upm.inesdata.spi.vocabulary.VocabularySharedService; +import org.upm.inesdata.spi.vocabulary.domain.ConnectorVocabulary; +import org.upm.inesdata.spi.vocabulary.domain.Vocabulary; +import org.upm.inesdata.vocabulary.service.VocabularyServiceImpl; + +import java.util.List; + +/** + * Implementation of the {@link VocabularySharedService} interface + */ +public class VocabularySharedServiceImpl extends VocabularyServiceImpl implements VocabularySharedService { + + /** + * Constructor + */ + public VocabularySharedServiceImpl(VocabularyIndex index, TransactionContext transactionContext) { + super(index, transactionContext); + } + + @Override + public ServiceResult> searchVocabulariesByConnector(ConnectorVocabulary connectorVocabulary) { + return transactionContext.execute(() -> { + try (var stream = index.searchVocabulariesByConnector(connectorVocabulary.getConnectorId())) { + return ServiceResult.success(stream.toList()); + } + }); + } +} diff --git a/extensions/vocabulary-shared-api/src/main/java/org/upm/inesdata/vocabulary/shared/api/transformer/JsonObjectToConnectorVocabularyTransformer.java b/extensions/vocabulary-shared-api/src/main/java/org/upm/inesdata/vocabulary/shared/api/transformer/JsonObjectToConnectorVocabularyTransformer.java new file mode 100644 index 0000000..9471195 --- /dev/null +++ b/extensions/vocabulary-shared-api/src/main/java/org/upm/inesdata/vocabulary/shared/api/transformer/JsonObjectToConnectorVocabularyTransformer.java @@ -0,0 +1,35 @@ +package org.upm.inesdata.vocabulary.shared.api.transformer; + +import jakarta.json.JsonObject; +import org.eclipse.edc.jsonld.spi.transformer.AbstractJsonLdTransformer; +import org.eclipse.edc.transform.spi.TransformerContext; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.upm.inesdata.spi.vocabulary.domain.ConnectorVocabulary; + +import static org.upm.inesdata.spi.vocabulary.domain.ConnectorVocabulary.PROPERTY_CONNECTOR_ID; + +/** + * Converts from an {@link ConnectorVocabulary} as a {@link JsonObject} in JSON-LD expanded form to an {@link ConnectorVocabulary}. + */ +public class JsonObjectToConnectorVocabularyTransformer extends AbstractJsonLdTransformer { + /** + * Constructor + */ + public JsonObjectToConnectorVocabularyTransformer() { + super(JsonObject.class, ConnectorVocabulary.class); + } + + @Override + public @Nullable ConnectorVocabulary transform(@NotNull JsonObject jsonObject, @NotNull TransformerContext context) { + var builder = ConnectorVocabulary.Builder.newInstance(); + + visitProperties(jsonObject, key -> value -> { + if (PROPERTY_CONNECTOR_ID.equals(key)) { + builder.connectorId(transformString(value, context)); + } + }); + + return builderResult(builder::build, context); + } +} diff --git a/extensions/vocabulary-shared-api/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/vocabulary-shared-api/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 0000000..02e4542 --- /dev/null +++ b/extensions/vocabulary-shared-api/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1 @@ +org.upm.inesdata.vocabulary.shared.api.VocabularySharedApiExtension \ No newline at end of file diff --git a/extensions/vocabulary-shared-retrieval/README.md b/extensions/vocabulary-shared-retrieval/README.md new file mode 100644 index 0000000..e69de29 diff --git a/extensions/vocabulary-shared-retrieval/build.gradle.kts b/extensions/vocabulary-shared-retrieval/build.gradle.kts new file mode 100644 index 0000000..a7f8129 --- /dev/null +++ b/extensions/vocabulary-shared-retrieval/build.gradle.kts @@ -0,0 +1,12 @@ +plugins { + `java-library` + id("com.gmv.inesdata.edc-application") +} + +dependencies { + api(libs.edc.web.spi) + api(libs.edc.auth.spi) + api(libs.edc.iam.oauth2.service) + api(libs.edc.spi.jsonld) + api(libs.edc.jersey.providers.lib) +} \ No newline at end of file diff --git a/extensions/vocabulary-shared-retrieval/src/main/java/org/upm/inesdata/vocabulary/shared/retrieval/VocabularySharedRetrievalExtension.java b/extensions/vocabulary-shared-retrieval/src/main/java/org/upm/inesdata/vocabulary/shared/retrieval/VocabularySharedRetrievalExtension.java new file mode 100644 index 0000000..44fbf9b --- /dev/null +++ b/extensions/vocabulary-shared-retrieval/src/main/java/org/upm/inesdata/vocabulary/shared/retrieval/VocabularySharedRetrievalExtension.java @@ -0,0 +1,86 @@ +package org.upm.inesdata.vocabulary.shared.retrieval; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.eclipse.edc.api.auth.spi.AuthenticationRequestFilter; +import org.eclipse.edc.api.auth.spi.registry.ApiAuthenticationRegistry; +import org.eclipse.edc.jsonld.spi.JsonLd; +import org.eclipse.edc.runtime.metamodel.annotation.Extension; +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.runtime.metamodel.annotation.Provides; +import org.eclipse.edc.runtime.metamodel.annotation.Setting; +import org.eclipse.edc.runtime.metamodel.annotation.SettingContext; +import org.eclipse.edc.spi.EdcException; +import org.eclipse.edc.spi.iam.IdentityService; +import org.eclipse.edc.spi.iam.TokenParameters; +import org.eclipse.edc.spi.iam.TokenRepresentation; +import org.eclipse.edc.spi.result.Result; +import org.eclipse.edc.spi.system.ExecutorInstrumentation; +import org.eclipse.edc.spi.system.Hostname; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; +import org.eclipse.edc.spi.types.TypeManager; +import org.eclipse.edc.web.jersey.providers.jsonld.JerseyJsonLdInterceptor; +import org.eclipse.edc.web.jersey.providers.jsonld.ObjectMapperProvider; +import org.eclipse.edc.web.spi.WebServer; +import org.eclipse.edc.web.spi.WebService; +import org.eclipse.edc.web.spi.configuration.WebServiceConfiguration; +import org.eclipse.edc.web.spi.configuration.WebServiceConfigurer; +import org.eclipse.edc.web.spi.configuration.WebServiceSettings; + +import java.net.URI; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import static java.lang.String.format; +import static org.eclipse.edc.policy.model.OdrlNamespace.ODRL_PREFIX; +import static org.eclipse.edc.policy.model.OdrlNamespace.ODRL_SCHEMA; +import static org.eclipse.edc.spi.constants.CoreConstants.JSON_LD; + +/** + * This extension launch a task to retrieve the vocabularies from other connectors . + */ +@Extension(value = VocabularySharedRetrievalExtension.NAME) +public class VocabularySharedRetrievalExtension implements ServiceExtension { + public static final String NAME = "Vocabulary Shared Retrieval Extension"; + + @Setting("The time to elapse between two crawl runs") + public static final String EXECUTION_PLAN_PERIOD_SECONDS = "edc.participants.cache.execution.period.seconds"; + + private final int DEFAULT_EXECUTION_PERIOD_SECONDS = 60; + + @Inject + private IdentityService identityService; + + private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); + + @Override + public void initialize(ServiceExtensionContext context) { + var periodSeconds = context.getSetting(EXECUTION_PLAN_PERIOD_SECONDS, DEFAULT_EXECUTION_PERIOD_SECONDS); + var monitor = context.getMonitor(); + //var participantRegistrationService = new ParticipantRegistrationService(monitor, new ObjectMapper()); + + // Schedule periodic updates + scheduler.scheduleAtFixedRate(() -> { + try { + // updateTargetNodeDirectory(context, participantRegistrationService); + } catch (Exception e) { + monitor.severe("Error updating TargetNodeDirectory", e); + } + }, periodSeconds, periodSeconds, TimeUnit.SECONDS); + } + + /*private void retrieveVocabularies(ServiceExtensionContext context, ParticipantRegistrationService participantRegistrationService) { + + Result tokenRepresentationResult = identityService.obtainClientCredentials( + TokenParameters.Builder.newInstance().build()); + + for (var target : participantRegistrationService.getTargetNodes(context.getConfig(), tokenRepresentationResult)) { + // skipping null target nodes + if (target != null){ + newDir.insert(target); + } + } + + }*/ +} diff --git a/extensions/vocabulary-shared-retrieval/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/vocabulary-shared-retrieval/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 0000000..e2ffb24 --- /dev/null +++ b/extensions/vocabulary-shared-retrieval/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1 @@ +org.upm.inesdata.vocabulary.shared.retrieval.VocabularySharedRetrievalExtension \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index deb5e1f..549d58b 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -47,12 +47,13 @@ edc-edr-cache-api = { module = "org.eclipse.edc:edr-cache-api", version.ref = "e edc-edr-store-core = { module = "org.eclipse.edc:edr-store-core", version.ref = "edc" } edc-edr-store-receiver = { module = "org.eclipse.edc:edr-store-receiver", version.ref = "edc" } edc-iam-oauth2-core = { module = "org.eclipse.edc:oauth2-core", version.ref = "edc" } -edc-iam-oauth2-service = { module = "org.eclipse.edc:oauth2-service", version.ref = "edc" } +edc-iam-oauth2-service = { module = "org.eclipse.edc:oauth2-service", version.ref = "edc"} +edc-jersey-providers-lib = { module = "org.eclipse.edc:jersey-providers-lib", version.ref = "edc" } edc-json-ld-lib = { module = "org.eclipse.edc:json-ld-lib", version.ref = "edc" } edc-lib-transform = { module = "org.eclipse.edc:transform-lib", version.ref = "edc" } edc-lib-util = { module = "org.eclipse.edc:util-lib", version.ref = "edc" } edc-lib-validator = { module = "org.eclipse.edc:validator-lib", version.ref = "edc" } -edc-management-api = { module = "org.eclipse.edc:management-api", version.ref = "edc" } +edc-management-api = { module = "org.eclipse.edc:management-api", version.ref = "edc"} edc-policy-engine-spi = { module = "org.eclipse.edc:policy-engine-spi", version.ref = "edc" } edc-spi-asset = { module = "org.eclipse.edc:asset-spi", version.ref = "edc" } edc-spi-core = { module = "org.eclipse.edc:core-spi", version.ref = "edc" } @@ -64,6 +65,7 @@ edc-transaction-spi = { module = "org.eclipse.edc:transaction-spi", version.ref edc-transfer-data-plane-signaling = { module = "org.eclipse.edc:transfer-data-plane-signaling", version.ref = "edc" } edc-transfer-pull-http-receiver = { module = "org.eclipse.edc:transfer-pull-http-dynamic-receiver", version.ref = "edc" } edc-transfer-spi = { module = "org.eclipse.edc:transfer-spi", version.ref = "edc" } +edc-token-core = { module = "org.eclipse.edc:token-core", version.ref = "edc" } edc-validator-data-address-http-data = { module = "org.eclipse.edc:validator-data-address-http-data", version.ref = "edc" } edc-validator-spi = { module = "org.eclipse.edc:validator-spi", version.ref = "edc" } edc-vault-hashicorp = { module = "org.eclipse.edc:vault-hashicorp", version.ref = "edc" } diff --git a/launchers/connector/build.gradle.kts b/launchers/connector/build.gradle.kts index 232b27c..af543ba 100644 --- a/launchers/connector/build.gradle.kts +++ b/launchers/connector/build.gradle.kts @@ -44,6 +44,7 @@ dependencies { // Vocabularios implementation(project(":extensions:vocabulary-api")) + implementation(project(":extensions:vocabulary-shared-api")) // Policies implementation(project(":extensions:policy-always-true")) @@ -97,6 +98,9 @@ dependencies { implementation(project(":extensions:audit-configuration")) implementation(project(":extensions:audit-event-configuration")) + // Shared API + implementation(project(":extensions:shared-api-configuration")) + runtimeOnly(libs.edc.transaction.local) runtimeOnly(libs.postgres) } diff --git a/settings.gradle.kts b/settings.gradle.kts index c140da8..308da7c 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -9,6 +9,7 @@ rootProject.name = "inesdata-connector" include(":spi:vocabulary-spi") include(":spi:federated-catalog-cache-spi") include(":spi:count-elements-spi") +include(":spi:control-plane-vocabulary-spi") // Extensions include(":extensions:auth-oauth2-jwt") @@ -26,6 +27,14 @@ include(":extensions:extended-data-plane-public-api") include(":extensions:audit-configuration") include(":extensions:audit-event-configuration") include(":extensions:inesdata-search-extension") +include(":extensions:vocabulary-retrieval-schedule") +include(":extensions:dsp-vocabulary-http-dispatcher") +include(":extensions:dsp-vocabulary-http-api") +include(":extensions:control-plane-agreggate-vocabulary-services") +include(":extensions:dsp-vocabulary-http") +include(":extensions:shared-api-configuration") +include(":extensions:vocabulary-shared-api") +include(":extensions:vocabulary-shared-retrieval") // Connector include(":launchers:connector") diff --git a/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/VocabularyIndex.java b/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/VocabularyIndex.java index 8a89aeb..fbe5764 100644 --- a/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/VocabularyIndex.java +++ b/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/VocabularyIndex.java @@ -58,4 +58,12 @@ public interface VocabularyIndex { */ StoreResult updateVocabulary(Vocabulary vocabulary); + /** + * Finds all stored vocabularies from a connector + * + * @param connectorId the connector ID + * @return A potentially empty collection of {@link Vocabulary}, never null. + */ + Stream searchVocabulariesByConnector(String connectorId); + } diff --git a/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/VocabularySharedService.java b/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/VocabularySharedService.java new file mode 100644 index 0000000..f77d248 --- /dev/null +++ b/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/VocabularySharedService.java @@ -0,0 +1,18 @@ +package org.upm.inesdata.spi.vocabulary; + +import org.eclipse.edc.spi.result.ServiceResult; +import org.upm.inesdata.spi.vocabulary.domain.ConnectorVocabulary; +import org.upm.inesdata.spi.vocabulary.domain.Vocabulary; + +import java.util.List; + +public interface VocabularySharedService extends VocabularyService { + + /** + * Get Vocabularies from a connector + * + * @param connectorVocabulary connector id + * @return the collection of vocabularies stored from a connector + */ + ServiceResult> searchVocabulariesByConnector(ConnectorVocabulary connectorVocabulary); +} diff --git a/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/domain/ConnectorVocabulary.java b/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/domain/ConnectorVocabulary.java new file mode 100644 index 0000000..c173299 --- /dev/null +++ b/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/domain/ConnectorVocabulary.java @@ -0,0 +1,55 @@ +package org.upm.inesdata.spi.vocabulary.domain; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; + +import static org.eclipse.edc.spi.constants.CoreConstants.EDC_NAMESPACE; + +@JsonDeserialize(builder = ConnectorVocabulary.Builder.class) +public class ConnectorVocabulary { + + public static final String PROPERTY_CONNECTOR_ID = EDC_NAMESPACE + "connectorId"; + + private String connectorId; + + public ConnectorVocabulary() { + } + + public String getConnectorId() { + return connectorId; + } + + public ConnectorVocabulary.Builder toBuilder() { + return ConnectorVocabulary.Builder.newInstance() + .connectorId(connectorId); + } + + + @JsonPOJOBuilder(withPrefix = "") + public static class Builder { + protected final ConnectorVocabulary connectorVocabulary; + + protected Builder(ConnectorVocabulary connectorVocabulary) { + this.connectorVocabulary = connectorVocabulary; + } + + @JsonCreator + public static ConnectorVocabulary.Builder newInstance() { + return new ConnectorVocabulary.Builder(new ConnectorVocabulary()); + } + + public ConnectorVocabulary.Builder connectorId(String connectorId) { + connectorVocabulary.connectorId = connectorId; + return self(); + } + + public ConnectorVocabulary.Builder self() { + return this; + } + + public ConnectorVocabulary build() { + return connectorVocabulary; + } + } +} diff --git a/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/domain/Vocabulary.java b/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/domain/Vocabulary.java index 9fece22..f3b6be4 100644 --- a/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/domain/Vocabulary.java +++ b/spi/vocabulary-spi/src/main/java/org/upm/inesdata/spi/vocabulary/domain/Vocabulary.java @@ -21,12 +21,14 @@ public class Vocabulary extends Entity { public static final String PROPERTY_NAME = EDC_NAMESPACE + "name"; public static final String PROPERTY_JSON_SCHEMA = EDC_NAMESPACE + "jsonSchema"; public static final String PROPERTY_CATEGORY = EDC_NAMESPACE + "category"; + public static final String PROPERTY_CONNECTOR_ID = EDC_NAMESPACE + "connectorId"; public static final String EDC_VOCABULARY_TYPE = EDC_NAMESPACE + "Vocabulary"; private String name; private String jsonSchema; private String category; + private String connectorId; private Vocabulary() { } @@ -42,12 +44,16 @@ public String getJsonSchema() { public String getCategory() { return category; } + public String getConnectorId() { + return connectorId; + } public Builder toBuilder() { return Vocabulary.Builder.newInstance() .id(id) .name(name) + .connectorId(connectorId) .jsonSchema(jsonSchema) .category(category) .createdAt(createdAt); @@ -76,6 +82,11 @@ public Builder name(String name) { return self(); } + public Builder connectorId(String connectorId) { + entity.connectorId = connectorId; + return self(); + } + public Builder jsonSchema(String jsonSchema) { Objects.requireNonNull(jsonSchema); entity.jsonSchema = jsonSchema; From d5fea2f0a39abfabaa53f20a23cc18544be42d8d Mon Sep 17 00:00:00 2001 From: ppel Date: Tue, 23 Jul 2024 11:19:59 +0200 Subject: [PATCH 128/174] #436844 - Created scheduled task for retrieving vocabularies --- docker/Dockerfile | 2 +- .../ParticipantRegistrationService.java | 53 +++++++++--- .../catalog/SharedUrlParticipant.java | 31 +++++++ .../storage/InMemoryVocabularyIndex.java | 26 +++++- extensions/vocabulary-index-sql/README.md | 4 +- extensions/vocabulary-index-sql/docs/er.puml | 3 +- .../vocabulary-index-sql/docs/schema.sql | 2 +- .../sql/index/SqlVocabularyIndex.java | 16 +++- .../schema/BaseSqlDialectStatements.java | 6 ++ .../index/schema/VocabularyStatements.java | 5 ++ extensions/vocabulary-shared-api/README.md | 3 + .../api/controller/VocabularySharedApi.java | 17 +++- .../VocabularySharedApiController.java | 16 +++- .../service/VocabularySharedServiceImpl.java | 8 ++ .../vocabulary-shared-retrieval/README.md | 3 + .../build.gradle.kts | 9 +- .../SharedUrlRegistrationService.java | 51 ++++++++++++ .../VocabularySharedRetrievalExtension.java | 57 ++++--------- .../VocabularySharedRetrievalService.java | 83 +++++++++++++++++++ .../spi/vocabulary/VocabularyIndex.java | 9 ++ .../vocabulary/VocabularySharedService.java | 7 ++ 21 files changed, 341 insertions(+), 70 deletions(-) create mode 100644 extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/SharedUrlParticipant.java create mode 100644 extensions/vocabulary-shared-retrieval/src/main/java/org/upm/inesdata/vocabulary/shared/retrieval/SharedUrlRegistrationService.java create mode 100644 extensions/vocabulary-shared-retrieval/src/main/java/org/upm/inesdata/vocabulary/shared/retrieval/VocabularySharedRetrievalService.java diff --git a/docker/Dockerfile b/docker/Dockerfile index 35eb42b..5b574b9 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -35,4 +35,4 @@ COPY resources/* ${CONNECTOR_RESOURCES}/ # Use "exec" for Kubernetes graceful termination (SIGINT) to reach JVM. ENTRYPOINT [ "sh", "-c", \ - "exec java $JVM_ARGS -jar connector.jar"] + "exec java $JVM_ARGS -agentlib:jdwp=transport=dt_socket,address=0.0.0.0:18787,server=y,suspend=y -jar connector.jar"] diff --git a/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/ParticipantRegistrationService.java b/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/ParticipantRegistrationService.java index cb9e026..659c324 100644 --- a/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/ParticipantRegistrationService.java +++ b/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/ParticipantRegistrationService.java @@ -47,9 +47,6 @@ public ParticipantRegistrationService(Monitor monitor, ObjectMapper objectMapper this.objectMapper = objectMapper; } - - - /** * Makes an HTTP GET request to the specified URL and returns the response as a string. * @@ -72,29 +69,61 @@ public String makeHttpGetRequest(String url, Result tokenRe * @return list of TargetNodes from configuration */ public List getTargetNodes(Config baseConfig, Result tokenRepresentationResult) { + var response = getNodes(baseConfig, tokenRepresentationResult); + if(response != null){ + return processResponseToTargetNodes(response); + }else{ + return new ArrayList<>(); + } + } + + /** + * Retrieve SharedUrlParticipant from configuration + * + * @param baseConfig EDC Configuration + * @param tokenRepresentationResult token + * @return list of SharedUrlParticipant from configuration + */ + public List getSharedUrlParticipantNodes(Config baseConfig, Result tokenRepresentationResult) { + var response = getNodes(baseConfig, tokenRepresentationResult); + if(response != null){ + return processResponseToSharedUrlParticipant(response); + }else{ + return new ArrayList<>(); + } + } + + private String getNodes(Config baseConfig, Result tokenRepresentationResult) { var participantsConfig = baseConfig.getConfig(EDC_CATALOG_REGISTRATION_SERVICE_HOST); if (participantsConfig.getEntries().isEmpty()) { monitor.severe("Error processing url registration service."); - return new ArrayList<>(); + return null; } else { var url = participantsConfig.getEntries().get(EDC_CATALOG_REGISTRATION_SERVICE_HOST) + RESOURCE_URL; try { - String response = makeHttpGetRequest(url, tokenRepresentationResult); - if(response==null){ - return new ArrayList<>(); - } - // Process the response and convert it to TargetNodes - // Assuming a method processResponseToTargetNodes(response) - return processResponseToTargetNodes(response); + return makeHttpGetRequest(url, tokenRepresentationResult); } catch (Exception e) { monitor.severe("Exception occurred while making HTTP GET request: " + e.getMessage()); - return new ArrayList<>(); + return null; } } } + + private List processResponseToSharedUrlParticipant(String response) { + List sharedUrlParticipants = new ArrayList<>(); + + try { + List nodes = objectMapper.readValue(response, new TypeReference>() {}); + } catch (Exception e) { + monitor.severe("Failed to deserialize the registration service response"); + } + + return sharedUrlParticipants; + } + private List processResponseToTargetNodes(String response) { List targetNodes = new ArrayList<>(); diff --git a/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/SharedUrlParticipant.java b/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/SharedUrlParticipant.java new file mode 100644 index 0000000..255adb9 --- /dev/null +++ b/extensions/participants-from-registration-service/src/main/java/org/upm/inesdata/catalog/SharedUrlParticipant.java @@ -0,0 +1,31 @@ +package org.upm.inesdata.catalog; + +public class SharedUrlParticipant { + private String participantId; + private String url; + private String sharedUrl; + + public String getParticipantId() { + return participantId; + } + + public void setParticipantId(String participantId) { + this.participantId = participantId; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getSharedUrl() { + return sharedUrl; + } + + public void setSharedUrl(String sharedUrl) { + this.sharedUrl = sharedUrl; + } +} diff --git a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/storage/InMemoryVocabularyIndex.java b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/storage/InMemoryVocabularyIndex.java index 4ecf051..c063863 100644 --- a/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/storage/InMemoryVocabularyIndex.java +++ b/extensions/vocabulary-api/src/main/java/org/upm/inesdata/vocabulary/storage/InMemoryVocabularyIndex.java @@ -99,13 +99,31 @@ public StoreResult updateVocabulary(Vocabulary vocabulary) { @Override public Stream searchVocabulariesByConnector(String connectorId) { - return null; + lock.readLock().lock(); + try { + return cache.values().stream() + .filter(vocabulary -> vocabulary.getConnectorId().equals(connectorId)); + } finally { + lock.readLock().unlock(); + } + } + + @Override + public StoreResult deleteByConnectorId(String connectorId) { + lock.writeLock().lock(); + try { + delete(connectorId); + } finally { + lock.writeLock().unlock(); + } + + return StoreResult.success(); } /** - * Remove a vocabulary from cache based on its id + * Remove a vocabulary from cache based on a key */ - private Vocabulary delete(String vocabularyId) { - return cache.remove(vocabularyId); + private Vocabulary delete(String key) { + return cache.remove(key); } } diff --git a/extensions/vocabulary-index-sql/README.md b/extensions/vocabulary-index-sql/README.md index 2e8e55c..0aa1547 100644 --- a/extensions/vocabulary-index-sql/README.md +++ b/extensions/vocabulary-index-sql/README.md @@ -8,13 +8,15 @@ Please apply this [schema](docs/schema.sql) to your SQL database. ## Entity Diagram -![ER Diagram](//www.plantuml.com/plantuml/png/SoWkIImgAStDuKhDAyaigLH8JKcEByjFJamgpKaigbIevb9Gq5B8JB5IA2ufoinBLx2n2V2simEBvYNcfiB4mG9PnVbvmSaPgRc9ACB9HQc99QafZYLM2ZdvO35TNQvQBeVKl1IWnG00) +![ER Diagram](//https://www.plantuml.com/plantuml/png/RSun2iCm38NXtQVGNCW5GWZ9MBeKUe2YsY9riIMGbO0flNj3GeT0r-_zmnkAeTgSaoEsQ1Ke-FiY7XzpGgtmTW0dYA65OXfvWgwxNlf-Ko_Cv4tq_7TcpFJplKUZIRGUy5M4R_v96O-jqbg7qLf8ibdJk8yRYCDwzWi0)