From 08bc3c95b2861ece647ad281f14695e7c25f1a45 Mon Sep 17 00:00:00 2001 From: gintarasm Date: Fri, 4 Aug 2023 14:33:18 +0300 Subject: [PATCH] first commit --- .github/workflows/gradle.yml | 35 +++ .gitignore | 4 + LICENSE | 21 ++ README.md | 137 +++++++++ build.gradle | 80 +++++ gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 59536 bytes gradle/wrapper/gradle-wrapper.properties | 5 + gradlew | 234 ++++++++++++++ gradlew.bat | 89 ++++++ jitpack.yml | 1 + settings.gradle | 1 + .../bigquery/BigQueryStreamProcessor.java | 32 ++ .../flink/bigquery/BigQueryStreamSink.java | 60 ++++ .../client/BigQueryJsonClientProvider.java | 78 +++++ .../client/BigQueryProtoClientProvider.java | 82 +++++ .../flink/bigquery/client/ClientProvider.java | 15 + .../metric/BigQueryStreamMetrics.java | 45 +++ .../flink/bigquery/model/BigQueryRecord.java | 10 + .../com/vinted/flink/bigquery/model/Rows.java | 75 +++++ .../bigquery/model/config/Credentials.java | 19 ++ .../model/config/DefaultCredentials.java | 11 + .../model/config/FileCredentialsProvider.java | 30 ++ .../model/config/JsonCredentialsProvider.java | 29 ++ .../bigquery/model/config/WriterSettings.java | 147 +++++++++ .../flink/bigquery/process/BatchTrigger.java | 128 ++++++++ .../flink/bigquery/process/RowBatcher.java | 36 +++ .../flink/bigquery/process/StreamState.java | 59 ++++ .../bigquery/process/StreamStateHandler.java | 156 ++++++++++ .../flink/bigquery/schema/SchemaTools.java | 185 ++++++++++++ .../bigquery/schema/SchemaTransformer.java | 82 +++++ .../serializer/JsonRowValueSerializer.java | 4 + .../serializer/NoOpRowSerializer.java | 8 + .../serializer/ProtoValueSerializer.java | 4 + .../serializer/RowValueSerializer.java | 7 + .../flink/bigquery/sink/AppendException.java | 29 ++ .../bigquery/sink/BigQuerySinkWriter.java | 116 +++++++ .../flink/bigquery/sink/ExecutorProvider.java | 9 + .../sink/buffered/BigQueryBufferedSink.java | 50 +++ .../buffered/BigQueryBufferedSinkWriter.java | 221 ++++++++++++++ .../sink/buffered/BigQueryCommittable.java | 30 ++ .../BigQueryCommittableSerializer.java | 31 ++ .../BigQueryJsonBufferedSinkWriter.java | 33 ++ .../BigQueryProtoBufferedSinkWriter.java | 44 +++ .../sink/buffered/BigQuerySinkCommitter.java | 62 ++++ .../BigQueryDefaultJsonSinkWriter.java | 44 +++ .../BigQueryDefaultProtoSinkWriter.java | 49 +++ .../defaultStream/BigQueryDefaultSink.java | 41 +++ .../BigQueryDefaultSinkWriter.java | 140 +++++++++ .../flink/bigquery/BatchTriggerTest.java | 170 +++++++++++ .../bigquery/BigQueryBufferedSinkTest.java | 95 ++++++ .../bigquery/BigQueryDefaultSinkTest.java | 224 ++++++++++++++ .../vinted/flink/bigquery/RowBatcherTest.java | 82 +++++ .../flink/bigquery/SchemaToolsTest.java | 120 ++++++++ .../flink/bigquery/StreamHandlerTest.java | 285 ++++++++++++++++++ .../vinted/flink/bigquery/util/FlinkTest.java | 183 +++++++++++ .../vinted/flink/bigquery/util/MockClock.java | 30 ++ .../bigquery/util/MockJsonClientProvider.java | 158 ++++++++++ .../util/MockProtoClientProvider.java | 161 ++++++++++ .../util/ProcessFunctionWithError.java | 30 ++ .../vinted/flink/bigquery/util/TestSink.java | 24 ++ src/test/resources/log4j2-test.properties | 15 + 61 files changed, 4385 insertions(+) create mode 100644 .github/workflows/gradle.yml create mode 100755 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100755 build.gradle create mode 100755 gradle/wrapper/gradle-wrapper.jar create mode 100755 gradle/wrapper/gradle-wrapper.properties create mode 100755 gradlew create mode 100755 gradlew.bat create mode 100644 jitpack.yml create mode 100755 settings.gradle create mode 100644 src/main/java/com/vinted/flink/bigquery/BigQueryStreamProcessor.java create mode 100644 src/main/java/com/vinted/flink/bigquery/BigQueryStreamSink.java create mode 100644 src/main/java/com/vinted/flink/bigquery/client/BigQueryJsonClientProvider.java create mode 100644 src/main/java/com/vinted/flink/bigquery/client/BigQueryProtoClientProvider.java create mode 100644 src/main/java/com/vinted/flink/bigquery/client/ClientProvider.java create mode 100644 src/main/java/com/vinted/flink/bigquery/metric/BigQueryStreamMetrics.java create mode 100644 src/main/java/com/vinted/flink/bigquery/model/BigQueryRecord.java create mode 100644 src/main/java/com/vinted/flink/bigquery/model/Rows.java create mode 100644 src/main/java/com/vinted/flink/bigquery/model/config/Credentials.java create mode 100644 src/main/java/com/vinted/flink/bigquery/model/config/DefaultCredentials.java create mode 100644 src/main/java/com/vinted/flink/bigquery/model/config/FileCredentialsProvider.java create mode 100644 src/main/java/com/vinted/flink/bigquery/model/config/JsonCredentialsProvider.java create mode 100644 src/main/java/com/vinted/flink/bigquery/model/config/WriterSettings.java create mode 100644 src/main/java/com/vinted/flink/bigquery/process/BatchTrigger.java create mode 100644 src/main/java/com/vinted/flink/bigquery/process/RowBatcher.java create mode 100644 src/main/java/com/vinted/flink/bigquery/process/StreamState.java create mode 100644 src/main/java/com/vinted/flink/bigquery/process/StreamStateHandler.java create mode 100644 src/main/java/com/vinted/flink/bigquery/schema/SchemaTools.java create mode 100644 src/main/java/com/vinted/flink/bigquery/schema/SchemaTransformer.java create mode 100644 src/main/java/com/vinted/flink/bigquery/serializer/JsonRowValueSerializer.java create mode 100644 src/main/java/com/vinted/flink/bigquery/serializer/NoOpRowSerializer.java create mode 100644 src/main/java/com/vinted/flink/bigquery/serializer/ProtoValueSerializer.java create mode 100644 src/main/java/com/vinted/flink/bigquery/serializer/RowValueSerializer.java create mode 100644 src/main/java/com/vinted/flink/bigquery/sink/AppendException.java create mode 100644 src/main/java/com/vinted/flink/bigquery/sink/BigQuerySinkWriter.java create mode 100644 src/main/java/com/vinted/flink/bigquery/sink/ExecutorProvider.java create mode 100644 src/main/java/com/vinted/flink/bigquery/sink/buffered/BigQueryBufferedSink.java create mode 100644 src/main/java/com/vinted/flink/bigquery/sink/buffered/BigQueryBufferedSinkWriter.java create mode 100644 src/main/java/com/vinted/flink/bigquery/sink/buffered/BigQueryCommittable.java create mode 100644 src/main/java/com/vinted/flink/bigquery/sink/buffered/BigQueryCommittableSerializer.java create mode 100644 src/main/java/com/vinted/flink/bigquery/sink/buffered/BigQueryJsonBufferedSinkWriter.java create mode 100644 src/main/java/com/vinted/flink/bigquery/sink/buffered/BigQueryProtoBufferedSinkWriter.java create mode 100644 src/main/java/com/vinted/flink/bigquery/sink/buffered/BigQuerySinkCommitter.java create mode 100644 src/main/java/com/vinted/flink/bigquery/sink/defaultStream/BigQueryDefaultJsonSinkWriter.java create mode 100644 src/main/java/com/vinted/flink/bigquery/sink/defaultStream/BigQueryDefaultProtoSinkWriter.java create mode 100644 src/main/java/com/vinted/flink/bigquery/sink/defaultStream/BigQueryDefaultSink.java create mode 100644 src/main/java/com/vinted/flink/bigquery/sink/defaultStream/BigQueryDefaultSinkWriter.java create mode 100644 src/test/java/com/vinted/flink/bigquery/BatchTriggerTest.java create mode 100644 src/test/java/com/vinted/flink/bigquery/BigQueryBufferedSinkTest.java create mode 100644 src/test/java/com/vinted/flink/bigquery/BigQueryDefaultSinkTest.java create mode 100644 src/test/java/com/vinted/flink/bigquery/RowBatcherTest.java create mode 100644 src/test/java/com/vinted/flink/bigquery/SchemaToolsTest.java create mode 100644 src/test/java/com/vinted/flink/bigquery/StreamHandlerTest.java create mode 100644 src/test/java/com/vinted/flink/bigquery/util/FlinkTest.java create mode 100644 src/test/java/com/vinted/flink/bigquery/util/MockClock.java create mode 100644 src/test/java/com/vinted/flink/bigquery/util/MockJsonClientProvider.java create mode 100644 src/test/java/com/vinted/flink/bigquery/util/MockProtoClientProvider.java create mode 100644 src/test/java/com/vinted/flink/bigquery/util/ProcessFunctionWithError.java create mode 100644 src/test/java/com/vinted/flink/bigquery/util/TestSink.java create mode 100644 src/test/resources/log4j2-test.properties diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml new file mode 100644 index 0000000..421824e --- /dev/null +++ b/.github/workflows/gradle.yml @@ -0,0 +1,35 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. +# This workflow will build a Java project with Gradle and cache/restore any dependencies to improve the workflow execution time +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-gradle + +name: Java CI with Gradle + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +permissions: + contents: read + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'temurin' + - name: Build and test + uses: gradle/gradle-build-action@v2.5.0 + with: + arguments: build + diff --git a/.gitignore b/.gitignore new file mode 100755 index 0000000..eaedac0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.gradle +build +.idea +**/.DS_Store diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..bfea04f --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 vinted + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..1a1a444 --- /dev/null +++ b/README.md @@ -0,0 +1,137 @@ +# Flink BigQuery Connector ![Build](https://github.com/vinted/flink-big-query-connector/actions/workflows/gradle.yml/badge.svg) [![](https://jitpack.io/v/com.vinted/flink-big-query-connector.svg)](https://jitpack.io/#com.vinted/flink-big-query-connector) + + +This project provides a BigQuery sink that allows writing data with exactly-once or at-least guarantees. + +## Usage + +There are builder classes to simplify constructing a BigQuery sink. The code snippet below shows an example of building a BigQuery sink in Java: + +```java +var credentials = new JsonCredentialsProvider("key"); + + var clientProvider = new BigQueryProtoClientProvider(credentials, + WriterSettings + .newBuilder() + .build() + ); + + var bigQuerySink = BigQueryStreamSink + .newProto() + .withClientProvider(clientProvider) + .withDeliveryGuarantee(DeliveryGuarantee.EXACTLY_ONCE) + .withRowValueSerializer(new NoOpRowSerializer<>()) + .build(); +``` + +The sink takes in a batch of records. Batching happens outside the sink by opening a window. Batched records need to implement the BigQueryRecord interface. + +```java + var trigger = BatchTrigger.builder() + .withCount(100) + .withTimeout(Duration.ofSeconds(1)) + .withSizeInMb(1) + .withResetTimerOnNewRecord(true) + .build(); + + var processor = new BigQueryStreamProcessor().withDeliveryGuarantee(DeliveryGuarantee.AT_LEAST_ONCE).build(); + + source + .key(s -> s) + .window(GlobalWindows.create()) + .trigger(trigger) + .process(processor) +``` + + +To write to BigQuery, you need to: + +* Define credentials +* Create a client provider +* Batch records +* Create a value serializer +* Sink to BigQuery + +# Credentials + +There are two types of credentials: + +* Loading from a file +```java + new FileCredentialsProvider("/path/to/file") +``` +* Passing as a JSON string +```java +new JsonCredentialsProvider("key") +``` + +# Types of Streams + +BigQuery supports two types of data formats: json and proto. When creating a stream, you can choose these types by creating the appropriate client and using the builder methods. + +* JSON +```java + var clientProvider = new BigQueryJsonClientProvider(credentials, + WriterSettings + .newBuilder() + .build() + ); + + var bigQuerySink = BigQueryStreamSink + .newJson() +``` +* Proto +```java + var clientProvider = new BigQueryProtoClientProvider(credentials, + WriterSettings + .newBuilder() + .build() + ); + + var bigQuerySink = BigQueryStreamSink + .newProto() +``` + +# Exactly once +It utilizes a [buffered stream](https://cloud.google.com/bigquery/docs/write-api#buffered_type), managed by the BigQueryStreamProcessor, to assign and process data batches. If a stream is inactive or closed, a new stream is created automatically. The BigQuery sink writer appends and flushes data to the latest offset upon checkpoint commit. +# At least once +Data is written to the [default stream](https://cloud.google.com/bigquery/docs/write-api#default_stream) and handled by the BigQueryStreamProcessor, which batches and sends rows to the sink for processing. +# Serializers + +For the proto stream, you need to implement `ProtoValueSerializer`, and for the JSON stream, you need to implement `JsonRowValueSerializer`. + +# Metrics + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ScopeMetricsDescriptionType
Streamstream_offsetCurrent offset for the stream. When using at least once, the offset is always 0Gauge
batch_countNumber of records in the appended batchGauge
batch_size_mbAppended batch size in mbGauge
split_batch_countNumber of times the batch hit the BigQuery limit and was split into two partsGauge
+ diff --git a/build.gradle b/build.gradle new file mode 100755 index 0000000..0a89b4d --- /dev/null +++ b/build.gradle @@ -0,0 +1,80 @@ +plugins { + id 'com.github.johnrengelman.shadow' version '7.1.2' + id 'maven-publish' + id 'java-library' + id 'me.qoomon.git-versioning' version '4.3.0' +} + +sourceCompatibility = JavaVersion.VERSION_11 +targetCompatibility = JavaVersion.VERSION_11 + +repositories { + mavenCentral() + mavenLocal() + maven { url 'https://packages.confluent.io/maven/' } + maven { url 'https://nexus.vinted.net/repository/maven-proxy-oom/' } +} + +group = "com.github.vinted" + +gitVersioning.apply { + tag { + pattern = '(?\\d+\\.\\d+\\.\\d+$)' + versionFormat = '${tagVersion}' + } + branch { + pattern = '.+' + versionFormat = '${branch}-SNAPSHOT' + } + preferTags = false +} + +ext { + flinkVersion = '1.17.0' + bigqueryVersion = '2.27.0' + bigqueryStorageVersion = '2.37.2' + json4sVersion = '4.0.3' +} + +dependencies { + // Flink provided dependencies + compileOnly "org.apache.flink:flink-connector-base:$flinkVersion" + compileOnly "org.apache.flink:flink-streaming-java:$flinkVersion" + compileOnly "org.apache.flink:flink-core:$flinkVersion" + + implementation "com.google.cloud:google-cloud-bigquery:$bigqueryVersion" + implementation "com.google.cloud:google-cloud-bigquerystorage:$bigqueryStorageVersion" + + testImplementation platform('org.junit:junit-bom:5.9.1') + testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine" + testImplementation 'org.mockito:mockito-junit-jupiter:4.11.0' + testImplementation "org.assertj:assertj-core:3.24.0" + testImplementation "org.apache.flink:flink-connector-base:$flinkVersion" + testImplementation "org.mockito:mockito-inline:4.7.0" + testImplementation "org.apache.flink:flink-test-utils:$flinkVersion" + testImplementation 'commons-io:commons-io:2.11.0' +} + +test { + useJUnitPlatform() + testLogging { + events "passed", "skipped", "failed" + } +} + +shadowJar { + classifier = null + relocate 'io.grpc', 'com.vinted.flink.bigquery.shaded.io.grpc' + relocate 'io.netty', 'com.vinted.flink.bigquery.shaded.io.netty' + mergeServiceFiles() +} + +publishing { + publications { + java(MavenPublication) { + artifactId = project.name + artifact shadowJar + } + } +} + diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100755 index 0000000000000000000000000000000000000000..7454180f2ae8848c63b8b4dea2cb829da983f2fa GIT binary patch literal 59536 zcma&NbC71ylI~qywr$(CZQJHswz}-9F59+k+g;UV+cs{`J?GrGXYR~=-ydruB3JCa zB64N^cILAcWk5iofq)<(fq;O7{th4@;QxID0)qN`mJ?GIqLY#rX8-|G{5M0pdVW5^ zzXk$-2kQTAC?_N@B`&6-N-rmVFE=$QD?>*=4<|!MJu@}isLc4AW#{m2if&A5T5g&~ ziuMQeS*U5sL6J698wOd)K@oK@1{peP5&Esut<#VH^u)gp`9H4)`uE!2$>RTctN+^u z=ASkePDZA-X8)rp%D;p*~P?*a_=*Kwc<^>QSH|^<0>o37lt^+Mj1;4YvJ(JR-Y+?%Nu}JAYj5 z_Qc5%Ao#F?q32i?ZaN2OSNhWL;2oDEw_({7ZbgUjna!Fqn3NzLM@-EWFPZVmc>(fZ z0&bF-Ch#p9C{YJT9Rcr3+Y_uR^At1^BxZ#eo>$PLJF3=;t_$2|t+_6gg5(j{TmjYU zK12c&lE?Eh+2u2&6Gf*IdKS&6?rYbSEKBN!rv{YCm|Rt=UlPcW9j`0o6{66#y5t9C zruFA2iKd=H%jHf%ypOkxLnO8#H}#Zt{8p!oi6)7#NqoF({t6|J^?1e*oxqng9Q2Cc zg%5Vu!em)}Yuj?kaP!D?b?(C*w!1;>R=j90+RTkyEXz+9CufZ$C^umX^+4|JYaO<5 zmIM3#dv`DGM;@F6;(t!WngZSYzHx?9&$xEF70D1BvfVj<%+b#)vz)2iLCrTeYzUcL z(OBnNoG6Le%M+@2oo)&jdOg=iCszzv59e zDRCeaX8l1hC=8LbBt|k5?CXgep=3r9BXx1uR8!p%Z|0+4Xro=xi0G!e{c4U~1j6!) zH6adq0}#l{%*1U(Cb%4AJ}VLWKBPi0MoKFaQH6x?^hQ!6em@993xdtS%_dmevzeNl z(o?YlOI=jl(`L9^ z0O+H9k$_@`6L13eTT8ci-V0ljDMD|0ifUw|Q-Hep$xYj0hTO@0%IS^TD4b4n6EKDG z??uM;MEx`s98KYN(K0>c!C3HZdZ{+_53DO%9k5W%pr6yJusQAv_;IA}925Y%;+!tY z%2k!YQmLLOr{rF~!s<3-WEUs)`ix_mSU|cNRBIWxOox_Yb7Z=~Q45ZNe*u|m^|)d* zog=i>`=bTe!|;8F+#H>EjIMcgWcG2ORD`w0WD;YZAy5#s{65~qfI6o$+Ty&-hyMyJ z3Ra~t>R!p=5ZpxA;QkDAoPi4sYOP6>LT+}{xp}tk+<0k^CKCFdNYG(Es>p0gqD)jP zWOeX5G;9(m@?GOG7g;e74i_|SmE?`B2i;sLYwRWKLy0RLW!Hx`=!LH3&k=FuCsM=9M4|GqzA)anEHfxkB z?2iK-u(DC_T1};KaUT@3nP~LEcENT^UgPvp!QC@Dw&PVAhaEYrPey{nkcn(ro|r7XUz z%#(=$7D8uP_uU-oPHhd>>^adbCSQetgSG`e$U|7mr!`|bU0aHl_cmL)na-5x1#OsVE#m*+k84Y^+UMeSAa zbrVZHU=mFwXEaGHtXQq`2ZtjfS!B2H{5A<3(nb-6ARVV8kEmOkx6D2x7~-6hl;*-*}2Xz;J#a8Wn;_B5=m zl3dY;%krf?i-Ok^Pal-}4F`{F@TYPTwTEhxpZK5WCpfD^UmM_iYPe}wpE!Djai6_{ z*pGO=WB47#Xjb7!n2Ma)s^yeR*1rTxp`Mt4sfA+`HwZf%!7ZqGosPkw69`Ix5Ku6G z@Pa;pjzV&dn{M=QDx89t?p?d9gna*}jBly*#1!6}5K<*xDPJ{wv4& zM$17DFd~L*Te3A%yD;Dp9UGWTjRxAvMu!j^Tbc}2v~q^59d4bz zvu#!IJCy(BcWTc`;v$9tH;J%oiSJ_i7s;2`JXZF+qd4C)vY!hyCtl)sJIC{ebI*0> z@x>;EzyBv>AI-~{D6l6{ST=em*U( z(r$nuXY-#CCi^8Z2#v#UXOt`dbYN1z5jzNF2 z411?w)whZrfA20;nl&C1Gi+gk<`JSm+{|*2o<< zqM#@z_D`Cn|0H^9$|Tah)0M_X4c37|KQ*PmoT@%xHc3L1ZY6(p(sNXHa&49Frzto& zR`c~ClHpE~4Z=uKa5S(-?M8EJ$zt0&fJk~p$M#fGN1-y$7!37hld`Uw>Urri(DxLa;=#rK0g4J)pXMC zxzraOVw1+kNWpi#P=6(qxf`zSdUC?D$i`8ZI@F>k6k zz21?d+dw7b&i*>Kv5L(LH-?J%@WnqT7j#qZ9B>|Zl+=> z^U-pV@1y_ptHo4hl^cPRWewbLQ#g6XYQ@EkiP z;(=SU!yhjHp%1&MsU`FV1Z_#K1&(|5n(7IHbx&gG28HNT)*~-BQi372@|->2Aw5It z0CBpUcMA*QvsPy)#lr!lIdCi@1k4V2m!NH)%Px(vu-r(Q)HYc!p zJ^$|)j^E#q#QOgcb^pd74^JUi7fUmMiNP_o*lvx*q%_odv49Dsv$NV;6J z9GOXKomA{2Pb{w}&+yHtH?IkJJu~}Z?{Uk++2mB8zyvh*xhHKE``99>y#TdD z&(MH^^JHf;g(Tbb^&8P*;_i*2&fS$7${3WJtV7K&&(MBV2~)2KB3%cWg#1!VE~k#C z!;A;?p$s{ihyojEZz+$I1)L}&G~ml=udD9qh>Tu(ylv)?YcJT3ihapi!zgPtWb*CP zlLLJSRCj-^w?@;RU9aL2zDZY1`I3d<&OMuW=c3$o0#STpv_p3b9Wtbql>w^bBi~u4 z3D8KyF?YE?=HcKk!xcp@Cigvzy=lnFgc^9c%(^F22BWYNAYRSho@~*~S)4%AhEttv zvq>7X!!EWKG?mOd9&n>vvH1p4VzE?HCuxT-u+F&mnsfDI^}*-d00-KAauEaXqg3k@ zy#)MGX!X;&3&0s}F3q40ZmVM$(H3CLfpdL?hB6nVqMxX)q=1b}o_PG%r~hZ4gUfSp zOH4qlEOW4OMUc)_m)fMR_rl^pCfXc{$fQbI*E&mV77}kRF z&{<06AJyJ!e863o-V>FA1a9Eemx6>^F$~9ppt()ZbPGfg_NdRXBWoZnDy2;#ODgf! zgl?iOcF7Meo|{AF>KDwTgYrJLb$L2%%BEtO>T$C?|9bAB&}s;gI?lY#^tttY&hfr# zKhC+&b-rpg_?~uVK%S@mQleU#_xCsvIPK*<`E0fHE1&!J7!xD#IB|SSPW6-PyuqGn3^M^Rz%WT{e?OI^svARX&SAdU77V(C~ zM$H{Kg59op{<|8ry9ecfP%=kFm(-!W&?U0@<%z*+!*<e0XesMxRFu9QnGqun6R_%T+B%&9Dtk?*d$Q zb~>84jEAPi@&F@3wAa^Lzc(AJz5gsfZ7J53;@D<;Klpl?sK&u@gie`~vTsbOE~Cd4 z%kr56mI|#b(Jk&;p6plVwmNB0H@0SmgdmjIn5Ne@)}7Vty(yb2t3ev@22AE^s!KaN zyQ>j+F3w=wnx7w@FVCRe+`vUH)3gW%_72fxzqX!S&!dchdkRiHbXW1FMrIIBwjsai8`CB2r4mAbwp%rrO>3B$Zw;9=%fXI9B{d(UzVap7u z6piC-FQ)>}VOEuPpuqznpY`hN4dGa_1Xz9rVg(;H$5Te^F0dDv*gz9JS<|>>U0J^# z6)(4ICh+N_Q`Ft0hF|3fSHs*?a=XC;e`sJaU9&d>X4l?1W=|fr!5ShD|nv$GK;j46@BV6+{oRbWfqOBRb!ir88XD*SbC(LF}I1h#6@dvK%Toe%@ zhDyG$93H8Eu&gCYddP58iF3oQH*zLbNI;rN@E{T9%A8!=v#JLxKyUe}e}BJpB{~uN zqgxRgo0*-@-iaHPV8bTOH(rS(huwK1Xg0u+e!`(Irzu@Bld&s5&bWgVc@m7;JgELd zimVs`>vQ}B_1(2#rv#N9O`fJpVfPc7V2nv34PC);Dzbb;p!6pqHzvy?2pD&1NE)?A zt(t-ucqy@wn9`^MN5apa7K|L=9>ISC>xoc#>{@e}m#YAAa1*8-RUMKwbm|;5p>T`Z zNf*ph@tnF{gmDa3uwwN(g=`Rh)4!&)^oOy@VJaK4lMT&5#YbXkl`q?<*XtsqD z9PRK6bqb)fJw0g-^a@nu`^?71k|m3RPRjt;pIkCo1{*pdqbVs-Yl>4E>3fZx3Sv44grW=*qdSoiZ9?X0wWyO4`yDHh2E!9I!ZFi zVL8|VtW38}BOJHW(Ax#KL_KQzarbuE{(%TA)AY)@tY4%A%P%SqIU~8~-Lp3qY;U-} z`h_Gel7;K1h}7$_5ZZT0&%$Lxxr-<89V&&TCsu}LL#!xpQ1O31jaa{U34~^le*Y%L za?7$>Jk^k^pS^_M&cDs}NgXlR>16AHkSK-4TRaJSh#h&p!-!vQY%f+bmn6x`4fwTp z$727L^y`~!exvmE^W&#@uY!NxJi`g!i#(++!)?iJ(1)2Wk;RN zFK&O4eTkP$Xn~4bB|q8y(btx$R#D`O@epi4ofcETrx!IM(kWNEe42Qh(8*KqfP(c0 zouBl6>Fc_zM+V;F3znbo{x#%!?mH3`_ANJ?y7ppxS@glg#S9^MXu|FM&ynpz3o&Qh z2ujAHLF3($pH}0jXQsa#?t--TnF1P73b?4`KeJ9^qK-USHE)4!IYgMn-7z|=ALF5SNGkrtPG@Y~niUQV2?g$vzJN3nZ{7;HZHzWAeQ;5P|@Tl3YHpyznGG4-f4=XflwSJY+58-+wf?~Fg@1p1wkzuu-RF3j2JX37SQUc? zQ4v%`V8z9ZVZVqS8h|@@RpD?n0W<=hk=3Cf8R?d^9YK&e9ZybFY%jdnA)PeHvtBe- zhMLD+SSteHBq*q)d6x{)s1UrsO!byyLS$58WK;sqip$Mk{l)Y(_6hEIBsIjCr5t>( z7CdKUrJTrW%qZ#1z^n*Lb8#VdfzPw~OIL76aC+Rhr<~;4Tl!sw?Rj6hXj4XWa#6Tp z@)kJ~qOV)^Rh*-?aG>ic2*NlC2M7&LUzc9RT6WM%Cpe78`iAowe!>(T0jo&ivn8-7 zs{Qa@cGy$rE-3AY0V(l8wjI^uB8Lchj@?L}fYal^>T9z;8juH@?rG&g-t+R2dVDBe zq!K%{e-rT5jX19`(bP23LUN4+_zh2KD~EAYzhpEO3MUG8@}uBHH@4J zd`>_(K4q&>*k82(dDuC)X6JuPrBBubOg7qZ{?x!r@{%0);*`h*^F|%o?&1wX?Wr4b z1~&cy#PUuES{C#xJ84!z<1tp9sfrR(i%Tu^jnXy;4`Xk;AQCdFC@?V%|; zySdC7qS|uQRcH}EFZH%mMB~7gi}a0utE}ZE_}8PQH8f;H%PN41Cb9R%w5Oi5el^fd z$n{3SqLCnrF##x?4sa^r!O$7NX!}&}V;0ZGQ&K&i%6$3C_dR%I7%gdQ;KT6YZiQrW zk%q<74oVBV>@}CvJ4Wj!d^?#Zwq(b$E1ze4$99DuNg?6t9H}k_|D7KWD7i0-g*EO7 z;5{hSIYE4DMOK3H%|f5Edx+S0VI0Yw!tsaRS2&Il2)ea^8R5TG72BrJue|f_{2UHa z@w;^c|K3da#$TB0P3;MPlF7RuQeXT$ zS<<|C0OF(k)>fr&wOB=gP8!Qm>F41u;3esv7_0l%QHt(~+n; zf!G6%hp;Gfa9L9=AceiZs~tK+Tf*Wof=4!u{nIO90jH@iS0l+#%8=~%ASzFv7zqSB^?!@N7)kp0t&tCGLmzXSRMRyxCmCYUD2!B`? zhs$4%KO~m=VFk3Buv9osha{v+mAEq=ik3RdK@;WWTV_g&-$U4IM{1IhGX{pAu%Z&H zFfwCpUsX%RKg);B@7OUzZ{Hn{q6Vv!3#8fAg!P$IEx<0vAx;GU%}0{VIsmFBPq_mb zpe^BChDK>sc-WLKl<6 zwbW|e&d&dv9Wu0goueyu>(JyPx1mz0v4E?cJjFuKF71Q1)AL8jHO$!fYT3(;U3Re* zPPOe%*O+@JYt1bW`!W_1!mN&=w3G9ru1XsmwfS~BJ))PhD(+_J_^N6j)sx5VwbWK| zwRyC?W<`pOCY)b#AS?rluxuuGf-AJ=D!M36l{ua?@SJ5>e!IBr3CXIxWw5xUZ@Xrw z_R@%?{>d%Ld4p}nEsiA@v*nc6Ah!MUs?GA7e5Q5lPpp0@`%5xY$C;{%rz24$;vR#* zBP=a{)K#CwIY%p} zXVdxTQ^HS@O&~eIftU+Qt^~(DGxrdi3k}DdT^I7Iy5SMOp$QuD8s;+93YQ!OY{eB24%xY7ml@|M7I(Nb@K_-?F;2?et|CKkuZK_>+>Lvg!>JE~wN`BI|_h6$qi!P)+K-1Hh(1;a`os z55)4Q{oJiA(lQM#;w#Ta%T0jDNXIPM_bgESMCDEg6rM33anEr}=|Fn6)|jBP6Y}u{ zv9@%7*#RI9;fv;Yii5CI+KrRdr0DKh=L>)eO4q$1zmcSmglsV`*N(x=&Wx`*v!!hn6X-l0 zP_m;X??O(skcj+oS$cIdKhfT%ABAzz3w^la-Ucw?yBPEC+=Pe_vU8nd-HV5YX6X8r zZih&j^eLU=%*;VzhUyoLF;#8QsEfmByk+Y~caBqSvQaaWf2a{JKB9B>V&r?l^rXaC z8)6AdR@Qy_BxQrE2Fk?ewD!SwLuMj@&d_n5RZFf7=>O>hzVE*seW3U?_p|R^CfoY`?|#x9)-*yjv#lo&zP=uI`M?J zbzC<^3x7GfXA4{FZ72{PE*-mNHyy59Q;kYG@BB~NhTd6pm2Oj=_ zizmD?MKVRkT^KmXuhsk?eRQllPo2Ubk=uCKiZ&u3Xjj~<(!M94c)Tez@9M1Gfs5JV z->@II)CDJOXTtPrQudNjE}Eltbjq>6KiwAwqvAKd^|g!exgLG3;wP+#mZYr`cy3#39e653d=jrR-ulW|h#ddHu(m9mFoW~2yE zz5?dB%6vF}+`-&-W8vy^OCxm3_{02royjvmwjlp+eQDzFVEUiyO#gLv%QdDSI#3W* z?3!lL8clTaNo-DVJw@ynq?q!%6hTQi35&^>P85G$TqNt78%9_sSJt2RThO|JzM$iL zg|wjxdMC2|Icc5rX*qPL(coL!u>-xxz-rFiC!6hD1IR%|HSRsV3>Kq~&vJ=s3M5y8SG%YBQ|{^l#LGlg!D?E>2yR*eV%9m$_J6VGQ~AIh&P$_aFbh zULr0Z$QE!QpkP=aAeR4ny<#3Fwyw@rZf4?Ewq`;mCVv}xaz+3ni+}a=k~P+yaWt^L z@w67!DqVf7D%7XtXX5xBW;Co|HvQ8WR1k?r2cZD%U;2$bsM%u8{JUJ5Z0k= zZJARv^vFkmWx15CB=rb=D4${+#DVqy5$C%bf`!T0+epLJLnh1jwCdb*zuCL}eEFvE z{rO1%gxg>1!W(I!owu*mJZ0@6FM(?C+d*CeceZRW_4id*D9p5nzMY&{mWqrJomjIZ z97ZNnZ3_%Hx8dn;H>p8m7F#^2;T%yZ3H;a&N7tm=Lvs&lgJLW{V1@h&6Vy~!+Ffbb zv(n3+v)_D$}dqd!2>Y2B)#<+o}LH#%ogGi2-?xRIH)1!SD)u-L65B&bsJTC=LiaF+YOCif2dUX6uAA|#+vNR z>U+KQekVGon)Yi<93(d!(yw1h3&X0N(PxN2{%vn}cnV?rYw z$N^}_o!XUB!mckL`yO1rnUaI4wrOeQ(+&k?2mi47hzxSD`N#-byqd1IhEoh!PGq>t z_MRy{5B0eKY>;Ao3z$RUU7U+i?iX^&r739F)itdrTpAi-NN0=?^m%?{A9Ly2pVv>Lqs6moTP?T2-AHqFD-o_ znVr|7OAS#AEH}h8SRPQ@NGG47dO}l=t07__+iK8nHw^(AHx&Wb<%jPc$$jl6_p(b$ z)!pi(0fQodCHfM)KMEMUR&UID>}m^(!{C^U7sBDOA)$VThRCI0_+2=( zV8mMq0R(#z;C|7$m>$>`tX+T|xGt(+Y48@ZYu#z;0pCgYgmMVbFb!$?%yhZqP_nhn zy4<#3P1oQ#2b51NU1mGnHP$cf0j-YOgAA}A$QoL6JVLcmExs(kU{4z;PBHJD%_=0F z>+sQV`mzijSIT7xn%PiDKHOujX;n|M&qr1T@rOxTdxtZ!&u&3HHFLYD5$RLQ=heur zb>+AFokUVQeJy-#LP*^)spt{mb@Mqe=A~-4p0b+Bt|pZ+@CY+%x}9f}izU5;4&QFE zO1bhg&A4uC1)Zb67kuowWY4xbo&J=%yoXlFB)&$d*-}kjBu|w!^zbD1YPc0-#XTJr z)pm2RDy%J3jlqSMq|o%xGS$bPwn4AqitC6&e?pqWcjWPt{3I{>CBy;hg0Umh#c;hU3RhCUX=8aR>rmd` z7Orw(5tcM{|-^J?ZAA9KP|)X6n9$-kvr#j5YDecTM6n z&07(nD^qb8hpF0B^z^pQ*%5ePYkv&FabrlI61ntiVp!!C8y^}|<2xgAd#FY=8b*y( zuQOuvy2`Ii^`VBNJB&R!0{hABYX55ooCAJSSevl4RPqEGb)iy_0H}v@vFwFzD%>#I>)3PsouQ+_Kkbqy*kKdHdfkN7NBcq%V{x^fSxgXpg7$bF& zj!6AQbDY(1u#1_A#1UO9AxiZaCVN2F0wGXdY*g@x$ByvUA?ePdide0dmr#}udE%K| z3*k}Vv2Ew2u1FXBaVA6aerI36R&rzEZeDDCl5!t0J=ug6kuNZzH>3i_VN`%BsaVB3 zQYw|Xub_SGf{)F{$ZX5`Jc!X!;eybjP+o$I{Z^Hsj@D=E{MnnL+TbC@HEU2DjG{3-LDGIbq()U87x4eS;JXnSh;lRlJ z>EL3D>wHt-+wTjQF$fGyDO$>d+(fq@bPpLBS~xA~R=3JPbS{tzN(u~m#Po!?H;IYv zE;?8%^vle|%#oux(Lj!YzBKv+Fd}*Ur-dCBoX*t{KeNM*n~ZPYJ4NNKkI^MFbz9!v z4(Bvm*Kc!-$%VFEewYJKz-CQN{`2}KX4*CeJEs+Q(!kI%hN1!1P6iOq?ovz}X0IOi z)YfWpwW@pK08^69#wSyCZkX9?uZD?C^@rw^Y?gLS_xmFKkooyx$*^5#cPqntNTtSG zlP>XLMj2!VF^0k#ole7`-c~*~+_T5ls?x4)ah(j8vo_ zwb%S8qoaZqY0-$ZI+ViIA_1~~rAH7K_+yFS{0rT@eQtTAdz#8E5VpwnW!zJ_^{Utv zlW5Iar3V5t&H4D6A=>?mq;G92;1cg9a2sf;gY9pJDVKn$DYdQlvfXq}zz8#LyPGq@ z+`YUMD;^-6w&r-82JL7mA8&M~Pj@aK!m{0+^v<|t%APYf7`}jGEhdYLqsHW-Le9TL z_hZZ1gbrz7$f9^fAzVIP30^KIz!!#+DRLL+qMszvI_BpOSmjtl$hh;&UeM{ER@INV zcI}VbiVTPoN|iSna@=7XkP&-4#06C};8ajbxJ4Gcq8(vWv4*&X8bM^T$mBk75Q92j z1v&%a;OSKc8EIrodmIiw$lOES2hzGDcjjB`kEDfJe{r}yE6`eZL zEB`9u>Cl0IsQ+t}`-cx}{6jqcANucqIB>Qmga_&<+80E2Q|VHHQ$YlAt{6`Qu`HA3 z03s0-sSlwbvgi&_R8s={6<~M^pGvBNjKOa>tWenzS8s zR>L7R5aZ=mSU{f?ib4Grx$AeFvtO5N|D>9#)ChH#Fny2maHWHOf2G=#<9Myot#+4u zWVa6d^Vseq_0=#AYS(-m$Lp;*8nC_6jXIjEM`omUmtH@QDs3|G)i4j*#_?#UYVZvJ z?YjT-?!4Q{BNun;dKBWLEw2C-VeAz`%?A>p;)PL}TAZn5j~HK>v1W&anteARlE+~+ zj>c(F;?qO3pXBb|#OZdQnm<4xWmn~;DR5SDMxt0UK_F^&eD|KZ=O;tO3vy4@4h^;2 zUL~-z`-P1aOe?|ZC1BgVsL)2^J-&vIFI%q@40w0{jjEfeVl)i9(~bt2z#2Vm)p`V_ z1;6$Ae7=YXk#=Qkd24Y23t&GvRxaOoad~NbJ+6pxqzJ>FY#Td7@`N5xp!n(c!=RE& z&<<@^a$_Ys8jqz4|5Nk#FY$~|FPC0`*a5HH!|Gssa9=~66&xG9)|=pOOJ2KE5|YrR zw!w6K2aC=J$t?L-;}5hn6mHd%hC;p8P|Dgh6D>hGnXPgi;6r+eA=?f72y9(Cf_ho{ zH6#)uD&R=73^$$NE;5piWX2bzR67fQ)`b=85o0eOLGI4c-Tb@-KNi2pz=Ke@SDcPn za$AxXib84`!Sf;Z3B@TSo`Dz7GM5Kf(@PR>Ghzi=BBxK8wRp>YQoXm+iL>H*Jo9M3 z6w&E?BC8AFTFT&Tv8zf+m9<&S&%dIaZ)Aoqkak_$r-2{$d~0g2oLETx9Y`eOAf14QXEQw3tJne;fdzl@wV#TFXSLXM2428F-Q}t+n2g%vPRMUzYPvzQ9f# zu(liiJem9P*?0%V@RwA7F53r~|I!Ty)<*AsMX3J{_4&}{6pT%Tpw>)^|DJ)>gpS~1rNEh z0$D?uO8mG?H;2BwM5a*26^7YO$XjUm40XmBsb63MoR;bJh63J;OngS5sSI+o2HA;W zdZV#8pDpC9Oez&L8loZO)MClRz!_!WD&QRtQxnazhT%Vj6Wl4G11nUk8*vSeVab@N#oJ}`KyJv+8Mo@T1-pqZ1t|?cnaVOd;1(h9 z!$DrN=jcGsVYE-0-n?oCJ^4x)F}E;UaD-LZUIzcD?W^ficqJWM%QLy6QikrM1aKZC zi{?;oKwq^Vsr|&`i{jIphA8S6G4)$KGvpULjH%9u(Dq247;R#l&I0{IhcC|oBF*Al zvLo7Xte=C{aIt*otJD}BUq)|_pdR>{zBMT< z(^1RpZv*l*m*OV^8>9&asGBo8h*_4q*)-eCv*|Pq=XNGrZE)^(SF7^{QE_~4VDB(o zVcPA_!G+2CAtLbl+`=Q~9iW`4ZRLku!uB?;tWqVjB0lEOf}2RD7dJ=BExy=<9wkb- z9&7{XFA%n#JsHYN8t5d~=T~5DcW4$B%3M+nNvC2`0!#@sckqlzo5;hhGi(D9=*A4` z5ynobawSPRtWn&CDLEs3Xf`(8^zDP=NdF~F^s&={l7(aw&EG}KWpMjtmz7j_VLO;@ zM2NVLDxZ@GIv7*gzl1 zjq78tv*8#WSY`}Su0&C;2F$Ze(q>F(@Wm^Gw!)(j;dk9Ad{STaxn)IV9FZhm*n+U} zi;4y*3v%A`_c7a__DJ8D1b@dl0Std3F||4Wtvi)fCcBRh!X9$1x!_VzUh>*S5s!oq z;qd{J_r79EL2wIeiGAqFstWtkfIJpjVh%zFo*=55B9Zq~y0=^iqHWfQl@O!Ak;(o*m!pZqe9 z%U2oDOhR)BvW8&F70L;2TpkzIutIvNQaTjjs5V#8mV4!NQ}zN=i`i@WI1z0eN-iCS z;vL-Wxc^Vc_qK<5RPh(}*8dLT{~GzE{w2o$2kMFaEl&q zP{V=>&3kW7tWaK-Exy{~`v4J0U#OZBk{a9{&)&QG18L@6=bsZ1zC_d{{pKZ-Ey>I> z;8H0t4bwyQqgu4hmO`3|4K{R*5>qnQ&gOfdy?z`XD%e5+pTDzUt3`k^u~SaL&XMe= z9*h#kT(*Q9jO#w2Hd|Mr-%DV8i_1{J1MU~XJ3!WUplhXDYBpJH><0OU`**nIvPIof z|N8@I=wA)sf45SAvx||f?Z5uB$kz1qL3Ky_{%RPdP5iN-D2!p5scq}buuC00C@jom zhfGKm3|f?Z0iQ|K$Z~!`8{nmAS1r+fp6r#YDOS8V*;K&Gs7Lc&f^$RC66O|)28oh`NHy&vq zJh+hAw8+ybTB0@VhWN^0iiTnLsCWbS_y`^gs!LX!Lw{yE``!UVzrV24tP8o;I6-65 z1MUiHw^{bB15tmrVT*7-#sj6cs~z`wk52YQJ*TG{SE;KTm#Hf#a~|<(|ImHH17nNM z`Ub{+J3dMD!)mzC8b(2tZtokKW5pAwHa?NFiso~# z1*iaNh4lQ4TS)|@G)H4dZV@l*Vd;Rw;-;odDhW2&lJ%m@jz+Panv7LQm~2Js6rOW3 z0_&2cW^b^MYW3)@o;neZ<{B4c#m48dAl$GCc=$>ErDe|?y@z`$uq3xd(%aAsX)D%l z>y*SQ%My`yDP*zof|3@_w#cjaW_YW4BdA;#Glg1RQcJGY*CJ9`H{@|D+*e~*457kd z73p<%fB^PV!Ybw@)Dr%(ZJbX}xmCStCYv#K3O32ej{$9IzM^I{6FJ8!(=azt7RWf4 z7ib0UOPqN40X!wOnFOoddd8`!_IN~9O)#HRTyjfc#&MCZ zZAMzOVB=;qwt8gV?{Y2?b=iSZG~RF~uyx18K)IDFLl})G1v@$(s{O4@RJ%OTJyF+Cpcx4jmy|F3euCnMK!P2WTDu5j z{{gD$=M*pH!GGzL%P)V2*ROm>!$Y=z|D`!_yY6e7SU$~a5q8?hZGgaYqaiLnkK%?0 zs#oI%;zOxF@g*@(V4p!$7dS1rOr6GVs6uYCTt2h)eB4?(&w8{#o)s#%gN@BBosRUe z)@P@8_Zm89pr~)b>e{tbPC~&_MR--iB{=)y;INU5#)@Gix-YpgP<-c2Ms{9zuCX|3 z!p(?VaXww&(w&uBHzoT%!A2=3HAP>SDxcljrego7rY|%hxy3XlODWffO_%g|l+7Y_ zqV(xbu)s4lV=l7M;f>vJl{`6qBm>#ZeMA}kXb97Z)?R97EkoI?x6Lp0yu1Z>PS?2{ z0QQ(8D)|lc9CO3B~e(pQM&5(1y&y=e>C^X$`)_&XuaI!IgDTVqt31wX#n+@!a_A0ZQkA zCJ2@M_4Gb5MfCrm5UPggeyh)8 zO9?`B0J#rkoCx(R0I!ko_2?iO@|oRf1;3r+i)w-2&j?=;NVIdPFsB)`|IC0zk6r9c zRrkfxWsiJ(#8QndNJj@{@WP2Ackr|r1VxV{7S&rSU(^)-M8gV>@UzOLXu9K<{6e{T zXJ6b92r$!|lwjhmgqkdswY&}c)KW4A)-ac%sU;2^fvq7gfUW4Bw$b!i@duy1CAxSn z(pyh$^Z=&O-q<{bZUP+$U}=*#M9uVc>CQVgDs4swy5&8RAHZ~$)hrTF4W zPsSa~qYv_0mJnF89RnnJTH`3}w4?~epFl=D(35$ zWa07ON$`OMBOHgCmfO(9RFc<)?$x)N}Jd2A(<*Ll7+4jrRt9w zwGxExUXd9VB#I|DwfxvJ;HZ8Q{37^wDhaZ%O!oO(HpcqfLH%#a#!~;Jl7F5>EX_=8 z{()l2NqPz>La3qJR;_v+wlK>GsHl;uRA8%j`A|yH@k5r%55S9{*Cp%uw6t`qc1!*T za2OeqtQj7sAp#Q~=5Fs&aCR9v>5V+s&RdNvo&H~6FJOjvaj--2sYYBvMq;55%z8^o z|BJDA4vzfow#DO#ZQHh;Oq_{r+qP{R9ox2TOgwQiv7Ow!zjN+A@BN;0tA2lUb#+zO z(^b89eV)D7UVE+h{mcNc6&GtpOqDn_?VAQ)Vob$hlFwW%xh>D#wml{t&Ofmm_d_+; zKDxzdr}`n2Rw`DtyIjrG)eD0vut$}dJAZ0AohZ+ZQdWXn_Z@dI_y=7t3q8x#pDI-K z2VVc&EGq445Rq-j0=U=Zx`oBaBjsefY;%)Co>J3v4l8V(T8H?49_@;K6q#r~Wwppc z4XW0(4k}cP=5ex>-Xt3oATZ~bBWKv)aw|I|Lx=9C1s~&b77idz({&q3T(Y(KbWO?+ zmcZ6?WeUsGk6>km*~234YC+2e6Zxdl~<_g2J|IE`GH%n<%PRv-50; zH{tnVts*S5*_RxFT9eM0z-pksIb^drUq4>QSww=u;UFCv2AhOuXE*V4z?MM`|ABOC4P;OfhS(M{1|c%QZ=!%rQTDFx`+}?Kdx$&FU?Y<$x;j7z=(;Lyz+?EE>ov!8vvMtSzG!nMie zsBa9t8as#2nH}n8xzN%W%U$#MHNXmDUVr@GX{?(=yI=4vks|V)!-W5jHsU|h_&+kY zS_8^kd3jlYqOoiI`ZqBVY!(UfnAGny!FowZWY_@YR0z!nG7m{{)4OS$q&YDyw6vC$ zm4!$h>*|!2LbMbxS+VM6&DIrL*X4DeMO!@#EzMVfr)e4Tagn~AQHIU8?e61TuhcKD zr!F4(kEebk(Wdk-?4oXM(rJwanS>Jc%<>R(siF+>+5*CqJLecP_we33iTFTXr6W^G z7M?LPC-qFHK;E!fxCP)`8rkxZyFk{EV;G-|kwf4b$c1k0atD?85+|4V%YATWMG|?K zLyLrws36p%Qz6{}>7b>)$pe>mR+=IWuGrX{3ZPZXF3plvuv5Huax86}KX*lbPVr}L z{C#lDjdDeHr~?l|)Vp_}T|%$qF&q#U;ClHEPVuS+Jg~NjC1RP=17=aQKGOcJ6B3mp z8?4*-fAD~}sX*=E6!}^u8)+m2j<&FSW%pYr_d|p_{28DZ#Cz0@NF=gC-o$MY?8Ca8 zr5Y8DSR^*urS~rhpX^05r30Ik#2>*dIOGxRm0#0YX@YQ%Mg5b6dXlS!4{7O_kdaW8PFSdj1=ryI-=5$fiieGK{LZ+SX(1b=MNL!q#lN zv98?fqqTUH8r8C7v(cx#BQ5P9W>- zmW93;eH6T`vuJ~rqtIBg%A6>q>gnWb3X!r0wh_q;211+Om&?nvYzL1hhtjB zK_7G3!n7PL>d!kj){HQE zE8(%J%dWLh1_k%gVXTZt zEdT09XSKAx27Ncaq|(vzL3gm83q>6CAw<$fTnMU05*xAe&rDfCiu`u^1)CD<>sx0i z*hr^N_TeN89G(nunZoLBf^81#pmM}>JgD@Nn1l*lN#a=B=9pN%tmvYFjFIoKe_(GF z-26x{(KXdfsQL7Uv6UtDuYwV`;8V3w>oT_I<`Ccz3QqK9tYT5ZQzbop{=I=!pMOCb zCU68`n?^DT%^&m>A%+-~#lvF!7`L7a{z<3JqIlk1$<||_J}vW1U9Y&eX<}l8##6i( zZcTT@2`9(Mecptm@{3A_Y(X`w9K0EwtPq~O!16bq{7c0f7#(3wn-^)h zxV&M~iiF!{-6A@>o;$RzQ5A50kxXYj!tcgme=Qjrbje~;5X2xryU;vH|6bE(8z^<7 zQ>BG7_c*JG8~K7Oe68i#0~C$v?-t@~@r3t2inUnLT(c=URpA9kA8uq9PKU(Ps(LVH zqgcqW>Gm?6oV#AldDPKVRcEyQIdTT`Qa1j~vS{<;SwyTdr&3*t?J)y=M7q*CzucZ&B0M=joT zBbj@*SY;o2^_h*>R0e({!QHF0=)0hOj^B^d*m>SnRrwq>MolNSgl^~r8GR#mDWGYEIJA8B<|{{j?-7p zVnV$zancW3&JVDtVpIlI|5djKq0(w$KxEFzEiiL=h5Jw~4Le23@s(mYyXWL9SX6Ot zmb)sZaly_P%BeX_9 zw&{yBef8tFm+%=--m*J|o~+Xg3N+$IH)t)=fqD+|fEk4AAZ&!wcN5=mi~Vvo^i`}> z#_3ahR}Ju)(Px7kev#JGcSwPXJ2id9%Qd2A#Uc@t8~egZ8;iC{e! z%=CGJOD1}j!HW_sgbi_8suYnn4#Ou}%9u)dXd3huFIb!ytlX>Denx@pCS-Nj$`VO&j@(z!kKSP0hE4;YIP#w9ta=3DO$7f*x zc9M4&NK%IrVmZAe=r@skWD`AEWH=g+r|*13Ss$+{c_R!b?>?UaGXlw*8qDmY#xlR= z<0XFbs2t?8i^G~m?b|!Hal^ZjRjt<@a? z%({Gn14b4-a|#uY^=@iiKH+k?~~wTj5K1A&hU z2^9-HTC)7zpoWK|$JXaBL6C z#qSNYtY>65T@Zs&-0cHeu|RX(Pxz6vTITdzJdYippF zC-EB+n4}#lM7`2Ry~SO>FxhKboIAF#Z{1wqxaCb{#yEFhLuX;Rx(Lz%T`Xo1+a2M}7D+@wol2)OJs$TwtRNJ={( zD@#zTUEE}#Fz#&(EoD|SV#bayvr&E0vzmb%H?o~46|FAcx?r4$N z&67W3mdip-T1RIxwSm_&(%U|+WvtGBj*}t69XVd&ebn>KOuL(7Y8cV?THd-(+9>G7*Nt%T zcH;`p={`SOjaf7hNd(=37Lz3-51;58JffzIPgGs_7xIOsB5p2t&@v1mKS$2D$*GQ6 zM(IR*j4{nri7NMK9xlDy-hJW6sW|ZiDRaFiayj%;(%51DN!ZCCCXz+0Vm#};70nOx zJ#yA0P3p^1DED;jGdPbQWo0WATN=&2(QybbVdhd=Vq*liDk`c7iZ?*AKEYC#SY&2g z&Q(Ci)MJ{mEat$ZdSwTjf6h~roanYh2?9j$CF@4hjj_f35kTKuGHvIs9}Re@iKMxS-OI*`0S z6s)fOtz}O$T?PLFVSeOjSO26$@u`e<>k(OSP!&YstH3ANh>)mzmKGNOwOawq-MPXe zy4xbeUAl6tamnx))-`Gi2uV5>9n(73yS)Ukma4*7fI8PaEwa)dWHs6QA6>$}7?(L8 ztN8M}?{Tf!Zu22J5?2@95&rQ|F7=FK-hihT-vDp!5JCcWrVogEnp;CHenAZ)+E+K5 z$Cffk5sNwD_?4+ymgcHR(5xgt20Z8M`2*;MzOM#>yhk{r3x=EyM226wb&!+j`W<%* zSc&|`8!>dn9D@!pYow~(DsY_naSx7(Z4i>cu#hA5=;IuI88}7f%)bRkuY2B;+9Uep zpXcvFWkJ!mQai63BgNXG26$5kyhZ2&*3Q_tk)Ii4M>@p~_~q_cE!|^A;_MHB;7s#9 zKzMzK{lIxotjc};k67^Xsl-gS!^*m*m6kn|sbdun`O?dUkJ{0cmI0-_2y=lTAfn*Y zKg*A-2sJq)CCJgY0LF-VQvl&6HIXZyxo2#!O&6fOhbHXC?%1cMc6y^*dOS{f$=137Ds1m01qs`>iUQ49JijsaQ( zksqV9@&?il$|4Ua%4!O15>Zy&%gBY&wgqB>XA3!EldQ%1CRSM(pp#k~-pkcCg4LAT zXE=puHbgsw)!xtc@P4r~Z}nTF=D2~j(6D%gTBw$(`Fc=OOQ0kiW$_RDd=hcO0t97h zb86S5r=>(@VGy1&#S$Kg_H@7G^;8Ue)X5Y+IWUi`o;mpvoV)`fcVk4FpcT|;EG!;? zHG^zrVVZOm>1KFaHlaogcWj(v!S)O(Aa|Vo?S|P z5|6b{qkH(USa*Z7-y_Uvty_Z1|B{rTS^qmEMLEYUSk03_Fg&!O3BMo{b^*`3SHvl0 zhnLTe^_vVIdcSHe)SQE}r~2dq)VZJ!aSKR?RS<(9lzkYo&dQ?mubnWmgMM37Nudwo z3Vz@R{=m2gENUE3V4NbIzAA$H1z0pagz94-PTJyX{b$yndsdKptmlKQKaaHj@3=ED zc7L?p@%ui|RegVYutK$64q4pe9+5sv34QUpo)u{1ci?)_7gXQd{PL>b0l(LI#rJmN zGuO+%GO`xneFOOr4EU(Wg}_%bhzUf;d@TU+V*2#}!2OLwg~%D;1FAu=Un>OgjPb3S z7l(riiCwgghC=Lm5hWGf5NdGp#01xQ59`HJcLXbUR3&n%P(+W2q$h2Qd z*6+-QXJ*&Kvk9ht0f0*rO_|FMBALen{j7T1l%=Q>gf#kma zQlg#I9+HB+z*5BMxdesMND`_W;q5|FaEURFk|~&{@qY32N$G$2B=&Po{=!)x5b!#n zxLzblkq{yj05#O7(GRuT39(06FJlalyv<#K4m}+vs>9@q-&31@1(QBv82{}Zkns~K ze{eHC_RDX0#^A*JQTwF`a=IkE6Ze@j#-8Q`tTT?k9`^ZhA~3eCZJ-Jr{~7Cx;H4A3 zcZ+Zj{mzFZbVvQ6U~n>$U2ZotGsERZ@}VKrgGh0xM;Jzt29%TX6_&CWzg+YYMozrM z`nutuS)_0dCM8UVaKRj804J4i%z2BA_8A4OJRQ$N(P9Mfn-gF;4#q788C@9XR0O3< zsoS4wIoyt046d+LnSCJOy@B@Uz*#GGd#+Ln1ek5Dv>(ZtD@tgZlPnZZJGBLr^JK+!$$?A_fA3LOrkoDRH&l7 zcMcD$Hsjko3`-{bn)jPL6E9Ds{WskMrivsUu5apD z?grQO@W7i5+%X&E&p|RBaEZ(sGLR@~(y^BI@lDMot^Ll?!`90KT!JXUhYS`ZgX3jnu@Ja^seA*M5R@f`=`ynQV4rc$uT1mvE?@tz)TN<=&H1%Z?5yjxcpO+6y_R z6EPuPKM5uxKpmZfT(WKjRRNHs@ib)F5WAP7QCADvmCSD#hPz$V10wiD&{NXyEwx5S z6NE`3z!IS^$s7m}PCwQutVQ#~w+V z=+~->DI*bR2j0^@dMr9`p>q^Ny~NrAVxrJtX2DUveic5vM%#N*XO|?YAWwNI$Q)_) zvE|L(L1jP@F%gOGtnlXtIv2&1i8q<)Xfz8O3G^Ea~e*HJsQgBxWL(yuLY+jqUK zRE~`-zklrGog(X}$9@ZVUw!8*=l`6mzYLtsg`AvBYz(cxmAhr^j0~(rzXdiOEeu_p zE$sf2(w(BPAvO5DlaN&uQ$4@p-b?fRs}d7&2UQ4Fh?1Hzu*YVjcndqJLw0#q@fR4u zJCJ}>_7-|QbvOfylj+e^_L`5Ep9gqd>XI3-O?Wp z-gt*P29f$Tx(mtS`0d05nHH=gm~Po_^OxxUwV294BDKT>PHVlC5bndncxGR!n(OOm znsNt@Q&N{TLrmsoKFw0&_M9$&+C24`sIXGWgQaz=kY;S{?w`z^Q0JXXBKFLj0w0U6P*+jPKyZHX9F#b0D1$&(- zrm8PJd?+SrVf^JlfTM^qGDK&-p2Kdfg?f>^%>1n8bu&byH(huaocL>l@f%c*QkX2i znl}VZ4R1en4S&Bcqw?$=Zi7ohqB$Jw9x`aM#>pHc0x z0$!q7iFu zZ`tryM70qBI6JWWTF9EjgG@>6SRzsd}3h+4D8d~@CR07P$LJ}MFsYi-*O%XVvD@yT|rJ+Mk zDllJ7$n0V&A!0flbOf)HE6P_afPWZmbhpliqJuw=-h+r;WGk|ntkWN(8tKlYpq5Ow z(@%s>IN8nHRaYb*^d;M(D$zGCv5C|uqmsDjwy4g=Lz>*OhO3z=)VD}C<65;`89Ye} zSCxrv#ILzIpEx1KdLPlM&%Cctf@FqTKvNPXC&`*H9=l=D3r!GLM?UV zOxa(8ZsB`&+76S-_xuj?G#wXBfDY@Z_tMpXJS7^mp z@YX&u0jYw2A+Z+bD#6sgVK5ZgdPSJV3>{K^4~%HV?rn~4D)*2H!67Y>0aOmzup`{D zzDp3c9yEbGCY$U<8biJ_gB*`jluz1ShUd!QUIQJ$*1;MXCMApJ^m*Fiv88RZ zFopLViw}{$Tyhh_{MLGIE2~sZ)t0VvoW%=8qKZ>h=adTe3QM$&$PO2lfqH@brt!9j ziePM8$!CgE9iz6B<6_wyTQj?qYa;eC^{x_0wuwV~W+^fZmFco-o%wsKSnjXFEx02V zF5C2t)T6Gw$Kf^_c;Ei3G~uC8SM-xyycmXyC2hAVi-IfXqhu$$-C=*|X?R0~hu z8`J6TdgflslhrmDZq1f?GXF7*ALeMmOEpRDg(s*H`4>_NAr`2uqF;k;JQ+8>A|_6ZNsNLECC%NNEb1Y1dP zbIEmNpK)#XagtL4R6BC{C5T(+=yA-(Z|Ap}U-AfZM#gwVpus3(gPn}Q$CExObJ5AC z)ff9Yk?wZ}dZ-^)?cbb9Fw#EjqQ8jxF4G3=L?Ra zg_)0QDMV1y^A^>HRI$x?Op@t;oj&H@1xt4SZ9(kifQ zb59B*`M99Td7@aZ3UWvj1rD0sE)d=BsBuW*KwkCds7ay(7*01_+L}b~7)VHI>F_!{ zyxg-&nCO?v#KOUec0{OOKy+sjWA;8rTE|Lv6I9H?CI?H(mUm8VXGwU$49LGpz&{nQp2}dinE1@lZ1iox6{ghN&v^GZv9J${7WaXj)<0S4g_uiJ&JCZ zr8-hsu`U%N;+9N^@&Q0^kVPB3)wY(rr}p7{p0qFHb3NUUHJb672+wRZs`gd1UjKPX z4o6zljKKA+Kkj?H>Ew63o%QjyBk&1!P22;MkD>sM0=z_s-G{mTixJCT9@_|*(p^bz zJ8?ZZ&;pzV+7#6Mn`_U-)k8Pjg?a;|Oe^us^PoPY$Va~yi8|?+&=y$f+lABT<*pZr zP}D{~Pq1Qyni+@|aP;ixO~mbEW9#c0OU#YbDZIaw=_&$K%Ep2f%hO^&P67hApZe`x zv8b`Mz@?M_7-)b!lkQKk)JXXUuT|B8kJlvqRmRpxtQDgvrHMXC1B$M@Y%Me!BSx3P z#2Eawl$HleZhhTS6Txm>lN_+I`>eV$&v9fOg)%zVn3O5mI*lAl>QcHuW6!Kixmq`X zBCZ*Ck6OYtDiK!N47>jxI&O2a9x7M|i^IagRr-fmrmikEQGgw%J7bO|)*$2FW95O4 zeBs>KR)izRG1gRVL;F*sr8A}aRHO0gc$$j&ds8CIO1=Gwq1%_~E)CWNn9pCtBE}+`Jelk4{>S)M)`Ll=!~gnn1yq^EX(+y*ik@3Ou0qU`IgYi3*doM+5&dU!cho$pZ zn%lhKeZkS72P?Cf68<#kll_6OAO26bIbueZx**j6o;I0cS^XiL`y+>{cD}gd%lux} z)3N>MaE24WBZ}s0ApfdM;5J_Ny}rfUyxfkC``Awo2#sgLnGPewK};dORuT?@I6(5~ z?kE)Qh$L&fwJXzK){iYx!l5$Tt|^D~MkGZPA}(o6f7w~O2G6Vvzdo*a;iXzk$B66$ zwF#;wM7A+(;uFG4+UAY(2`*3XXx|V$K8AYu#ECJYSl@S=uZW$ksfC$~qrrbQj4??z-)uz0QL}>k^?fPnJTPw% zGz)~?B4}u0CzOf@l^um}HZzbaIwPmb<)< zi_3@E9lc)Qe2_`*Z^HH;1CXOceL=CHpHS{HySy3T%<^NrWQ}G0i4e1xm_K3(+~oi$ zoHl9wzb?Z4j#90DtURtjtgvi7uw8DzHYmtPb;?%8vb9n@bszT=1qr)V_>R%s!92_` zfnHQPANx z<#hIjIMm#*(v*!OXtF+w8kLu`o?VZ5k7{`vw{Yc^qYclpUGIM_PBN1+c{#Vxv&E*@ zxg=W2W~JuV{IuRYw3>LSI1)a!thID@R=bU+cU@DbR^_SXY`MC7HOsCN z!dO4OKV7(E_Z8T#8MA1H`99?Z!r0)qKW_#|29X3#Jb+5+>qUidbeP1NJ@)(qi2S-X zao|f0_tl(O+$R|Qwd$H{_ig|~I1fbp_$NkI!0E;Y z6JrnU{1Ra6^on{9gUUB0mwzP3S%B#h0fjo>JvV~#+X0P~JV=IG=yHG$O+p5O3NUgG zEQ}z6BTp^Fie)Sg<){Z&I8NwPR(=mO4joTLHkJ>|Tnk23E(Bo`FSbPc05lF2-+)X? z6vV3*m~IBHTy*^E!<0nA(tCOJW2G4DsH7)BxLV8kICn5lu6@U*R`w)o9;Ro$i8=Q^V%uH8n3q=+Yf;SFRZu z!+F&PKcH#8cG?aSK_Tl@K9P#8o+jry@gdexz&d(Q=47<7nw@e@FFfIRNL9^)1i@;A z28+$Z#rjv-wj#heI|<&J_DiJ*s}xd-f!{J8jfqOHE`TiHHZVIA8CjkNQ_u;Ery^^t zl1I75&u^`1_q)crO+JT4rx|z2ToSC>)Or@-D zy3S>jW*sNIZR-EBsfyaJ+Jq4BQE4?SePtD2+jY8*%FsSLZ9MY>+wk?}}}AFAw)vr{ml)8LUG-y9>^t!{~|sgpxYc0Gnkg`&~R z-pilJZjr@y5$>B=VMdZ73svct%##v%wdX~9fz6i3Q-zOKJ9wso+h?VME7}SjL=!NUG{J?M&i!>ma`eoEa@IX`5G>B1(7;%}M*%-# zfhJ(W{y;>MRz!Ic8=S}VaBKqh;~7KdnGEHxcL$kA-6E~=!hrN*zw9N+_=odt<$_H_8dbo;0=42wcAETPCVGUr~v(`Uai zb{=D!Qc!dOEU6v)2eHSZq%5iqK?B(JlCq%T6av$Cb4Rko6onlG&?CqaX7Y_C_cOC3 zYZ;_oI(}=>_07}Oep&Ws7x7-R)cc8zfe!SYxJYP``pi$FDS)4Fvw5HH=FiU6xfVqIM!hJ;Rx8c0cB7~aPtNH(Nmm5Vh{ibAoU#J6 zImRCr?(iyu_4W_6AWo3*vxTPUw@vPwy@E0`(>1Qi=%>5eSIrp^`` zK*Y?fK_6F1W>-7UsB)RPC4>>Ps9)f+^MqM}8AUm@tZ->j%&h1M8s*s!LX5&WxQcAh z8mciQej@RPm?660%>{_D+7er>%zX_{s|$Z+;G7_sfNfBgY(zLB4Ey}J9F>zX#K0f6 z?dVNIeEh?EIShmP6>M+d|0wMM85Sa4diw1hrg|ITJ}JDg@o8y>(rF9mXk5M z2@D|NA)-7>wD&wF;S_$KS=eE84`BGw3g0?6wGxu8ys4rwI?9U=*^VF22t3%mbGeOh z`!O-OpF7#Vceu~F`${bW0nYVU9ecmk31V{tF%iv&5hWofC>I~cqAt@u6|R+|HLMMX zVxuSlMFOK_EQ86#E8&KwxIr8S9tj_goWtLv4f@!&h8;Ov41{J~496vp9vX=(LK#j! zAwi*21RAV-LD>9Cw3bV_9X(X3)Kr0-UaB*7Y>t82EQ%!)(&(XuAYtTsYy-dz+w=$ir)VJpe!_$ z6SGpX^i(af3{o=VlFPC);|J8#(=_8#vdxDe|Cok+ANhYwbE*FO`Su2m1~w+&9<_9~ z-|tTU_ACGN`~CNW5WYYBn^B#SwZ(t4%3aPp z;o)|L6Rk569KGxFLUPx@!6OOa+5OjQLK5w&nAmwxkC5rZ|m&HT8G%GVZxB_@ME z>>{rnXUqyiJrT(8GMj_ap#yN_!9-lO5e8mR3cJiK3NE{_UM&=*vIU`YkiL$1%kf+1 z4=jk@7EEj`u(jy$HnzE33ZVW_J4bj}K;vT?T91YlO(|Y0FU4r+VdbmQ97%(J5 zkK*Bed8+C}FcZ@HIgdCMioV%A<*4pw_n}l*{Cr4}a(lq|injK#O?$tyvyE`S%(1`H z_wwRvk#13ElkZvij2MFGOj`fhy?nC^8`Zyo%yVcUAfEr8x&J#A{|moUBAV_^f$hpaUuyQeY3da^ zS9iRgf87YBwfe}>BO+T&Fl%rfpZh#+AM?Dq-k$Bq`vG6G_b4z%Kbd&v>qFjow*mBl z-OylnqOpLg}or7_VNwRg2za3VBK6FUfFX{|TD z`Wt0Vm2H$vdlRWYQJqDmM?JUbVqL*ZQY|5&sY*?!&%P8qhA~5+Af<{MaGo(dl&C5t zE%t!J0 zh6jqANt4ABdPxSTrVV}fLsRQal*)l&_*rFq(Ez}ClEH6LHv{J#v?+H-BZ2)Wy{K@9 z+ovXHq~DiDvm>O~r$LJo!cOuwL+Oa--6;UFE2q@g3N8Qkw5E>ytz^(&($!O47+i~$ zKM+tkAd-RbmP{s_rh+ugTD;lriL~`Xwkad#;_aM?nQ7L_muEFI}U_4$phjvYgleK~`Fo`;GiC07&Hq1F<%p;9Q;tv5b?*QnR%8DYJH3P>Svmv47Y>*LPZJy8_{9H`g6kQpyZU{oJ`m%&p~D=K#KpfoJ@ zn-3cqmHsdtN!f?~w+(t+I`*7GQA#EQC^lUA9(i6=i1PqSAc|ha91I%X&nXzjYaM{8$s&wEx@aVkQ6M{E2 zfzId#&r(XwUNtPcq4Ngze^+XaJA1EK-%&C9j>^9(secqe{}z>hR5CFNveMsVA)m#S zk)_%SidkY-XmMWlVnQ(mNJ>)ooszQ#vaK;!rPmGKXV7am^_F!Lz>;~{VrIO$;!#30XRhE1QqO_~#+Ux;B_D{Nk=grn z8Y0oR^4RqtcYM)7a%@B(XdbZCOqnX#fD{BQTeLvRHd(irHKq=4*jq34`6@VAQR8WG z^%)@5CXnD_T#f%@-l${>y$tfb>2LPmc{~5A82|16mH)R?&r#KKLs7xpN-D`=&Cm^R zvMA6#Ahr<3X>Q7|-qfTY)}32HkAz$_mibYV!I)u>bmjK`qwBe(>za^0Kt*HnFbSdO z1>+ryKCNxmm^)*$XfiDOF2|{-v3KKB?&!(S_Y=Ht@|ir^hLd978xuI&N{k>?(*f8H z=ClxVJK_%_z1TH0eUwm2J+2To7FK4o+n_na)&#VLn1m;!+CX+~WC+qg1?PA~KdOlC zW)C@pw75_xoe=w7i|r9KGIvQ$+3K?L{7TGHwrQM{dCp=Z*D}3kX7E-@sZnup!BImw z*T#a=+WcTwL78exTgBn|iNE3#EsOorO z*kt)gDzHiPt07fmisA2LWN?AymkdqTgr?=loT7z@d`wnlr6oN}@o|&JX!yPzC*Y8d zu6kWlTzE1)ckyBn+0Y^HMN+GA$wUO_LN6W>mxCo!0?oiQvT`z$jbSEu&{UHRU0E8# z%B^wOc@S!yhMT49Y)ww(Xta^8pmPCe@eI5C*ed96)AX9<>))nKx0(sci8gwob_1}4 z0DIL&vsJ1_s%<@y%U*-eX z5rN&(zef-5G~?@r79oZGW1d!WaTqQn0F6RIOa9tJ=0(kdd{d1{<*tHT#cCvl*i>YY zH+L7jq8xZNcTUBqj(S)ztTU!TM!RQ}In*n&Gn<>(60G7}4%WQL!o>hbJqNDSGwl#H z`4k+twp0cj%PsS+NKaxslAEu9!#U3xT1|_KB6`h=PI0SW`P9GTa7caD1}vKEglV8# zjKZR`pluCW19c2fM&ZG)c3T3Um;ir3y(tSCJ7Agl6|b524dy5El{^EQBG?E61H0XY z`bqg!;zhGhyMFl&(o=JWEJ8n~z)xI}A@C0d2hQGvw7nGv)?POU@(kS1m=%`|+^ika zXl8zjS?xqW$WlO?Ewa;vF~XbybHBor$f<%I&*t$F5fynwZlTGj|IjZtVfGa7l&tK} zW>I<69w(cZLu)QIVG|M2xzW@S+70NinQzk&Y0+3WT*cC)rx~04O-^<{JohU_&HL5XdUKW!uFy|i$FB|EMu0eUyW;gsf`XfIc!Z0V zeK&*hPL}f_cX=@iv>K%S5kL;cl_$v?n(Q9f_cChk8Lq$glT|=e+T*8O4H2n<=NGmn z+2*h+v;kBvF>}&0RDS>)B{1!_*XuE8A$Y=G8w^qGMtfudDBsD5>T5SB;Qo}fSkkiV ze^K^M(UthkwrD!&*tTsu>Dacdj_q`~V%r_twr$(Ct&_dKeeXE?fA&4&yASJWJ*}~- zel=@W)tusynfC_YqH4ll>4Eg`Xjs5F7Tj>tTLz<0N3)X<1px_d2yUY>X~y>>93*$) z5PuNMQLf9Bu?AAGO~a_|J2akO1M*@VYN^VxvP0F$2>;Zb9;d5Yfd8P%oFCCoZE$ z4#N$^J8rxYjUE_6{T%Y>MmWfHgScpuGv59#4u6fpTF%~KB^Ae`t1TD_^Ud#DhL+Dm zbY^VAM#MrAmFj{3-BpVSWph2b_Y6gCnCAombVa|1S@DU)2r9W<> zT5L8BB^er3zxKt1v(y&OYk!^aoQisqU zH(g@_o)D~BufUXcPt!Ydom)e|aW{XiMnes2z&rE?og>7|G+tp7&^;q?Qz5S5^yd$i z8lWr4g5nctBHtigX%0%XzIAB8U|T6&JsC4&^hZBw^*aIcuNO47de?|pGXJ4t}BB`L^d8tD`H`i zqrP8?#J@8T#;{^B!KO6J=@OWKhAerih(phML`(Rg7N1XWf1TN>=Z3Do{l_!d~DND&)O)D>ta20}@Lt77qSnVsA7>)uZAaT9bsB>u&aUQl+7GiY2|dAEg@%Al3i316y;&IhQL^8fw_nwS>f60M_-m+!5)S_6EPM7Y)(Nq^8gL7(3 zOiot`6Wy6%vw~a_H?1hLVzIT^i1;HedHgW9-P#)}Y6vF%C=P70X0Tk^z9Te@kPILI z_(gk!k+0%CG)%!WnBjjw*kAKs_lf#=5HXC00s-}oM-Q1aXYLj)(1d!_a7 z*Gg4Fe6F$*ujVjI|79Z5+Pr`us%zW@ln++2l+0hsngv<{mJ%?OfSo_3HJXOCys{Ug z00*YR-(fv<=&%Q!j%b-_ppA$JsTm^_L4x`$k{VpfLI(FMCap%LFAyq;#ns5bR7V+x zO!o;c5y~DyBPqdVQX)8G^G&jWkBy2|oWTw>)?5u}SAsI$RjT#)lTV&Rf8;>u*qXnb z8F%Xb=7#$m)83z%`E;49)t3fHInhtc#kx4wSLLms!*~Z$V?bTyUGiS&m>1P(952(H zuHdv=;o*{;5#X-uAyon`hP}d#U{uDlV?W?_5UjJvf%11hKwe&(&9_~{W)*y1nR5f_ z!N(R74nNK`y8>B!0Bt_Vr!;nc3W>~RiKtGSBkNlsR#-t^&;$W#)f9tTlZz>n*+Fjz z3zXZ;jf(sTM(oDzJt4FJS*8c&;PLTW(IQDFs_5QPy+7yhi1syPCarvqrHFcf&yTy)^O<1EBx;Ir`5W{TIM>{8w&PB>ro4;YD<5LF^TjTb0!zAP|QijA+1Vg>{Afv^% zmrkc4o6rvBI;Q8rj4*=AZacy*n8B{&G3VJc)so4$XUoie0)vr;qzPZVbb<#Fc=j+8CGBWe$n|3K& z_@%?{l|TzKSlUEO{U{{%Fz_pVDxs7i9H#bnbCw7@4DR=}r_qV!Zo~CvD4ZI*+j3kO zW6_=|S`)(*gM0Z;;}nj`73OigF4p6_NPZQ-Od~e$c_);;4-7sR>+2u$6m$Gf%T{aq zle>e3(*Rt(TPD}03n5)!Ca8Pu!V}m6v0o1;5<1h$*|7z|^(3$Y&;KHKTT}hV056wuF0Xo@mK-52~r=6^SI1NC%c~CC?n>yX6wPTgiWYVz!Sx^atLby9YNn1Rk{g?|pJaxD4|9cUf|V1_I*w zzxK)hRh9%zOl=*$?XUjly5z8?jPMy%vEN)f%T*|WO|bp5NWv@B(K3D6LMl!-6dQg0 zXNE&O>Oyf%K@`ngCvbGPR>HRg5!1IV$_}m@3dWB7x3t&KFyOJn9pxRXCAzFr&%37wXG;z^xaO$ekR=LJG ztIHpY8F5xBP{mtQidqNRoz= z@){+N3(VO5bD+VrmS^YjG@+JO{EOIW)9=F4v_$Ed8rZtHvjpiEp{r^c4F6Ic#ChlC zJX^DtSK+v(YdCW)^EFcs=XP7S>Y!4=xgmv>{S$~@h=xW-G4FF9?I@zYN$e5oF9g$# zb!eVU#J+NjLyX;yb)%SY)xJdvGhsnE*JEkuOVo^k5PyS=o#vq!KD46UTW_%R=Y&0G zFj6bV{`Y6)YoKgqnir2&+sl+i6foAn-**Zd1{_;Zb7Ki=u394C5J{l^H@XN`_6XTKY%X1AgQM6KycJ+= zYO=&t#5oSKB^pYhNdzPgH~aEGW2=ec1O#s-KG z71}LOg@4UEFtp3GY1PBemXpNs6UK-ax*)#$J^pC_me;Z$Je(OqLoh|ZrW*mAMBFn< zHttjwC&fkVfMnQeen8`Rvy^$pNRFVaiEN4Pih*Y3@jo!T0nsClN)pdrr9AYLcZxZ| zJ5Wlj+4q~($hbtuY zVQ7hl>4-+@6g1i`1a)rvtp-;b0>^`Dloy(#{z~ytgv=j4q^Kl}wD>K_Y!l~ zp(_&7sh`vfO(1*MO!B%<6E_bx1)&s+Ae`O)a|X=J9y~XDa@UB`m)`tSG4AUhoM=5& znWoHlA-(z@3n0=l{E)R-p8sB9XkV zZ#D8wietfHL?J5X0%&fGg@MH~(rNS2`GHS4xTo7L$>TPme+Is~!|79=^}QbPF>m%J zFMkGzSndiPO|E~hrhCeo@&Ea{M(ieIgRWMf)E}qeTxT8Q#g-!Lu*x$v8W^M^>?-g= zwMJ$dThI|~M06rG$Sv@C@tWR>_YgaG&!BAbkGggVQa#KdtDB)lMLNVLN|51C@F^y8 zCRvMB^{GO@j=cHfmy}_pCGbP%xb{pNN>? z?7tBz$1^zVaP|uaatYaIN+#xEN4jBzwZ|YI_)p(4CUAz1ZEbDk>J~Y|63SZaak~#0 zoYKruYsWHoOlC1(MhTnsdUOwQfz5p6-D0}4;DO$B;7#M{3lSE^jnTT;ns`>!G%i*F?@pR1JO{QTuD0U+~SlZxcc8~>IB{)@8p`P&+nDxNj`*gh|u?yrv$phpQcW)Us)bi`kT%qLj(fi{dWRZ%Es2!=3mI~UxiW0$-v3vUl?#g{p6eF zMEUAqo5-L0Ar(s{VlR9g=j7+lt!gP!UN2ICMokAZ5(Agd>})#gkA2w|5+<%-CuEP# zqgcM}u@3(QIC^Gx<2dbLj?cFSws_f3e%f4jeR?4M^M3cx1f+Qr6ydQ>n)kz1s##2w zk}UyQc+Z5G-d-1}{WzjkLXgS-2P7auWSJ%pSnD|Uivj5u!xk0 z_^-N9r9o;(rFDt~q1PvE#iJZ_f>J3gcP$)SOqhE~pD2|$=GvpL^d!r z6u=sp-CrMoF7;)}Zd7XO4XihC4ji?>V&(t^?@3Q&t9Mx=qex6C9d%{FE6dvU6%d94 zIE;hJ1J)cCqjv?F``7I*6bc#X)JW2b4f$L^>j{*$R`%5VHFi*+Q$2;nyieduE}qdS{L8y8F08yLs?w}{>8>$3236T-VMh@B zq-nujsb_1aUv_7g#)*rf9h%sFj*^mIcImRV*k~Vmw;%;YH(&ylYpy!&UjUVqqtfG` zox3esju?`unJJA_zKXRJP)rA3nXc$m^{S&-p|v|-0x9LHJm;XIww7C#R$?00l&Yyj z=e}gKUOpsImwW?N)+E(awoF@HyP^EhL+GlNB#k?R<2>95hz!h9sF@U20DHSB3~WMa zk90+858r@-+vWwkawJ)8ougd(i#1m3GLN{iSTylYz$brAsP%=&m$mQQrH$g%3-^VR zE%B`Vi&m8f3T~&myTEK28BDWCVzfWir1I?03;pX))|kY5ClO^+bae z*7E?g=3g7EiisYOrE+lA)2?Ln6q2*HLNpZEWMB|O-JI_oaHZB%CvYB(%=tU= zE*OY%QY58fW#RG5=gm0NR#iMB=EuNF@)%oZJ}nmm=tsJ?eGjia{e{yuU0l3{d^D@)kVDt=1PE)&tf_hHC%0MB znL|CRCPC}SeuVTdf>-QV70`0(EHizc21s^sU>y%hW0t!0&y<7}Wi-wGy>m%(-jsDj zP?mF|>p_K>liZ6ZP(w5(|9Ga%>tLgb$|doDDfkdW>Z z`)>V2XC?NJT26mL^@ zf+IKr27TfM!UbZ@?zRddC7#6ss1sw%CXJ4FWC+t3lHZupzM77m^=9 z&(a?-LxIq}*nvv)y?27lZ{j zifdl9hyJudyP2LpU$-kXctshbJDKS{WfulP5Dk~xU4Le4c#h^(YjJit4#R8_khheS z|8(>2ibaHES4+J|DBM7I#QF5u-*EdN{n=Kt@4Zt?@Tv{JZA{`4 zU#kYOv{#A&gGPwT+$Ud}AXlK3K7hYzo$(fBSFjrP{QQ zeaKg--L&jh$9N}`pu{Bs>?eDFPaWY4|9|foN%}i;3%;@4{dc+iw>m}{3rELqH21G! z`8@;w-zsJ1H(N3%|1B@#ioLOjib)j`EiJqPQVSbPSPVHCj6t5J&(NcWzBrzCiDt{4 zdlPAUKldz%6x5II1H_+jv)(xVL+a;P+-1hv_pM>gMRr%04@k;DTokASSKKhU1Qms| zrWh3a!b(J3n0>-tipg{a?UaKsP7?+|@A+1WPDiQIW1Sf@qDU~M_P65_s}7(gjTn0X zucyEm)o;f8UyshMy&>^SC3I|C6jR*R_GFwGranWZe*I>K+0k}pBuET&M~ z;Odo*ZcT?ZpduHyrf8E%IBFtv;JQ!N_m>!sV6ly$_1D{(&nO~w)G~Y`7sD3#hQk%^ zp}ucDF_$!6DAz*PM8yE(&~;%|=+h(Rn-=1Wykas_-@d&z#=S}rDf`4w(rVlcF&lF! z=1)M3YVz7orwk^BXhslJ8jR);sh^knJW(Qmm(QdSgIAIdlN4Te5KJisifjr?eB{FjAX1a0AB>d?qY4Wx>BZ8&}5K0fA+d{l8 z?^s&l8#j7pR&ijD?0b%;lL9l$P_mi2^*_OL+b}4kuLR$GAf85sOo02?Y#90}CCDiS zZ%rbCw>=H~CBO=C_JVV=xgDe%b4FaEFtuS7Q1##y686r%F6I)s-~2(}PWK|Z8M+Gu zl$y~5@#0Ka%$M<&Cv%L`a8X^@tY&T7<0|(6dNT=EsRe0%kp1Qyq!^43VAKYnr*A5~ zsI%lK1ewqO;0TpLrT9v}!@vJK{QoVa_+N4FYT#h?Y8rS1S&-G+m$FNMP?(8N`MZP zels(*?kK{{^g9DOzkuZXJ2;SrOQsp9T$hwRB1(phw1c7`!Q!by?Q#YsSM#I12RhU{$Q+{xj83axHcftEc$mNJ8_T7A-BQc*k(sZ+~NsO~xAA zxnbb%dam_fZlHvW7fKXrB~F&jS<4FD2FqY?VG?ix*r~MDXCE^WQ|W|WM;gsIA4lQP zJ2hAK@CF*3*VqPr2eeg6GzWFlICi8S>nO>5HvWzyZTE)hlkdC_>pBej*>o0EOHR|) z$?};&I4+_?wvL*g#PJ9)!bc#9BJu1(*RdNEn>#Oxta(VWeM40ola<0aOe2kSS~{^P zDJBd}0L-P#O-CzX*%+$#v;(x%<*SPgAje=F{Zh-@ucd2DA(yC|N_|ocs*|-!H%wEw z@Q!>siv2W;C^^j^59OAX03&}&D*W4EjCvfi(ygcL#~t8XGa#|NPO+*M@Y-)ctFA@I z-p7npT1#5zOLo>7q?aZpCZ=iecn3QYklP;gF0bq@>oyBq94f6C=;Csw3PkZ|5q=(c zfs`aw?II0e(h=|7o&T+hq&m$; zBrE09Twxd9BJ2P+QPN}*OdZ-JZV7%av@OM7v!!NL8R;%WFq*?{9T3{ct@2EKgc8h) zMxoM$SaF#p<`65BwIDfmXG6+OiK0e)`I=!A3E`+K@61f}0e z!2a*FOaDrOe>U`q%K!QN`&=&0C~)CaL3R4VY(NDt{Xz(Xpqru5=r#uQN1L$Je1*dkdqQ*=lofQaN%lO!<5z9ZlHgxt|`THd>2 zsWfU$9=p;yLyJyM^t zS2w9w?Bpto`@H^xJpZDKR1@~^30Il6oFGfk5%g6w*C+VM)+%R@gfIwNprOV5{F^M2 zO?n3DEzpT+EoSV-%OdvZvNF+pDd-ZVZ&d8 zKeIyrrfPN=EcFRCPEDCVflX#3-)Ik_HCkL(ejmY8vzcf-MTA{oHk!R2*36`O68$7J zf}zJC+bbQk--9Xm!u#lgLvx8TXx2J258E5^*IZ(FXMpq$2LUUvhWQPs((z1+2{Op% z?J}9k5^N=z;7ja~zi8a_-exIqWUBJwohe#4QJ`|FF*$C{lM18z^#hX6!5B8KAkLUX ziP=oti-gpV(BsLD{0(3*dw}4JxK23Y7M{BeFPucw!sHpY&l%Ws4pSm`+~V7;bZ%Dx zeI)MK=4vC&5#;2MT7fS?^ch9?2;%<8Jlu-IB&N~gg8t;6S-#C@!NU{`p7M8@2iGc& zg|JPg%@gCoCQ&s6JvDU&`X2S<57f(k8nJ1wvBu{8r?;q3_kpZZ${?|( z+^)UvR33sjSd)aT!UPkA;ylO6{aE3MQa{g%Mcf$1KONcjO@&g5zPHWtzM1rYC{_K> zgQNcs<{&X{OA=cEWw5JGqpr0O>x*Tfak2PE9?FuWtz^DDNI}rwAaT0(bdo-<+SJ6A z&}S%boGMWIS0L}=S>|-#kRX;e^sUsotry(MjE|3_9duvfc|nwF#NHuM-w7ZU!5ei8 z6Mkf>2)WunY2eU@C-Uj-A zG(z0Tz2YoBk>zCz_9-)4a>T46$(~kF+Y{#sA9MWH%5z#zNoz)sdXq7ZR_+`RZ%0(q zC7&GyS_|BGHNFl8Xa%@>iWh%Gr?=J5<(!OEjauj5jyrA-QXBjn0OAhJJ9+v=!LK`` z@g(`^*84Q4jcDL`OA&ZV60djgwG`|bcD*i50O}Q{9_noRg|~?dj%VtKOnyRs$Uzqg z191aWoR^rDX#@iSq0n z?9Sg$WSRPqSeI<}&n1T3!6%Wj@5iw5`*`Btni~G=&;J+4`7g#OQTa>u`{4ZZ(c@s$ zK0y;ySOGD-UTjREKbru{QaS>HjN<2)R%Nn-TZiQ(Twe4p@-saNa3~p{?^V9Nixz@a zykPv~<@lu6-Ng9i$Lrk(xi2Tri3q=RW`BJYOPC;S0Yly%77c727Yj-d1vF!Fuk{Xh z)lMbA69y7*5ufET>P*gXQrxsW+ zz)*MbHZv*eJPEXYE<6g6_M7N%#%mR{#awV3i^PafNv(zyI)&bH?F}2s8_rR(6%!V4SOWlup`TKAb@ee>!9JKPM=&8g#BeYRH9FpFybxBXQI2|g}FGJfJ+ zY-*2hB?o{TVL;Wt_ek;AP5PBqfDR4@Z->_182W z{P@Mc27j6jE*9xG{R$>6_;i=y{qf(c`5w9fa*`rEzX6t!KJ(p1H|>J1pC-2zqWENF zmm=Z5B4u{cY2XYl(PfrInB*~WGWik3@1oRhiMOS|D;acnf-Bs(QCm#wR;@Vf!hOPJ zgjhDCfDj$HcyVLJ=AaTbQ{@vIv14LWWF$=i-BDoC11}V;2V8A`S>_x)vIq44-VB-v z*w-d}$G+Ql?En8j!~ZkCpQ$|cA0|+rrY>tiCeWxkRGPoarxlGU2?7%k#F693RHT24 z-?JsiXlT2PTqZqNb&sSc>$d;O4V@|b6VKSWQb~bUaWn1Cf0+K%`Q&Wc<>mQ>*iEGB zbZ;aYOotBZ{vH3y<0A*L0QVM|#rf*LIsGx(O*-7)r@yyBIzJnBFSKBUSl1e|8lxU* zzFL+YDVVkIuzFWeJ8AbgN&w(4-7zbiaMn{5!JQXu)SELk*CNL+Fro|2v|YO)1l15t zs(0^&EB6DPMyaqvY>=KL>)tEpsn;N5Q#yJj<9}ImL((SqErWN3Q=;tBO~ExTCs9hB z2E$7eN#5wX4<3m^5pdjm#5o>s#eS_Q^P)tm$@SawTqF*1dj_i#)3};JslbLKHXl_N z)Fxzf>FN)EK&Rz&*|6&%Hs-^f{V|+_vL1S;-1K-l$5xiC@}%uDuwHYhmsV?YcOUlk zOYkG5v2+`+UWqpn0aaaqrD3lYdh0*!L`3FAsNKu=Q!vJu?Yc8n|CoYyDo_`r0mPoo z8>XCo$W4>l(==h?2~PoRR*kEe)&IH{1sM41mO#-36`02m#nTX{r*r`Q5rZ2-sE|nA zhnn5T#s#v`52T5|?GNS`%HgS2;R(*|^egNPDzzH_z^W)-Q98~$#YAe)cEZ%vge965AS_am#DK#pjPRr-!^za8>`kksCAUj(Xr*1NW5~e zpypt_eJpD&4_bl_y?G%>^L}=>xAaV>KR6;^aBytqpiHe%!j;&MzI_>Sx7O%F%D*8s zSN}cS^<{iiK)=Ji`FpO#^zY!_|D)qeRNAtgmH)m;qC|mq^j(|hL`7uBz+ULUj37gj zksdbnU+LSVo35riSX_4z{UX=%n&}7s0{WuZYoSfwAP`8aKN9P@%e=~1`~1ASL-z%# zw>DO&ixr}c9%4InGc*_y42bdEk)ZdG7-mTu0bD@_vGAr*NcFoMW;@r?@LUhRI zCUJgHb`O?M3!w)|CPu~ej%fddw20lod?Ufp8Dmt0PbnA0J%KE^2~AIcnKP()025V> zG>noSM3$5Btmc$GZoyP^v1@Poz0FD(6YSTH@aD0}BXva?LphAiSz9f&Y(aDAzBnUh z?d2m``~{z;{}kZJ>a^wYI?ry(V9hIoh;|EFc0*-#*`$T0DRQ1;WsqInG;YPS+I4{g zJGpKk%%Sdc5xBa$Q^_I~(F97eqDO7AN3EN0u)PNBAb+n+ zWBTxQx^;O9o0`=g+Zrt_{lP!sgWZHW?8bLYS$;1a@&7w9rD9|Ge;Gb?sEjFoF9-6v z#!2)t{DMHZ2@0W*fCx;62d#;jouz`R5Y(t{BT=$N4yr^^o$ON8d{PQ=!O zX17^CrdM~7D-;ZrC!||<+FEOxI_WI3CA<35va%4v>gc zEX-@h8esj=a4szW7x{0g$hwoWRQG$yK{@3mqd-jYiVofJE!Wok1* znV7Gm&Ssq#hFuvj1sRyHg(6PFA5U*Q8Rx>-blOs=lb`qa{zFy&n4xY;sd$fE+<3EI z##W$P9M{B3c3Si9gw^jlPU-JqD~Cye;wr=XkV7BSv#6}DrsXWFJ3eUNrc%7{=^sP> zrp)BWKA9<}^R9g!0q7yWlh;gr_TEOD|#BmGq<@IV;ueg+D2}cjpp+dPf&Q(36sFU&K8}hA85U61faW&{ zlB`9HUl-WWCG|<1XANN3JVAkRYvr5U4q6;!G*MTdSUt*Mi=z_y3B1A9j-@aK{lNvx zK%p23>M&=KTCgR!Ee8c?DAO2_R?B zkaqr6^BSP!8dHXxj%N1l+V$_%vzHjqvu7p@%Nl6;>y*S}M!B=pz=aqUV#`;h%M0rU zHfcog>kv3UZAEB*g7Er@t6CF8kHDmKTjO@rejA^ULqn!`LwrEwOVmHx^;g|5PHm#B zZ+jjWgjJ!043F+&#_;D*mz%Q60=L9Ove|$gU&~As5^uz@2-BfQ!bW)Khn}G+Wyjw- z19qI#oB(RSNydn0t~;tAmK!P-d{b-@@E5|cdgOS#!>%#Rj6ynkMvaW@37E>@hJP^8 z2zk8VXx|>#R^JCcWdBCy{0nPmYFOxN55#^-rlqobe0#L6)bi?E?SPymF*a5oDDeSd zO0gx?#KMoOd&G(2O@*W)HgX6y_aa6iMCl^~`{@UR`nMQE`>n_{_aY5nA}vqU8mt8H z`oa=g0SyiLd~BxAj2~l$zRSDHxvDs;I4>+M$W`HbJ|g&P+$!U7-PHX4RAcR0szJ*( ze-417=bO2q{492SWrqDK+L3#ChUHtz*@MP)e^%@>_&#Yk^1|tv@j4%3T)diEX zATx4K*hcO`sY$jk#jN5WD<=C3nvuVsRh||qDHnc~;Kf59zr0;c7VkVSUPD%NnnJC_ zl3F^#f_rDu8l}l8qcAz0FFa)EAt32IUy_JLIhU_J^l~FRH&6-ivSpG2PRqzDdMWft>Zc(c)#tb%wgmWN%>IOPm zZi-noqS!^Ftb81pRcQi`X#UhWK70hy4tGW1mz|+vI8c*h@ zfFGJtW3r>qV>1Z0r|L>7I3un^gcep$AAWfZHRvB|E*kktY$qQP_$YG60C@X~tTQjB3%@`uz!qxtxF+LE!+=nrS^07hn` zEgAp!h|r03h7B!$#OZW#ACD+M;-5J!W+{h|6I;5cNnE(Y863%1(oH}_FTW})8zYb$7czP zg~Szk1+_NTm6SJ0MS_|oSz%e(S~P-&SFp;!k?uFayytV$8HPwuyELSXOs^27XvK-D zOx-Dl!P|28DK6iX>p#Yb%3`A&CG0X2S43FjN%IB}q(!hC$fG}yl1y9W&W&I@KTg6@ zK^kpH8=yFuP+vI^+59|3%Zqnb5lTDAykf z9S#X`3N(X^SpdMyWQGOQRjhiwlj!0W-yD<3aEj^&X%=?`6lCy~?`&WSWt z?U~EKFcCG_RJ(Qp7j=$I%H8t)Z@6VjA#>1f@EYiS8MRHZphp zMA_5`znM=pzUpBPO)pXGYpQ6gkine{6u_o!P@Q+NKJ}k!_X7u|qfpAyIJb$_#3@wJ z<1SE2Edkfk9C!0t%}8Yio09^F`YGzpaJHGk*-ffsn85@)%4@`;Fv^8q(-Wk7r=Q8p zT&hD`5(f?M{gfzGbbwh8(}G#|#fDuk7v1W)5H9wkorE0ZZjL0Q1=NRGY>zwgfm81DdoaVwNH;or{{eSyybt)m<=zXoA^RALYG-2t zouH|L*BLvmm9cdMmn+KGopyR@4*=&0&4g|FLoreZOhRmh=)R0bg~ zT2(8V_q7~42-zvb)+y959OAv!V$u(O3)%Es0M@CRFmG{5sovIq4%8Ahjk#*5w{+)+ zMWQoJI_r$HxL5km1#6(e@{lK3Udc~n0@g`g$s?VrnQJ$!oPnb?IHh-1qA`Rz$)Ai< z6w$-MJW-gKNvOhL+XMbE7&mFt`x1KY>k4(!KbbpZ`>`K@1J<(#vVbjx@Z@(6Q}MF# zMnbr-f55(cTa^q4+#)=s+ThMaV~E`B8V=|W_fZWDwiso8tNMTNse)RNBGi=gVwgg% zbOg8>mbRN%7^Um-7oj4=6`$|(K7!+t^90a{$18Z>}<#!bm%ZEFQ{X(yBZMc>lCz0f1I2w9Sq zuGh<9<=AO&g6BZte6hn>Qmvv;Rt)*cJfTr2=~EnGD8P$v3R|&1RCl&7)b+`=QGapi zPbLg_pxm`+HZurtFZ;wZ=`Vk*do~$wB zxoW&=j0OTbQ=Q%S8XJ%~qoa3Ea|au5o}_(P;=!y-AjFrERh%8la!z6Fn@lR?^E~H12D?8#ht=1F;7@o4$Q8GDj;sSC%Jfn01xgL&%F2 zwG1|5ikb^qHv&9hT8w83+yv&BQXOQyMVJSBL(Ky~p)gU3#%|blG?IR9rP^zUbs7rOA0X52Ao=GRt@C&zlyjNLv-} z9?*x{y(`509qhCV*B47f2hLrGl^<@SuRGR!KwHei?!CM10Tq*YDIoBNyRuO*>3FU? zHjipIE#B~y3FSfOsMfj~F9PNr*H?0oHyYB^G(YyNh{SxcE(Y-`x5jFMKb~HO*m+R% zrq|ic4fzJ#USpTm;X7K+E%xsT_3VHKe?*uc4-FsILUH;kL>_okY(w`VU*8+l>o>Jm ziU#?2^`>arnsl#)*R&nf_%>A+qwl%o{l(u)M?DK1^mf260_oteV3#E_>6Y4!_hhVD zM8AI6MM2V*^_M^sQ0dmHu11fy^kOqXqzpr?K$`}BKWG`=Es(9&S@K@)ZjA{lj3ea7_MBP zk(|hBFRjHVMN!sNUkrB;(cTP)T97M$0Dtc&UXSec<+q?y>5=)}S~{Z@ua;1xt@=T5 zI7{`Z=z_X*no8s>mY;>BvEXK%b`a6(DTS6t&b!vf_z#HM{Uoy_5fiB(zpkF{})ruka$iX*~pq1ZxD?q68dIo zIZSVls9kFGsTwvr4{T_LidcWtt$u{kJlW7moRaH6+A5hW&;;2O#$oKyEN8kx`LmG)Wfq4ykh+q{I3|RfVpkR&QH_x;t41Uw z`P+tft^E2B$domKT@|nNW`EHwyj>&}K;eDpe z1bNOh=fvIfk`&B61+S8ND<(KC%>y&?>opCnY*r5M+!UrWKxv0_QvTlJc>X#AaI^xo zaRXL}t5Ej_Z$y*|w*$6D+A?Lw-CO-$itm^{2Ct82-<0IW)0KMNvJHgBrdsIR0v~=H z?n6^}l{D``Me90`^o|q!olsF?UX3YSq^6Vu>Ijm>>PaZI8G@<^NGw{Cx&%|PwYrfw zR!gX_%AR=L3BFsf8LxI|K^J}deh0ZdV?$3r--FEX`#INxsOG6_=!v)DI>0q|BxT)z z-G6kzA01M?rba+G_mwNMQD1mbVbNTWmBi*{s_v_Ft9m2Avg!^78(QFu&n6mbRJ2bA zv!b;%yo{g*9l2)>tsZJOOp}U~8VUH`}$ z8p_}t*XIOehezolNa-a2x0BS})Y9}&*TPgua{Ewn-=wVrmJUeU39EKx+%w%=ixQWK zDLpwaNJs65#6o7Ln7~~X+p_o2BR1g~VCfxLzxA{HlWAI6^H;`juI=&r1jQrUv_q0Z z1Ja-tjdktrrP>GOC*#p?*xfQU5MqjMsBe!9lh(u8)w$e@Z|>aUHI5o;MGw*|Myiz3 z-f0;pHg~Q#%*Kx8MxH%AluVXjG2C$)WL-K63@Q`#y9_k_+}eR(x4~dp7oV-ek0H>I zgy8p#i4GN{>#v=pFYUQT(g&b$OeTy-X_#FDgNF8XyfGY6R!>inYn8IR2RDa&O!(6< znXs{W!bkP|s_YI*Yx%4stI`=ZO45IK6rBs`g7sP40ic}GZ58s?Mc$&i`kq_tfci>N zIHrC0H+Qpam1bNa=(`SRKjixBTtm&e`j9porEci!zdlg1RI0Jw#b(_Tb@RQK1Zxr_ z%7SUeH6=TrXt3J@js`4iDD0=IoHhK~I7^W8^Rcp~Yaf>2wVe|Hh1bUpX9ATD#moByY57-f2Ef1TP^lBi&p5_s7WGG9|0T}dlfxOx zXvScJO1Cnq`c`~{Dp;{;l<-KkCDE+pmexJkd}zCgE{eF=)K``-qC~IT6GcRog_)!X z?fK^F8UDz$(zFUrwuR$qro5>qqn>+Z%<5>;_*3pZ8QM|yv9CAtrAx;($>4l^_$_-L z*&?(77!-=zvnCVW&kUcZMb6;2!83si518Y%R*A3JZ8Is|kUCMu`!vxDgaWjs7^0j( ziTaS4HhQ)ldR=r)_7vYFUr%THE}cPF{0H45FJ5MQW^+W>P+eEX2kLp3zzFe*-pFVA zdDZRybv?H|>`9f$AKVjFWJ=wegO7hOOIYCtd?Vj{EYLT*^gl35|HQ`R=ti+ADm{jyQE7K@kdjuqJhWVSks>b^ zxha88-h3s;%3_5b1TqFCPTxVjvuB5U>v=HyZ$?JSk+&I%)M7KE*wOg<)1-Iy)8-K! z^XpIt|0ibmk9RtMmlUd7#Ap3Q!q9N4atQy)TmrhrFhfx1DAN`^vq@Q_SRl|V z#lU<~n67$mT)NvHh`%als+G-)x1`Y%4Bp*6Un5Ri9h=_Db zA-AdP!f>f0m@~>7X#uBM?diI@)Egjuz@jXKvm zJo+==juc9_<;CqeRaU9_Mz@;3e=E4=6TK+c`|uu#pIqhSyNm`G(X)&)B`8q0RBv#> z`gGlw(Q=1Xmf55VHj%C#^1lpc>LY8kfA@|rlC1EA<1#`iuyNO z(=;irt{_&K=i4)^x%;U(Xv<)+o=dczC5H3W~+e|f~{*ucxj@{Yi-cw^MqYr3fN zF5D+~!wd$#al?UfMnz(@K#wn`_5na@rRr8XqN@&M&FGEC@`+OEv}sI1hw>Up0qAWf zL#e4~&oM;TVfjRE+10B_gFlLEP9?Q-dARr3xi6nQqnw>k-S;~b z;!0s2VS4}W8b&pGuK=7im+t(`nz@FnT#VD|!)eQNp-W6)@>aA+j~K*H{$G`y2|QHY z|Hmy+CR@#jWY4~)lr1qBJB_RfHJFfP<}pK5(#ZZGSqcpyS&}01LnTWk5fzmXMGHkJ zTP6L^B+uj;lmB_W<~4=${+v0>z31M!-_O@o-O9GyW)j_mjx}!0@br_LE-7SIuPP84 z;5=O(U*g_um0tyG|61N@d9lEuOeiRd+#NY^{nd5;-CVlw&Ap7J?qwM^?E29wvS}2d zbzar4Fz&RSR(-|s!Z6+za&Z zY#D<5q_JUktIzvL0)yq_kLWG6DO{ri=?c!y!f(Dk%G{8)k`Gym%j#!OgXVDD3;$&v@qy#ISJfp=Vm>pls@9-mapVQChAHHd-x+OGx)(*Yr zC1qDUTZ6mM(b_hi!TuFF2k#8uI2;kD70AQ&di$L*4P*Y-@p`jdm%_c3f)XhYD^6M8&#Y$ZpzQMcR|6nsH>b=*R_Von!$BTRj7yGCXokoAQ z&ANvx0-Epw`QIEPgI(^cS2f(Y85yV@ygI{ewyv5Frng)e}KCZF7JbR(&W618_dcEh(#+^zZFY;o<815<5sOHQdeax9_!PyM&;{P zkBa5xymca0#)c#tke@3KNEM8a_mT&1gm;p&&JlMGH(cL(b)BckgMQ^9&vRwj!~3@l zY?L5}=Jzr080OGKb|y`ee(+`flQg|!lo6>=H)X4`$Gz~hLmu2a%kYW_Uu8x09Pa0J zKZ`E$BKJ=2GPj_3l*TEcZ*uYRr<*J^#5pILTT;k_cgto1ZL-%slyc16J~OH-(RgDA z%;EjEnoUkZ&acS{Q8`{i6T5^nywgqQI5bDIymoa7CSZG|WWVk>GM9)zy*bNih|QIm z%0+(Nnc*a_xo;$=!HQYaapLms>J1ToyjtFByY`C2H1wT#178#4+|{H0BBqtCdd$L% z_3Hc60j@{t9~MjM@LBalR&6@>B;9?r<7J~F+WXyYu*y3?px*=8MAK@EA+jRX8{CG?GI-< z54?Dc9CAh>QTAvyOEm0^+x;r2BWX|{3$Y7)L5l*qVE*y0`7J>l2wCmW zL1?|a`pJ-l{fb_N;R(Z9UMiSj6pQjOvQ^%DvhIJF!+Th7jO2~1f1N+(-TyCFYQZYw z4)>7caf^Ki_KJ^Zx2JUb z&$3zJy!*+rCV4%jqwyuNY3j1ZEiltS0xTzd+=itTb;IPYpaf?8Y+RSdVdpacB(bVQ zC(JupLfFp8y43%PMj2}T|VS@%LVp>hv4Y!RPMF?pp8U_$xCJ)S zQx!69>bphNTIb9yn*_yfj{N%bY)t{L1cs8<8|!f$;UQ*}IN=2<6lA;x^(`8t?;+ST zh)z4qeYYgZkIy{$4x28O-pugO&gauRh3;lti9)9Pvw+^)0!h~%m&8Q!AKX%urEMnl z?yEz?g#ODn$UM`+Q#$Q!6|zsq_`dLO5YK-6bJM6ya>}H+vnW^h?o$z;V&wvuM$dR& zeEq;uUUh$XR`TWeC$$c&Jjau2it3#%J-y}Qm>nW*s?En?R&6w@sDXMEr#8~$=b(gk zwDC3)NtAP;M2BW_lL^5ShpK$D%@|BnD{=!Tq)o(5@z3i7Z){} zGr}Exom_qDO{kAVkZ*MbLNHE666Kina#D{&>Jy%~w7yX$oj;cYCd^p9zy z8*+wgSEcj$4{WxKmCF(5o7U4jqwEvO&dm1H#7z}%VXAbW&W24v-tS6N3}qrm1OnE)fUkoE8yMMn9S$?IswS88tQWm4#Oid#ckgr6 zRtHm!mfNl-`d>O*1~d7%;~n+{Rph6BBy^95zqI{K((E!iFQ+h*C3EsbxNo_aRm5gj zKYug($r*Q#W9`p%Bf{bi6;IY0v`pB^^qu)gbg9QHQ7 zWBj(a1YSu)~2RK8Pi#C>{DMlrqFb9e_RehEHyI{n?e3vL_}L>kYJC z_ly$$)zFi*SFyNrnOt(B*7E$??s67EO%DgoZL2XNk8iVx~X_)o++4oaK1M|ou73vA0K^503j@uuVmLcHH4ya-kOIDfM%5%(E z+Xpt~#7y2!KB&)PoyCA+$~DXqxPxxALy!g-O?<9+9KTk4Pgq4AIdUkl`1<1#j^cJg zgU3`0hkHj_jxV>`Y~%LAZl^3o0}`Sm@iw7kwff{M%VwtN)|~!p{AsfA6vB5UolF~d zHWS%*uBDt<9y!9v2Xe|au&1j&iR1HXCdyCjxSgG*L{wmTD4(NQ=mFjpa~xooc6kju z`~+d{j7$h-;HAB04H!Zscu^hZffL#9!p$)9>sRI|Yovm)g@F>ZnosF2EgkU3ln0bR zTA}|+E(tt)!SG)-bEJi_0m{l+(cAz^pi}`9=~n?y&;2eG;d9{M6nj>BHGn(KA2n|O zt}$=FPq!j`p&kQ8>cirSzkU0c08%8{^Qyqi-w2LoO8)^E7;;I1;HQ6B$u0nNaX2CY zSmfi)F`m94zL8>#zu;8|{aBui@RzRKBlP1&mfFxEC@%cjl?NBs`cr^nm){>;$g?rhKr$AO&6qV_Wbn^}5tfFBry^e1`%du2~o zs$~dN;S_#%iwwA_QvmMjh%Qo?0?rR~6liyN5Xmej8(*V9ym*T`xAhHih-v$7U}8=dfXi2i*aAB!xM(Xekg*ix@r|ymDw*{*s0?dlVys2e)z62u1 z+k3esbJE=-P5S$&KdFp+2H7_2e=}OKDrf( z9-207?6$@f4m4B+9E*e((Y89!q?zH|mz_vM>kp*HGXldO0Hg#!EtFhRuOm$u8e~a9 z5(roy7m$Kh+zjW6@zw{&20u?1f2uP&boD}$#Zy)4o&T;vyBoqFiF2t;*g=|1=)PxB z8eM3Mp=l_obbc?I^xyLz?4Y1YDWPa+nm;O<$Cn;@ane616`J9OO2r=rZr{I_Kizyc zP#^^WCdIEp*()rRT+*YZK>V@^Zs=ht32x>Kwe zab)@ZEffz;VM4{XA6e421^h~`ji5r%)B{wZu#hD}f3$y@L0JV9f3g{-RK!A?vBUA}${YF(vO4)@`6f1 z-A|}e#LN{)(eXloDnX4Vs7eH|<@{r#LodP@Nz--$Dg_Par%DCpu2>2jUnqy~|J?eZ zBG4FVsz_A+ibdwv>mLp>P!(t}E>$JGaK$R~;fb{O3($y1ssQQo|5M;^JqC?7qe|hg zu0ZOqeFcp?qVn&Qu7FQJ4hcFi&|nR!*j)MF#b}QO^lN%5)4p*D^H+B){n8%VPUzi! zDihoGcP71a6!ab`l^hK&*dYrVYzJ0)#}xVrp!e;lI!+x+bfCN0KXwUAPU9@#l7@0& QuEJmfE|#`Dqx|px0L@K;Y5)KL literal 0 HcmV?d00001 diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100755 index 0000000..2e6e589 --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew new file mode 100755 index 0000000..1b6c787 --- /dev/null +++ b/gradlew @@ -0,0 +1,234 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit + +APP_NAME="Gradle" +APP_BASE_NAME=${0##*/} + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100755 index 0000000..ac1b06f --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/jitpack.yml b/jitpack.yml new file mode 100644 index 0000000..f78f664 --- /dev/null +++ b/jitpack.yml @@ -0,0 +1 @@ +jdk: openjdk11 \ No newline at end of file diff --git a/settings.gradle b/settings.gradle new file mode 100755 index 0000000..3b4581d --- /dev/null +++ b/settings.gradle @@ -0,0 +1 @@ +rootProject.name = 'flink-big-query-connector' diff --git a/src/main/java/com/vinted/flink/bigquery/BigQueryStreamProcessor.java b/src/main/java/com/vinted/flink/bigquery/BigQueryStreamProcessor.java new file mode 100644 index 0000000..96bb219 --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/BigQueryStreamProcessor.java @@ -0,0 +1,32 @@ +package com.vinted.flink.bigquery; + +import com.vinted.flink.bigquery.client.ClientProvider; +import com.vinted.flink.bigquery.model.BigQueryRecord; +import com.vinted.flink.bigquery.model.Rows; +import com.vinted.flink.bigquery.process.RowBatcher; +import com.vinted.flink.bigquery.process.StreamStateHandler; +import org.apache.flink.connector.base.DeliveryGuarantee; +import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction; +import org.apache.flink.streaming.api.windowing.windows.Window; + +public class BigQueryStreamProcessor { + private ClientProvider clientProvider; + private DeliveryGuarantee deliveryGuarantee; + + public BigQueryStreamProcessor withClientProvider(ClientProvider clientProvider) { + this.clientProvider = clientProvider; + return this; + } + + public BigQueryStreamProcessor withDeliveryGuarantee(DeliveryGuarantee deliveryGuarantee) { + this.deliveryGuarantee = deliveryGuarantee; + return this; + } + + public ProcessWindowFunction, K, W> build() { + if (deliveryGuarantee == DeliveryGuarantee.EXACTLY_ONCE) { + return new StreamStateHandler<>(this.clientProvider); + } + return new RowBatcher<>(); + } +} diff --git a/src/main/java/com/vinted/flink/bigquery/BigQueryStreamSink.java b/src/main/java/com/vinted/flink/bigquery/BigQueryStreamSink.java new file mode 100644 index 0000000..c210a14 --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/BigQueryStreamSink.java @@ -0,0 +1,60 @@ +package com.vinted.flink.bigquery; + +import com.google.cloud.bigquery.storage.v1.JsonStreamWriter; +import com.google.cloud.bigquery.storage.v1.StreamWriter; +import com.google.common.util.concurrent.MoreExecutors; +import org.apache.flink.api.connector.sink2.Sink; +import com.vinted.flink.bigquery.client.ClientProvider; +import com.vinted.flink.bigquery.model.Rows; +import com.vinted.flink.bigquery.serializer.NoOpRowSerializer; +import com.vinted.flink.bigquery.serializer.RowValueSerializer; +import com.vinted.flink.bigquery.sink.ExecutorProvider; +import com.vinted.flink.bigquery.sink.buffered.BigQueryBufferedSink; +import com.vinted.flink.bigquery.sink.defaultStream.BigQueryDefaultSink; +import org.apache.flink.connector.base.DeliveryGuarantee; + +public class BigQueryStreamSink { + private RowValueSerializer rowValueSerializer = new NoOpRowSerializer<>(); + private ClientProvider clientProvider = null; + + private ExecutorProvider executorProvider = MoreExecutors::directExecutor; + + private DeliveryGuarantee deliveryGuarantee = DeliveryGuarantee.AT_LEAST_ONCE; + private BigQueryStreamSink() { + } + + public static BigQueryStreamSink newProto() { + return new BigQueryStreamSink<>(); + } + + public static BigQueryStreamSink newJson() { + return new BigQueryStreamSink<>(); + } + + public BigQueryStreamSink withRowValueSerializer(RowValueSerializer serializer) { + this.rowValueSerializer = serializer; + return this; + } + + public BigQueryStreamSink withClientProvider(ClientProvider clientProvider) { + this.clientProvider = clientProvider; + return this; + } + + public BigQueryStreamSink withExecutorProvider(ExecutorProvider executorProvider) { + this.executorProvider = executorProvider; + return this; + } + + public BigQueryStreamSink withDeliveryGuarantee(DeliveryGuarantee deliveryGuarantee) { + this.deliveryGuarantee = deliveryGuarantee; + return this; + } + + public Sink> build() { + if (deliveryGuarantee == DeliveryGuarantee.AT_LEAST_ONCE) { + return new BigQueryDefaultSink<>(this.rowValueSerializer, this.clientProvider, executorProvider); + } + return new BigQueryBufferedSink<>(this.rowValueSerializer, this.clientProvider, executorProvider); + } +} diff --git a/src/main/java/com/vinted/flink/bigquery/client/BigQueryJsonClientProvider.java b/src/main/java/com/vinted/flink/bigquery/client/BigQueryJsonClientProvider.java new file mode 100644 index 0000000..23b2455 --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/client/BigQueryJsonClientProvider.java @@ -0,0 +1,78 @@ +package com.vinted.flink.bigquery.client; + +import com.google.api.gax.core.FixedExecutorProvider; +import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient; +import com.google.cloud.bigquery.storage.v1.BigQueryWriteSettings; +import com.google.cloud.bigquery.storage.v1.JsonStreamWriter; +import com.google.cloud.bigquery.storage.v1.TableSchema; +import com.google.protobuf.Descriptors; +import com.vinted.flink.bigquery.model.config.Credentials; +import com.vinted.flink.bigquery.model.config.WriterSettings; +import com.vinted.flink.bigquery.schema.SchemaTransformer; + +import java.io.IOException; +import java.util.concurrent.Executors; + +public class BigQueryJsonClientProvider implements ClientProvider { + private Credentials credentials; + private WriterSettings writerSettings; + + private transient BigQueryWriteClient bigQueryWriteClient; + + public BigQueryJsonClientProvider(Credentials credentials, WriterSettings writerSettings) { + this.credentials = credentials; + this.writerSettings = writerSettings; + } + + @Override + public BigQueryWriteClient getClient() { + if (this.bigQueryWriteClient == null) { + try { + bigQueryWriteClient = BigQueryWriteClient + .create(this.writerSettings.toBqWriteSettings(credentials)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + return bigQueryWriteClient; + } + + @Override + public JsonStreamWriter getWriter(String streamName, TableId table) { + try { + var executorProvider = this.writerSettings.getWriterThreads() > 1 ? + FixedExecutorProvider.create(Executors.newScheduledThreadPool(writerSettings.getWriterThreads())) : + BigQueryWriteSettings.defaultExecutorProviderBuilder().build(); + return JsonStreamWriter + .newBuilder(streamName, getTableSchema(table), this.getClient()) + .setEnableConnectionPool(this.writerSettings.getEnableConnectionPool()) + .setExecutorProvider(executorProvider) + .build(); + } catch (Descriptors.DescriptorValidationException | IOException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + @Override + public WriterSettings writeSettings() { + return this.writerSettings; + } + + TableSchema getTableSchema(TableId table) { + var schema = BigQueryOptions + .newBuilder() + .setProjectId(table.getProject()) + .setCredentials(credentials.getCredentials()) + .build() + .getService() + .getTable(table.getDataset(), table.getTable()) + .getDefinition() + .getSchema(); + + return SchemaTransformer.convertTableSchema(schema); + } + +} diff --git a/src/main/java/com/vinted/flink/bigquery/client/BigQueryProtoClientProvider.java b/src/main/java/com/vinted/flink/bigquery/client/BigQueryProtoClientProvider.java new file mode 100644 index 0000000..c4fd18e --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/client/BigQueryProtoClientProvider.java @@ -0,0 +1,82 @@ +package com.vinted.flink.bigquery.client; + +import com.google.api.gax.core.FixedExecutorProvider; +import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.storage.v1.*; +import com.google.protobuf.Descriptors; +import com.vinted.flink.bigquery.model.config.Credentials; +import com.vinted.flink.bigquery.model.config.WriterSettings; +import com.vinted.flink.bigquery.schema.SchemaTransformer; + +import java.io.IOException; +import java.util.concurrent.Executors; + +public class BigQueryProtoClientProvider implements ClientProvider { + private Credentials credentials; + private WriterSettings writerSettings; + + private transient BigQueryWriteClient bigQueryWriteClient; + + public BigQueryProtoClientProvider(Credentials credentials, WriterSettings writerSettings) { + this.credentials = credentials; + this.writerSettings = writerSettings; + } + + @Override + public BigQueryWriteClient getClient() { + if (this.bigQueryWriteClient == null) { + try { + bigQueryWriteClient = BigQueryWriteClient + .create(this.writerSettings.toBqWriteSettings(credentials)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + return bigQueryWriteClient; + } + + @Override + public StreamWriter getWriter(String streamName, TableId table) { + try { + var descriptor = BQTableSchemaToProtoDescriptor.convertBQTableSchemaToProtoDescriptor(getTableSchema(table)); + var protoSchema = ProtoSchemaConverter.convert(descriptor); + var executorProvider = this.writerSettings.getWriterThreads() > 1 ? + FixedExecutorProvider.create(Executors.newScheduledThreadPool(writerSettings.getWriterThreads())) : + BigQueryWriteSettings.defaultExecutorProviderBuilder().build(); + var streamWriterBuilder = StreamWriter + .newBuilder(streamName, getClient()) + .setMaxInflightRequests(this.writerSettings.getMaxInflightRequests()) + .setMaxInflightBytes(this.writerSettings.getMaxInflightBytes()) + .setMaxRetryDuration(this.writerSettings.getMaxRetryDuration()) + .setEnableConnectionPool(this.writerSettings.getEnableConnectionPool()) + .setExecutorProvider(executorProvider) + .setLocation(table.getProject()) + .setWriterSchema(protoSchema); + return streamWriterBuilder.build(); + } catch (IOException | Descriptors.DescriptorValidationException e) { + throw new RuntimeException(e); + } + } + + @Override + public WriterSettings writeSettings() { + return this.writerSettings; + } + + TableSchema getTableSchema(TableId table) { + var schema = BigQueryOptions + .newBuilder() + .setProjectId(table.getProject()) + .setCredentials(credentials.getCredentials()) + .build() + .getService() + .getTable(table.getDataset(), table.getTable()) + .getDefinition() + .getSchema(); + + return SchemaTransformer.convertTableSchema(schema); + } + +} diff --git a/src/main/java/com/vinted/flink/bigquery/client/ClientProvider.java b/src/main/java/com/vinted/flink/bigquery/client/ClientProvider.java new file mode 100644 index 0000000..c922f63 --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/client/ClientProvider.java @@ -0,0 +1,15 @@ +package com.vinted.flink.bigquery.client; + +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient; +import com.vinted.flink.bigquery.model.config.WriterSettings; + +import java.io.Serializable; + +public interface ClientProvider extends Serializable { + BigQueryWriteClient getClient(); + + A getWriter(String streamName, TableId table); + + WriterSettings writeSettings(); +} diff --git a/src/main/java/com/vinted/flink/bigquery/metric/BigQueryStreamMetrics.java b/src/main/java/com/vinted/flink/bigquery/metric/BigQueryStreamMetrics.java new file mode 100644 index 0000000..7a3223e --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/metric/BigQueryStreamMetrics.java @@ -0,0 +1,45 @@ +package com.vinted.flink.bigquery.metric; + +public class BigQueryStreamMetrics { + private String streamName; + + private long offset = 0; + private long batchCount = 0; + private double batchSizeInMb = 0.0; + private long splitBatchCount = 0; + + public BigQueryStreamMetrics(String streamName) { + this.streamName = streamName; + } + + public void incSplitCount() { + splitBatchCount += 1; + } + public void updateSize(long sizeInBytes) { + batchSizeInMb = sizeInBytes / 1000000.0; + } + + public long getOffset() { + return offset; + } + + public void setOffset(long offset) { + this.offset = offset; + } + + public long getBatchCount() { + return batchCount; + } + + public void setBatchCount(long batchCount) { + this.batchCount = batchCount; + } + + public double getBatchSizeInMb() { + return batchSizeInMb; + } + + public long getSplitBatchCount() { + return splitBatchCount; + } +} diff --git a/src/main/java/com/vinted/flink/bigquery/model/BigQueryRecord.java b/src/main/java/com/vinted/flink/bigquery/model/BigQueryRecord.java new file mode 100644 index 0000000..463652c --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/model/BigQueryRecord.java @@ -0,0 +1,10 @@ +package com.vinted.flink.bigquery.model; + + +import com.google.cloud.bigquery.TableId; + +public interface BigQueryRecord { + TableId getTable(); + + long getSize(); +} diff --git a/src/main/java/com/vinted/flink/bigquery/model/Rows.java b/src/main/java/com/vinted/flink/bigquery/model/Rows.java new file mode 100644 index 0000000..7796ec5 --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/model/Rows.java @@ -0,0 +1,75 @@ +package com.vinted.flink.bigquery.model; + +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.storage.v1.TableName; + +import java.io.Serializable; +import java.util.List; + +public class Rows implements Serializable { + private List data; + private long offset; + private String stream; + private TableId table; + + public Rows updateBatch(List data, long offset) { + return new Rows<>(data, offset, stream, table); + } + + public static Rows defaultStream(List data, TableId table) { + var fullPath = TableName.of(table.getProject(), table.getDataset(), table.getTable()).toString(); + return new Rows<>(data, -1, String.format("%s/_default", fullPath), table); + } + + public Rows() { + } + + public Rows(List data, long offset, String stream, TableId table) { + this.data = data; + this.offset = offset; + this.stream = stream; + this.table = table; + } + + public List getData() { + return data; + } + + public void setData(List data) { + this.data = data; + } + + public long getOffset() { + return offset; + } + + public void setOffset(long offset) { + this.offset = offset; + } + + public String getStream() { + return stream; + } + + public void setStream(String stream) { + this.stream = stream; + } + + public TableId getTable() { + return table; + } + + public void setTable(TableId table) { + this.table = table; + } + + @Override + public String toString() { + return "Rows{" + + "dataCount=" + data.size() + + ", offset=" + offset + + ", stream='" + stream + '\'' + + ", table=" + table + + '}'; + } +} diff --git a/src/main/java/com/vinted/flink/bigquery/model/config/Credentials.java b/src/main/java/com/vinted/flink/bigquery/model/config/Credentials.java new file mode 100644 index 0000000..536b7af --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/model/config/Credentials.java @@ -0,0 +1,19 @@ +package com.vinted.flink.bigquery.model.config; + +import com.google.api.gax.core.CredentialsProvider; + +import java.io.IOException; +import java.io.Serializable; + +public interface Credentials extends Serializable { + CredentialsProvider toProvider(); + + default com.google.auth.Credentials getCredentials() { + try { + return this.toProvider().getCredentials(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/src/main/java/com/vinted/flink/bigquery/model/config/DefaultCredentials.java b/src/main/java/com/vinted/flink/bigquery/model/config/DefaultCredentials.java new file mode 100644 index 0000000..8fa7ffc --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/model/config/DefaultCredentials.java @@ -0,0 +1,11 @@ +package com.vinted.flink.bigquery.model.config; + +import com.google.api.gax.core.CredentialsProvider; +import com.google.cloud.bigquery.storage.v1.BigQueryWriteSettings; + +public class DefaultCredentials implements Credentials { + @Override + public CredentialsProvider toProvider() { + return BigQueryWriteSettings.defaultCredentialsProviderBuilder().build(); + } +} diff --git a/src/main/java/com/vinted/flink/bigquery/model/config/FileCredentialsProvider.java b/src/main/java/com/vinted/flink/bigquery/model/config/FileCredentialsProvider.java new file mode 100644 index 0000000..4799032 --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/model/config/FileCredentialsProvider.java @@ -0,0 +1,30 @@ +package com.vinted.flink.bigquery.model.config; + +import com.google.api.gax.core.CredentialsProvider; +import com.google.auth.oauth2.GoogleCredentials; + +import java.io.BufferedInputStream; +import java.io.FileInputStream; + +public class FileCredentialsProvider implements Credentials { + private String path; + + public FileCredentialsProvider(String path) { + this.path = path; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + @Override + public CredentialsProvider toProvider() { + return () -> GoogleCredentials + .fromStream(new BufferedInputStream(new FileInputStream(path))) + .createScoped("https://www.googleapis.com/auth/cloud-platform"); + } +} diff --git a/src/main/java/com/vinted/flink/bigquery/model/config/JsonCredentialsProvider.java b/src/main/java/com/vinted/flink/bigquery/model/config/JsonCredentialsProvider.java new file mode 100644 index 0000000..3b53c41 --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/model/config/JsonCredentialsProvider.java @@ -0,0 +1,29 @@ +package com.vinted.flink.bigquery.model.config; + +import com.google.api.gax.core.CredentialsProvider; +import com.google.auth.oauth2.GoogleCredentials; + +import java.io.ByteArrayInputStream; + +public class JsonCredentialsProvider implements Credentials { + private String serviceAccountKey; + + public JsonCredentialsProvider(String serviceAccountKey) { + this.serviceAccountKey = serviceAccountKey; + } + + public String getServiceAccountKey() { + return serviceAccountKey; + } + + public void setServiceAccountKey(String serviceAccountKey) { + this.serviceAccountKey = serviceAccountKey; + } + + @Override + public CredentialsProvider toProvider() { + return () -> GoogleCredentials + .fromStream(new ByteArrayInputStream(serviceAccountKey.getBytes())) + .createScoped("https://www.googleapis.com/auth/cloud-platform"); + } +} diff --git a/src/main/java/com/vinted/flink/bigquery/model/config/WriterSettings.java b/src/main/java/com/vinted/flink/bigquery/model/config/WriterSettings.java new file mode 100644 index 0000000..e3d3ecb --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/model/config/WriterSettings.java @@ -0,0 +1,147 @@ +package com.vinted.flink.bigquery.model.config; + +import com.google.cloud.bigquery.storage.v1.BigQueryWriteSettings; + +import java.io.IOException; +import java.io.Serializable; +import java.time.Duration; + +public class WriterSettings implements Serializable { + + private int streamsPerTable; + private int writerThreads; + private Duration timeout; + private int retryCount; + private Duration retryPause; + private Long maxInflightRequests; + private Long maxInflightBytes; + private Duration maxRetryDuration; + private Boolean enableConnectionPool; + + public int getStreamsPerTable() { + return streamsPerTable; + } + + public int getWriterThreads() { + return writerThreads; + } + + public Duration getTimeout() { + return timeout; + } + + public int getRetryCount() { + return retryCount; + } + + public Duration getRetryPause() { + return retryPause; + } + + public Long getMaxInflightRequests() { + return maxInflightRequests; + } + + public Long getMaxInflightBytes() { + return maxInflightBytes; + } + + public Duration getMaxRetryDuration() { + return maxRetryDuration; + } + + public Boolean getEnableConnectionPool() { + return enableConnectionPool; + } + + public BigQueryWriteSettings toBqWriteSettings(Credentials credentials) { + try { + return BigQueryWriteSettings + .newBuilder() + .setCredentialsProvider(credentials.toProvider()) + .build(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static WriterSettingsBuilder newBuilder() { + return new WriterSettingsBuilder(); + } + + public static final class WriterSettingsBuilder implements Serializable { + private int streamsPerTable = 1; + private int writerThreads = 1; + private Duration timeout = Duration.ofSeconds(10); + private int retryCount = 5; + private Duration retryPause = Duration.ofSeconds(5); + private Long maxInflightRequests = 1000L; + private Long maxInflightBytes = 100L * 1024L * 1024L; // 100Mb. + private Duration maxRetryDuration = Duration.ofMinutes(5); + private Boolean enableConnectionPool = false; + + private WriterSettingsBuilder() { + } + + public WriterSettingsBuilder withStreamsPerTable(int streamsPerTable) { + this.streamsPerTable = streamsPerTable; + return this; + } + + public WriterSettingsBuilder withWriterThreads(int writerThreads) { + this.writerThreads = writerThreads; + return this; + } + + public WriterSettingsBuilder withTimeout(Duration timeout) { + this.timeout = timeout; + return this; + } + + public WriterSettingsBuilder withRetryCount(int retryCount) { + this.retryCount = retryCount; + return this; + } + + public WriterSettingsBuilder withRetryPause(Duration retryPause) { + this.retryPause = retryPause; + return this; + } + + public WriterSettingsBuilder withMaxInflightRequests(Long maxInflightRequests) { + this.maxInflightRequests = maxInflightRequests; + return this; + } + + public WriterSettingsBuilder withMaxInflightBytes(Long maxInflightBytes) { + this.maxInflightBytes = maxInflightBytes; + return this; + } + + public WriterSettingsBuilder withMaxRetryDuration(Duration maxRetryDuration) { + this.maxRetryDuration = maxRetryDuration; + return this; + } + + public WriterSettingsBuilder withEnableConnectionPool(Boolean enableConnectionPool) { + this.enableConnectionPool = enableConnectionPool; + return this; + } + + public WriterSettings build() { + WriterSettings writerSettings = new WriterSettings(); + writerSettings.writerThreads = this.writerThreads; + writerSettings.timeout = this.timeout; + writerSettings.streamsPerTable = this.streamsPerTable; + writerSettings.retryCount = this.retryCount; + writerSettings.enableConnectionPool = this.enableConnectionPool; + writerSettings.maxInflightBytes = this.maxInflightBytes; + writerSettings.maxInflightRequests = this.maxInflightRequests; + writerSettings.retryPause = this.retryPause; + writerSettings.maxRetryDuration = this.maxRetryDuration; + return writerSettings; + } + } +} + + diff --git a/src/main/java/com/vinted/flink/bigquery/process/BatchTrigger.java b/src/main/java/com/vinted/flink/bigquery/process/BatchTrigger.java new file mode 100644 index 0000000..c1160cc --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/process/BatchTrigger.java @@ -0,0 +1,128 @@ +package com.vinted.flink.bigquery.process; + +import com.vinted.flink.bigquery.model.BigQueryRecord; +import org.apache.flink.api.common.functions.ReduceFunction; +import org.apache.flink.api.common.state.ReducingStateDescriptor; +import org.apache.flink.api.common.typeutils.base.LongSerializer; +import org.apache.flink.streaming.api.windowing.triggers.*; +import org.apache.flink.streaming.api.windowing.windows.Window; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; + +public class BatchTrigger extends Trigger { + private static final Logger logger = LoggerFactory.getLogger(BatchTrigger.class); + private final ReducingStateDescriptor sizeStateDesc = + new ReducingStateDescriptor<>("size", new BatchTrigger.Sum(), LongSerializer.INSTANCE); + private CountTrigger countTrigger; + private int maxSizeMb; + + public static Builder builder() { + return new Builder<>(); + } + + public BatchTrigger(long maxCount, int maxSizeMb) { + this.maxSizeMb = maxSizeMb; + this.countTrigger = CountTrigger.of(maxCount); + } + + @Override + public TriggerResult onElement(A element, long timestamp, W window, TriggerContext ctx) throws Exception { + var countStatus = countTrigger.onElement(element, timestamp, window, ctx); + var currentSize = ctx.getPartitionedState(sizeStateDesc); + + TriggerResult finalStatus; + + if (countStatus.isFire()) { + finalStatus = countStatus; + } else { + var size = element.getSize(); + currentSize.add(size); + var currentSizeInMbs = currentSize.get() / 1_000_000; + if (currentSizeInMbs >= maxSizeMb) { + logger.debug("Batch size limit ({}) reached. Releasing batch of size {}", maxSizeMb, currentSizeInMbs); + finalStatus = TriggerResult.FIRE; + } else { + finalStatus = TriggerResult.CONTINUE; + } + } + + if (finalStatus.isFire()) { + currentSize.clear(); + } + + return finalStatus; + } + + @Override + public TriggerResult onProcessingTime(long time, W window, TriggerContext ctx) throws Exception { + return countTrigger.onProcessingTime(time, window, ctx); + } + + @Override + public TriggerResult onEventTime(long time, W window, TriggerContext ctx) throws Exception { + return countTrigger.onEventTime(time, window, ctx); + } + + @Override + public void clear(W window, TriggerContext ctx) throws Exception { + countTrigger.clear(window, ctx); + ctx.getPartitionedState(sizeStateDesc).clear(); + } + + @Override + public boolean canMerge() { + return countTrigger.canMerge(); + } + + @Override + public void onMerge(W window, OnMergeContext ctx) throws Exception { + ctx.mergePartitionedState(sizeStateDesc); + countTrigger.onMerge(window, ctx); + } + + static private class Sum implements ReduceFunction { + public Long reduce(Long value1, Long value2) { + return value1 + value2; + } + } + + public static final class Builder { + private int count; + private int sizeInMb; + private Duration timeout; + private boolean resetTimerOnNewRecord = false; + + public Builder withCount(int count) { + this.count = count; + return this; + } + + public Builder withSizeInMb(int sizeInMb) { + this.sizeInMb = sizeInMb; + return this; + } + + public Builder withTimeout(Duration timeout) { + this.timeout = timeout; + return this; + } + + public Builder withResetTimerOnNewRecord(boolean reset) { + this.resetTimerOnNewRecord = reset; + return this; + } + + public Trigger build() { + return PurgingTrigger.of( + ProcessingTimeoutTrigger.of( + new BatchTrigger<>(this.count, this.sizeInMb), + timeout, + resetTimerOnNewRecord, + true + ) + ); + } + } +} diff --git a/src/main/java/com/vinted/flink/bigquery/process/RowBatcher.java b/src/main/java/com/vinted/flink/bigquery/process/RowBatcher.java new file mode 100644 index 0000000..43ffe2b --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/process/RowBatcher.java @@ -0,0 +1,36 @@ +package com.vinted.flink.bigquery.process; + +import com.google.cloud.bigquery.TableId; +import com.vinted.flink.bigquery.model.BigQueryRecord; +import com.vinted.flink.bigquery.model.Rows; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction; +import org.apache.flink.streaming.api.windowing.windows.Window; +import org.apache.flink.util.Collector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +public class RowBatcher extends ProcessWindowFunction, K, W> { + private static final Logger logger = LoggerFactory.getLogger(RowBatcher.class); + + @Override + public void open(Configuration parameters) throws Exception { + super.open(parameters); + } + + @Override + public void process(K k, ProcessWindowFunction, K, W>.Context context, Iterable batch, Collector> out) throws Exception { + var table = getTable(batch.iterator().next()); + + var data = StreamSupport.stream(batch.spliterator(), false).collect(Collectors.toList()); + var result = Rows.defaultStream(data, table); + out.collect(result); + } + + private TableId getTable(A data) { + return data.getTable(); + } +} diff --git a/src/main/java/com/vinted/flink/bigquery/process/StreamState.java b/src/main/java/com/vinted/flink/bigquery/process/StreamState.java new file mode 100644 index 0000000..89eec11 --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/process/StreamState.java @@ -0,0 +1,59 @@ +package com.vinted.flink.bigquery.process; + +import java.util.concurrent.TimeUnit; + +public class StreamState { + private String name; + private long offset; + private long lastUpdate; + + public StreamState() { + } + + public StreamState(String name, long offset, long lastUpdate) { + this.name = name; + this.offset = offset; + this.lastUpdate = lastUpdate; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public long getOffset() { + return offset; + } + + public void setOffset(long offset) { + this.offset = offset; + } + + public long getLastUpdate() { + return lastUpdate; + } + + public void setLastUpdate(long lastUpdate) { + this.lastUpdate = lastUpdate; + } + + public boolean expired(int ttlDays, long currentTime) { + return currentTime >= lastUpdate + TimeUnit.DAYS.toMillis(ttlDays); + } + + public StreamState update(long batchSize, long lastUpdate) { + return new StreamState(name, offset + batchSize, lastUpdate); + } + + @Override + public String toString() { + return "StreamState{" + + "name='" + name + '\'' + + ", offset=" + offset + + ", lastUpdate=" + lastUpdate + + '}'; + } +} diff --git a/src/main/java/com/vinted/flink/bigquery/process/StreamStateHandler.java b/src/main/java/com/vinted/flink/bigquery/process/StreamStateHandler.java new file mode 100644 index 0000000..9df41bd --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/process/StreamStateHandler.java @@ -0,0 +1,156 @@ +package com.vinted.flink.bigquery.process; + +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.storage.v1.*; +import com.vinted.flink.bigquery.client.ClientProvider; +import com.vinted.flink.bigquery.model.BigQueryRecord; +import com.vinted.flink.bigquery.model.Rows; +import io.grpc.Status; +import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.ValueStateDescriptor; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction; +import org.apache.flink.streaming.api.windowing.windows.Window; +import org.apache.flink.util.Collector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.time.Clock; +import java.util.HashSet; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +public class StreamStateHandler extends ProcessWindowFunction, K, W> { + private static final Logger logger = LoggerFactory.getLogger(StreamStateHandler.class); + private final ClientProvider clientProvider; + private transient Clock clock; + private transient HashSet localStreamState; + private transient BigQueryWriteClient client; + private transient ValueState streamState; + private final int streamTTLDays = 7; + + + public StreamStateHandler(ClientProvider clientProvider) { + this.clientProvider = clientProvider; + } + + @Override + public void open(Configuration parameters) throws Exception { + super.open(parameters); + this.streamState = this.getRuntimeContext().getState(new ValueStateDescriptor<>( + "streamState", + TypeInformation.of(StreamState.class) + )); + client = clientProvider.getClient(); + localStreamState = new HashSet<>(); + clock = Clock.systemUTC(); + } + + @Override + public void process(K k, ProcessWindowFunction, K, W>.Context context, Iterable batch, Collector> out) throws Exception { + var table = getTable(batch.iterator().next()); + var value = streamState.value(); + + var state = getState(table, value); + var data = StreamSupport.stream(batch.spliterator(), false).collect(Collectors.toList()); + var result = new Rows<>(data, state.getOffset(), state.getName(), table); + out.collect(result); + + updateState(state.update(data.size(), getClock().millis())); + } + + private StreamState getState(TableId table, StreamState value) { + switch (resolveStatus(value)) { + case NOT_EXISTS: { + logger.info("Stream for {} never existed. Creating new stream", table); + return createStream(table, this::updateState); + } + case NOT_CACHED: { + logger.info("Stream for {} is not in local cache. Getting stream from BigQuery", table); + try { + client.getWriteStream(value.getName()); + return updateState(value); + } catch (Throwable error) { + switch (Status.fromThrowable(error).getCode()) { + case INVALID_ARGUMENT: { + if (error.getCause() instanceof Exceptions.StreamFinalizedException) { + logger.warn("Stream for {} is finished. Creating new stream", table, error); + return createStream(table, this::updateState); + } + throw error; + } + case NOT_FOUND: { + logger.warn("Stream for {} not found. Creating new stream", table, error); + return createStream(table, this::updateState); + } + default: + throw error; + } + + } + } + case EXPIRED: { + logger.warn("Stream for {} is old and might be finished due to TTL. Creating new stream", table); + return createStream(table, this::updateState); + } + default: + return value; + } + } + + + private StreamState updateState(StreamState newState) { + localStreamState.add(newState.getName()); + + try { + streamState.update(newState); + } catch (IOException e) { + throw new RuntimeException(e); + } + return newState; + } + + private StreamState createStream(TableId table, Function stateUpdateFunc) { + var fullPath = TableName.of(table.getProject(), table.getDataset(), table.getTable()).toString(); + logger.info("Creating new stream for: {}", table); + var createWriteStreamRequest = CreateWriteStreamRequest + .newBuilder() + .setParent(fullPath) + .setWriteStream(WriteStream + .newBuilder() + .setType(WriteStream.Type.BUFFERED) + .build() + ).build(); + var stream = client.createWriteStream(createWriteStreamRequest); + return stateUpdateFunc.apply(new StreamState(stream.getName(), 0L, getClock().millis())); + } + + private StreamStateStatus resolveStatus(StreamState state) { + if (state == null) { + return StreamStateStatus.NOT_EXISTS; + } + + if (state.expired(streamTTLDays, getClock().millis())) { + return StreamStateStatus.EXPIRED; + } + + if (!localStreamState.contains(state.getName())) { + return StreamStateStatus.NOT_CACHED; + } + + return StreamStateStatus.CACHED; + } + + private TableId getTable(A data) { + return data.getTable(); + } + + protected Clock getClock() { + return clock; + } + + enum StreamStateStatus {NOT_CACHED, EXPIRED, CACHED, NOT_EXISTS} +} diff --git a/src/main/java/com/vinted/flink/bigquery/schema/SchemaTools.java b/src/main/java/com/vinted/flink/bigquery/schema/SchemaTools.java new file mode 100644 index 0000000..c2b9074 --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/schema/SchemaTools.java @@ -0,0 +1,185 @@ +package com.vinted.flink.bigquery.schema; + +import com.google.api.services.bigquery.model.TableFieldSchema; +import com.google.api.services.bigquery.model.TableReference; +import com.google.cloud.bigquery.*; +import com.google.cloud.bigquery.spi.v2.BigQueryRpc; +import com.google.cloud.bigquery.spi.v2.HttpBigQueryRpc; +import com.google.api.services.bigquery.model.TableSchema; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class SchemaTools implements Serializable { + private static final Logger logger = LoggerFactory.getLogger(SchemaTools.class); + private BigQueryOptions bigQueryOptions; + private transient BigQuery bigQuery; + private transient BigQueryRpc bigQueryRpc; + + public BigQuery getBigQuery() { + if (bigQuery == null) { + bigQuery = bigQueryOptions.getService(); + } + return bigQuery; + } + + public BigQueryRpc getBigQueryRpc() { + if (bigQueryRpc == null) { + bigQueryRpc = (HttpBigQueryRpc) bigQueryOptions.getRpc(); + } + return bigQueryRpc; + } + + public SchemaTools(BigQueryOptions bigQueryOptions) { + this.bigQueryOptions = bigQueryOptions; + } + + public void createTable(TableInfo tableInfo) { + try { + getBigQuery().create(tableInfo); + logger.info("Table {} created successfully", tableInfo.getTableId()); + } catch (BigQueryException e) { + logger.error("Unable to create table " + tableInfo.getTableId(), e); + throw e; + } + } + + public void updateView(TableId tableId, String query) { + try { + var viewMetadata = getBigQuery().getTable(tableId); + ViewDefinition definition = viewMetadata.getDefinition(); + var updatedViewDefinition = definition.toBuilder().setQuery(query).build(); + getBigQuery().update(viewMetadata.toBuilder().setDefinition(updatedViewDefinition).build()); + logger.info("View query {} updated successfully", tableId); + } catch (BigQueryException e) { + logger.error("View query " + tableId + " was not updated.", e); + throw e; + } + } + + public boolean exists(TableId tableId) { + var table = getBigQuery().getTable(tableId); + return table != null && table.exists(); + } + + public com.google.cloud.bigquery.Table getTable(TableId tableId, BigQuery.TableOption... bigQueryOptions) { + return getBigQuery().getTable(tableId, bigQueryOptions); + } + + public List listTables(DatasetId datasetId) { + var data = getBigQuery().listTables(datasetId).iterateAll(); + var list = new ArrayList
(); + data.forEach(list::add); + return list; + } + + public List listTableIds(DatasetId datasetId) { + return listTables(datasetId).stream().map(TableInfo::getTableId).collect(Collectors.toList()); + } + + public void updateTable(TableId tableId, Schema schema) { + var table = getBigQuery().getTable(tableId); + TableDefinition definition = table.getDefinition(); + var currentBigQueryTableFields = definition.getSchema().getFields(); + var newFields = schema.getFields(); + if (!newFields.containsAll(currentBigQueryTableFields)) { + var fieldsToRemove = topLevelColumnsToRemove(newFields, currentBigQueryTableFields); + var newFieldsWithFieldsToRemove = FieldList.of(Stream.concat(newFields.stream(), fieldsToRemove.stream()).collect(Collectors.toList())); + updateWithRpc(tableId, newFieldsWithFieldsToRemove); + deleteWithQuery(tableId, fieldsToRemove); + } else { + logger.info("No schema changes for {} are needed.", tableId); + } + } + + private TableReference getTableReference(TableId tableId) { + var tableReference = new TableReference(); + tableReference.setTableId(tableId.getTable()); + tableReference.setDatasetId(tableId.getDataset()); + tableReference.setProjectId(tableId.getProject()); + return tableReference; + } + + private List fieldsToTableFieldSchema(FieldList fields) { + return fields.stream().map(field -> { + var tableFieldSchema = new TableFieldSchema(); + tableFieldSchema.setName(field.getName()); + if (field.getMode() != null) { + tableFieldSchema.setMode(field.getMode().toString()); + } + if (field.getType() != null) { + tableFieldSchema.setType(field.getType().toString()); + } + tableFieldSchema.setDescription(field.getDescription()); + tableFieldSchema.setPrecision(field.getPrecision()); + tableFieldSchema.setScale(field.getScale()); + tableFieldSchema.setMaxLength(field.getMaxLength()); + + if (field.getPolicyTags() != null) { + tableFieldSchema.setPolicyTags(new TableFieldSchema.PolicyTags().setNames(field.getPolicyTags().getNames())); + } + + if (field.getType() == LegacySQLTypeName.RECORD) { + tableFieldSchema.setFields(fieldsToTableFieldSchema(field.getSubFields())); + } + + return tableFieldSchema; + }).collect(Collectors.toList()); + } + + private TableSchema getTableSchema(FieldList fields) { + var tableSchema = new TableSchema(); + return tableSchema.setFields(fieldsToTableFieldSchema(fields)); + } + + private List topLevelColumnsToRemove(FieldList newFields, FieldList bqTableFields) { + var newSchemaNames = newFields.stream().map(Field::getName).collect(Collectors.toSet()); + return bqTableFields.stream().filter(f -> !newSchemaNames.contains(f.getName())).collect(Collectors.toList()); + } + + private void updateWithRpc(TableId tableId, FieldList fields) { + var table = new com.google.api.services.bigquery.model.Table(); + table.setTableReference(getTableReference(tableId)); + table.setSchema(getTableSchema(fields)); + + try { + getBigQueryRpc().patch(table, Map.of()); + } catch (Exception e) { + logger.error("Unable to alter bigquery schema using rpc", e); + throw e; + } + } + + private Optional buildQuery(TableId tableId, List removeColumns) { + var queryStatements = removeColumns.stream().map(f -> String.format("DROP COLUMN IF EXISTS %s", f.getName())).collect(Collectors.toList()); + if (!queryStatements.isEmpty()) + return Optional.of(String.format("ALTER TABLE %s.%s %s", tableId.getDataset(), tableId.getTable(), String.join(",", queryStatements))); + else { + return Optional.empty(); + } + } + + private void deleteWithQuery(TableId tableId, List columnsToRemove) { + var combinedQuery = buildQuery(tableId, columnsToRemove); + if (combinedQuery.isPresent()) { + try { + var queryConfig = QueryJobConfiguration.newBuilder(combinedQuery.get()).build(); + getBigQuery().query(queryConfig); + logger.info("Table {} updated successfully using query", tableId); + } catch (Exception e) { + logger.error("Unable to alter bigquery schema using a query", e); + throw new RuntimeException(e); + } + } else { + logger.debug("Update using query was not performed."); + } + } + +} diff --git a/src/main/java/com/vinted/flink/bigquery/schema/SchemaTransformer.java b/src/main/java/com/vinted/flink/bigquery/schema/SchemaTransformer.java new file mode 100644 index 0000000..32eddd7 --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/schema/SchemaTransformer.java @@ -0,0 +1,82 @@ +package com.vinted.flink.bigquery.schema; + +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.Schema; +import com.google.cloud.bigquery.StandardSQLTypeName; +import com.google.cloud.bigquery.storage.v1.TableFieldSchema; +import com.google.cloud.bigquery.storage.v1.TableSchema; +import com.google.common.collect.ImmutableMap; + +import java.util.stream.IntStream; + +public class SchemaTransformer { + private static ImmutableMap BQTableSchemaModeMap = ImmutableMap.of( + Field.Mode.NULLABLE, + TableFieldSchema.Mode.NULLABLE, + Field.Mode.REPEATED, + TableFieldSchema.Mode.REPEATED, + Field.Mode.REQUIRED, + TableFieldSchema.Mode.REQUIRED + ); + + private static ImmutableMap BQTableSchemaTypeMap = new ImmutableMap.Builder() + .put(StandardSQLTypeName.BOOL, TableFieldSchema.Type.BOOL) + .put(StandardSQLTypeName.BYTES, TableFieldSchema.Type.BYTES) + .put(StandardSQLTypeName.DATE, TableFieldSchema.Type.DATE) + .put(StandardSQLTypeName.DATETIME, TableFieldSchema.Type.DATETIME) + .put(StandardSQLTypeName.FLOAT64, TableFieldSchema.Type.DOUBLE) + .put(StandardSQLTypeName.GEOGRAPHY, TableFieldSchema.Type.GEOGRAPHY) + .put(StandardSQLTypeName.INT64, TableFieldSchema.Type.INT64) + .put(StandardSQLTypeName.NUMERIC, TableFieldSchema.Type.NUMERIC) + .put(StandardSQLTypeName.BIGNUMERIC, TableFieldSchema.Type.BIGNUMERIC) + .put(StandardSQLTypeName.JSON, TableFieldSchema.Type.JSON) + .put(StandardSQLTypeName.STRING, TableFieldSchema.Type.STRING) + .put(StandardSQLTypeName.INTERVAL, TableFieldSchema.Type.INTERVAL) + .put(StandardSQLTypeName.STRUCT, TableFieldSchema.Type.STRUCT) + .put(StandardSQLTypeName.TIME, TableFieldSchema.Type.TIME) + .put(StandardSQLTypeName.TIMESTAMP, TableFieldSchema.Type.TIMESTAMP) + .build(); + + /** + * Converts from BigQuery client Table Schema to bigquery storage API Table Schema. + * + * @param schema the BigQuery client Table Schema + * @return the bigquery storage API Table Schema + */ + public static TableSchema convertTableSchema(Schema schema) { + var result = TableSchema.newBuilder(); + + IntStream.range(0, schema.getFields().size()).forEach(i -> { + result.addFields(i, convertFieldSchema(schema.getFields().get(i))); + }); + + return result.build(); + } + + /** + * Converts from bigquery v2 Field Schema to bigquery storage API Field Schema. + * + * @param field the BigQuery client Field Schema + * @return the bigquery storage API Field Schema + */ + public static TableFieldSchema convertFieldSchema(Field field) { + var fieldVar = field; + var result = TableFieldSchema.newBuilder(); + if (field.getMode() == null) { + fieldVar = field.toBuilder().setMode(Field.Mode.NULLABLE).build(); + } + result.setMode(BQTableSchemaModeMap.get(fieldVar.getMode())); + result.setName(field.getName()); + result.setType(BQTableSchemaTypeMap.get(fieldVar.getType().getStandardType())); + if (fieldVar.getDescription() != null) { + result.setDescription(fieldVar.getDescription()); + } + if (fieldVar.getSubFields() != null) { + Field finalFieldVar = fieldVar; + IntStream.range(0, fieldVar.getSubFields().size()).forEach(i -> { + result.addFields(i, convertFieldSchema(finalFieldVar.getSubFields().get(i))); + }); + } + return result.build(); + } +} diff --git a/src/main/java/com/vinted/flink/bigquery/serializer/JsonRowValueSerializer.java b/src/main/java/com/vinted/flink/bigquery/serializer/JsonRowValueSerializer.java new file mode 100644 index 0000000..e829867 --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/serializer/JsonRowValueSerializer.java @@ -0,0 +1,4 @@ +package com.vinted.flink.bigquery.serializer; + +public interface JsonRowValueSerializer extends RowValueSerializer { +} diff --git a/src/main/java/com/vinted/flink/bigquery/serializer/NoOpRowSerializer.java b/src/main/java/com/vinted/flink/bigquery/serializer/NoOpRowSerializer.java new file mode 100644 index 0000000..9e579cc --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/serializer/NoOpRowSerializer.java @@ -0,0 +1,8 @@ +package com.vinted.flink.bigquery.serializer; + +public class NoOpRowSerializer implements RowValueSerializer{ + @Override + public byte[] serialize(A value) { + throw new RuntimeException("Not supported"); + } +} diff --git a/src/main/java/com/vinted/flink/bigquery/serializer/ProtoValueSerializer.java b/src/main/java/com/vinted/flink/bigquery/serializer/ProtoValueSerializer.java new file mode 100644 index 0000000..56469e4 --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/serializer/ProtoValueSerializer.java @@ -0,0 +1,4 @@ +package com.vinted.flink.bigquery.serializer; + +public interface ProtoValueSerializer extends RowValueSerializer { +} diff --git a/src/main/java/com/vinted/flink/bigquery/serializer/RowValueSerializer.java b/src/main/java/com/vinted/flink/bigquery/serializer/RowValueSerializer.java new file mode 100644 index 0000000..b2f2445 --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/serializer/RowValueSerializer.java @@ -0,0 +1,7 @@ +package com.vinted.flink.bigquery.serializer; + +import java.io.Serializable; + +public interface RowValueSerializer extends Serializable { + byte[] serialize(A value); +} diff --git a/src/main/java/com/vinted/flink/bigquery/sink/AppendException.java b/src/main/java/com/vinted/flink/bigquery/sink/AppendException.java new file mode 100644 index 0000000..11fad7a --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/sink/AppendException.java @@ -0,0 +1,29 @@ +package com.vinted.flink.bigquery.sink; + +import com.vinted.flink.bigquery.model.Rows; + +public class AppendException extends RuntimeException { + private final Rows rows; + + private final String traceId; + + private final Throwable error; + + public AppendException(String traceId, Rows rows, Throwable error) { + this.traceId = traceId; + this.rows = rows; + this.error = error; + } + + public String getTraceId() { + return traceId; + } + + public Rows getRows() { + return (Rows) rows; + } + + public Throwable getError() { + return error; + } +} diff --git a/src/main/java/com/vinted/flink/bigquery/sink/BigQuerySinkWriter.java b/src/main/java/com/vinted/flink/bigquery/sink/BigQuerySinkWriter.java new file mode 100644 index 0000000..d9a28cb --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/sink/BigQuerySinkWriter.java @@ -0,0 +1,116 @@ +package com.vinted.flink.bigquery.sink; + +import com.google.api.core.ApiFuture; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.storage.v1.AppendRowsResponse; +import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient; +import com.google.common.collect.Iterators; +import com.vinted.flink.bigquery.client.ClientProvider; +import com.vinted.flink.bigquery.metric.BigQueryStreamMetrics; +import io.grpc.Status; +import org.apache.flink.api.connector.sink2.Sink; +import org.apache.flink.api.connector.sink2.SinkWriter; +import com.vinted.flink.bigquery.model.Rows; +import com.vinted.flink.bigquery.serializer.RowValueSerializer; +import org.apache.flink.metrics.Counter; +import org.apache.flink.metrics.Gauge; +import org.apache.flink.metrics.groups.SinkWriterMetricGroup; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public abstract class BigQuerySinkWriter implements SinkWriter> { + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + private final Iterator streamIndexIterator; + private final SinkWriterMetricGroup metricGroup; + + protected ClientProvider clientProvider; + protected transient Map streamMap = new ConcurrentHashMap<>(); + protected Sink.InitContext sinkInitContext; + protected RowValueSerializer rowSerializer; + + protected BigQueryWriteClient client; + protected final Executor appendExecutor; + protected Counter numBytesOutCounter; + protected Counter numRecordsOutCounter; + protected transient Map metrics = new HashMap<>(); + + protected abstract ApiFuture append(String traceId, Rows rows); + + public BigQuerySinkWriter( + Sink.InitContext sinkInitContext, + RowValueSerializer rowSerializer, + ClientProvider clientProvider, + ExecutorProvider executorProvider) { + + this.sinkInitContext = sinkInitContext; + this.rowSerializer = rowSerializer; + this.clientProvider = clientProvider; + this.appendExecutor = executorProvider.create(); + this.client = this.clientProvider.getClient(); + this.metricGroup = this.sinkInitContext.metricGroup(); + this.numBytesOutCounter = this.metricGroup.getIOMetricGroup().getNumBytesOutCounter(); + this.numRecordsOutCounter = this.metricGroup.getIOMetricGroup().getNumRecordsOutCounter(); + this.streamIndexIterator = Iterators.cycle(IntStream + .range(0, this.clientProvider.writeSettings().getStreamsPerTable()) + .boxed() + .collect(Collectors.toList())); + + } + + protected final StreamT streamWriter(String traceId, String streamName, TableId table) { + var streamWithIndex = String.format("%s-%s",streamName, streamIndexIterator.next()); + return streamMap.computeIfAbsent(streamWithIndex, name -> { + logger.trace("Trace-id {} Stream not found {}. Creating new stream", traceId, streamWithIndex); + // Stream name can't contain index + return this.clientProvider.getWriter(streamName, table); + }); + } + + @Override + public void write(Rows rows, Context context) { + numRecordsOutCounter.inc(rows.getData().size()); + metrics.computeIfAbsent(rows.getStream(), s -> { + var metric = new BigQueryStreamMetrics(rows.getStream()); + var group = metricGroup + .addGroup("table", rows.getTable().getTable()) + .addGroup("stream", rows.getStream()); + group.gauge("stream_offset", (Gauge) metric::getOffset); + group.gauge("batch_count", metric::getBatchCount); + group.gauge("batch_size_mb", metric::getBatchSizeInMb); + group.gauge("split_batch_count", metric::getSplitBatchCount); + + return metric; + }); + var traceId = UUID.randomUUID().toString(); + try { + writeWithRetry(traceId, rows, clientProvider.writeSettings().getRetryCount()); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + protected abstract void writeWithRetry(String traceId, Rows rows, int retryCount) throws Throwable; + + protected String createLogMessage(String title, String errorTraceId, Status status, Throwable error, Rows errorRows) { + return String.format("Trace-id: %s %s \nstatus: %s\nerror: %s\nstream: %s\ntable: %s\nactual offset: %s\nsize: %s", + errorTraceId, + title, + status.getCode(), + error.getMessage(), + errorRows.getStream(), + errorRows.getTable(), + errorRows.getOffset(), + errorRows.getData().size() + ); + } + +} diff --git a/src/main/java/com/vinted/flink/bigquery/sink/ExecutorProvider.java b/src/main/java/com/vinted/flink/bigquery/sink/ExecutorProvider.java new file mode 100644 index 0000000..54ea9ab --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/sink/ExecutorProvider.java @@ -0,0 +1,9 @@ +package com.vinted.flink.bigquery.sink; + +import java.io.Serializable; +import java.util.concurrent.Executor; + +public interface ExecutorProvider extends Serializable { + + Executor create(); +} diff --git a/src/main/java/com/vinted/flink/bigquery/sink/buffered/BigQueryBufferedSink.java b/src/main/java/com/vinted/flink/bigquery/sink/buffered/BigQueryBufferedSink.java new file mode 100644 index 0000000..411ca9f --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/sink/buffered/BigQueryBufferedSink.java @@ -0,0 +1,50 @@ +package com.vinted.flink.bigquery.sink.buffered; + +import com.google.cloud.bigquery.storage.v1.JsonStreamWriter; +import com.google.cloud.bigquery.storage.v1.StreamWriter; +import org.apache.flink.api.connector.sink2.Committer; +import org.apache.flink.api.connector.sink2.TwoPhaseCommittingSink; +import com.vinted.flink.bigquery.client.ClientProvider; +import com.vinted.flink.bigquery.model.Rows; +import com.vinted.flink.bigquery.serializer.JsonRowValueSerializer; +import com.vinted.flink.bigquery.serializer.ProtoValueSerializer; +import com.vinted.flink.bigquery.serializer.RowValueSerializer; +import com.vinted.flink.bigquery.sink.ExecutorProvider; +import org.apache.flink.core.io.SimpleVersionedSerializer; + +import java.io.IOException; + +public class BigQueryBufferedSink implements TwoPhaseCommittingSink, BigQueryCommittable> { + private final RowValueSerializer rowValueSerializer; + private final ClientProvider clientProvider; + + private final ExecutorProvider executorProvider; + + public BigQueryBufferedSink(RowValueSerializer rowValueSerializer, ClientProvider clientProvider, ExecutorProvider executorProvider) { + this.rowValueSerializer = rowValueSerializer; + this.clientProvider = clientProvider; + this.executorProvider = executorProvider; + } + + @Override + public PrecommittingSinkWriter, BigQueryCommittable> createWriter(InitContext context) throws IOException { + if (rowValueSerializer instanceof JsonRowValueSerializer) { + return new BigQueryJsonBufferedSinkWriter<>(context, rowValueSerializer, (ClientProvider) clientProvider, executorProvider); + } else if (rowValueSerializer instanceof ProtoValueSerializer) { + return new BigQueryProtoBufferedSinkWriter<>(context, rowValueSerializer, (ClientProvider) clientProvider, executorProvider); + } else { + throw new RuntimeException("Not supported serializer"); + } + } + + @Override + public Committer createCommitter() throws IOException { + return new BigQuerySinkCommitter(clientProvider); + } + + @Override + public SimpleVersionedSerializer getCommittableSerializer() { + return new BigQueryCommittableSerializer(); + } + +} diff --git a/src/main/java/com/vinted/flink/bigquery/sink/buffered/BigQueryBufferedSinkWriter.java b/src/main/java/com/vinted/flink/bigquery/sink/buffered/BigQueryBufferedSinkWriter.java new file mode 100644 index 0000000..bc006ee --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/sink/buffered/BigQueryBufferedSinkWriter.java @@ -0,0 +1,221 @@ +package com.vinted.flink.bigquery.sink.buffered; + +import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutureCallback; +import com.google.api.core.ApiFutures; +import com.google.cloud.bigquery.storage.v1.AppendRowsResponse; +import com.google.cloud.bigquery.storage.v1.Exceptions; +import io.grpc.Status; +import org.apache.flink.api.connector.sink2.Sink; +import org.apache.flink.api.connector.sink2.TwoPhaseCommittingSink; +import com.vinted.flink.bigquery.client.ClientProvider; +import com.vinted.flink.bigquery.metric.BigQueryStreamMetrics; +import com.vinted.flink.bigquery.model.Rows; +import com.vinted.flink.bigquery.serializer.RowValueSerializer; +import com.vinted.flink.bigquery.sink.AppendException; +import com.vinted.flink.bigquery.sink.BigQuerySinkWriter; +import com.vinted.flink.bigquery.sink.ExecutorProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.function.Function; + +public abstract class BigQueryBufferedSinkWriter + extends BigQuerySinkWriter + implements TwoPhaseCommittingSink.PrecommittingSinkWriter, BigQueryCommittable> { + private static final Logger logger = LoggerFactory.getLogger(BigQueryBufferedSinkWriter.class); + private Map streamOffsets = new ConcurrentHashMap<>(); + public BigQueryBufferedSinkWriter( + Sink.InitContext sinkInitContext, + RowValueSerializer rowSerializer, + ClientProvider clientProvider, + ExecutorProvider executorProvider) { + super(sinkInitContext, rowSerializer, clientProvider, executorProvider); + } + + protected abstract ApiFuture append(String traceId, Rows rows); + + @Override + protected void writeWithRetry(String traceId, Rows rows, int retryCount) throws Throwable { + try { + logger.debug( + "Trace-id: {} Appending rows \nstream: {}\ntable: {}\noffset: {}\nsize: {}\nretries: {}", + traceId, rows.getStream(), rows.getTable(), rows.getOffset(), rows.getData().size(), retryCount + ); + var response = append(traceId, rows); + var callback = new AppendCallBack<>(this, rows, retryCount, traceId); + ApiFutures.addCallback(response, callback, appendExecutor); + try { + callback.future.get(); + } catch (ExecutionException e) { + throw e.getCause(); + } + } catch (AppendException exception) { + var error = exception.getError(); + var errorRows = exception.getRows(); + var errorTraceId = exception.getTraceId(); + var status = Status.fromThrowable(error); + Function createLogMessage = (title) -> + this.createLogMessage(title, errorTraceId, status, error, errorRows); + switch (status.getCode()) { + case INTERNAL: + case CANCELLED: + case ABORTED: { + logger.warn(createLogMessage.apply("Recoverable error. Retrying.., "), error); + try { + Thread.sleep(clientProvider.writeSettings().getRetryPause().toMillis()); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + if (retryCount > 0) { + writeWithRetry(errorTraceId, errorRows, retryCount - 1); + } else { + throw error; + } + break; + } + // ALREADY_EXISTS: The row was already written. + // This error can happen when you provide stream offsets. + // It indicates that a duplicate record was detected. + // It's caused by retrying some batch in case more rows are flushed and offset commit is not stored in checkpoint. + // Sometimes whole batch can be skipped, sometimes only part of the batch is skipped and new recores are appended. + case ALREADY_EXISTS: { + if (error instanceof Exceptions.OffsetAlreadyExists) { + var o = (Exceptions.OffsetAlreadyExists) error; + var offsetToSkip = (int) (o.getExpectedOffset() - o.getActualOffset()); + + if (offsetToSkip >= errorRows.getData().size()) { + logger.info( + createLogMessage.apply("Whole batch was already stored. Expected offset {}, skipping..."), o.getExpectedOffset() + ); + } else { + var batchToStore = errorRows.getData().subList(offsetToSkip, errorRows.getData().size()); + logger.warn( + createLogMessage.apply("Skipping {} items from batch. Offsets:({}-{})"), + offsetToSkip, o.getExpectedOffset(), o.getActualOffset() + ); + logger.info(createLogMessage.apply("Storing {} items with offset {}"), batchToStore.size(), o.getExpectedOffset()); + writeWithRetry(errorTraceId, errorRows.updateBatch(batchToStore, o.getExpectedOffset()), clientProvider.writeSettings().getRetryCount()); + } + } else { + logger.error( + createLogMessage.apply("Unable to parse expected and actual offset. Failed to write this batch."), error); + throw error; + } + + break; + } + // OUT_OF_RANGE Returned when the specified offset in the stream is beyond the current end of the stream. + // This is non recoverable exception. Wrapping message with debug info and throwing it. + case OUT_OF_RANGE: { + if (error instanceof Exceptions.OffsetOutOfRange) { + var o = (Exceptions.OffsetOutOfRange) error; + logger.error(createLogMessage.apply("Actual offset " + o.getActualOffset() + " is out range. Expected " + o.getExpectedOffset()), error); + } else { + logger.error(createLogMessage.apply(error.getMessage()), error); + } + + throw error; + } + // INVALID_ARGUMENT Stream is already finalized. + case INVALID_ARGUMENT: { + if (error instanceof Exceptions.StreamFinalizedException) { + logger.error(createLogMessage.apply( + "Stream is already finalized actualOffset: {}."), errorRows.getOffset(), error); + throw error; + } else if (error.getMessage().contains("INVALID_ARGUMENT: MessageSize is too large.")) { + Optional.ofNullable(metrics.get(errorRows.getStream())).ifPresent(BigQueryStreamMetrics::incSplitCount); + logger.warn(createLogMessage.apply("MessageSize is too large. Splitting batch")); + var first = errorRows.getData().subList(0, errorRows.getData().size() / 2); + var second = errorRows.getData().subList(errorRows.getData().size() / 2, errorRows.getData().size()); + writeWithRetry(errorTraceId, errorRows.updateBatch(first, errorRows.getOffset()), clientProvider.writeSettings().getRetryCount()); + writeWithRetry(errorTraceId, errorRows.updateBatch(second, errorRows.getOffset() + first.size()), clientProvider.writeSettings().getRetryCount()); + } else { + logger.error(createLogMessage.apply(error.getMessage()), error); + throw error; + } + break; + } + default: { + logger.error(createLogMessage.apply("Non recoverable BigQuery stream error for:"), error); + throw error; + } + } + } catch (Throwable t) { + logger.error("Non recoverable BigQuery stream error for:", t); + throw t; + } + + } + + @Override + public List prepareCommit() { + var result = new ArrayList(); + streamOffsets.entrySet().stream() + .filter(entry -> entry.getValue() > 0) + .forEach(entry -> { + result.add(new BigQueryCommittable(entry.getKey(), entry.getValue())); + }); + streamOffsets.clear(); + return result; + } + + @Override + public void close() { + logger.info("Closing BigQuery write stream"); + streamMap.values().forEach(stream -> { + try { + stream.close(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + client.close(); + } + + @Override + public void flush(boolean endOfInput) { + } + + static class AppendCallBack implements ApiFutureCallback { + private final BigQueryBufferedSinkWriter parent; + private final Rows rows; + private final String traceId; + private final int retryCount; + + private final CompletableFuture future = new CompletableFuture<>(); + + public AppendCallBack(BigQueryBufferedSinkWriter parent, Rows rows, int retryCount, String traceId) { + this.parent = parent; + this.rows = rows; + this.traceId = traceId; + this.retryCount = retryCount; + } + + @Override + public void onFailure(Throwable t) { + logger.info("Trace-id {} Received error {}", t.getMessage(), traceId); + future.completeExceptionally(new AppendException(traceId, rows, t)); + } + + @Override + public void onSuccess(AppendRowsResponse result) { + this.parent.streamOffsets.put(rows.getStream(), rows.getOffset() + rows.getData().size()); + var streamOffset = this.parent.streamOffsets.get(rows.getStream()); + logger.debug( + "Trace-id {} Stream offset updated \nstream: {}\ntable: {}\nnew offset: {}\nsize: {}\nretries: {}\nresponse offset: {}", + this.traceId, rows.getStream(), rows.getTable(), streamOffset, rows.getData().size(), retryCount, result.getAppendResult().getOffset() + ); + Optional.ofNullable(this.parent.metrics.get(rows.getStream())).ifPresent(m -> { + m.setBatchCount(rows.getData().size()); + m.setOffset(rows.getOffset()); + }); + future.complete(result); + } + } +} diff --git a/src/main/java/com/vinted/flink/bigquery/sink/buffered/BigQueryCommittable.java b/src/main/java/com/vinted/flink/bigquery/sink/buffered/BigQueryCommittable.java new file mode 100644 index 0000000..275c7e6 --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/sink/buffered/BigQueryCommittable.java @@ -0,0 +1,30 @@ +package com.vinted.flink.bigquery.sink.buffered; + +public class BigQueryCommittable { + private String streamName; + private long offset; + + public BigQueryCommittable() { + } + + public BigQueryCommittable(String streamName, long offset) { + this.streamName = streamName; + this.offset = offset; + } + + public String getStreamName() { + return streamName; + } + + public void setStreamName(String streamName) { + this.streamName = streamName; + } + + public long getOffset() { + return offset; + } + + public void setOffset(long offset) { + this.offset = offset; + } +} diff --git a/src/main/java/com/vinted/flink/bigquery/sink/buffered/BigQueryCommittableSerializer.java b/src/main/java/com/vinted/flink/bigquery/sink/buffered/BigQueryCommittableSerializer.java new file mode 100644 index 0000000..13cad3d --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/sink/buffered/BigQueryCommittableSerializer.java @@ -0,0 +1,31 @@ +package com.vinted.flink.bigquery.sink.buffered; + +import org.apache.flink.core.io.SimpleVersionedSerializer; + +import java.io.*; + +public class BigQueryCommittableSerializer implements SimpleVersionedSerializer { + @Override + public int getVersion() { + return 1; + } + + @Override + public byte[] serialize(BigQueryCommittable data) throws IOException { + try (var baos = new ByteArrayOutputStream(); var out = new DataOutputStream(baos)) { + out.writeUTF(data.getStreamName()); + out.writeLong(data.getOffset()); + out.flush(); + return baos.toByteArray(); + } + } + + @Override + public BigQueryCommittable deserialize(int version, byte[] serialized) throws IOException { + try (var bais = new ByteArrayInputStream(serialized); var in = new DataInputStream(bais)) { + var name = in.readUTF(); + var offset = in.readLong(); + return new BigQueryCommittable(name, offset); + } + } +} diff --git a/src/main/java/com/vinted/flink/bigquery/sink/buffered/BigQueryJsonBufferedSinkWriter.java b/src/main/java/com/vinted/flink/bigquery/sink/buffered/BigQueryJsonBufferedSinkWriter.java new file mode 100644 index 0000000..81ab379 --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/sink/buffered/BigQueryJsonBufferedSinkWriter.java @@ -0,0 +1,33 @@ +package com.vinted.flink.bigquery.sink.buffered; + +import com.google.api.core.ApiFuture; +import com.google.cloud.bigquery.storage.v1.AppendRowsResponse; +import com.google.cloud.bigquery.storage.v1.JsonStreamWriter; +import com.google.protobuf.Descriptors; +import org.apache.flink.api.connector.sink2.Sink; +import com.vinted.flink.bigquery.client.ClientProvider; +import com.vinted.flink.bigquery.model.Rows; +import com.vinted.flink.bigquery.serializer.RowValueSerializer; +import com.vinted.flink.bigquery.sink.ExecutorProvider; +import org.json.JSONArray; +import org.json.JSONObject; + +import java.io.IOException; + +public class BigQueryJsonBufferedSinkWriter extends BigQueryBufferedSinkWriter { + public BigQueryJsonBufferedSinkWriter(Sink.InitContext sinkInitContext, RowValueSerializer rowSerializer, ClientProvider clientProvider, ExecutorProvider executorProvider) { + super(sinkInitContext, rowSerializer, clientProvider, executorProvider); + } + + @Override + protected ApiFuture append(String traceId, Rows rows) { + var rowArray = new JSONArray(); + rows.getData().forEach(row -> rowArray.put(new JSONObject(new String(rowSerializer.serialize(row))))); + + try { + return streamWriter(traceId, rows.getStream(), rows.getTable()).append(rowArray, rows.getOffset()); + } catch (IOException | Descriptors.DescriptorValidationException e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/main/java/com/vinted/flink/bigquery/sink/buffered/BigQueryProtoBufferedSinkWriter.java b/src/main/java/com/vinted/flink/bigquery/sink/buffered/BigQueryProtoBufferedSinkWriter.java new file mode 100644 index 0000000..da801c9 --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/sink/buffered/BigQueryProtoBufferedSinkWriter.java @@ -0,0 +1,44 @@ +package com.vinted.flink.bigquery.sink.buffered; + +import com.google.api.core.ApiFuture; +import com.google.cloud.bigquery.storage.v1.AppendRowsResponse; +import com.google.cloud.bigquery.storage.v1.ProtoRows; +import com.google.cloud.bigquery.storage.v1.StreamWriter; +import com.google.protobuf.ByteString; +import org.apache.flink.api.connector.sink2.Sink; +import com.vinted.flink.bigquery.client.ClientProvider; +import com.vinted.flink.bigquery.model.Rows; +import com.vinted.flink.bigquery.serializer.RowValueSerializer; +import com.vinted.flink.bigquery.sink.ExecutorProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Optional; +import java.util.stream.Collectors; + +public class BigQueryProtoBufferedSinkWriter extends BigQueryBufferedSinkWriter { + private static final Logger logger = LoggerFactory.getLogger(BigQueryProtoBufferedSinkWriter.class); + public BigQueryProtoBufferedSinkWriter(Sink.InitContext sinkInitContext, RowValueSerializer rowSerializer, ClientProvider clientProvider, ExecutorProvider executorProvider) { + super(sinkInitContext, rowSerializer, clientProvider, executorProvider); + } + + @Override + protected ApiFuture append(String traceId, Rows rows) { + var prows = ProtoRows + .newBuilder() + .addAllSerializedRows(rows.getData().stream().map(r -> ByteString.copyFrom(rowSerializer.serialize(r))).collect(Collectors.toList())) + .build(); + var size = prows.getSerializedSize(); + numBytesOutCounter.inc(size); + Optional.ofNullable(metrics.get(rows.getStream())).ifPresent(s -> s.updateSize(size)); + var writer = streamWriter(traceId, rows.getStream(), rows.getTable()); + + if (writer.isClosed() || writer.isUserClosed()) { + logger.warn("Trace-id {}, StreamWrite is closed. Recreating stream for {}", traceId, rows.getStream()); + } + + logger.trace("Trace-id {}, Writing rows stream {} to steamWriter for {} writer id {}", traceId, rows.getStream(), writer.getStreamName(), writer.getWriterId()); + + return writer.append(prows, rows.getOffset()); + } +} diff --git a/src/main/java/com/vinted/flink/bigquery/sink/buffered/BigQuerySinkCommitter.java b/src/main/java/com/vinted/flink/bigquery/sink/buffered/BigQuerySinkCommitter.java new file mode 100644 index 0000000..6edd310 --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/sink/buffered/BigQuerySinkCommitter.java @@ -0,0 +1,62 @@ +package com.vinted.flink.bigquery.sink.buffered; + +import com.google.cloud.bigquery.storage.v1.FlushRowsRequest; +import com.google.protobuf.Int64Value; +import com.vinted.flink.bigquery.client.ClientProvider; +import io.grpc.Status; +import org.apache.flink.api.connector.sink2.Committer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.UUID; + +public class BigQuerySinkCommitter implements Committer { + private static final Logger logger = LoggerFactory.getLogger(BigQuerySinkCommitter.class); + private ClientProvider clientProvider; + + public BigQuerySinkCommitter(ClientProvider clientProvider) { + this.clientProvider = clientProvider; + } + + @Override + public void commit(Collection> committables) { + var traceId = UUID.randomUUID().toString(); + committables.forEach(committable -> { + var offsetToCommit = committable.getCommittable().getOffset() - 1; // Advance the cursor to the latest record. + logger.debug(createLogMessage(traceId, committable.getCommittable(), "Committing offset")); + var request = FlushRowsRequest.newBuilder() + .setWriteStream(committable.getCommittable().getStreamName()) + .setOffset(Int64Value.of(offsetToCommit)) + .build(); + try { + clientProvider.getClient().flushRows(request); + } catch (Throwable t) { + var status = Status.fromThrowable(t); + switch (status.getCode()) { + case ALREADY_EXISTS: { + logger.warn("Trace-id {} Rows offset already exists",traceId, t); + committable.signalAlreadyCommitted(); + break; + } + default: { + logger.error(createLogMessage(traceId, committable.getCommittable(), "Commit failed. " + t.getMessage()), t); + committable.signalFailedWithUnknownReason(t); + break; + } + } + } + }); + } + + private String createLogMessage(String traceId, BigQueryCommittable commit, String title) { + return String.format("Trace-id %s %s\nstream: %s\ncommit offset: %s", + traceId, title, commit.getStreamName(), commit.getOffset() + ); + } + + + @Override + public void close() throws Exception { + } +} diff --git a/src/main/java/com/vinted/flink/bigquery/sink/defaultStream/BigQueryDefaultJsonSinkWriter.java b/src/main/java/com/vinted/flink/bigquery/sink/defaultStream/BigQueryDefaultJsonSinkWriter.java new file mode 100644 index 0000000..436851a --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/sink/defaultStream/BigQueryDefaultJsonSinkWriter.java @@ -0,0 +1,44 @@ +package com.vinted.flink.bigquery.sink.defaultStream; + +import com.google.api.core.ApiFuture; +import com.google.cloud.bigquery.storage.v1.AppendRowsResponse; +import com.google.cloud.bigquery.storage.v1.JsonStreamWriter; +import com.google.protobuf.Descriptors; +import org.apache.flink.api.connector.sink2.Sink; +import com.vinted.flink.bigquery.client.ClientProvider; +import com.vinted.flink.bigquery.model.Rows; +import com.vinted.flink.bigquery.serializer.RowValueSerializer; +import com.vinted.flink.bigquery.sink.ExecutorProvider; +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +public class BigQueryDefaultJsonSinkWriter extends BigQueryDefaultSinkWriter { + private static final Logger logger = LoggerFactory.getLogger(BigQueryDefaultJsonSinkWriter.class); + public BigQueryDefaultJsonSinkWriter(Sink.InitContext sinkInitContext, RowValueSerializer rowSerializer, ClientProvider clientProvider, ExecutorProvider executorProvider) { + super(sinkInitContext, rowSerializer, clientProvider, executorProvider); + } + + @Override + protected ApiFuture append(String traceId, Rows rows) { + var rowArray = new JSONArray(); + rows.getData().forEach(row -> rowArray.put(new JSONObject(new String(rowSerializer.serialize(row))))); + var writer = streamWriter(traceId, rows.getStream(), rows.getTable()); + + if (writer.isClosed() || writer.isUserClosed()) { + logger.warn("Trace-id {}, StreamWrite is closed. Recreating stream for {}", traceId, rows.getStream()); + } + + logger.trace("Trace-id {}, Writing rows stream {} to steamWriter for {} writer id {}", traceId, rows.getStream(), writer.getStreamName(), writer.getWriterId()); + + try { + return writer.append(rowArray); + } catch (IOException | Descriptors.DescriptorValidationException e) { + logger.error("Trace-id {}, StreamWriter failed to append {}", traceId, e.getMessage()); + throw new RuntimeException(e); + } + } +} diff --git a/src/main/java/com/vinted/flink/bigquery/sink/defaultStream/BigQueryDefaultProtoSinkWriter.java b/src/main/java/com/vinted/flink/bigquery/sink/defaultStream/BigQueryDefaultProtoSinkWriter.java new file mode 100644 index 0000000..89e44ca --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/sink/defaultStream/BigQueryDefaultProtoSinkWriter.java @@ -0,0 +1,49 @@ +package com.vinted.flink.bigquery.sink.defaultStream; + +import com.google.api.core.ApiFuture; +import com.google.cloud.bigquery.storage.v1.AppendRowsResponse; +import com.google.cloud.bigquery.storage.v1.ProtoRows; +import com.google.cloud.bigquery.storage.v1.StreamWriter; +import com.google.protobuf.ByteString; +import org.apache.flink.api.connector.sink2.Sink; +import com.vinted.flink.bigquery.client.ClientProvider; +import com.vinted.flink.bigquery.model.Rows; +import com.vinted.flink.bigquery.serializer.RowValueSerializer; +import com.vinted.flink.bigquery.sink.ExecutorProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Optional; +import java.util.stream.Collectors; + +public class BigQueryDefaultProtoSinkWriter extends BigQueryDefaultSinkWriter { + private static final Logger logger = LoggerFactory.getLogger(BigQueryDefaultProtoSinkWriter.class); + + public BigQueryDefaultProtoSinkWriter( + Sink.InitContext sinkInitContext, + RowValueSerializer rowSerializer, + ClientProvider clientProvider, + ExecutorProvider executorProvider) { + super(sinkInitContext, rowSerializer, clientProvider, executorProvider); + } + + @Override + protected ApiFuture append(String traceId, Rows rows) { + var prows = ProtoRows + .newBuilder() + .addAllSerializedRows(rows.getData().stream().map(r -> ByteString.copyFrom(rowSerializer.serialize(r))).collect(Collectors.toList())) + .build(); + var size = prows.getSerializedSize(); + numBytesOutCounter.inc(size); + numRecordsOutCounter.inc(rows.getData().size()); + Optional.ofNullable(metrics.get(rows.getStream())).ifPresent(s -> s.updateSize(size)); + var writer = streamWriter(traceId, rows.getStream(), rows.getTable()); + + if (writer.isClosed() || writer.isUserClosed()) { + logger.warn("Trace-id {}, StreamWrite is closed. Recreating stream for {}", traceId, rows.getStream()); + } + + logger.trace("Trace-id {}, Writing rows stream {} to steamWriter for {} writer id {}", traceId, rows.getStream(), writer.getStreamName(), writer.getWriterId()); + return writer.append(prows); + } +} diff --git a/src/main/java/com/vinted/flink/bigquery/sink/defaultStream/BigQueryDefaultSink.java b/src/main/java/com/vinted/flink/bigquery/sink/defaultStream/BigQueryDefaultSink.java new file mode 100644 index 0000000..85baddf --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/sink/defaultStream/BigQueryDefaultSink.java @@ -0,0 +1,41 @@ +package com.vinted.flink.bigquery.sink.defaultStream; + +import com.google.cloud.bigquery.storage.v1.JsonStreamWriter; +import com.google.cloud.bigquery.storage.v1.StreamWriter; +import com.vinted.flink.bigquery.client.ClientProvider; +import org.apache.flink.api.connector.sink2.Sink; +import org.apache.flink.api.connector.sink2.SinkWriter; +import com.vinted.flink.bigquery.model.Rows; +import com.vinted.flink.bigquery.serializer.JsonRowValueSerializer; +import com.vinted.flink.bigquery.serializer.ProtoValueSerializer; +import com.vinted.flink.bigquery.serializer.RowValueSerializer; +import com.vinted.flink.bigquery.sink.ExecutorProvider; + +import java.io.IOException; + +public class BigQueryDefaultSink implements Sink> { + private final RowValueSerializer rowValueSerializer; + private final ClientProvider clientProvider; + private final ExecutorProvider executorProvider; + + public BigQueryDefaultSink( + RowValueSerializer rowValueSerializer, + ClientProvider clientProvider, + ExecutorProvider executorProvider) { + this.rowValueSerializer = rowValueSerializer; + this.clientProvider = clientProvider; + this.executorProvider = executorProvider; + } + + @Override + public SinkWriter> createWriter(InitContext context) throws IOException { + if (rowValueSerializer instanceof JsonRowValueSerializer) { + return new BigQueryDefaultJsonSinkWriter(context, rowValueSerializer, (ClientProvider) clientProvider, executorProvider); + } else if (rowValueSerializer instanceof ProtoValueSerializer) { + return new BigQueryDefaultProtoSinkWriter(context, rowValueSerializer, (ClientProvider) clientProvider, executorProvider); + } else { + throw new RuntimeException("Not supported serializer"); + } + } + +} diff --git a/src/main/java/com/vinted/flink/bigquery/sink/defaultStream/BigQueryDefaultSinkWriter.java b/src/main/java/com/vinted/flink/bigquery/sink/defaultStream/BigQueryDefaultSinkWriter.java new file mode 100644 index 0000000..c0099e3 --- /dev/null +++ b/src/main/java/com/vinted/flink/bigquery/sink/defaultStream/BigQueryDefaultSinkWriter.java @@ -0,0 +1,140 @@ +package com.vinted.flink.bigquery.sink.defaultStream; + +import com.google.api.core.ApiFutureCallback; +import com.google.api.core.ApiFutures; +import com.google.cloud.bigquery.storage.v1.AppendRowsResponse; +import com.vinted.flink.bigquery.metric.BigQueryStreamMetrics; +import io.grpc.Status; +import org.apache.flink.api.connector.sink2.Sink; +import com.vinted.flink.bigquery.client.ClientProvider; +import com.vinted.flink.bigquery.model.Rows; +import com.vinted.flink.bigquery.serializer.RowValueSerializer; +import com.vinted.flink.bigquery.sink.AppendException; +import com.vinted.flink.bigquery.sink.BigQuerySinkWriter; +import com.vinted.flink.bigquery.sink.ExecutorProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Optional; +import java.util.concurrent.Phaser; +import java.util.function.Function; + +public abstract class BigQueryDefaultSinkWriter + extends BigQuerySinkWriter { + private static final Logger logger = LoggerFactory.getLogger(BigQueryDefaultSinkWriter.class); + + private final Phaser inflightRequestCount = new Phaser(1); + + private volatile AppendException appendAsyncException = null; + + public BigQueryDefaultSinkWriter( + Sink.InitContext sinkInitContext, + RowValueSerializer rowSerializer, + ClientProvider clientProvider, + ExecutorProvider executorProvider) { + super(sinkInitContext, rowSerializer, clientProvider, executorProvider); + } + + private void checkAsyncException() { + // reset this exception since we could close the writer later on + RuntimeException e = appendAsyncException; + if (e != null) { + appendAsyncException = null; + throw e; + } + } + + @Override + protected void writeWithRetry(String traceId, Rows rows, int retryCount) throws Throwable { + try { + checkAsyncException(); + logger.debug( + "Trace-id: {} Appending rows \nstream: {}\ntable: {}\noffset: {}\nsize: {}\nretries: {}", + traceId, rows.getStream(), rows.getTable(), rows.getOffset(), rows.getData().size(), retryCount + ); + var response = append(traceId, rows); + var callback = new AppendCallBack<>(this, traceId, rows, retryCount); + ApiFutures.addCallback(response, callback, appendExecutor); + inflightRequestCount.register(); + } catch (AppendException exception) { + var error = exception.getError(); + var errorRows = exception.getRows(); + var errorTraceId = exception.getTraceId(); + var status = Status.fromThrowable(error); + Function createLogMessage = (title) -> + this.createLogMessage(title, errorTraceId, status, error, errorRows); + logger.error(createLogMessage.apply("Non recoverable BigQuery stream error for:"), error); + throw error; + } catch (Throwable t) { + logger.error("Non recoverable BigQuery stream error for:", t); + throw t; + } + } + + @Override + public void close() { + logger.info("Closing BigQuery write stream"); + inflightRequestCount.arriveAndAwaitAdvance(); + streamMap.values().forEach(stream -> { + try { + stream.close(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + client.close(); + } + + @Override + public void flush(boolean endOfInput) { + if (endOfInput) { + inflightRequestCount.arriveAndAwaitAdvance(); + } + checkAsyncException(); + } + + static class AppendCallBack implements ApiFutureCallback { + private final BigQueryDefaultSinkWriter parent; + private final Rows rows; + private final String traceId; + private final int retryCount; + + public AppendCallBack(BigQueryDefaultSinkWriter parent, String traceId, Rows rows, int retryCount) { + this.parent = parent; + this.traceId = traceId; + this.rows = rows; + this.retryCount = retryCount; + } + + @Override + public void onFailure(Throwable t) { + logger.info("Trace-id {} Received error {}", t.getMessage(), traceId); + var status = Status.fromThrowable(t); + if (status.getCode() == Status.Code.INVALID_ARGUMENT && t.getMessage().contains("INVALID_ARGUMENT: MessageSize is too large.")) { + Optional.ofNullable(this.parent.metrics.get(rows.getStream())).ifPresent(BigQueryStreamMetrics::incSplitCount); + logger.warn("Trace-id {} MessageSize is too large. Splitting batch", traceId); + var first = rows.getData().subList(0, rows.getData().size() / 2); + var second = rows.getData().subList(rows.getData().size() / 2, rows.getData().size()); + try { + this.parent.writeWithRetry(traceId, rows.updateBatch(first, rows.getOffset()), retryCount); + this.parent.writeWithRetry(traceId, rows.updateBatch(second, rows.getOffset() + first.size()), retryCount); + } catch (Throwable e) { + this.parent.appendAsyncException = new AppendException(traceId, rows, t); + this.parent.inflightRequestCount.arriveAndDeregister(); + } + } else { + this.parent.appendAsyncException = new AppendException(traceId, rows, t); + this.parent.inflightRequestCount.arriveAndDeregister(); + } + } + + @Override + public void onSuccess(AppendRowsResponse result) { + Optional.ofNullable(this.parent.metrics.get(rows.getStream())).ifPresent(m -> { + m.setBatchCount(rows.getData().size()); + m.setOffset(result.getAppendResult().getOffset().getValue()); + }); + this.parent.inflightRequestCount.arriveAndDeregister(); + } + } +} diff --git a/src/test/java/com/vinted/flink/bigquery/BatchTriggerTest.java b/src/test/java/com/vinted/flink/bigquery/BatchTriggerTest.java new file mode 100644 index 0000000..a40a1c1 --- /dev/null +++ b/src/test/java/com/vinted/flink/bigquery/BatchTriggerTest.java @@ -0,0 +1,170 @@ +package com.vinted.flink.bigquery; + + +import com.google.cloud.bigquery.TableId; +import com.vinted.flink.bigquery.model.BigQueryRecord; +import com.vinted.flink.bigquery.util.FlinkTest; +import com.vinted.flink.bigquery.process.BatchTrigger; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.api.functions.windowing.ProcessAllWindowFunction; +import org.apache.flink.streaming.api.windowing.assigners.GlobalWindows; +import org.apache.flink.streaming.api.windowing.triggers.Trigger; +import org.apache.flink.streaming.api.windowing.windows.GlobalWindow; +import org.apache.flink.util.Collector; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +import java.io.Serializable; +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.assertj.core.api.Assertions.assertThat; + +@ExtendWith(FlinkTest.class) +public class BatchTriggerTest { + @Test + public void shouldTriggerWindowWhenCountIsReached(@FlinkTest.FlinkParam FlinkTest.PipelineRunner runner) throws Exception { + Trigger trigger = BatchTrigger.builder() + .withCount(2) + .withTimeout(Duration.ofMinutes(1)) + .withResetTimerOnNewRecord(true) + .withSizeInMb(1) + .build(); + + List> result = runner.run(pipeline(trigger, List.of("1", "2", "3", "4", "5"))); + + assertThat(result).hasSize(2); + assertThat(result.get(0).stream().map(s -> s.value).collect(Collectors.toList())).containsOnly("1", "2"); + assertThat(result.get(1).stream().map(s -> s.value).collect(Collectors.toList())).containsOnly("3", "4"); + } + + @Test + public void shouldNotReleaseDataBeforeTimeout(@FlinkTest.FlinkParam FlinkTest.PipelineRunner runner) throws Exception { + Trigger trigger = BatchTrigger.builder() + .withCount(3) + .withTimeout(Duration.ofMinutes(1)) + .withResetTimerOnNewRecord(true) + .withSizeInMb(1) + .build(); + + List> result = runner.run(pipeline(trigger, List.of("1", "2"))); + + assertThat(result).isEmpty(); + } + + @Test + public void shouldReleaseDataAfterTimeout(@FlinkTest.FlinkParam FlinkTest.PipelineRunner runner) throws Exception { + Trigger trigger = BatchTrigger.builder() + .withCount(3) + .withTimeout(Duration.ofSeconds(1)) + .withResetTimerOnNewRecord(true) + .withSizeInMb(1) + .build(); + + List> result = runner.run(pipeline(trigger, List.of("1", "2", "await"))); + + + assertThat(result).hasSize(1); + assertThat(result.get(0).stream().map(s -> s.value).collect(Collectors.toList())).containsOnly("1", "2"); + } + + @Test + public void shouldReleaseDataWhenBatchHitsSizeLimit(@FlinkTest.FlinkParam FlinkTest.PipelineRunner runner) throws Exception { + Trigger trigger = BatchTrigger.builder() + .withCount(30) + .withTimeout(Duration.ofSeconds(1)) + .withResetTimerOnNewRecord(true) + .withSizeInMb(1) + .build(); + + var string1 = givenRandomStringWithSize(400000, "a"); + var string2 = givenRandomStringWithSize(400000, "b"); + var string3 = givenRandomStringWithSize(400000, "c"); + + List> result = runner.run(pipeline(trigger, List.of(string1, string2, string3, "end"))); + + assertThat(result).hasSize(1); + assertThat(result.get(0).stream().map(s -> s.value).collect(Collectors.toList())).containsOnly(string1, string2, string3); + } + + private Function>> pipeline(Trigger trigger, List elements) { + return (env) -> + env.fromCollection(new IteratorWithWait(elements.stream().map(Record::new).collect(Collectors.toList())), Record.class) + .windowAll(GlobalWindows.create()) + .trigger(trigger) + .process(new Batching()) + .uid("batching"); + + } + + private String givenRandomStringWithSize(int size, String value) { + var builder = new StringBuilder(size); + IntStream.rangeClosed(0, size).forEach(a -> { + builder.append(value); + }); + + return builder.toString(); + } + + static class IteratorWithWait implements Iterator, Serializable { + private ArrayDeque data; + + public IteratorWithWait(List data) { + this.data = new ArrayDeque<>(data); + } + + @Override + public boolean hasNext() { + return !data.isEmpty(); + } + + @Override + public Record next() { + var i = data.pop(); + if (i.value.equals("await")) { + try { + Thread.sleep(Duration.ofSeconds(2).toMillis()); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + return i; + } + } + + static class Batching extends ProcessAllWindowFunction, GlobalWindow> { + @Override + public void process(ProcessAllWindowFunction, GlobalWindow>.Context context, Iterable elements, Collector> out) throws Exception { + var list = new ArrayList(); + elements.forEach(list::add); + out.collect(list); + } + + } + + public static class Record implements Serializable, BigQueryRecord { + + private String value; + + public Record(String value) { + this.value = value; + } + + @Override + public TableId getTable() { + return TableId.of("test-project", "test-dataset", "test-table"); + } + + @Override + public long getSize() { + return value.getBytes().length; + } + } +} diff --git a/src/test/java/com/vinted/flink/bigquery/BigQueryBufferedSinkTest.java b/src/test/java/com/vinted/flink/bigquery/BigQueryBufferedSinkTest.java new file mode 100644 index 0000000..1d3d33c --- /dev/null +++ b/src/test/java/com/vinted/flink/bigquery/BigQueryBufferedSinkTest.java @@ -0,0 +1,95 @@ +package com.vinted.flink.bigquery; + +import com.google.cloud.bigquery.TableId; +import io.grpc.Status; +import com.vinted.flink.bigquery.model.Rows; +import com.vinted.flink.bigquery.serializer.JsonRowValueSerializer; +import com.vinted.flink.bigquery.util.FlinkTest; +import com.vinted.flink.bigquery.util.FlinkTest.FlinkParam; +import com.vinted.flink.bigquery.util.FlinkTest.PipelineRunner; +import com.vinted.flink.bigquery.util.MockJsonClientProvider; +import org.apache.flink.connector.base.DeliveryGuarantee; +import org.apache.flink.runtime.client.JobExecutionException; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.datastream.DataStreamSink; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import java.util.stream.IntStream; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.*; + + +@ExtendWith(FlinkTest.class) +public class BigQueryBufferedSinkTest { + TableId testTable = TableId.of("test-project", "test-dataset", "test-table"); + String stream = "projects/test/datasets/test/tables/test/streams/stream1"; + + @Test + public void shouldAppendRows(@FlinkParam PipelineRunner runner, @FlinkParam MockJsonClientProvider mockClientProvider) throws Exception { + mockClientProvider.givenSuccessfulAppend(); + + runner.runWithCustomSink(withBigQuerySink(mockClientProvider, pipeline(List.of( + givenRows(1) + )))); + + verify(mockClientProvider.getMockJsonWriter(), times(1)).append(any()); + } + + @Test + public void shouldSplitTheBatchWhenAppendingTooLargeBatch(@FlinkParam PipelineRunner runner, @FlinkParam MockJsonClientProvider mockClientProvider) throws Exception { + mockClientProvider.givenAppendingTooLargeBatch(); + + runner + .withRetryCount(0) + .runWithCustomSink(withBigQuerySink(mockClientProvider, pipeline(List.of( + givenRows(6) + )))); + + + verify(mockClientProvider.getMockJsonWriter(), times(3)).append(any()); + } + + @Test + public void shouldNotRetryOnException(@FlinkParam PipelineRunner runner, @FlinkParam MockJsonClientProvider mockClientProvider) throws Exception { + mockClientProvider.givenFailingAppendWithStatus(Status.INTERNAL); + + assertThatThrownBy(() -> { + runner + .withRetryCount(0) + .runWithCustomSink(withBigQuerySink(mockClientProvider, pipeline(List.of( + givenRows(1) + )))); + }).isInstanceOf(JobExecutionException.class); + + + verify(mockClientProvider.getMockJsonWriter(), times(1)).append(any()); + } + + private Rows givenRows(int count) { + var data = new ArrayList(count); + IntStream.rangeClosed(1, count) + .forEach(i -> data.add("{\"value\": " + i + "}")); + + return Rows.defaultStream(data, testTable); + } + + private Function>> pipeline(List> data) { + return env -> env.fromCollection(data); + } + + private Function>> withBigQuerySink(MockJsonClientProvider mockClientProvider, Function>> pipeline) { + var sink = BigQueryStreamSink.newJson() + .withClientProvider(mockClientProvider) + .withDeliveryGuarantee(DeliveryGuarantee.AT_LEAST_ONCE) + .withRowValueSerializer((JsonRowValueSerializer) String::getBytes) + .build(); + + return pipeline.andThen(s -> s.sinkTo(sink)); + } +} diff --git a/src/test/java/com/vinted/flink/bigquery/BigQueryDefaultSinkTest.java b/src/test/java/com/vinted/flink/bigquery/BigQueryDefaultSinkTest.java new file mode 100644 index 0000000..e7ee66e --- /dev/null +++ b/src/test/java/com/vinted/flink/bigquery/BigQueryDefaultSinkTest.java @@ -0,0 +1,224 @@ +package com.vinted.flink.bigquery; + +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.storage.v1.FlushRowsRequest; +import com.google.protobuf.Int64Value; +import com.vinted.flink.bigquery.model.Rows; +import com.vinted.flink.bigquery.util.FlinkTest; +import com.vinted.flink.bigquery.util.MockJsonClientProvider; +import io.grpc.Status; +import com.vinted.flink.bigquery.serializer.JsonRowValueSerializer; +import org.apache.flink.connector.base.DeliveryGuarantee; +import org.apache.flink.runtime.client.JobExecutionException; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.datastream.DataStreamSink; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import java.util.stream.IntStream; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.*; + + +@ExtendWith(FlinkTest.class) +public class BigQueryDefaultSinkTest { + TableId testTable = TableId.of("test-project", "test-dataset", "test-table"); + String stream = "projects/test/datasets/test/tables/test/streams/stream1"; + + @Test + public void shouldAppendRows(@FlinkTest.FlinkParam FlinkTest.PipelineRunner runner, @FlinkTest.FlinkParam MockJsonClientProvider mockClientProvider) throws Exception { + mockClientProvider.givenSuccessfulAppend(); + + runner.runWithCustomSink(withBigQuerySink(mockClientProvider, pipeline(List.of( + givenRowWithOffset(1, 0) + )))); + + verify(mockClientProvider.getMockJsonWriter(), times(1)).append(any(), eq(0L)); + } + + @Test + public void shouldFlushRowsWhenExactlyOnceDeliveryEnabled(@FlinkTest.FlinkParam FlinkTest.PipelineRunner runner, @FlinkTest.FlinkParam MockJsonClientProvider mockClientProvider) throws Exception { + mockClientProvider.givenSuccessfulAppend(); + + runner.runWithCustomSink(withBigQuerySink(mockClientProvider, pipeline(List.of( + givenRowWithOffset(1, 1) + )))); + + verify(mockClientProvider.getClient(), times(1)).flushRows( + FlushRowsRequest.newBuilder() + .setWriteStream(stream) + .setOffset(Int64Value.of(1)) + .build() + ); + } + + @Test + public void shouldRetryAppendWhenFailingWithInternalError(@FlinkTest.FlinkParam FlinkTest.PipelineRunner runner, @FlinkTest.FlinkParam MockJsonClientProvider mockClientProvider) throws Exception { + mockClientProvider.givenFailingAppendWithStatus(Status.INTERNAL); + + assertThatThrownBy(() -> { + runner + .withRetryCount(0) + .runWithCustomSink(withBigQuerySink(mockClientProvider, pipeline(List.of( + givenRowWithOffset(1, 1) + )))); + }).isInstanceOf(JobExecutionException.class); + + + verify(mockClientProvider.getMockJsonWriter(), times(6)).append(any(), anyLong()); + } + + @Test + @Disabled("Retry causes out of order exception in committer and later in writer") + public void shouldRetryOnTimeoutException(@FlinkTest.FlinkParam FlinkTest.PipelineRunner runner, @FlinkTest.FlinkParam MockJsonClientProvider mockClientProvider) throws Exception { + mockClientProvider.givenTimeoutForAppend(); + + assertThatThrownBy(() -> { + runner + .withRetryCount(0) + .runWithCustomSink(withBigQuerySink(mockClientProvider, pipeline(List.of( + givenRowWithOffset(1, 1) + )))); + }).isInstanceOf(JobExecutionException.class); + + + verify(mockClientProvider.getMockJsonWriter(), times(6)).append(any(), anyLong()); + } + + + @Test + public void shouldDoNothingWhenFullBatchWasAlreadyAppended(@FlinkTest.FlinkParam FlinkTest.PipelineRunner runner, @FlinkTest.FlinkParam MockJsonClientProvider mockClientProvider) throws Exception { + mockClientProvider.givenAppendingExistingOffset(16, 4, stream); + + runner + .withRetryCount(0) + .runWithCustomSink(withBigQuerySink(mockClientProvider, pipeline(List.of( + givenRowWithOffset(4, 4) + )))); + + + verify(mockClientProvider.getMockJsonWriter(), times(1)).append(any(), anyLong()); + } + + @Test + public void shouldSplitBatchWhenAppendingBatchWhereNotAllRowsAreAppended(@FlinkTest.FlinkParam FlinkTest.PipelineRunner runner, @FlinkTest.FlinkParam MockJsonClientProvider mockClientProvider) throws Exception { + mockClientProvider.givenAppendingExistingOffset(4, 2, stream); + + runner + .withRetryCount(0) + .runWithCustomSink(withBigQuerySink(mockClientProvider, pipeline(List.of( + givenRowWithOffset(6, 2) + )))); + + + verify(mockClientProvider.getMockJsonWriter(), times(1)).append(any(), eq(2L)); + verify(mockClientProvider.getMockJsonWriter(), times(1)).append(any(), eq(4L)); + } + + @Test + public void shouldFailAndNotRetryWhenFailedWithOutOfRange(@FlinkTest.FlinkParam FlinkTest.PipelineRunner runner, @FlinkTest.FlinkParam MockJsonClientProvider mockClientProvider) throws Exception { + mockClientProvider.givenFailingAppendWithStatus(Status.OUT_OF_RANGE); + + assertThatThrownBy(() -> { + runner + .withRetryCount(0) + .runWithCustomSink(withBigQuerySink(mockClientProvider, pipeline(List.of( + givenRowWithOffset(1, 0) + )))); + }).isInstanceOf(JobExecutionException.class); + + + verify(mockClientProvider.getMockJsonWriter(), times(1)).append(any(), anyLong()); + } + + @Test + public void shouldFailAndNotRetryWhenAppendingFailedWithAlreadyExistsWithoutOffsetInformation(@FlinkTest.FlinkParam FlinkTest.PipelineRunner runner, @FlinkTest.FlinkParam MockJsonClientProvider mockClientProvider) throws Exception { + mockClientProvider.givenFailingAppendWithStatus(Status.ALREADY_EXISTS); + + assertThatThrownBy(() -> { + runner + .withRetryCount(0) + .runWithCustomSink(withBigQuerySink(mockClientProvider, pipeline(List.of( + givenRowWithOffset(1, 0) + )))); + }).isInstanceOf(JobExecutionException.class); + + + verify(mockClientProvider.getMockJsonWriter(), times(1)).append(any(), anyLong()); + } + + @Test + public void shouldFailAndNotRetryWhenAppendingFailedWithInvalidArgument(@FlinkTest.FlinkParam FlinkTest.PipelineRunner runner, @FlinkTest.FlinkParam MockJsonClientProvider mockClientProvider) throws Exception { + mockClientProvider.givenFailingAppendWithStatus(Status.INVALID_ARGUMENT); + + assertThatThrownBy(() -> { + runner + .withRetryCount(0) + .runWithCustomSink(withBigQuerySink(mockClientProvider, pipeline(List.of( + givenRowWithOffset(1, 0) + )))); + }).isInstanceOf(JobExecutionException.class); + + + verify(mockClientProvider.getMockJsonWriter(), times(1)).append(any(), anyLong()); + } + + @Test + public void shouldFailAndNotRetryWhenAppendingToFinalizedStream(@FlinkTest.FlinkParam FlinkTest.PipelineRunner runner, @FlinkTest.FlinkParam MockJsonClientProvider mockClientProvider) throws Exception { + mockClientProvider.givenStreamIsFinalized(stream); + + assertThatThrownBy(() -> { + runner + .withRetryCount(0) + .runWithCustomSink(withBigQuerySink(mockClientProvider, pipeline(List.of( + givenRowWithOffset(1, 0) + )))); + }).isInstanceOf(JobExecutionException.class); + + + verify(mockClientProvider.getMockJsonWriter(), times(1)).append(any(), anyLong()); + } + + @Test + public void shouldSplitTheBatchWhenAppendingTooLargeBatch(@FlinkTest.FlinkParam FlinkTest.PipelineRunner runner, @FlinkTest.FlinkParam MockJsonClientProvider mockClientProvider) throws Exception { + mockClientProvider.givenAppendingTooLargeBatch(); + + runner + .withRetryCount(0) + .runWithCustomSink(withBigQuerySink(mockClientProvider, pipeline(List.of( + givenRowWithOffset(6, 4) + )))); + + + verify(mockClientProvider.getMockJsonWriter(), times(2)).append(any(), eq(4L)); + verify(mockClientProvider.getMockJsonWriter(), times(1)).append(any(), eq(7L)); + } + + private Rows givenRowWithOffset(int count, int offset) { + var data = new ArrayList(count); + IntStream.rangeClosed(1, count) + .forEach(i -> data.add("{\"value\": " + i + "}")); + + return new Rows<>(data, offset, stream, testTable); + } + + private Function>> pipeline(List> data) { + return env -> env.fromCollection(data); + } + + private Function>> withBigQuerySink(MockJsonClientProvider mockClientProvider, Function>> pipeline) { + var sink = BigQueryStreamSink.newJson() + .withClientProvider(mockClientProvider) + .withDeliveryGuarantee(DeliveryGuarantee.EXACTLY_ONCE) + .withRowValueSerializer((JsonRowValueSerializer) String::getBytes) + .build(); + + return pipeline.andThen(s -> s.sinkTo(sink)); + } +} diff --git a/src/test/java/com/vinted/flink/bigquery/RowBatcherTest.java b/src/test/java/com/vinted/flink/bigquery/RowBatcherTest.java new file mode 100644 index 0000000..4617b15 --- /dev/null +++ b/src/test/java/com/vinted/flink/bigquery/RowBatcherTest.java @@ -0,0 +1,82 @@ +package com.vinted.flink.bigquery; + +import com.google.cloud.bigquery.TableId; +import com.vinted.flink.bigquery.model.BigQueryRecord; +import com.vinted.flink.bigquery.model.Rows; +import com.vinted.flink.bigquery.util.FlinkTest; +import com.vinted.flink.bigquery.util.MockClock; +import com.vinted.flink.bigquery.process.BatchTrigger; +import com.vinted.flink.bigquery.process.RowBatcher; +import com.vinted.flink.bigquery.util.MockJsonClientProvider; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.api.windowing.assigners.GlobalWindows; +import org.apache.flink.streaming.api.windowing.triggers.Trigger; +import org.apache.flink.streaming.api.windowing.windows.GlobalWindow; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +import java.io.Serializable; +import java.time.Duration; +import java.util.List; +import java.util.function.Function; + +import static org.assertj.core.api.Assertions.assertThat; + +@ExtendWith(FlinkTest.class) +public class RowBatcherTest { + TableId testTable = TableId.of("test-project", "test-dataset", "test-table"); + + @Test + public void shouldReleaseTwoBatches(@FlinkTest.FlinkParam FlinkTest.PipelineRunner runner, @FlinkTest.FlinkParam MockJsonClientProvider clientProvider, @FlinkTest.FlinkParam MockClock clock) throws Exception { + List> result = runner.run(pipeline(clientProvider, clock, List.of(new Record("key", "1"), new Record("key", "2"), new Record("key", "3"), new Record("key", "4")))); + + assertThat(result) + .usingRecursiveComparison() + .isEqualTo(List.of( + new Rows<>(List.of(new Record("key", "1"), new Record("key", "2")), -1, "projects/test-project/datasets/test-dataset/tables/test-table/_default", testTable), + new Rows<>(List.of(new Record("key", "3"), new Record("key", "4")), -1, "projects/test-project/datasets/test-dataset/tables/test-table/_default", testTable) + )); + } + + + private Function>> pipeline(MockJsonClientProvider provider, MockClock clock, List elements) { + return (env) -> { + Trigger trigger = BatchTrigger.builder() + .withCount(2) + .withTimeout(Duration.ofMinutes(1)) + .withResetTimerOnNewRecord(true) + .withSizeInMb(1) + .build(); + + //var table = new Table(testTable.getProject(), testTable.getDataset(), testTable.getTable()); + return env.fromCollection(elements) + .keyBy(s -> s.key) + .window(GlobalWindows.create()) + .trigger(trigger) + .process(new RowBatcher<>() { + }); + }; + + } + + static class Record implements Serializable, BigQueryRecord { + public String key; + public String value; + + public Record(String key, String value) { + this.key = key; + this.value = value; + } + + @Override + public TableId getTable() { + return TableId.of("test-project", "test-dataset", "test-table"); + } + + @Override + public long getSize() { + return this.value.getBytes().length; + } + } +} diff --git a/src/test/java/com/vinted/flink/bigquery/SchemaToolsTest.java b/src/test/java/com/vinted/flink/bigquery/SchemaToolsTest.java new file mode 100644 index 0000000..6c08329 --- /dev/null +++ b/src/test/java/com/vinted/flink/bigquery/SchemaToolsTest.java @@ -0,0 +1,120 @@ +package com.vinted.flink.bigquery; + +import com.google.api.services.bigquery.model.TableFieldSchema; +import com.google.cloud.bigquery.*; +import com.google.cloud.bigquery.spi.v2.HttpBigQueryRpc; +import com.vinted.flink.bigquery.schema.SchemaTools; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +public class SchemaToolsTest { + @Mock + BigQuery aBqClient; + @Mock + HttpBigQueryRpc aBqRpcClient; + + @Mock + BigQueryOptions aBqOptions; + @Mock + Table aTable; + @Mock + TableDefinition aTableDefinition; + + TableId tableId = TableId.of("test-dwh-poc", "cdc", "test1"); + + @BeforeEach + public void init() { + Mockito.reset(aBqClient); + Mockito.reset(aBqRpcClient); + Mockito.reset(aBqOptions); + Mockito.reset(aTable); + Mockito.reset(aTableDefinition); + + lenient().doReturn(aBqClient).when(aBqOptions).getService(); + lenient().doReturn(aBqRpcClient).when(aBqOptions).getRpc(); + lenient().doReturn(aTable).when(aBqClient).getTable(any(TableId.class), any()); + lenient().doReturn(aTableDefinition).when(aTable).getDefinition(); + } + + @Test + public void shouldAddAndRemoveFields() throws InterruptedException { + var bqSchema = Schema.of(FieldList.of(Field.of("a1", LegacySQLTypeName.STRING))); + doReturn(bqSchema).when(aTableDefinition).getSchema(); + var schemaTools = new SchemaTools(aBqOptions); + + var newSchema = Schema.of( + FieldList.of( + Field.of("b1", LegacySQLTypeName.INTEGER), + Field.of("b2", LegacySQLTypeName.RECORD, FieldList.of(Field.of("c1", LegacySQLTypeName.STRING))) + )); + + var bqTableFieldSchema = List.of( + new TableFieldSchema().setName("a1").setType(LegacySQLTypeName.STRING.toString()), + new TableFieldSchema().setName("b1").setType(LegacySQLTypeName.INTEGER.toString()), + new TableFieldSchema().setName("b2").setType(LegacySQLTypeName.RECORD.toString()) + .setFields(List.of(new TableFieldSchema().setName("c1").setType(LegacySQLTypeName.STRING.toString()))) + ); + + var bqClientCaptor = ArgumentCaptor.forClass(QueryJobConfiguration.class); + var bqRpcClientCaptor = ArgumentCaptor.forClass(com.google.api.services.bigquery.model.Table.class); + schemaTools.updateTable(tableId, newSchema); + + verify(aBqClient, times(1)).query(bqClientCaptor.capture()); + verify(aBqRpcClient, times(1)).patch(bqRpcClientCaptor.capture(), anyMap()); + assertThat(bqClientCaptor.getValue().getQuery()).isEqualTo("ALTER TABLE cdc.test1 DROP COLUMN IF EXISTS a1"); + assertThat(bqRpcClientCaptor.getValue().getSchema().getFields()).containsExactlyInAnyOrderElementsOf(bqTableFieldSchema); + } + + @Test + public void shouldCreateNewTable() throws InterruptedException { + var bqSchema = Schema.of(); + lenient().doReturn(bqSchema).when(aTableDefinition).getSchema(); + var schemaTools = new SchemaTools(aBqOptions); + + var newSchema = Schema.of(FieldList.of(Field.of("b1", StandardSQLTypeName.INT64))); + + var captor = ArgumentCaptor.forClass(TableInfo.class); + schemaTools.createTable(TableInfo.newBuilder(tableId, StandardTableDefinition.of(newSchema)).build()); + + verify(aBqClient, times(1)).create(captor.capture()); + assertThat(captor.getValue().getDefinition().getSchema().getFields()).containsExactlyInAnyOrderElementsOf(newSchema.getFields()); + + } + + @Test + public void shouldDoNothingWhenSchemasAreTheSame() throws InterruptedException { + var bqSchema = Schema.of( + FieldList.of( + Field.of("b1", LegacySQLTypeName.INTEGER), + Field.of("a1", LegacySQLTypeName.STRING) + ) + ); + var newSchema = Schema.of( + FieldList.of( + Field.of("a1", LegacySQLTypeName.STRING), + Field.of("b1", LegacySQLTypeName.INTEGER) + ) + ); + + lenient().doReturn(bqSchema).when(aTableDefinition).getSchema(); + var schemaTools = new SchemaTools(aBqOptions); + + var bqClientCaptor = ArgumentCaptor.forClass(QueryJobConfiguration.class); + var bqRpcClientCaptor = ArgumentCaptor.forClass(com.google.api.services.bigquery.model.Table.class); + schemaTools.updateTable(tableId, newSchema); + + verify(aBqClient, times(0)).query(bqClientCaptor.capture()); + verify(aBqRpcClient, times(0)).patch(bqRpcClientCaptor.capture(), anyMap()); + } +} diff --git a/src/test/java/com/vinted/flink/bigquery/StreamHandlerTest.java b/src/test/java/com/vinted/flink/bigquery/StreamHandlerTest.java new file mode 100644 index 0000000..c0cc2e5 --- /dev/null +++ b/src/test/java/com/vinted/flink/bigquery/StreamHandlerTest.java @@ -0,0 +1,285 @@ +package com.vinted.flink.bigquery; + +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.storage.v1.CreateWriteStreamRequest; +import com.vinted.flink.bigquery.model.BigQueryRecord; +import com.vinted.flink.bigquery.model.Rows; +import com.vinted.flink.bigquery.util.FlinkTest; +import com.vinted.flink.bigquery.util.MockClock; +import com.vinted.flink.bigquery.util.MockJsonClientProvider; +import com.vinted.flink.bigquery.process.BatchTrigger; +import com.vinted.flink.bigquery.process.StreamStateHandler; +import org.apache.flink.runtime.client.JobExecutionException; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.api.windowing.assigners.GlobalWindows; +import org.apache.flink.streaming.api.windowing.triggers.Trigger; +import org.apache.flink.streaming.api.windowing.windows.GlobalWindow; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; + +import java.io.Serializable; +import java.time.Clock; +import java.time.Duration; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +@ExtendWith(FlinkTest.class) +public class StreamHandlerTest { + TableId testTable = TableId.of("test-project", "test-dataset", "test-table"); + + @Test + public void shouldReleaseTwoBatches(@FlinkTest.FlinkParam FlinkTest.PipelineRunner runner, @FlinkTest.FlinkParam MockJsonClientProvider clientProvider, @FlinkTest.FlinkParam MockClock clock) throws Exception { + clientProvider.givenCreateStream("stream1", "stream2", "stream3"); + + List> result = runner.run(pipeline(clientProvider, clock, List.of(new Record("key", "1"), new Record("key", "2"), new Record("key", "3"), new Record("key", "4")))); + + assertThat(result) + .usingRecursiveComparison() + .isEqualTo(List.of( + new Rows<>(List.of(new Record("key", "1"), new Record("key", "2")), 0, "stream1", testTable), + new Rows<>(List.of(new Record("key", "3"), new Record("key", "4")), 2, "stream1", testTable) + )); + } + + @Test + public void shouldTrackOffsetsByKey(@FlinkTest.FlinkParam FlinkTest.PipelineRunner runner, @FlinkTest.FlinkParam MockJsonClientProvider clientProvider, @FlinkTest.FlinkParam MockClock clock) throws Exception { + clientProvider.givenCreateStream("stream1", "stream2", "stream3"); + + List> result = runner.run(pipeline( + clientProvider, clock, List.of( + new Record("key", "1"), new Record("key", "2"), + new Record("key2", "3"), new Record("key2", "4"), + new Record("key2", "5"), new Record("key2", "6") + ))); + + assertThat(result) + .usingRecursiveComparison() + .isEqualTo(List.of( + new Rows<>(List.of(new Record("key", "1"), new Record("key", "2")), 0, "stream1", testTable), + new Rows<>(List.of(new Record("key2", "3"), new Record("key2", "4")), 0, "stream2", testTable), + new Rows<>(List.of(new Record("key2", "5"), new Record("key2", "6")), 2, "stream2", testTable) + )); + } + + @Test + public void shouldCreateStreamWhenReceivingNewBatch(@FlinkTest.FlinkParam FlinkTest.PipelineRunner runner, @FlinkTest.FlinkParam MockJsonClientProvider clientProvider, @FlinkTest.FlinkParam MockClock clock) throws Exception { + clientProvider.givenCreateStream("stream1", "stream2", "stream3"); + + runner.run(pipeline(clientProvider, clock, List.of(new Record("key", "1"), new Record("key", "2")))); + + var bqClientCaptor = ArgumentCaptor.forClass(CreateWriteStreamRequest.class); + verify(clientProvider.getClient(), times(1)).createWriteStream(bqClientCaptor.capture()); + } + + @Test + public void shouldUseSameStreamForOtherBatchesWithSameKeys(@FlinkTest.FlinkParam FlinkTest.PipelineRunner runner, @FlinkTest.FlinkParam MockJsonClientProvider clientProvider, @FlinkTest.FlinkParam MockClock clock) throws Exception { + clientProvider.givenCreateStream("stream1", "stream2", "stream3"); + + runner.run(pipeline(clientProvider, clock, List.of(new Record("key", "1"), new Record("key", "2"), new Record("key", "3"), new Record("key", "4")))); + + var bqClientCaptor = ArgumentCaptor.forClass(CreateWriteStreamRequest.class); + verify(clientProvider.getClient(), times(1)).createWriteStream(bqClientCaptor.capture()); + } + + @Test + public void shouldReuseStreamWhenRecoveringFromCheckpointAndStreamNotExpired(@FlinkTest.FlinkParam FlinkTest.PipelineRunner runner, @FlinkTest.FlinkParam MockJsonClientProvider clientProvider, @FlinkTest.FlinkParam MockClock clock) throws Exception { + clientProvider.givenCreateStream("stream1", "stream2", "stream3"); + + runner + .withErrorAfter(5) + .run(pipeline(clientProvider, clock, List.of( + new Record("key", "1"), new Record("key", "2"), + new Record("key", "3"), new Record("key", "4"), + new Record("key", "5"), new Record("key", "6"), + new Record("key", "7"), new Record("key", "8"), + new Record("key", "9"), new Record("key", "10"), + new Record("key", "11"), new Record("key", "12") + ))); + + var bqClientCaptor = ArgumentCaptor.forClass(CreateWriteStreamRequest.class); + + verify(clientProvider.getClient(), times(1)).createWriteStream(bqClientCaptor.capture()); + verify(clientProvider.getClient(), times(1)).getWriteStream("stream1"); + } + + @Test + public void shouldReuseStreamWhenRecoveringFromCheckpointAndStreamIsExpired(@FlinkTest.FlinkParam FlinkTest.PipelineRunner runner, @FlinkTest.FlinkParam MockJsonClientProvider clientProvider, @FlinkTest.FlinkParam MockClock clock) throws Exception { + clientProvider.givenCreateStream("stream2", "stream3", "stream4"); + clock.givenCurrentMillis(10, 20, 30, 40, 50, TimeUnit.DAYS.toMillis(20)); + + var result = runner + .withErrorAfter(2) + .run(pipeline(clientProvider, clock, List.of( + new Record("key", "1"), new Record("key", "2"), + new Record("key", "3"), new Record("key", "4"), + new Record("key", "5"), new Record("key", "6"), + new Record("key", "7"), new Record("key", "8"), + new Record("key", "9"), new Record("key", "10") + ))); + + + var streams = result.stream().collect(Collectors.groupingBy(Rows::getStream)); + assertThat(streams).hasSize(2); + + + var stream1Records = streams.get("stream2").stream().map(Rows::getData).collect(Collectors.toList()); + var stream2Records = streams.get("stream3").stream().map(Rows::getData).collect(Collectors.toList()); + + assertThat(stream1Records) + .usingRecursiveComparison() + .asList() + .doesNotContain(new Record("key", "9"), new Record("key", "10")); + + assertThat(stream2Records) + .usingRecursiveComparison() + .asList() + .doesNotContain(new Record("key", "9"), new Record("key", "10")); + + + var bqClientCaptor = ArgumentCaptor.forClass(CreateWriteStreamRequest.class); + verify(clientProvider.getClient(), times(2)).createWriteStream(bqClientCaptor.capture()); + } + + @Test + public void shouldRecreateStreamWhenNotFound(@FlinkTest.FlinkParam FlinkTest.PipelineRunner runner, @FlinkTest.FlinkParam MockJsonClientProvider clientProvider, @FlinkTest.FlinkParam MockClock clock) throws Exception { + clientProvider.givenCreateStream("stream1", "stream2"); + clientProvider.givenStreamDoesNotExist("stream1"); + + var result = runner + .withErrorAfter(5) + .run(pipeline(clientProvider, clock, List.of( + new Record("key", "1"), new Record("key", "2"), + new Record("key", "3"), new Record("key", "4"), + new Record("key", "5"), new Record("key", "6"), + new Record("key", "7"), new Record("key", "8"), + new Record("key", "9"), new Record("key", "10"), + new Record("key", "11"), new Record("key", "12") + ))); + + + var streams = result.stream().collect(Collectors.groupingBy(Rows::getStream)); + assertThat(streams).hasSize(2); + + var stream1Records = streams.get("stream1").stream().map(Rows::getData).collect(Collectors.toList()); + var stream2Records = streams.get("stream2").stream().map(Rows::getData).collect(Collectors.toList()); + + assertThat(stream1Records) + .usingRecursiveComparison() + .asList() + .doesNotContain(new Record("key", "11"), new Record("key", "12")); + + assertThat(stream2Records) + .usingRecursiveComparison() + .asList() + .doesNotContain(new Record("key", "11"), new Record("key", "12")); + + + var bqClientCaptor = ArgumentCaptor.forClass(CreateWriteStreamRequest.class); + verify(clientProvider.getClient(), times(2)).createWriteStream(bqClientCaptor.capture()); + } + + @Test + public void shouldRecreateStreamWhenStreamIsFinalized(@FlinkTest.FlinkParam FlinkTest.PipelineRunner runner, @FlinkTest.FlinkParam MockJsonClientProvider clientProvider, @FlinkTest.FlinkParam MockClock clock) throws Exception { + clientProvider.givenCreateStream("stream1", "stream2"); + clientProvider.givenStreamIsFinalized("stream1"); + + var result = runner + .withErrorAfter(5) + .run(pipeline(clientProvider, clock, List.of( + new Record("key", "1"), new Record("key", "2"), + new Record("key", "3"), new Record("key", "4"), + new Record("key", "5"), new Record("key", "6"), + new Record("key", "7"), new Record("key", "8"), + new Record("key", "9"), new Record("key", "10"), + new Record("key", "11"), new Record("key", "12") + ))); + + + var streams = result.stream().collect(Collectors.groupingBy(Rows::getStream)); + assertThat(streams).hasSize(2); + + var stream1Records = streams.get("stream1").stream().map(Rows::getData).collect(Collectors.toList()); + var stream2Records = streams.get("stream2").stream().map(Rows::getData).collect(Collectors.toList()); + + assertThat(stream1Records) + .usingRecursiveComparison() + .asList() + .doesNotContain(new Record("key", "11"), new Record("key", "12")); + + assertThat(stream2Records) + .usingRecursiveComparison() + .asList() + .doesNotContain(new Record("key", "11"), new Record("key", "12")); + + var bqClientCaptor = ArgumentCaptor.forClass(CreateWriteStreamRequest.class); + verify(clientProvider.getClient(), times(2)).createWriteStream(bqClientCaptor.capture()); + } + + @Test + public void shouldFailWhenUnhandledExceptionIsThrown(@FlinkTest.FlinkParam FlinkTest.PipelineRunner runner, @FlinkTest.FlinkParam MockJsonClientProvider clientProvider, @FlinkTest.FlinkParam MockClock clock) throws Exception { + clientProvider.givenCreateStream("stream1", "stream2"); + clientProvider.givenGettingStreamFails("stream1"); + assertThatThrownBy(() -> { + runner + .withErrorAfter(3) + .run(pipeline(clientProvider, clock, List.of( + new Record("key", "1"), new Record("key", "2"), + new Record("key", "3"), new Record("key", "4"), + new Record("key", "5"), new Record("key", "6"), + new Record("key", "7"), new Record("key", "8") + ))); + }).isInstanceOf(JobExecutionException.class); + } + + private Function>> pipeline(MockJsonClientProvider provider, MockClock clock, List elements) { + return (env) -> { + Trigger trigger = BatchTrigger.builder() + .withCount(2) + .withTimeout(Duration.ofMinutes(1)) + .withResetTimerOnNewRecord(true) + .withSizeInMb(1) + .build(); + + return env.fromCollection(elements) + .keyBy(s -> s.key) + .window(GlobalWindows.create()) + .trigger(trigger) + .process(new StreamStateHandler<>(provider) { + @Override + protected Clock getClock() { + return clock.get(); + } + }).uid("unique-state"); + }; + + } + + static class Record implements Serializable, BigQueryRecord { + public String key; + public String value; + + public Record(String key, String value) { + this.key = key; + this.value = value; + } + + @Override + public TableId getTable() { + return TableId.of("test-project", "test-dataset", "test-table"); + } + + @Override + public long getSize() { + return this.value.getBytes().length; + } + } +} diff --git a/src/test/java/com/vinted/flink/bigquery/util/FlinkTest.java b/src/test/java/com/vinted/flink/bigquery/util/FlinkTest.java new file mode 100644 index 0000000..132d62f --- /dev/null +++ b/src/test/java/com/vinted/flink/bigquery/util/FlinkTest.java @@ -0,0 +1,183 @@ +package com.vinted.flink.bigquery.util; + +import com.vinted.flink.bigquery.model.Rows; +import org.apache.flink.api.common.restartstrategy.RestartStrategies; +import org.apache.flink.api.common.time.Time; +import com.vinted.flink.bigquery.process.StreamState; +import org.apache.flink.configuration.CheckpointingOptions; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration; +import org.apache.flink.streaming.api.CheckpointingMode; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.datastream.DataStreamSink; +import org.apache.flink.streaming.api.environment.CheckpointConfig; +import org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.test.util.MiniClusterWithClientResource; +import org.junit.jupiter.api.extension.*; +import org.testcontainers.shaded.org.apache.commons.io.FileUtils; + +import java.io.IOException; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; + +public class FlinkTest implements AfterAllCallback, AfterEachCallback, BeforeEachCallback, ParameterResolver { + private Path tempDir; + + private int defaultParallelism = 1; + + { + try { + tempDir = Files.createTempDirectory("flink-checkpoints"); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private Configuration config = new Configuration(); + + { + config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, tempDir.toUri().toString()); + config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, tempDir.toUri().toString()); + config.setBoolean(ExecutionCheckpointingOptions.ENABLE_CHECKPOINTS_AFTER_TASKS_FINISH, true); + + } + + private MiniClusterWithClientResource flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder() + .setNumberSlotsPerTaskManager(defaultParallelism) + .setNumberTaskManagers(1) + .setConfiguration(config) + .build()); + + @Override + public void afterAll(ExtensionContext context) throws Exception { + FileUtils.deleteDirectory(tempDir.toFile()); + } + @Override + public void afterEach(ExtensionContext context) throws Exception { + flinkCluster.after(); + MockClock.reset(); + MockJsonClientProvider.reset(); + ProcessFunctionWithError.clear(); + TestSink.clear(); + } + + @Override + public void beforeEach(ExtensionContext context) throws Exception { + flinkCluster.before(); + MockClock.reset(); + MockJsonClientProvider.reset(); + MockProtoClientProvider.reset(); + ProcessFunctionWithError.clear(); + TestSink.clear(); + } + + @Override + public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException { + return parameterContext.isAnnotated(FlinkParam.class); + } + + @Override + public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException { + Class type = parameterContext.getParameter().getType(); + if (MockClock.class.equals(type)) { + return new MockClock(); + } + if (MockJsonClientProvider.class.equals(type)) { + return new MockJsonClientProvider(); + } + + if (MockProtoClientProvider.class.equals(type)) { + return new MockProtoClientProvider(); + } + + if (PipelineRunner.class.equals(type)) { + return new PipelineRunner(); + } + throw new ParameterResolutionException("No random generator implemented for " + type); + } + @Retention(RetentionPolicy.RUNTIME) + @Target({ElementType.FIELD, ElementType.PARAMETER}) + public @interface FlinkParam { + + } + + + public class PipelineRunner { + private int defaultParallelism = 1; + private int retryCount = 1; + + private boolean error = false; + private int errorAfter = 0; + + public PipelineRunner withRetryCount(int count) { + this.retryCount = count; + return this; + } + + public PipelineRunner withDefaultParallelism(int value) { + this.defaultParallelism = value; + return this; + } + + public PipelineRunner withErrorAfter(int records) { + this.error = true; + this.errorAfter = records; + return this; + } + + + public List run(Function> execute) throws Exception { + var env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(defaultParallelism); + env.enableCheckpointing(10); + env.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE); + env.getCheckpointConfig().setMinPauseBetweenCheckpoints(10); + env.getCheckpointConfig().enableUnalignedCheckpoints(); + env.getCheckpointConfig().setCheckpointStorage(tempDir.toUri()); + env.getCheckpointConfig().setExternalizedCheckpointCleanup(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION); + env.configure(config); + env.setRestartStrategy(RestartStrategies.fixedDelayRestart( + retryCount, // number of restart attempts + Time.of(5, TimeUnit.MILLISECONDS) // delay + )); + + env.getConfig().registerKryoType(StreamState.class); + env.getConfig().registerKryoType(Rows.class); + + var testSink = new TestSink(); + var stream = error ? execute.apply(env).process(new ProcessFunctionWithError<>(errorAfter)) : execute.apply(env); + stream.addSink(testSink); + var result = env.execute(); + return testSink.getResults(result); + } + + public void runWithCustomSink(Function> execute) throws Exception { + var env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(defaultParallelism); + env.enableCheckpointing(10); + env.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE); + env.getCheckpointConfig().setMinPauseBetweenCheckpoints(10); + env.getCheckpointConfig().enableUnalignedCheckpoints(); + env.getCheckpointConfig().setCheckpointStorage(tempDir.toUri()); + env.getCheckpointConfig().setExternalizedCheckpointCleanup(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION); + env.configure(config); + env.setRestartStrategy(RestartStrategies.fixedDelayRestart( + retryCount, // number of restart attempts + Time.of(5, TimeUnit.MILLISECONDS) // delay + )); + + env.getConfig().registerKryoType(StreamState.class); + env.getConfig().registerKryoType(Rows.class); + execute.apply(env); + env.execute(); + } + } +} diff --git a/src/test/java/com/vinted/flink/bigquery/util/MockClock.java b/src/test/java/com/vinted/flink/bigquery/util/MockClock.java new file mode 100644 index 0000000..daa4e15 --- /dev/null +++ b/src/test/java/com/vinted/flink/bigquery/util/MockClock.java @@ -0,0 +1,30 @@ +package com.vinted.flink.bigquery.util; + +import org.mockito.Mockito; + +import java.io.Serializable; +import java.time.Clock; +import java.util.ArrayList; + +public class MockClock implements Serializable { + private static Clock clock = Mockito.mock(Clock.class); + + public static void reset() { + Mockito.reset(clock); + } + + public Clock get() { + return MockClock.clock; + } + + public void givenCurrentMillis(long... ms) { + var list = new ArrayList(); + for (long i : ms) { + list.add(i); + } + + long first = list.get(0); + Long[] rest = list.subList(1, list.size()).toArray(Long[]::new); + Mockito.when(clock.millis()).thenReturn(first, rest); + } +} diff --git a/src/test/java/com/vinted/flink/bigquery/util/MockJsonClientProvider.java b/src/test/java/com/vinted/flink/bigquery/util/MockJsonClientProvider.java new file mode 100644 index 0000000..8bddc1b --- /dev/null +++ b/src/test/java/com/vinted/flink/bigquery/util/MockJsonClientProvider.java @@ -0,0 +1,158 @@ +package com.vinted.flink.bigquery.util; + +import com.google.api.core.SettableApiFuture; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.storage.v1.*; +import com.google.protobuf.Descriptors; +import com.vinted.flink.bigquery.client.ClientProvider; +import com.vinted.flink.bigquery.model.config.WriterSettings; +import io.grpc.Status; +import io.grpc.StatusException; +import org.mockito.Mockito; + +import java.io.IOException; +import java.io.Serializable; +import java.util.List; + +public class MockJsonClientProvider implements ClientProvider, Serializable { + private static BigQueryWriteClient mockClient = Mockito.mock(BigQueryWriteClient.class); + private static JsonStreamWriter writer = Mockito.mock(JsonStreamWriter.class); + + public void givenStreamDoesNotExist(String streamName) { + Mockito.doThrow(new RuntimeException(new StatusException(Status.NOT_FOUND))) + .when(MockJsonClientProvider.mockClient).getWriteStream(streamName); + } + + public void givenStreamIsFinalized(String streamName) throws Descriptors.DescriptorValidationException, IOException { + var exception = createFinalizedStreamException(); + var ex = new RuntimeException(exception); + Mockito.when(MockJsonClientProvider.mockClient.getWriteStream(streamName)) + .thenThrow(ex); + + Mockito.when(MockJsonClientProvider.writer.append(Mockito.any(), Mockito.anyLong())) + .thenReturn(createAppendRowsResponseError(exception)); + } + + public void givenGettingStreamFails(String streamName) { + Mockito.doThrow(new RuntimeException(new StatusException(Status.INTERNAL))) + .when(MockJsonClientProvider.mockClient).getWriteStream(streamName); + } + + public void givenCreateStream(String... streamNames) { + var list = List.of(streamNames); + var mock = Mockito.when(MockJsonClientProvider.mockClient.createWriteStream(Mockito.nullable(CreateWriteStreamRequest.class))) + .thenReturn(WriteStream + .newBuilder() + .setName(list.get(0)) + .buildPartial() + ); + list.subList(1, list.size()).forEach(name -> { + mock.thenReturn(WriteStream + .newBuilder() + .setName(name) + .buildPartial()); + }); + } + + public void givenSuccessfulAppend() throws Descriptors.DescriptorValidationException, IOException { + Mockito.when(MockJsonClientProvider.writer.append(Mockito.any(), Mockito.anyLong())) + .thenReturn(createAppendRowsResponse()); + + Mockito.when(MockJsonClientProvider.writer.append(Mockito.any())) + .thenReturn(createAppendRowsResponse()); + } + + public void givenFailingAppendWithStatus(Status status) throws Descriptors.DescriptorValidationException, IOException { + Mockito.when(MockJsonClientProvider.writer.append(Mockito.any(), Mockito.anyLong())) + .thenReturn(createAppendRowsResponseError(new StatusException(status))); + + Mockito.when(MockJsonClientProvider.writer.append(Mockito.any())) + .thenReturn(createAppendRowsResponseError(new StatusException(status))); + } + + public void givenTimeoutForAppend() throws Descriptors.DescriptorValidationException, IOException { + Mockito.when(MockJsonClientProvider.writer.append(Mockito.any(), Mockito.anyLong())) + .thenReturn(createTimeoutAppendRowsResponse()); + } + + + public void givenAppendingExistingOffset(int expected, int actual, String streamName) throws Descriptors.DescriptorValidationException, IOException { + var offsetMock = createOffsetAlreadyExistsException(expected, actual, streamName); + + Mockito.when(MockJsonClientProvider.writer.append(Mockito.any(), Mockito.anyLong())) + .thenReturn(createAppendRowsResponseError(offsetMock)) + .thenReturn(createAppendRowsResponse()); + } + + public void givenAppendingTooLargeBatch() throws Descriptors.DescriptorValidationException, IOException { + var ex = new StatusException(Status.INVALID_ARGUMENT + .augmentDescription("MessageSize is too large. Max allow: 10000000 Actual: 12040940 status: INVALID_ARGUMENT stream: project")); + + Mockito.when(MockJsonClientProvider.writer.append(Mockito.any(), Mockito.anyLong())) + .thenReturn(createAppendRowsResponseError(ex)) + .thenReturn(createAppendRowsResponse()); + + Mockito.when(MockJsonClientProvider.writer.append(Mockito.any())) + .thenReturn(createAppendRowsResponseError(ex)) + .thenReturn(createAppendRowsResponse()); + } + + public static void reset() { + Mockito.reset(MockJsonClientProvider.mockClient); + Mockito.reset(MockJsonClientProvider.writer); + } + + private static Exceptions.StreamFinalizedException createFinalizedStreamException() { + var exception = Mockito.mock(Exceptions.StreamFinalizedException.class); + Mockito.when(exception.getStatus()).thenReturn(Status.INVALID_ARGUMENT); + Mockito.when(exception.getCause()).thenReturn(new RuntimeException()); + return exception; + } + + private static Exceptions.OffsetAlreadyExists createOffsetAlreadyExistsException(long expected, long actual, String streamName) { + var offsetMock = Mockito.mock(Exceptions.OffsetAlreadyExists.class); + Mockito.when(offsetMock.getStatus()).thenReturn(Status.ALREADY_EXISTS); + Mockito.when(offsetMock.getStreamName()).thenReturn(streamName); + Mockito.when(offsetMock.getExpectedOffset()).thenReturn(expected); + Mockito.when(offsetMock.getActualOffset()).thenReturn(actual); + Mockito.when(offsetMock.getCause()).thenReturn(new RuntimeException()); + return offsetMock; + } + + private static SettableApiFuture createAppendRowsResponse() { + SettableApiFuture result = SettableApiFuture.create(); + result.set(AppendRowsResponse.newBuilder().buildPartial()); + return result; + } + + private static SettableApiFuture createTimeoutAppendRowsResponse() { + SettableApiFuture result = SettableApiFuture.create(); + return result; + } + + private static SettableApiFuture createAppendRowsResponseError(Throwable exception) { + SettableApiFuture result = SettableApiFuture.create(); + result.setException(exception); + return result; + } + + public JsonStreamWriter getMockJsonWriter() { + return MockJsonClientProvider.writer; + } + + @Override + public BigQueryWriteClient getClient() { + return MockJsonClientProvider.mockClient; + } + + @Override + public JsonStreamWriter getWriter(String streamName, TableId table) { + return MockJsonClientProvider.writer; + } + + + @Override + public WriterSettings writeSettings() { + return WriterSettings.newBuilder().build(); + } +} diff --git a/src/test/java/com/vinted/flink/bigquery/util/MockProtoClientProvider.java b/src/test/java/com/vinted/flink/bigquery/util/MockProtoClientProvider.java new file mode 100644 index 0000000..36eb555 --- /dev/null +++ b/src/test/java/com/vinted/flink/bigquery/util/MockProtoClientProvider.java @@ -0,0 +1,161 @@ +package com.vinted.flink.bigquery.util; + +import com.google.api.core.SettableApiFuture; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.storage.v1.*; +import com.google.protobuf.Descriptors; +import com.vinted.flink.bigquery.client.ClientProvider; +import com.vinted.flink.bigquery.model.config.WriterSettings; +import io.grpc.Status; +import io.grpc.StatusException; +import org.mockito.Mockito; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; + +public class MockProtoClientProvider implements ClientProvider, Serializable { + private static BigQueryWriteClient mockClient = Mockito.mock(BigQueryWriteClient.class); + private static StreamWriter protoWriter = Mockito.mock(StreamWriter.class); + + private static void givenExistingStream(String streamName) { + var stream = WriteStream + .newBuilder() + .setName(streamName) + .build(); + + Mockito.doReturn(stream).when(MockProtoClientProvider.mockClient).getWriteStream(Mockito.anyString()); + } + + public void givenStreamDoesNotExist(String streamName) { + Mockito.doThrow(new RuntimeException(new StatusException(Status.NOT_FOUND))) + .when(MockProtoClientProvider.mockClient).getWriteStream(streamName); + } + + public void givenStreamIsFinalized(String streamName) throws Descriptors.DescriptorValidationException, IOException { + var exception = createFinalizedStreamException(); + var ex = new RuntimeException(exception); + Mockito.when(MockProtoClientProvider.mockClient.getWriteStream(streamName)) + .thenThrow(ex); + + Mockito.when(MockProtoClientProvider.protoWriter.append(Mockito.any(), Mockito.anyLong())) + .thenReturn(createAppendRowsResponseError(exception)); + } + + public void givenGettingStreamFails(String streamName) { + Mockito.doThrow(new RuntimeException(new StatusException(Status.INTERNAL))) + .when(MockProtoClientProvider.mockClient).getWriteStream(streamName); + } + + public void givenCreateStream(String... streamNames) { + var list = new ArrayList(); + for (String i : streamNames) { + list.add(i); + } + + var mock = Mockito.when(MockProtoClientProvider.mockClient.createWriteStream(Mockito.nullable(CreateWriteStreamRequest.class))) + .thenReturn(WriteStream + .newBuilder() + .setName(list.get(0)) + .buildPartial() + ); + list.subList(1, list.size()).forEach(name -> { + mock.thenReturn(WriteStream + .newBuilder() + .setName(name) + .buildPartial()); + }); + } + + public void givenSuccessfulAppend() throws Descriptors.DescriptorValidationException, IOException { + Mockito.when(MockProtoClientProvider.protoWriter.append(Mockito.any(), Mockito.anyLong())) + .thenReturn(createAppendRowsResponse()); + } + + public void givenFailingAppendWithStatus(Status status) throws Descriptors.DescriptorValidationException, IOException { + Mockito.when(MockProtoClientProvider.protoWriter.append(Mockito.any(), Mockito.anyLong())) + .thenReturn(createAppendRowsResponseError(new StatusException(status))); + } + + public void givenTimeoutForAppend() throws Descriptors.DescriptorValidationException, IOException { + Mockito.when(MockProtoClientProvider.protoWriter.append(Mockito.any(), Mockito.anyLong())) + .thenReturn(createTimeoutAppendRowsResponse()); + } + + + public void givenAppendingExistingOffset(int expected, int actual, String streamName) throws Descriptors.DescriptorValidationException, IOException { + var offsetMock = createOffsetAlreadyExistsException(expected, actual, streamName); + + Mockito.when(MockProtoClientProvider.protoWriter.append(Mockito.any(), Mockito.anyLong())) + .thenReturn(createAppendRowsResponseError(offsetMock)) + .thenReturn(createAppendRowsResponse()); + } + + public void givenAppendingTooLargeBatch() throws Descriptors.DescriptorValidationException, IOException { + var ex = new StatusException(Status.INVALID_ARGUMENT + .augmentDescription("MessageSize is too large. Max allow: 10000000 Actual: 12040940 status: INVALID_ARGUMENT stream: project")); + + Mockito.when(MockProtoClientProvider.protoWriter.append(Mockito.any(), Mockito.anyLong())) + .thenReturn(createAppendRowsResponseError(ex)) + .thenReturn(createAppendRowsResponse()); + } + + public static void reset() { + Mockito.reset(MockProtoClientProvider.mockClient); + Mockito.reset(MockProtoClientProvider.protoWriter); + } + + private static Exceptions.StreamFinalizedException createFinalizedStreamException() { + var exception = Mockito.mock(Exceptions.StreamFinalizedException.class); + Mockito.when(exception.getStatus()).thenReturn(Status.INVALID_ARGUMENT); + Mockito.when(exception.getCause()).thenReturn(new RuntimeException()); + return exception; + } + + private static Exceptions.OffsetAlreadyExists createOffsetAlreadyExistsException(long expected, long actual, String streamName) { + var offsetMock = Mockito.mock(Exceptions.OffsetAlreadyExists.class); + Mockito.when(offsetMock.getStatus()).thenReturn(Status.ALREADY_EXISTS); + Mockito.when(offsetMock.getStreamName()).thenReturn(streamName); + Mockito.when(offsetMock.getExpectedOffset()).thenReturn(expected); + Mockito.when(offsetMock.getActualOffset()).thenReturn(actual); + Mockito.when(offsetMock.getCause()).thenReturn(new RuntimeException()); + return offsetMock; + } + + private static SettableApiFuture createAppendRowsResponse() { + SettableApiFuture result = SettableApiFuture.create(); + result.set(AppendRowsResponse.newBuilder().buildPartial()); + return result; + } + + private static SettableApiFuture createTimeoutAppendRowsResponse() { + SettableApiFuture result = SettableApiFuture.create(); + return result; + } + + private static SettableApiFuture createAppendRowsResponseError(Throwable exception) { + SettableApiFuture result = SettableApiFuture.create(); + result.setException(exception); + return result; + } + + public StreamWriter getMockProtoWriter() { + return MockProtoClientProvider.protoWriter; + } + + @Override + public BigQueryWriteClient getClient() { + return MockProtoClientProvider.mockClient; + } + + @Override + public StreamWriter getWriter(String streamName, TableId table) { + return MockProtoClientProvider.protoWriter; + } + + + @Override + public WriterSettings writeSettings() { + return WriterSettings.newBuilder().build(); + } +} diff --git a/src/test/java/com/vinted/flink/bigquery/util/ProcessFunctionWithError.java b/src/test/java/com/vinted/flink/bigquery/util/ProcessFunctionWithError.java new file mode 100644 index 0000000..a399971 --- /dev/null +++ b/src/test/java/com/vinted/flink/bigquery/util/ProcessFunctionWithError.java @@ -0,0 +1,30 @@ +package com.vinted.flink.bigquery.util; + +import org.apache.flink.streaming.api.functions.ProcessFunction; +import org.apache.flink.util.Collector; + +import java.util.concurrent.atomic.AtomicInteger; + +public class ProcessFunctionWithError extends ProcessFunction { + + private int errorAfterRecord; + + public ProcessFunctionWithError(int errorAfterRecord) { + this.errorAfterRecord = errorAfterRecord; + } + + private static AtomicInteger counter = new AtomicInteger(1); + + public static void clear() { + counter.set(0); + } + @Override + public void processElement(T value, ProcessFunction.Context ctx, Collector out) throws Exception { + var ct = ProcessFunctionWithError.counter.getAndIncrement(); + if (ct == errorAfterRecord) { + throw new RuntimeException("error"); + } + Thread.sleep(10); + out.collect(value); + } +} diff --git a/src/test/java/com/vinted/flink/bigquery/util/TestSink.java b/src/test/java/com/vinted/flink/bigquery/util/TestSink.java new file mode 100644 index 0000000..3160238 --- /dev/null +++ b/src/test/java/com/vinted/flink/bigquery/util/TestSink.java @@ -0,0 +1,24 @@ +package com.vinted.flink.bigquery.util; + +import org.apache.flink.api.common.JobExecutionResult; +import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; + +import java.util.ArrayList; +import java.util.List; + +public class TestSink extends RichSinkFunction { + private static ArrayList state = new ArrayList<>(); + + public static void clear() { + state.clear(); + } + + @Override + public void invoke(T value, Context context) throws Exception { + TestSink.state.add(value); + } + + public List getResults(JobExecutionResult jobResult) { + return (List) TestSink.state; + }; +} diff --git a/src/test/resources/log4j2-test.properties b/src/test/resources/log4j2-test.properties new file mode 100644 index 0000000..8f76b04 --- /dev/null +++ b/src/test/resources/log4j2-test.properties @@ -0,0 +1,15 @@ +status=warn +# Name of the configuration +name=ConsoleLogConfigDemo +# Console appender configuration +appender.console.type=Console +appender.console.name=consoleLogger +appender.console.layout.type=PatternLayout +appender.console.layout.pattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n +# Root logger level +rootLogger.level=TRACE +# Root logger referring to console appender +rootLogger.appenderRef.stdout.ref=consoleLogger +logger.flink.name=org.apache.flink +logger.flink.level=ERROR +logger.flink.appenderRef.stdout.ref=consoleLogger