From 8988c62d24bb06189b7443edadf9656bccce5fa7 Mon Sep 17 00:00:00 2001 From: martintrajanovski Date: Wed, 5 Mar 2025 09:12:35 +0100 Subject: [PATCH 1/5] metadata quantity_range valueSI conversion support and test --- src/common/utils.ts | 21 ++++++++++++++++++--- test/DatasetSimple.js | 42 +++++++++++++++++++++++++++++++++++++++++- test/TestData.js | 5 +++++ 3 files changed, 64 insertions(+), 4 deletions(-) diff --git a/src/common/utils.ts b/src/common/utils.ts index b035453fe..57213adc7 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -1,7 +1,7 @@ import { Logger } from "@nestjs/common"; import { inspect } from "util"; import { DateTime } from "luxon"; -import { format, unit, Unit, createUnit } from "mathjs"; +import { format, unit, Unit, createUnit, MathJSON } from "mathjs"; import { Expression, FilterQuery, Model, PipelineStage } from "mongoose"; import { IAxiosError, @@ -20,11 +20,26 @@ Unit.isValidAlpha = function (c) { createUnit("Å", "1 angstrom"); export const convertToSI = ( - inputValue: number, + inputValue: number | [number, number], inputUnit: string, -): { valueSI: number; unitSI: string } => { +): { valueSI: number | number[]; unitSI: string } => { try { const normalizedUnit = inputUnit.normalize("NFC"); // catch and normalize the different versions of Å in unicode + if (inputValue instanceof Array) { + const quantity: MathJSON[] = []; + inputValue.forEach((value) => { + quantity.push( + unit(value, normalizedUnit) + .to(unit(normalizedUnit).toSI().toJSON().unit) + .toJSON(), + ); + }); + + const valueSI = quantity.map((q) => Number(q.value)); + const unitSI = quantity[0].unit; + + return { valueSI, unitSI }; + } // Workaround related to a bug reported at https://github.com/josdejong/mathjs/issues/3097 and https://github.com/josdejong/mathjs/issues/2499 const quantity = unit(inputValue, normalizedUnit) .to(unit(normalizedUnit).toSI().toJSON().unit) diff --git a/test/DatasetSimple.js b/test/DatasetSimple.js index 897b79e9f..227ea3c49 100644 --- a/test/DatasetSimple.js +++ b/test/DatasetSimple.js @@ -16,7 +16,7 @@ describe("0200: Dataset Simple: Check different dataset types and their inherita db.collection("Dataset").deleteMany({}); }); - beforeEach(async() => { + beforeEach(async () => { accessTokenAdminIngestor = await utils.getToken(appUrl, { username: "adminIngestor", password: TestData.Accounts["adminIngestor"]["password"], @@ -199,6 +199,46 @@ describe("0200: Dataset Simple: Check different dataset types and their inherita }); }); + it("0081: raw dataset should contain scientific metadata quantity range and correct ValueSI and UnitSI", async () => { + return request(appUrl) + .get("/api/v3/Datasets/" + pidRaw1) + .set("Accept", "application/json") + .set({ Authorization: `Bearer ${accessTokenAdminIngestor}` }) + .expect(TestData.SuccessfulGetStatusCode) + .expect("Content-Type", /json/) + .then((res) => { + res.body.should.have + .property("scientificMetadata") + .and.to.be.a("object"); + + const scientificMetadata = res.body.scientificMetadata; + + scientificMetadata.should.have + .property("approx_distance_range") + .and.to.be.a("object"); + + const approxDistanceRange = scientificMetadata.approx_distance_range; + approxDistanceRange.should.have + .property("unitSI") + .and.to.be.a("string") + .and.equal("m"); + approxDistanceRange.should.have + .property("valueSI") + .and.to.be.a("array"); + approxDistanceRange.should.have.property("value").and.to.be.a("array"); + const valueSI = approxDistanceRange.valueSI; + const value = approxDistanceRange.value; + + value.should.have.lengthOf(2); + value[0].should.be.a("number").and.equal(1); + value[1].should.be.a("number").and.equal(2); + + valueSI.should.have.lengthOf(2); + valueSI[0].should.be.a("number").and.equal(0.01); + valueSI[1].should.be.a("number").and.equal(0.02); + }); + }); + // get counts again it("0090: check for correct new count of datasets", async () => { return request(appUrl) diff --git a/test/TestData.js b/test/TestData.js index 237f9e0ca..1db79a292 100644 --- a/test/TestData.js +++ b/test/TestData.js @@ -176,6 +176,11 @@ const TestData = { value: 8500, unit: "", }, + approx_distance_range: { + value: [1, 2], + unit: "cm", + type: "quantity_range", + }, beamlineParameters: { Monostripe: "Ru/C", "Ring current": { From 0daefe47036f5bec07a281c22e531468c3f8ae41 Mon Sep 17 00:00:00 2001 From: martintrajanovski Date: Fri, 7 Mar 2025 11:28:18 +0100 Subject: [PATCH 2/5] add extra check for quantity_range length of the value --- src/common/utils.ts | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/src/common/utils.ts b/src/common/utils.ts index 57213adc7..b59a28658 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -26,19 +26,27 @@ export const convertToSI = ( try { const normalizedUnit = inputUnit.normalize("NFC"); // catch and normalize the different versions of Å in unicode if (inputValue instanceof Array) { - const quantity: MathJSON[] = []; - inputValue.forEach((value) => { - quantity.push( - unit(value, normalizedUnit) - .to(unit(normalizedUnit).toSI().toJSON().unit) - .toJSON(), - ); - }); + if (inputValue.length === 2) { + const quantity: MathJSON[] = []; + inputValue.forEach((value) => { + quantity.push( + unit(value, normalizedUnit) + .to(unit(normalizedUnit).toSI().toJSON().unit) + .toJSON(), + ); + }); - const valueSI = quantity.map((q) => Number(q.value)); - const unitSI = quantity[0].unit; + const valueSI = quantity.map((q) => Number(q.value)); + const unitSI = quantity[0].unit; - return { valueSI, unitSI }; + return { valueSI, unitSI }; + } else { + console.error( + "More than two values provided in the quantity_range field", + JSON.stringify({ inputValue }), + ); + return { valueSI: inputValue, unitSI: inputUnit }; + } } // Workaround related to a bug reported at https://github.com/josdejong/mathjs/issues/3097 and https://github.com/josdejong/mathjs/issues/2499 const quantity = unit(inputValue, normalizedUnit) From 9f29cd1157826ec792ab457578451580235f91a9 Mon Sep 17 00:00:00 2001 From: martintrajanovski Date: Thu, 20 Mar 2025 15:37:40 +0100 Subject: [PATCH 3/5] feat: add count endpoint for instruments --- src/instruments/instruments.controller.ts | 18 ++++++++++++++++++ src/instruments/instruments.service.ts | 9 +++++++++ 2 files changed, 27 insertions(+) diff --git a/src/instruments/instruments.controller.ts b/src/instruments/instruments.controller.ts index 593c29ee8..cf98c234e 100644 --- a/src/instruments/instruments.controller.ts +++ b/src/instruments/instruments.controller.ts @@ -35,6 +35,7 @@ import { filterExample, replaceLikeOperator, } from "src/common/utils"; +import { CountApiResponse } from "src/common/types"; @ApiBearerAuth() @ApiTags("instruments") @@ -88,6 +89,23 @@ export class InstrumentsController { return this.instrumentsService.findAll(instrumentFilter); } + @UseGuards(PoliciesGuard) + @CheckPolicies("instruments", (ability: AppAbility) => + ability.can(Action.InstrumentRead, Instrument), + ) + @Get("/count") + @ApiQuery({ + name: "filter", + description: "Database filters to apply when retrieve instrument count", + required: false, + }) + async count(@Query("filter") filter?: string): Promise { + const instrumentFilter: IFilters = replaceLikeOperator( + JSON.parse(filter ?? "{}"), + ); + return this.instrumentsService.count(instrumentFilter); + } + // GET /instrument/findOne @UseGuards(PoliciesGuard) @CheckPolicies("instruments", (ability: AppAbility) => diff --git a/src/instruments/instruments.service.ts b/src/instruments/instruments.service.ts index a924d30ae..5055b5137 100644 --- a/src/instruments/instruments.service.ts +++ b/src/instruments/instruments.service.ts @@ -2,6 +2,7 @@ import { Injectable } from "@nestjs/common"; import { InjectModel } from "@nestjs/mongoose"; import { FilterQuery, Model } from "mongoose"; import { IFilters } from "src/common/interfaces/common.interface"; +import { CountApiResponse } from "src/common/types"; import { parseLimitFilters } from "src/common/utils"; import { CreateInstrumentDto } from "./dto/create-instrument.dto"; import { PartialUpdateInstrumentDto } from "./dto/update-instrument.dto"; @@ -36,6 +37,14 @@ export class InstrumentsService { return instruments; } + async count(filter: IFilters): Promise { + const whereFilter: FilterQuery = filter.where ?? {}; + + const count = await this.instrumentModel.countDocuments(whereFilter).exec(); + + return { count }; + } + async findOne( filter: FilterQuery, ): Promise { From caf87e92bcd4cc89f631b4a7bc68090a476d5d1a Mon Sep 17 00:00:00 2001 From: martintrajanovski Date: Mon, 31 Mar 2025 15:30:22 +0200 Subject: [PATCH 4/5] start introducing instrument complete endpoint to include data and totalCount --- src/app.module.ts | 1 + src/common/interfaces/common.interface.ts | 12 +++++ src/instruments/instruments.controller.ts | 26 +++++++++- src/instruments/instruments.service.ts | 63 +++++++++++++++++++++-- 4 files changed, 98 insertions(+), 4 deletions(-) diff --git a/src/app.module.ts b/src/app.module.ts index 467d9d148..9478da06d 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -144,6 +144,7 @@ import { MetricsModule } from "./metrics/metrics.module"; MongooseModule.forRootAsync({ useFactory: async (configService: ConfigService) => ({ uri: configService.get("mongodbUri"), + sanitizeFilter: true, }), inject: [ConfigService], }), diff --git a/src/common/interfaces/common.interface.ts b/src/common/interfaces/common.interface.ts index 63a9a4bfd..d9669b564 100644 --- a/src/common/interfaces/common.interface.ts +++ b/src/common/interfaces/common.interface.ts @@ -47,6 +47,18 @@ export interface IFilters { limits?: ILimitsFilter; } +export interface IFiltersNew { + textSearch?: string; + where?: FilterQuery; + limits?: ILimitsFilter; + fields?: string[]; +} + +export interface CompleteResponse { + data: T[]; + totalCount: number; +} + export interface IFacets { fields?: T; facets?: string[]; diff --git a/src/instruments/instruments.controller.ts b/src/instruments/instruments.controller.ts index cf98c234e..acbf8f447 100644 --- a/src/instruments/instruments.controller.ts +++ b/src/instruments/instruments.controller.ts @@ -29,7 +29,11 @@ import { AppAbility } from "src/casl/casl-ability.factory"; import { Action } from "src/casl/action.enum"; import { Instrument, InstrumentDocument } from "./schemas/instrument.schema"; import { FormatPhysicalQuantitiesInterceptor } from "src/common/interceptors/format-physical-quantities.interceptor"; -import { IFilters } from "src/common/interfaces/common.interface"; +import { + CompleteResponse, + IFilters, + IFiltersNew, +} from "src/common/interfaces/common.interface"; import { filterDescription, filterExample, @@ -89,6 +93,26 @@ export class InstrumentsController { return this.instrumentsService.findAll(instrumentFilter); } + @UseGuards(PoliciesGuard) + @CheckPolicies("instruments", (ability: AppAbility) => + ability.can(Action.InstrumentRead, Instrument), + ) + @Get("/complete") + @ApiQuery({ + name: "filters", + description: "Database filters to apply when retrieve all instruments", + required: false, + }) + async findAllComplete( + @Query("filters") filters?: string, + ): Promise> { + const instrumentFilter: IFiltersNew = JSON.parse( + filters ?? "{}", + ); + + return this.instrumentsService.findAllComplete(instrumentFilter); + } + @UseGuards(PoliciesGuard) @CheckPolicies("instruments", (ability: AppAbility) => ability.can(Action.InstrumentRead, Instrument), diff --git a/src/instruments/instruments.service.ts b/src/instruments/instruments.service.ts index 5055b5137..296845b72 100644 --- a/src/instruments/instruments.service.ts +++ b/src/instruments/instruments.service.ts @@ -1,12 +1,18 @@ import { Injectable } from "@nestjs/common"; import { InjectModel } from "@nestjs/mongoose"; -import { FilterQuery, Model } from "mongoose"; -import { IFilters } from "src/common/interfaces/common.interface"; +import { FilterQuery, Model, PipelineStage } from "mongoose"; +import { + CompleteResponse, + IFilters, + IFiltersNew, + ILimitsFilter, +} from "src/common/interfaces/common.interface"; import { CountApiResponse } from "src/common/types"; -import { parseLimitFilters } from "src/common/utils"; +import { parseLimitFilters, parsePipelineProjection } from "src/common/utils"; import { CreateInstrumentDto } from "./dto/create-instrument.dto"; import { PartialUpdateInstrumentDto } from "./dto/update-instrument.dto"; import { Instrument, InstrumentDocument } from "./schemas/instrument.schema"; +import { parsePipelineSort } from "src/common/utils"; @Injectable() export class InstrumentsService { @@ -37,6 +43,57 @@ export class InstrumentsService { return instruments; } + buildFacetPipeline( + limits: ILimitsFilter | undefined, + fields: string[] | undefined, + ) { + const { limit, skip, sort } = parseLimitFilters(limits); + + const facet: PipelineStage.Facet = { + $facet: { + data: [{ $skip: skip }, { $limit: limit }], + count: [{ $count: "totalCount" }], + }, + }; + + if (sort && typeof sort === "object") { + const sortParsed = parsePipelineSort(sort); + + facet.$facet.data.push({ $sort: sortParsed }); + } + + if (fields) { + const project: PipelineStage.Project["$project"] = + parsePipelineProjection(fields); + + facet.$facet.data.push({ $project: project }); + } + + return facet; + } + + async findAllComplete( + filter: IFiltersNew, + ): Promise> { + const whereFilter: FilterQuery = filter.where ?? {}; + + const $match: PipelineStage.Match = { $match: whereFilter }; + + const $facet = this.buildFacetPipeline(filter.limits, filter.fields); + const $project = { + $project: { + data: 1, + totalCount: { $arrayElemAt: ["$count.totalCount", 0] }, + }, + }; + + const pipeline: PipelineStage[] = [$match, $facet, $project]; + + const [result] = await this.instrumentModel.aggregate(pipeline).exec(); + + return result; + } + async count(filter: IFilters): Promise { const whereFilter: FilterQuery = filter.where ?? {}; From b4b8a175099ce7c277d5a6b36cfc51c05c481b91 Mon Sep 17 00:00:00 2001 From: martintrajanovski Date: Fri, 4 Apr 2025 11:49:24 +0200 Subject: [PATCH 5/5] add findAllComplete in the instruments which is better documented and returns pagginated data with totalCount --- .../api-data-count-response.decorator.ts | 35 ++++++++++ src/common/interfaces/common.interface.ts | 13 ++-- .../swagger-filter-content.ts} | 16 ++--- src/common/types.ts | 6 ++ src/common/utils.ts | 42 ++++++++++++ src/datasets/datasets-public.v4.controller.ts | 13 ++-- src/datasets/datasets.v4.controller.ts | 13 ++-- src/instruments/instruments.controller.ts | 24 ++++--- src/instruments/instruments.service.ts | 65 +++++-------------- 9 files changed, 136 insertions(+), 91 deletions(-) create mode 100644 src/common/decorators/api-data-count-response.decorator.ts rename src/{datasets/types/dataset-filter-content.ts => common/swagger-filter-content.ts} (92%) diff --git a/src/common/decorators/api-data-count-response.decorator.ts b/src/common/decorators/api-data-count-response.decorator.ts new file mode 100644 index 000000000..dbabfc3ca --- /dev/null +++ b/src/common/decorators/api-data-count-response.decorator.ts @@ -0,0 +1,35 @@ +import { applyDecorators, Type } from "@nestjs/common"; +import { HttpStatus } from "@nestjs/common/enums"; +import { ApiExtraModels, ApiOkResponse, getSchemaPath } from "@nestjs/swagger"; +import { DataCountOutputDto } from "../types"; + +export const ApiDataCountResponse = >( + dataDto: DataDto, + description = "List of items with total count. The list is limited by the limit and offset parameters.", + statusCode = HttpStatus.OK, +) => + applyDecorators( + ApiExtraModels(DataCountOutputDto, dataDto), + ApiOkResponse({ + schema: { + allOf: [ + { $ref: getSchemaPath(DataCountOutputDto) }, + { + properties: { + data: { + type: "array", + items: { $ref: getSchemaPath(dataDto) }, + }, + totalCount: { + type: "number", + default: 0, + }, + }, + }, + ], + required: ["data", "totalCount"], + }, + status: statusCode, + description: description, + }), + ); diff --git a/src/common/interfaces/common.interface.ts b/src/common/interfaces/common.interface.ts index d9669b564..3693e956f 100644 --- a/src/common/interfaces/common.interface.ts +++ b/src/common/interfaces/common.interface.ts @@ -47,16 +47,17 @@ export interface IFilters { limits?: ILimitsFilter; } -export interface IFiltersNew { - textSearch?: string; +export interface IFiltersV4 { where?: FilterQuery; - limits?: ILimitsFilter; + include?: string[]; fields?: string[]; + limits?: ILimitsFilterV4; } -export interface CompleteResponse { - data: T[]; - totalCount: number; +export interface ILimitsFilterV4 { + limit: number; + skip: number; + sort: Record; } export interface IFacets { diff --git a/src/datasets/types/dataset-filter-content.ts b/src/common/swagger-filter-content.ts similarity index 92% rename from src/datasets/types/dataset-filter-content.ts rename to src/common/swagger-filter-content.ts index 904bde038..10601642f 100644 --- a/src/datasets/types/dataset-filter-content.ts +++ b/src/common/swagger-filter-content.ts @@ -8,7 +8,7 @@ const FILTERS: Record<"limits" | "fields" | "where" | "include", object> = { where: { type: "object", example: { - datasetName: { $regex: "Dataset", $options: "i" }, + field1: { $regex: "Test", $options: "i" }, }, }, include: { @@ -22,24 +22,24 @@ const FILTERS: Record<"limits" | "fields" | "where" | "include", object> = { type: "array", items: { type: "string", - example: "datasetName", + example: "field1", }, }, limits: { type: "object", properties: { - limit: { - type: "number", - example: 10, - }, skip: { type: "number", example: 0, }, + limit: { + type: "number", + example: 10, + }, sort: { type: "object", properties: { - datasetName: { + field1: { type: "string", example: "asc | desc", }, @@ -54,7 +54,7 @@ const FILTERS: Record<"limits" | "fields" | "where" | "include", object> = { * But we want to have it when we run the application as it improves swagger documentation and usage a lot. * We use "content" property as it is described in the swagger specification: https://swagger.io/docs/specification/v3_0/describing-parameters/#schema-vs-content:~:text=explode%3A%20false-,content,-is%20used%20in */ -export const getSwaggerDatasetFilterContent = ( +export const getSwaggerFilterContent = ( filtersToInclude: Record = { where: true, include: true, diff --git a/src/common/types.ts b/src/common/types.ts index 48f9a325a..4178ab1ae 100644 --- a/src/common/types.ts +++ b/src/common/types.ts @@ -46,3 +46,9 @@ export class IsValidResponse { @ApiProperty({ type: Boolean }) isvalid: boolean; } + +export class DataCountOutputDto { + data: T[]; + + totalCount: number; +} diff --git a/src/common/utils.ts b/src/common/utils.ts index 51f10305a..d1ffd588c 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -7,10 +7,12 @@ import { IAxiosError, IFilters, ILimitsFilter, + ILimitsFilterV4, IScientificFilter, } from "./interfaces/common.interface"; import { ScientificRelation } from "./scientific-relation.enum"; import { DatasetType } from "src/datasets/types/dataset-type.enum"; +import { isEmpty } from "lodash"; // add Å to mathjs accepted units as equivalent to angstrom const isAlphaOriginal = Unit.isValidAlpha; @@ -364,6 +366,46 @@ export const parsePipelineProjection = (fieldsProjection: string[]) => { return pipelineProjection; }; +export const buildDataCountFacetPipeline = ( + limits: ILimitsFilterV4 | undefined, + fields: string[] | undefined, +) => { + const skip = limits?.skip || 0; + const limit = limits?.limit || 10; + + const facet: PipelineStage.Facet = { + $facet: { + data: [{ $skip: skip }, { $limit: limit }], + count: [{ $count: "totalCount" }], + }, + }; + + if (limits?.sort && !isEmpty(limits?.sort)) { + const sortParsed = parsePipelineSort(limits.sort); + + facet.$facet.data.push({ $sort: sortParsed }); + } + + if (fields) { + const project: PipelineStage.Project["$project"] = + parsePipelineProjection(fields); + + facet.$facet.data.push({ $project: project }); + } + + return facet; +}; + +// NOTE: This returns default projection of the data and totalCount fields for findAll complete endpoints +export const getDefaultDataCountProject = (): PipelineStage.Project => { + return { + $project: { + data: 1, + totalCount: { $arrayElemAt: ["$count.totalCount", 0] }, + }, + }; +}; + export const parseLimitFiltersForPipeline = ( limits: ILimitsFilter | undefined, ): PipelineStage[] => { diff --git a/src/datasets/datasets-public.v4.controller.ts b/src/datasets/datasets-public.v4.controller.ts index e87124cd2..c300c0cf9 100644 --- a/src/datasets/datasets-public.v4.controller.ts +++ b/src/datasets/datasets-public.v4.controller.ts @@ -27,8 +27,8 @@ import { import { DatasetLookupKeysEnum } from "./types/dataset-lookup"; import { IncludeValidationPipe } from "./pipes/include-validation.pipe"; import { FilterValidationPipe } from "./pipes/filter-validation.pipe"; -import { getSwaggerDatasetFilterContent } from "./types/dataset-filter-content"; import { AllowAny } from "src/auth/decorators/allow-any.decorator"; +import { getSwaggerFilterContent } from "src/common/swagger-filter-content"; @ApiExtraModels(HistoryClass, TechniqueClass, RelationshipClass) @ApiTags("datasets public v4") @@ -58,7 +58,7 @@ export class DatasetsPublicV4Controller { "Database filters to apply when retrieving the public datasets", required: false, type: String, - content: getSwaggerDatasetFilterContent(), + content: getSwaggerFilterContent(), }) @ApiResponse({ status: HttpStatus.OK, @@ -171,12 +171,7 @@ export class DatasetsPublicV4Controller { description: "Database filters to apply when retrieving public dataset", required: true, type: String, - content: getSwaggerDatasetFilterContent({ - where: true, - include: true, - fields: true, - limits: true, - }), + content: getSwaggerFilterContent(), }) @ApiResponse({ status: HttpStatus.OK, @@ -208,7 +203,7 @@ export class DatasetsPublicV4Controller { "Database filters to apply when retrieving count for public datasets", required: false, type: String, - content: getSwaggerDatasetFilterContent({ + content: getSwaggerFilterContent({ where: true, include: false, fields: false, diff --git a/src/datasets/datasets.v4.controller.ts b/src/datasets/datasets.v4.controller.ts index 7bf59c773..1b4d4ed30 100644 --- a/src/datasets/datasets.v4.controller.ts +++ b/src/datasets/datasets.v4.controller.ts @@ -69,8 +69,8 @@ import { DatasetLookupKeysEnum } from "./types/dataset-lookup"; import { IncludeValidationPipe } from "./pipes/include-validation.pipe"; import { PidValidationPipe } from "./pipes/pid-validation.pipe"; import { FilterValidationPipe } from "./pipes/filter-validation.pipe"; -import { getSwaggerDatasetFilterContent } from "./types/dataset-filter-content"; import { plainToInstance } from "class-transformer"; +import { getSwaggerFilterContent } from "src/common/swagger-filter-content"; @ApiBearerAuth() @ApiExtraModels( @@ -355,7 +355,7 @@ export class DatasetsV4Controller { description: "Database filters to apply when retrieving datasets", required: false, type: String, - content: getSwaggerDatasetFilterContent(), + content: getSwaggerFilterContent(), }) @ApiResponse({ status: HttpStatus.OK, @@ -523,12 +523,7 @@ export class DatasetsV4Controller { description: "Database filters to apply when retrieving dataset", required: true, type: String, - content: getSwaggerDatasetFilterContent({ - where: true, - include: true, - fields: true, - limits: true, - }), + content: getSwaggerFilterContent(), }) @ApiResponse({ status: HttpStatus.OK, @@ -566,7 +561,7 @@ export class DatasetsV4Controller { description: "Database filters to apply when retrieving count for datasets", required: false, type: String, - content: getSwaggerDatasetFilterContent({ + content: getSwaggerFilterContent({ where: true, include: false, fields: false, diff --git a/src/instruments/instruments.controller.ts b/src/instruments/instruments.controller.ts index acbf8f447..eaf296f32 100644 --- a/src/instruments/instruments.controller.ts +++ b/src/instruments/instruments.controller.ts @@ -29,17 +29,15 @@ import { AppAbility } from "src/casl/casl-ability.factory"; import { Action } from "src/casl/action.enum"; import { Instrument, InstrumentDocument } from "./schemas/instrument.schema"; import { FormatPhysicalQuantitiesInterceptor } from "src/common/interceptors/format-physical-quantities.interceptor"; -import { - CompleteResponse, - IFilters, - IFiltersNew, -} from "src/common/interfaces/common.interface"; +import { IFilters, IFiltersV4 } from "src/common/interfaces/common.interface"; import { filterDescription, filterExample, replaceLikeOperator, } from "src/common/utils"; -import { CountApiResponse } from "src/common/types"; +import { CountApiResponse, DataCountOutputDto } from "src/common/types"; +import { getSwaggerFilterContent } from "src/common/swagger-filter-content"; +import { ApiDataCountResponse } from "src/common/decorators/api-data-count-response.decorator"; @ApiBearerAuth() @ApiTags("instruments") @@ -102,15 +100,23 @@ export class InstrumentsController { name: "filters", description: "Database filters to apply when retrieve all instruments", required: false, + type: String, + content: getSwaggerFilterContent({ + where: true, + include: false, + fields: true, + limits: true, + }), }) + @ApiDataCountResponse(Instrument, "Return the instruments requested") async findAllComplete( @Query("filters") filters?: string, - ): Promise> { - const instrumentFilter: IFiltersNew = JSON.parse( + ): Promise> { + const parsedFilters: IFiltersV4 = JSON.parse( filters ?? "{}", ); - return this.instrumentsService.findAllComplete(instrumentFilter); + return this.instrumentsService.findAllComplete(parsedFilters); } @UseGuards(PoliciesGuard) diff --git a/src/instruments/instruments.service.ts b/src/instruments/instruments.service.ts index 296845b72..d0ebf4a79 100644 --- a/src/instruments/instruments.service.ts +++ b/src/instruments/instruments.service.ts @@ -1,18 +1,16 @@ import { Injectable } from "@nestjs/common"; import { InjectModel } from "@nestjs/mongoose"; import { FilterQuery, Model, PipelineStage } from "mongoose"; +import { IFilters, IFiltersV4 } from "src/common/interfaces/common.interface"; +import { CountApiResponse, DataCountOutputDto } from "src/common/types"; import { - CompleteResponse, - IFilters, - IFiltersNew, - ILimitsFilter, -} from "src/common/interfaces/common.interface"; -import { CountApiResponse } from "src/common/types"; -import { parseLimitFilters, parsePipelineProjection } from "src/common/utils"; + buildDataCountFacetPipeline, + getDefaultDataCountProject, + parseLimitFilters, +} from "src/common/utils"; import { CreateInstrumentDto } from "./dto/create-instrument.dto"; import { PartialUpdateInstrumentDto } from "./dto/update-instrument.dto"; import { Instrument, InstrumentDocument } from "./schemas/instrument.schema"; -import { parsePipelineSort } from "src/common/utils"; @Injectable() export class InstrumentsService { @@ -43,55 +41,22 @@ export class InstrumentsService { return instruments; } - buildFacetPipeline( - limits: ILimitsFilter | undefined, - fields: string[] | undefined, - ) { - const { limit, skip, sort } = parseLimitFilters(limits); - - const facet: PipelineStage.Facet = { - $facet: { - data: [{ $skip: skip }, { $limit: limit }], - count: [{ $count: "totalCount" }], - }, - }; - - if (sort && typeof sort === "object") { - const sortParsed = parsePipelineSort(sort); - - facet.$facet.data.push({ $sort: sortParsed }); - } - - if (fields) { - const project: PipelineStage.Project["$project"] = - parsePipelineProjection(fields); - - facet.$facet.data.push({ $project: project }); - } - - return facet; - } - async findAllComplete( - filter: IFiltersNew, - ): Promise> { - const whereFilter: FilterQuery = filter.where ?? {}; + filter: IFiltersV4, + ): Promise> { + const $match: PipelineStage.Match = { $match: filter.where ?? {} }; - const $match: PipelineStage.Match = { $match: whereFilter }; + const $facet = buildDataCountFacetPipeline(filter.limits, filter.fields); - const $facet = this.buildFacetPipeline(filter.limits, filter.fields); - const $project = { - $project: { - data: 1, - totalCount: { $arrayElemAt: ["$count.totalCount", 0] }, - }, - }; + const $project = getDefaultDataCountProject(); const pipeline: PipelineStage[] = [$match, $facet, $project]; - const [result] = await this.instrumentModel.aggregate(pipeline).exec(); + const [result] = await this.instrumentModel + .aggregate>(pipeline) + .exec(); - return result; + return { data: result.data, totalCount: result.totalCount || 0 }; } async count(filter: IFilters): Promise {