Merge pull request #10 from blenderskool/partial-query

Partial query middleware ✂

components/SecretInput.tsx

@@ -105,6 +105,7 @@ export default function SecretInput({ name, control, containerProps = {}, inputP
         }}
         autoComplete="off"
         spellCheck="false"
+        required={required}
         onPaste={syncScroll}
         onCut={syncScroll}
         onFocus={syncScroll}

lib/internals/send-response.ts

@@ -1,10 +1,11 @@
-import type { AxiosResponse } from 'axios';
 import type { NextApiResponse } from 'next';
 import { setAllHeaders } from './utils';
 
-export async function sendResponse(res: NextApiResponse, apiRes: AxiosResponse) {
-  res.statusMessage = apiRes.statusText;
-  res.status(apiRes.status);
+export async function sendResponse(res: NextApiResponse, apiRes: ResultResponse) {
+  if (apiRes.statusText) {
+    res.statusMessage = apiRes.statusText;
+    res.status(apiRes.status);
+  }
 
   setAllHeaders(res, apiRes.headers);
 

lib/middlewares/cache.ts

@@ -8,18 +8,19 @@ import redis from '../redis';
 import { setAllHeaders } from '../internals/utils';
 import { ApiRouteWithMiddlewares } from '../../pages/api/v1/_types';
 
-export type CachingOptions = {
-  enabled: boolean;
+export interface CachingOptions extends MiddlewareOptions {
   // Duration in seconds
   duration: number;
 };
 
+const createCacheKey = (req: NextApiRequest, apiRoute: ApiRouteWithMiddlewares) => `cache:${apiRoute.method}:${req.url}`;
+
 /**
  * Caches the result and headers from the API and returns it for some duration before refetching
  * @param apiRoute ApiRoute object
  * @returns middleware function
  */
-export default function cache(apiRoute: ApiRouteWithMiddlewares) {
+export function cacheRead(apiRoute: ApiRouteWithMiddlewares) {
   return async (req: NextApiRequest, res: NextApiResponse, next) => {
     const cachingOpts = apiRoute.caching as CachingOptions;
     // Caching is only supported on GET requests
@@ -28,7 +29,7 @@ export default function cache(apiRoute: ApiRouteWithMiddlewares) {
       return;
     }
 
-    const key = `cache:${apiRoute.method}:${req.url}`;
+    const key = createCacheKey(req, apiRoute);
 
     /**
      * try catch not applied here as the docs say that err is always null,
@@ -45,32 +46,48 @@ export default function cache(apiRoute: ApiRouteWithMiddlewares) {
       const headers: OutgoingHttpHeaders = JSON.parse(cachedHeaders);
 
       setAllHeaders(res, headers);
-      res.setHeader('cache-control', `max-age=${Math.max(0, cacheAge)}`);
-      res.status(200);
-      Readable.from(cachedResult).pipe(res);
+      res.setHeader('cache-control', `max-age=${Math.max(0, cacheAge)}`)
+      res.status(200).send(cachedResult);
+    } else {
+      next();
+    }
+  };
+}
+
+export function cacheWrite(apiRoute: ApiRouteWithMiddlewares) {
+  return async (req: NextApiRequest, res: NextApiResponse, next) => {
+    const cachingOpts = apiRoute.caching as CachingOptions;
+    // Caching is only supported on GET requests
+    if (!cachingOpts.enabled || apiRoute.method !== ApiMethod.GET) {
+      next();
       return;
     }
 
-    // Listen to data piped into response
-    res.on('pipe', async (apiData) => {
-      // Cache the data only if the request was a success
-      if (res.statusCode === 200) {
-        const { duration } = cachingOpts;
-        res.setHeader('cache-control', `max-age=${Math.max(0, duration)}`);
+    const key = createCacheKey(req, apiRoute);
+    const { duration } = cachingOpts;
 
-        const headers = JSON.stringify(res.getHeaders());
-        const buffer = await getStream.buffer(apiData);
-        // TODO: Handle errors from below commands
-        await redis
-          .pipeline()
-          .setex(`${key}:headers`, duration, headers)
-          .setex(`${key}:response`, duration, buffer)
-          .exec();
+    const headers = JSON.stringify(req.locals.result.headers);
+    const buffer = await getStream.buffer(req.locals.result.data);
+    // TODO: Handle errors from below commands
+    await redis
+      .pipeline()
+      .setex(`${key}:headers`, duration, headers)
+      .setex(`${key}:response`, duration, buffer)
+      .exec();
+
+    console.log("Cache middleware: Added to cache");
 
-        console.log("Cache middleware: Added to cache");
-      }
-    });
+    const resultReadable = new Readable();
+    resultReadable.push(buffer);
+    resultReadable.push(null);
 
+    req.locals.result = {
+      headers: {
+        ...req.locals.result.headers,
+        'cache-control': `max-age=${Math.max(0, duration)}`,
+      },
+      data: resultReadable,
+    };
     next();
-  };
+  }
 }

lib/middlewares/index.ts

@@ -1,3 +1,4 @@
-export { default as middlewareRatelimit } from './rate-limit';
-export { default as middlewareRestriction } from './restriction';
-export { default as middlewareCache } from './cache';
+export * from './rate-limit';
+export * from './restriction';
+export * from './cache';
+export * from './partial-query';

lib/middlewares/partial-query.ts

@@ -0,0 +1,29 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { Readable } from 'stream';
+import * as mask from 'json-mask';
+import getStream from 'get-stream';
+
+export interface PartialQueryOptions extends MiddlewareOptions {};
+
+export function partialJsonQuery(filterString: string) {
+  return async (req: NextApiRequest, res: NextApiResponse, next: Function) => {
+    const resultBuffer = await getStream.buffer(req.locals.result.data);
+    // Below code is prone to errors if the body did not contain valid JSON data
+    try {
+      const resultData = mask(JSON.parse(resultBuffer as any), filterString);
+
+      const resultReadable = new Readable();
+      resultReadable.push(JSON.stringify(resultData));
+      resultReadable.push(null);
+
+      req.locals.result = {
+        headers: { 'content-type': 'application/json' },
+        data: resultReadable,
+      };
+    } catch(_) {
+      // Handle errors here if _necessary_ in the future
+    } finally {
+      next();
+    }
+  };
+}

lib/middlewares/rate-limit.ts

@@ -4,8 +4,7 @@ import { ApiRouteWithMiddlewares } from '../../pages/api/v1/_types';
 
 import redis from '../redis';
 
-export type RateLimitingOptions = {
-  enabled: boolean;
+export interface RateLimitingOptions extends MiddlewareOptions {
   windowSize: number,
   maxRequests: number,
 };
@@ -15,7 +14,7 @@ export type RateLimitingOptions = {
  * @param apiRoute ApiRoute object
  * @returns middleware function
  */
-export default function rateLimit(apiRoute: ApiRouteWithMiddlewares) {
+export function rateLimit(apiRoute: ApiRouteWithMiddlewares) {
   return async (req: NextApiRequest, res: NextApiResponse, next) => {
     const rateLimiting = apiRoute.rateLimiting as RateLimitingOptions;
     if (!rateLimiting.enabled) {

lib/middlewares/restriction.ts

@@ -2,14 +2,12 @@ import { IpFilter } from 'express-ipfilter';
 import cors, { CorsOptions } from 'cors';
 import { ApiRouteWithMiddlewares } from '../../pages/api/v1/_types';
 
-export type RestrictionOptions = {
-  enabled: boolean;
+export interface RestrictionOptions extends MiddlewareOptions {
   type: 'HTTP' | 'IP';
   allowedOrigins: string[];
   allowedIps: string[];
 };
 
-
 function createCorsOptions(apiRoute: ApiRouteWithMiddlewares): CorsOptions {
   const { allowedOrigins } = apiRoute.restriction;
   return {
@@ -29,7 +27,7 @@ function createCorsOptions(apiRoute: ApiRouteWithMiddlewares): CorsOptions {
  * @param apiRoute ApiRoute object
  * @returns middleware function
  */
-export default function restriction(apiRoute: ApiRouteWithMiddlewares): Function {
+export function restriction(apiRoute: ApiRouteWithMiddlewares): Function {
   // No API restriction
   const options = apiRoute.restriction;
   if (!options.enabled) return cors();

package.json

@@ -26,6 +26,7 @@
     "get-stream": "^6.0.1",
     "ioredis": "^4.28.0",
     "ioredis-mock": "^5.8.0",
+    "json-mask": "^1.0.4",
     "micromustache": "^8.0.3",
     "nanoid": "^3.1.30",
     "next": "^11.1.2",

pages/api/v1/[..._path].ts

@@ -10,7 +10,7 @@ import type { ApiRouteWithProjectSecrets, QueryParams, ExpandedHeaders } from '.
 import getApiRoute from '@/lib/internals/get-api-route';
 import { sendResponse } from '@/lib/internals/send-response';
 import { addQueryParams, expandObjectEntries, mergeHeaders, movingAverage, substituteSecrets } from '@/lib/internals/utils';
-import { middlewareCache, middlewareRatelimit, middlewareRestriction } from '@/lib/middlewares';
+import * as middlewares from '@/lib/middlewares';
 import prisma from '@/lib/prisma';
 import { decryptSecret } from '@/lib/internals/secrets';
 
@@ -39,6 +39,8 @@ function runMiddleware(req: NextApiRequest, res: NextApiResponse, fn: Function):
  * @param res 
  */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  req.locals = { result: null };
+
   // Get ApiRoute object from database
   const { apiRoute, path }: { apiRoute: ApiRouteWithProjectSecrets, path: string[] } = await runMiddleware(req, res, getApiRoute);
 
@@ -49,9 +51,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
   }
 
   // Middleware plugins
-  await runMiddleware(req, res, middlewareRestriction(apiRoute));
-  await runMiddleware(req, res, middlewareRatelimit(apiRoute));
-  await runMiddleware(req, res, middlewareCache(apiRoute));
+  await runMiddleware(req, res, middlewares.restriction(apiRoute));
+  await runMiddleware(req, res, middlewares.rateLimit(apiRoute));
+  await runMiddleware(req, res, middlewares.cacheRead(apiRoute));
 
   // Decrypt the project secrets
   const secrets = Object.fromEntries(apiRoute.project.Secret.map(({ name, secret }) => [name, decryptSecret(secret)]));
@@ -66,32 +68,47 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
 
   // Add request headers
   delete req.headers.host;
+  delete req.headers['accept-encoding'];
   const currentHeaders: ExpandedHeaders = expandObjectEntries(req.headers);
   const requestHeaders = mergeHeaders(substituteSecrets(apiRoute.headers as ExpandedHeaders, secrets), currentHeaders);
 
   // Request made
   try {
     const startTime = performance.now();
+    const isPartialQueryEnabled = !!apiRoute.partialQuery.enabled && requestUrl.searchParams.has('diode-filter');
     const apiResponse = await axios.request({
       method: apiRoute.method,
       url: requestUrl.toString(),
       headers: requestHeaders,
 
       /**
-       * Get response as stream and prevent its decoding
-       * as proxy does not consume the result
+       * Get response as stream and decode it
+       * only if partial query middleware is enabled
        */
-      decompress: false,
+      decompress: isPartialQueryEnabled,
       responseType: 'stream',
 
       data: apiRoute.method === ApiMethod.GET ? undefined : req.body,
     });
     const timeTaken = performance.now() - startTime;
-    const newAverage = movingAverage(apiRoute, timeTaken);
+
+    req.locals.result = apiResponse;
+
+    if (isPartialQueryEnabled && apiResponse.headers['content-type'].includes('application/json')) {
+      /**
+       * get() is used instead of getAll() as only the filter configured
+       * either in dashboard or the incoming query param is used.
+       * Not both to avoid confusion
+       */
+      await runMiddleware(req, res, middlewares.partialJsonQuery(requestUrl.searchParams.get('diode-filter')));
+    }
 
+    await runMiddleware(req, res, middlewares.cacheWrite(apiRoute));
     // Response preparation
-    sendResponse(res, apiResponse);
-    await prisma.$executeRaw`UPDATE "public"."ApiRoute" SET "successes" = "successes" + 1, "avgResponseMs" = ${newAverage} WHERE "public"."ApiRoute"."id" = ${apiRoute.id}`;
+    sendResponse(res, req.locals.result);
+
+    const newResponseAverage = movingAverage(apiRoute, timeTaken);
+    await prisma.$executeRaw`UPDATE "public"."ApiRoute" SET "successes" = "successes" + 1, "avgResponseMs" = ${newResponseAverage} WHERE "public"."ApiRoute"."id" = ${apiRoute.id}`;
   } catch(err) {
     if (axios.isAxiosError(err)) {
       // Response preparation

pages/api/v1/_types.ts

@@ -1,16 +1,15 @@
 import { ApiRoute, Secret } from '@prisma/client';
-import { CachingOptions } from '@/lib/middlewares/cache';
-import { RateLimitingOptions } from '@/lib/middlewares/rate-limit';
-import { RestrictionOptions } from '@/lib/middlewares/restriction';
+import { RestrictionOptions, RateLimitingOptions, CachingOptions, PartialQueryOptions } from '@/lib/middlewares';
 
 export type QueryParams = [string, string][];
 
 export type ExpandedHeaders = [string, string][];
 
-export type ApiRouteWithMiddlewares = Omit<ApiRoute, 'restriction' | 'rateLimiting' | 'caching'> & {
-  restriction: RestrictionOptions,
-  rateLimiting: RateLimitingOptions,
-  caching: CachingOptions,
+export type ApiRouteWithMiddlewares = Omit<ApiRoute, 'restriction' | 'rateLimiting' | 'caching' | 'partialQuery'> & {
+  restriction: RestrictionOptions;
+  rateLimiting: RateLimitingOptions;
+  caching: CachingOptions;
+  partialQuery: PartialQueryOptions;
 };
 
 export type ApiRouteWithProjectSecrets = ApiRouteWithMiddlewares & {