305 update content persistence strategy

.env.example

@@ -1,11 +1,11 @@
 ORDO_ID_ALGORITHM=ECDSA:256
-ORDO_ID_ALLOW_ORIGIN=${ORDO_DT_HOST}, ${ORDO_WEB_HOST}
-ORDO_ID_AUD=${ORDO_ID_ALLOW_ORIGIN}
+ORDO_ID_ALLOW_ORIGIN=http://localhost:3002, http://localhost:3000
+ORDO_ID_AUD=http://localhost:3002, http://localhost:3000
 ORDO_ID_DEFAULT_FILE_LIMIT=1000
 ORDO_ID_DEFAULT_MAX_FUNCTIONS=10
 ORDO_ID_DEFAULT_MAX_UPLOAD_SIZE=1.5
-ORDO_ID_HOST=http://localhost:3000
-ORDO_ID_ISS=${ORDO_ID_HOST}
+ORDO_ID_HOST=http://localhost:3001
+ORDO_ID_ISS=http://localhost:3001
 ORDO_ID_PERSISTED_TOKEN_LIFETIME=2592000
 ORDO_ID_PORT=3000
 ORDO_ID_TOKEN_DB_PATH="var/srv/id/tokens.json"
@@ -14,45 +14,14 @@ ORDO_ID_TOKEN_PRIVATE_KEY=
 ORDO_ID_TOKEN_PUBLIC_KEY=
 ORDO_ID_USER_DB_PATH="var/srv/id/users.json"
 
-ORDO_ID_USER_TABLE_NAME=users
-ORDO_ID_TOKENS_TABLE_NAME=tokens
-ORDO_ID_EMAIL_API_KEY=
-ORDO_ID_USER_DYNAMODB_ENDPOINT=
-ORDO_ID_USER_DYNAMODB_ACCESS_KEY=
-ORDO_ID_USER_DYNAMODB_SECRET_KEY=
-ORDO_ID_USER_DYNAMODB_REGION=
-ORDO_ID_USER_DYNAMODB_USER_TABLE=
-ORDO_ID_TOKEN_DYNAMODB_ENDPOINT=
-ORDO_ID_TOKEN_DYNAMODB_ACCESS_KEY=
-ORDO_ID_TOKEN_DYNAMODB_SECRET_KEY=
-ORDO_ID_TOKEN_DYNAMODB_REGION=
-ORDO_ID_TOKEN_DYNAMODB_USER_TABLE=
-
-ORDO_DT_DATA_PERSISTENCE_STRATEGY=fs
-ORDO_DT_CONTENT_PERSISTENCE_STRATEGY=fs
-ORDO_DT_DATA_PATH=./var/srv/data/data
-ORDO_DT_CONTENT_PATH=./var/srv/data/content
 ORDO_DT_PORT=3002
-ORDO_DT_HOST=http://localhost:${ORDO_DT_PORT}
-ORDO_DT_DATA_S3_ACCESS_KEY=
-ORDO_DT_DATA_S3_SECRET_KEY=
-ORDO_DT_DATA_S3_REGION=
-ORDO_DT_DATA_S3_BUCKET_NAME=
-ORDO_DT_DATA_S3_ENDPOINT=
-ORDO_DT_CONTENT_S3_ACCESS_KEY=
-ORDO_DT_CONTENT_S3_SECRET_KEY=
-ORDO_DT_CONTENT_S3_REGION=
-ORDO_DT_CONTENT_S3_BUCKET_NAME=
-ORDO_DT_CONTENT_S3_ENDPOINT=
-ORDO_DT_ENV=development
-ORDO_DT_REGION=global
-ORDO_DT_DOCKER_REGISTRY=ordo.pink
-ORDO_DT_DOCKER_REGISTRY_SCOPE=global
-ORDO_DT_DOCKER_REGISTRY_USERNAME=
-ORDO_DT_DOCKER_REGISTRY_PASSWORD=
-ORDO_DT_VERSION=0.1.0
-ORDO_WORKSPACE_PORT=3004
-ORDO_WORKSPACE_HOST=http://localhost:${ORDO_WORKSPACE_PORT}
-ORDO_STATIC_PORT=3003
-ORDO_STATIC_HOST=http://localhost:${ORDO_STATIC_PORT}
-ORDO_STATIC_ROOT=./var/srv/static
+ORDO_ID_HOST=http://localhost:3002
+ORDO_DT_ALLOW_ORIGIN=http://localhost:3000
+ORDO_DT_DATA_PATH=var/srv/dt
+
+ORDO_PB_PORT=3003
+ORDO_PB_HOST=http://localhost:3003
+ORDO_PB_DATA_PATH=var/srv/pb
+
+ORDO_WEB_PORT=3000
+ORDO_WEB_HOST=http://localhost:3000

lib/backend-dt/index.ts

@@ -0,0 +1,7 @@
+/*
+ * SPDX-FileCopyrightText: Copyright 2025, 谢尔盖 ||↓ and the Ordo.pink contributors
+ * SPDX-License-Identifier: Unlicense
+ */
+
+export * from "./src/backend-dt.impl"
+export * from "./src/backend-dt.types"

lib/backend-dt/license

@@ -0,0 +1,19 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or distribute this software, either in
+source code form or as a compiled binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors of this software dedicate any
+and all copyright interest in the software to the public domain. We make this dedication for the
+benefit of the public at large and to the detriment of our heirs and successors. We intend this
+dedication to be an overt act of relinquishment in perpetuity of all present and future rights to
+this software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <http://unlicense.org/>

lib/backend-dt/readme.md

@@ -0,0 +1 @@
+# Backend Dt

lib/backend-dt/src/backend-dt.impl.ts

@@ -0,0 +1,219 @@
+/*
+ * SPDX-FileCopyrightText: Copyright 2025, 谢尔盖 ||↓ and the Ordo.pink contributors
+ * SPDX-License-Identifier: Unlicense
+ */
+
+import { CurrentUser, METADATA_CONTENT_FSID, Metadata, RRR } from "@ordo-pink/core"
+import { Oath, invokers0, ops0 } from "@ordo-pink/oath"
+import { Routary, TIntake } from "@ordo-pink/routary"
+import { create_json_response, create_response, status_from_rrr } from "@ordo-pink/backend-util-create-response"
+import { set_x_response_time_header, start_response_timer, stop_response_timer } from "@ordo-pink/backend-util-response-time"
+import { default_handler } from "@ordo-pink/backend-util-default-handler"
+import { extract_request_ip } from "@ordo-pink/backend-util-extract-request-ip"
+import { is_finite_non_negative_int } from "@ordo-pink/tau"
+import { log_request } from "@ordo-pink/backend-util-log-request"
+import { routary_cors } from "@ordo-pink/routary-cors"
+import { set_content_type_application_json_header } from "@ordo-pink/backend-util-set-header"
+
+import { type TDTChamber, type TDTContext } from "./backend-dt.types"
+
+// TODO Extract colonoscope from Routary
+// TODO WebSocket for dt-dt and dt-web notifications
+export const create_backend_dt = (chamber: TDTChamber) =>
+	Routary.Of<TDTContext>({ ...chamber, headers: new Headers(), request_ip: null, status: 200 })
+		.head("/:uid/:fsid", intake => {
+			const context = { ...intake, status: 204, request_ip: null, headers: intake.headers ?? new Headers() }
+
+			return Oath.Resolve(context)
+				.pipe(ops0.tap(start_response_timer))
+				.pipe(ops0.tap(extract_request_ip))
+				.pipe(ops0.chain(validate_request_params))
+				.pipe(ops0.chain(authenticate))
+				.pipe(ops0.chain(check_authorization(context)))
+				.pipe(ops0.map(extract_ids(context)))
+				.pipe(ops0.chain(check_file_exists(context)))
+				.pipe(ops0.chain(set_last_modified_header(context)))
+				.pipe(ops0.map(() => context))
+				.pipe(ops0.rejected_map(rrr => ({ rrr, intake: context })))
+				.fix(status_from_rrr)
+				.pipe(ops0.tap(stop_response_timer))
+				.pipe(ops0.tap(set_x_response_time_header))
+				.pipe(ops0.tap(log_request))
+				.pipe(ops0.map(create_response))
+				.invoke(invokers0.force_resolve)
+		})
+
+		.get("/:uid/:fsid", intake => {
+			const context = { ...intake, status: 200, request_ip: null, headers: intake.headers ?? new Headers() }
+
+			return Oath.Resolve(context)
+				.pipe(ops0.tap(start_response_timer))
+				.pipe(ops0.tap(extract_request_ip))
+				.pipe(ops0.chain(validate_request_params))
+				.pipe(ops0.chain(authenticate))
+				.pipe(ops0.chain(check_authorization(context)))
+				.pipe(ops0.map(extract_ids(context)))
+				.pipe(ops0.chain(check_file_exists(context)))
+				.pipe(ops0.chain(set_last_modified_header(context)))
+				.pipe(ops0.chain(({ uid, fsid }) => context.data_persistence_strategy.read(uid, fsid)))
+				.pipe(ops0.tap(file => void (context.payload = file)))
+				.pipe(ops0.map(() => context))
+				.pipe(ops0.tap(context => context.headers.set("Content-Type", "application/octet-stream")))
+				.pipe(ops0.rejected_map(rrr => ({ rrr, intake: context })))
+				.fix(status_from_rrr)
+				.pipe(ops0.tap(stop_response_timer))
+				.pipe(ops0.tap(set_x_response_time_header))
+				.pipe(ops0.tap(log_request))
+				.pipe(ops0.map(create_response))
+				.invoke(invokers0.force_resolve)
+		})
+
+		.post(
+			"/:uid/:fsid",
+			default_handler(intake =>
+				validate_request_params(intake)
+					.pipe(ops0.chain(validate_body_is_not_empty))
+					.pipe(ops0.chain(authenticate))
+					.pipe(ops0.chain(check_authorization(intake)))
+					.pipe(ops0.chain(check_can_create_files(intake)))
+					.pipe(ops0.chain(validate_file_size_limit(intake)))
+					.pipe(ops0.map(extract_ids(intake)))
+					.pipe(ops0.chain(check_file_does_not_exist(intake)))
+					.pipe(ops0.chain(({ uid, fsid }) => intake.data_persistence_strategy.create(uid, fsid, intake.req.body!)))
+					.pipe(ops0.map(() => intake))
+					.pipe(ops0.rejected_map(rrr => ({ rrr, intake }))),
+			),
+		)
+
+		.put(
+			"/:uid/:fsid",
+			default_handler(intake =>
+				validate_request_params(intake)
+					.pipe(ops0.chain(validate_body_is_not_empty))
+					.pipe(ops0.chain(authenticate))
+					.pipe(ops0.chain(check_authorization(intake)))
+					.pipe(ops0.chain(check_total_files_limit_if_file_does_not_exist(intake)))
+					.pipe(ops0.chain(validate_file_size_limit(intake)))
+					.pipe(ops0.map(extract_ids(intake)))
+					.pipe(ops0.chain(({ uid, fsid }) => intake.data_persistence_strategy.update(uid, fsid, intake.req.body!)))
+					.pipe(ops0.map(() => intake))
+					.pipe(ops0.map(() => intake))
+					.pipe(ops0.rejected_map(rrr => ({ rrr, intake }))),
+			),
+		)
+
+		.delete(
+			"/:uid/:fsid",
+			default_handler(intake =>
+				validate_request_params(intake)
+					.pipe(ops0.chain(authenticate))
+					.pipe(ops0.chain(check_authorization(intake)))
+					.pipe(ops0.map(extract_ids(intake)))
+					.pipe(ops0.chain(check_file_exists(intake)))
+					.pipe(ops0.chain(({ uid, fsid }) => intake.data_persistence_strategy.delete(uid, fsid)))
+					.pipe(ops0.map(() => intake))
+					.pipe(ops0.rejected_map(rrr => ({ rrr, intake }))),
+			),
+		)
+
+		.get("/healthcheck", () => new Response("OK"))
+
+		.use(routary_cors({ allow_origin: chamber.allow_origin, allow_headers: ["content-type", "authorization"] }))
+
+		.start(intake =>
+			Oath.Resolve<TIntake<TDTContext>>({ ...intake, headers: new Headers(), status: 404, request_ip: null })
+				.pipe(ops0.tap(start_response_timer))
+				.pipe(ops0.tap(extract_request_ip))
+				.pipe(ops0.tap(set_content_type_application_json_header))
+				.pipe(ops0.tap(stop_response_timer))
+				.pipe(ops0.tap(set_x_response_time_header))
+				.pipe(ops0.tap(log_request))
+				.pipe(ops0.tap(intake => void (intake.payload = "resource not found")))
+				.pipe(ops0.map(create_json_response))
+				.invoke(invokers0.force_resolve),
+		)
+
+export const validate_request_params = (intake: TIntake<TDTContext>) =>
+	Oath.Merge([
+		Oath.If(Metadata.Validations.is_fsid(intake.params.fsid)).pipe(ops0.rejected_map(() => RRR.codes.einval("Invalid FSID"))),
+		Oath.If(CurrentUser.Validations.is_id(intake.params.uid)).pipe(ops0.rejected_map(() => RRR.codes.einval("Invalid UID"))),
+	]).pipe(ops0.map(() => intake))
+
+export const authenticate = (intake: TIntake<TDTContext>) =>
+	Oath.FromNullable(intake.req.headers.get("Authorization"))
+		.pipe(ops0.map(Authorization => ({ Authorization })))
+		.pipe(ops0.map(headers => ({ ...headers, Origin: intake.dt_host }))) // TODO Move to env
+		.pipe(ops0.map(headers => ({ headers, method: "POST" })))
+		.pipe(ops0.chain(init => Oath.FromPromise(() => fetch(`${intake.id_host}/tokens/validate`, init)))) // TODO Move to env
+		.pipe(ops0.chain(res => Oath.FromPromise(() => res.json())))
+		.pipe(ops0.chain(body => Oath.If(body?.success, { T: () => body.payload })))
+		.pipe(ops0.chain(x => Oath.If(CurrentUser.Validations.is_dto(x), { T: () => x as Ordo.User.Current.DTO })))
+		.pipe(ops0.rejected_map(e => RRR.codes.eacces(e?.message ?? "Unauthorized")))
+
+// TODO checking permissions for editing files of other users
+export const check_authorization = (intake: TIntake<TDTContext>) => (user: Ordo.User.Current.DTO) =>
+	Oath.If(user.id === intake.params.uid)
+		.pipe(ops0.map(() => user))
+		.pipe(ops0.rejected_map(() => RRR.codes.eperm("Permission denied")))
+
+export const check_file_exists =
+	(intake: TIntake<TDTContext>) =>
+	({ uid, fsid }: TIDs) =>
+		intake.data_persistence_strategy
+			.exists(uid, fsid)
+			.pipe(ops0.chain(exists => Oath.If(exists)))
+			.pipe(ops0.map(() => ({ uid, fsid })))
+			.pipe(ops0.rejected_map(() => RRR.codes.enoent("File not found")))
+
+export type TIDs = { uid: Ordo.User.UID; fsid: Ordo.Metadata.FSID }
+export const extract_ids = (intake: TIntake<TDTContext>) => () => ({
+	uid: intake.params.uid as Ordo.User.UID,
+	fsid: intake.params.fsid as Ordo.Metadata.FSID,
+})
+
+const check_file_does_not_exist =
+	(intake: TIntake<TDTContext>) =>
+	({ uid, fsid }: TIDs) =>
+		intake.data_persistence_strategy
+			.exists(uid, fsid)
+			.pipe(ops0.chain(exists => Oath.If(!exists)))
+			.pipe(ops0.map(() => ({ uid, fsid })))
+			.pipe(ops0.rejected_map(() => RRR.codes.eexist("File already exists")))
+
+export const validate_body_is_not_empty = (intake: TIntake<TDTContext>) =>
+	Oath.FromNullable(intake.req.body)
+		.pipe(ops0.map(() => intake))
+		.pipe(ops0.rejected_map(() => RRR.codes.einval("Empty file body")))
+
+export const validate_file_size_limit = (intake: TIntake<TDTContext>) => (user: Ordo.User.Current.DTO) =>
+	Oath.FromNullable(intake.req.headers.get("content-length"))
+		.pipe(ops0.map(file_size => Number.parseInt(file_size, 10)))
+		.pipe(ops0.chain(file_size => Oath.If(is_finite_non_negative_int(file_size), { T: () => file_size })))
+		.pipe(ops0.chain(file_size => Oath.If(CurrentUser.FromDTO(user).can_upload(file_size))))
+		.pipe(ops0.rejected_map(() => RRR.codes.efbig("File too big")))
+
+// TODO check if attemted to create a file in other user's space
+export const check_can_create_files = (intake: TIntake<TDTContext>) => (user: Ordo.User.Current.DTO) =>
+	intake.data_persistence_strategy
+		.read(user.id, METADATA_CONTENT_FSID)
+		.pipe(ops0.map(metadata => metadata.length))
+		.fix(() => 0)
+		.pipe(ops0.map(total_files => CurrentUser.FromDTO(user).can_create_files(total_files)))
+		.pipe(ops0.chain(can_create => Oath.If(can_create)))
+		.pipe(ops0.map(() => user))
+		.pipe(ops0.rejected_map(() => RRR.codes.enospc("Too many files")))
+
+const check_total_files_limit_if_file_does_not_exist = (intake: TIntake<TDTContext>) => (user: Ordo.User.Current.DTO) =>
+	intake.data_persistence_strategy
+		.exists(intake.params.uid as Ordo.User.UID, intake.params.fsid as Ordo.Metadata.FSID)
+		.pipe(ops0.chain(exists => (exists ? Oath.Resolve(user) : check_can_create_files(intake)(user))))
+
+const set_last_modified_header =
+	(intake: TIntake<TDTContext>) =>
+	({ uid, fsid }: TIDs) =>
+		intake.data_persistence_strategy
+			.mtime(uid, fsid)
+			.pipe(ops0.map(mtime => new Date(mtime)))
+			.pipe(ops0.map(date => date.toUTCString()))
+			.pipe(ops0.tap(last_modified => intake.headers.set("Last-Modified", last_modified)))
+			.pipe(ops0.map(() => ({ uid, fsid })))

lib/backend-dt/src/backend-dt.test.ts

@@ -0,0 +1,11 @@
+/*
+ * SPDX-FileCopyrightText: Copyright 2025, 谢尔盖 ||↓ and the Ordo.pink contributors
+ * SPDX-License-Identifier: Unlicense
+ */
+
+import { expect, test } from "bun:test"
+import { backend_dt } from "./backend-dt.impl"
+
+test("backend-dt should pass", () => {
+	expect(backend_dt).toEqual("backend-dt")
+})

lib/backend-dt/src/backend-dt.types.ts

@@ -0,0 +1,17 @@
+/*
+ * SPDX-FileCopyrightText: Copyright 2025, 谢尔盖 ||↓ and the Ordo.pink contributors
+ * SPDX-License-Identifier: Unlicense
+ */
+
+import type { TDefaultContext } from "@ordo-pink/backend-util-default-handler"
+import type { TLogger } from "@ordo-pink/logger"
+
+export type TDTChamber = {
+	allow_origin: string[]
+	data_persistence_strategy: OrdoBackend.Data.PersistenceStrategy
+	logger: TLogger
+	id_host: string
+	dt_host: string
+}
+
+export type TDTContext = TDefaultContext & TDTChamber