From eb2f95a4e7407ad08b60b525f99b3fbdb9f6afa9 Mon Sep 17 00:00:00 2001 From: fadkeabhi <31249309+fadkeabhi@users.noreply.github.com> Date: Sat, 9 Nov 2024 22:31:04 +0530 Subject: [PATCH] Feat Add support for jsonl files: --- src/core/core.ts | 15 +++++++++++---- src/core/json_file.ts | 33 +++++++++++++++++++++++---------- src/core/json_object.ts | 23 ++++++++++++++++------- src/utils/console.ts | 9 +++++++++ 4 files changed, 59 insertions(+), 21 deletions(-) diff --git a/src/core/core.ts b/src/core/core.ts index 83daff2..98ce846 100644 --- a/src/core/core.ts +++ b/src/core/core.ts @@ -2,6 +2,7 @@ import * as fs from 'fs/promises'; import * as YAML from 'yaml'; import { matchYamlExt } from '../utils/yaml'; import { error, messages } from '../utils/console'; +import { getFileExt } from './json_file'; export async function getFile(objectPath: string) { let json_file: any = undefined; @@ -37,9 +38,15 @@ export function getRootFolder(path: string) { } export async function saveFilePublic(path: string, data: any) { - // When path extension is for YAML file, then stringify with YAML encoder. - // Otherwise, default JSON encoder is used. - var json = matchYamlExt(path) ? YAML.stringify(data) : JSON.stringify(data); + let json = "" + if (getFileExt(path) === "jsonl") { + // If the file extension is `.jsonl`, the data is serialized as JSON Lines (JSONL) format. + json = data.map((item: any) => JSON.stringify(item)).join('\n'); + } else { + // When path extension is for YAML file, then stringify with YAML encoder. + // Otherwise, default JSON encoder is used. + json = matchYamlExt(path) ? YAML.stringify(data) : JSON.stringify(data); + } await fs .writeFile(path, json, 'utf8') @@ -47,4 +54,4 @@ export async function saveFilePublic(path: string, data: any) { .catch(_ => { error(messages.file.cannot_save_file); }); -} +} \ No newline at end of file diff --git a/src/core/json_file.ts b/src/core/json_file.ts index 5324fda..46d4585 100644 --- a/src/core/json_file.ts +++ b/src/core/json_file.ts @@ -20,7 +20,12 @@ export async function fileTranslator( return; } - jsonObj = { data: JSON.parse(jsonObj) }; + // if file extension is .jsonl convert the readed file to json with array of each line of the json file + if (getFileExt(objectPath) === "jsonl") { + jsonObj = { data: jsonObj.split('\n').map((line: string) => JSON.parse(line)) }; + } else { + jsonObj = { data: JSON.parse(jsonObj) }; + } // step: check if translation file already exists, if exists save content of it in oldTranslations let oldTranslations = JSON.parse("{}") @@ -36,7 +41,7 @@ export async function fileTranslator( let response = await getFileFromPath(fileName); let oldTranslation = response?.jsonObj - try{ + try { if (oldTranslation === undefined) { // Old Translation not found oldTranslations[lang] = { data: {} }; @@ -44,15 +49,18 @@ export async function fileTranslator( oldTranslation = { data: JSON.parse(oldTranslation) }; oldTranslations[lang] = oldTranslation; } - } catch{ + } catch { // If error in parsing json skip it oldTranslations[lang] = { data: {} }; } - + } + // if jsonl file force to translate all keys + const reTraslateOldTranslations = getFileExt(objectPath) === "jsonl"; + // step: translate object - let newJsonObj = await objectTranslator(TranslationConfig, jsonObj, from, to, oldTranslations); + let newJsonObj = await objectTranslator(TranslationConfig, jsonObj, from, to, oldTranslations, reTraslateOldTranslations); if (newJsonObj === undefined) { error(messages.file.cannot_translate); return; @@ -91,13 +99,18 @@ export async function getFileFromPath( return { jsonObj, objectPath }; } -function getFileExt(latestPath: string): string { +export function getFileExt(latestPath: string): string { // Check if source file has YAML extension and return the extension ("yml" or "yaml"). const sourceFileMatchYamlExt = matchYamlExt(latestPath); - // When source file has "yml" or "yaml" extension, use the same in output file path. - // Otherwise, default "json" extension used. - const fileExt = sourceFileMatchYamlExt || 'json'; - + let fileExt = ""; + if (latestPath.toLowerCase().endsWith('.jsonl')) { + // If the latestPath file extension is jsonl, keep fileExt jsonl + fileExt = "jsonl"; + } else { + // When source file has "yml" or "yaml" extension, use the same in output file path. + // Otherwise, default "json" extension used. + fileExt = sourceFileMatchYamlExt || 'json'; + } return fileExt; } diff --git a/src/core/json_object.ts b/src/core/json_object.ts index 31c1b0f..a0e8ac2 100644 --- a/src/core/json_object.ts +++ b/src/core/json_object.ts @@ -13,18 +13,24 @@ export async function objectTranslator( object: translatedObject, from: string, to: string[], - oldTranslations: { [key: string]: any } + oldTranslations: { [key: string]: any }, + reTraslateOldTranslations: boolean = false ): Promise { queue.concurrency = TranslationConfig.concurrencyLimit; - if (object && from && to) { let generalObject: translatedObject[] | null[] = []; await Promise.all( Object.keys(to).map(async function (index) { const indexAsNum = Number(index); - // Remove the keys which are already translated - const copyObject = removeKeys(JSON.parse(JSON.stringify(object)), oldTranslations[to[indexAsNum]]); + + let copyObject = JSON.parse(JSON.stringify(object)) + if (!reTraslateOldTranslations) { + // Remove the keys which are already translated + copyObject = removeKeys(JSON.parse(JSON.stringify(object)), oldTranslations[to[indexAsNum]]); + } + + const newTranslations = await deepDiver( TranslationConfig, @@ -33,12 +39,15 @@ export async function objectTranslator( to[indexAsNum] ); - // Insert old translations that we removed into the generalObject - generalObject[indexAsNum] = mergeKeys(oldTranslations[to[indexAsNum]], newTranslations) + if (!reTraslateOldTranslations) { + // Insert old translations that we removed into the generalObject + generalObject[indexAsNum] = mergeKeys(oldTranslations[to[indexAsNum]], newTranslations); + } else { + generalObject[indexAsNum] = JSON.parse(JSON.stringify(newTranslations)); + } }) ); - return generalObject as translatedObject[]; } else { throw new Error( diff --git a/src/utils/console.ts b/src/utils/console.ts index 6384079..d8ceedd 100644 --- a/src/utils/console.ts +++ b/src/utils/console.ts @@ -110,6 +110,10 @@ export function removeKeys(fromDict: any, toDict: any): any { return fromDict; } + if (Array.isArray(fromDict)) { + return fromDict; + } + if (typeof fromDict !== 'object' || fromDict === null) { return fromDict; } @@ -144,6 +148,11 @@ export function mergeKeys(base: any, insert: any): any { return insert; } + // Handle only insert is array + if (Array.isArray(insert)) { + return insert; + } + // Handle objects const result = { ...base };