Feat: Add support for JSONL file translation

src/core/core.ts

@@ -2,6 +2,7 @@ import * as fs from 'fs/promises';
 import * as YAML from 'yaml';
 import { matchYamlExt } from '../utils/yaml';
 import { error, messages } from '../utils/console';
+import { getFileExt } from './json_file';
 
 export async function getFile(objectPath: string) {
   let json_file: any = undefined;
@@ -37,14 +38,20 @@ export function getRootFolder(path: string) {
 }
 
 export async function saveFilePublic(path: string, data: any) {
-  // When path extension is for YAML file, then stringify with YAML encoder.
-  // Otherwise, default JSON encoder is used.
-  var json = matchYamlExt(path) ? YAML.stringify(data) : JSON.stringify(data);
+  let json = ""
+  if (getFileExt(path) === "jsonl") {
+    // If the file extension is `.jsonl`, the data is serialized as JSON Lines (JSONL) format.
+    json = data.map((item: any) => JSON.stringify(item)).join('\n');
+  } else {
+    // When path extension is for YAML file, then stringify with YAML encoder.
+    // Otherwise, default JSON encoder is used.
+    json = matchYamlExt(path) ? YAML.stringify(data) : JSON.stringify(data);
+  }
 
   await fs
     .writeFile(path, json, 'utf8')
     .then(_ => {})
     .catch(_ => {
       error(messages.file.cannot_save_file);
     });
-}
+}

src/core/json_file.ts

@@ -20,7 +20,12 @@ export async function fileTranslator(
     return;
   }
 
-  jsonObj = { data: JSON.parse(jsonObj) };
+  // if file extension is .jsonl convert the readed file to json with array of each line of the json file
+  if (getFileExt(objectPath) === "jsonl") {
+    jsonObj = { data: jsonObj.split('\n').map((line: string) => JSON.parse(line)) };
+  } else {
+    jsonObj = { data: JSON.parse(jsonObj) };
+  }
 
   // step: check if translation file already exists, if exists save content of it in oldTranslations
   let oldTranslations = JSON.parse("{}")
@@ -36,23 +41,26 @@ export async function fileTranslator(
 
     let response = await getFileFromPath(fileName);
     let oldTranslation = response?.jsonObj
-    try{
+    try {
       if (oldTranslation === undefined) {
         // Old Translation not found
         oldTranslations[lang] = { data: {} };
       } else {
         oldTranslation = { data: JSON.parse(oldTranslation) };
         oldTranslations[lang] = oldTranslation;
       }
-    } catch{
+    } catch {
       // If error in parsing json skip it
       oldTranslations[lang] = { data: {} };
     }
-    
+
   }
 
+  // if jsonl file force to translate all keys
+  const reTraslateOldTranslations = getFileExt(objectPath) === "jsonl";
+
   // step: translate object
-  let newJsonObj = await objectTranslator(TranslationConfig, jsonObj, from, to, oldTranslations);
+  let newJsonObj = await objectTranslator(TranslationConfig, jsonObj, from, to, oldTranslations, reTraslateOldTranslations);
   if (newJsonObj === undefined) {
     error(messages.file.cannot_translate);
     return;
@@ -91,13 +99,18 @@ export async function getFileFromPath(
   return { jsonObj, objectPath };
 }
 
-function getFileExt(latestPath: string): string {
+export function getFileExt(latestPath: string): string {
   // Check if source file has YAML extension and return the extension ("yml" or "yaml").
   const sourceFileMatchYamlExt = matchYamlExt(latestPath);
 
-  // When source file has "yml" or "yaml" extension, use the same in output file path.
-  // Otherwise, default "json" extension used.
-  const fileExt = sourceFileMatchYamlExt || 'json';
-
+  let fileExt = "";
+  if (latestPath.toLowerCase().endsWith('.jsonl')) {
+    // If the latestPath file extension is jsonl, keep fileExt jsonl
+    fileExt = "jsonl";
+  } else {
+    // When source file has "yml" or "yaml" extension, use the same in output file path.
+    // Otherwise, default "json" extension used.
+    fileExt = sourceFileMatchYamlExt || 'json';
+  }
   return fileExt;
 }

src/core/json_object.ts

@@ -13,18 +13,24 @@ export async function objectTranslator(
   object: translatedObject,
   from: string,
   to: string[],
-  oldTranslations: { [key: string]: any }
+  oldTranslations: { [key: string]: any },
+  reTraslateOldTranslations: boolean = false
 ): Promise<translatedObject[]> {
   queue.concurrency = TranslationConfig.concurrencyLimit;
-
   if (object && from && to) {
     let generalObject: translatedObject[] | null[] = [];
 
     await Promise.all(
       Object.keys(to).map(async function (index) {
         const indexAsNum = Number(index);
-        // Remove the keys which are already translated
-        const copyObject = removeKeys(JSON.parse(JSON.stringify(object)), oldTranslations[to[indexAsNum]]);
+
+        let copyObject = JSON.parse(JSON.stringify(object))
+        if (!reTraslateOldTranslations) {
+          // Remove the keys which are already translated
+          copyObject = removeKeys(JSON.parse(JSON.stringify(object)), oldTranslations[to[indexAsNum]]);
+        }
+
+
 
         const newTranslations = await deepDiver(
           TranslationConfig,
@@ -33,12 +39,15 @@ export async function objectTranslator(
           to[indexAsNum]
         );
 
-        // Insert old translations that we removed into the generalObject
-        generalObject[indexAsNum] = mergeKeys(oldTranslations[to[indexAsNum]], newTranslations)
+        if (!reTraslateOldTranslations) {
+          // Insert old translations that we removed into the generalObject
+          generalObject[indexAsNum] = mergeKeys(oldTranslations[to[indexAsNum]], newTranslations);
+        } else {
+          generalObject[indexAsNum] = JSON.parse(JSON.stringify(newTranslations));
+        }
 
       })
     );
-
     return generalObject as translatedObject[];
   } else {
     throw new Error(

src/utils/console.ts

@@ -110,6 +110,10 @@ export function removeKeys(fromDict: any, toDict: any): any {
     return fromDict;
   }
 
+  if (Array.isArray(fromDict)) {
+    return fromDict;
+  }
+
   if (typeof fromDict !== 'object' || fromDict === null) {
     return fromDict;
   }
@@ -144,6 +148,11 @@ export function mergeKeys(base: any, insert: any): any {
     return insert;
   }
 
+  // Handle only insert is array
+  if (Array.isArray(insert)) {
+    return insert;
+  }
+
   // Handle objects
   const result = { ...base };