Implementation of the Maroun-style query more retriever #448

Author: VinceFINET

force-app/main/default/lwc/orgcheckApp/api/core/orgcheck-api-datafactory.js

@@ -98,14 +98,14 @@ export class OrgCheckDataFactory2 {
 export class OrgCheckDataFactory {
 
     #allValidations;
-    #needDepencencies;
+    #needDependencies;
     #instances;
 
     constructor(sfdcManager) {
 
         const currentApiVersion = sfdcManager.getApiVersion();
 
-        this.#allValidations = [
+        this.#allValidations = [ // START:ALL_VALIDATIONS
             { 
                 description: 'Not referenced anywhere',
                 formula: (d) => IS_EMPTY(d.dependencies?.referenced), 
@@ -359,7 +359,8 @@ export class OrgCheckDataFactory {
                 badField: 'usedPercentage',
                 applicable: [ SFDC_Limit ]
             }
-        ].map((v, i) => { 
+        ] // END:ALL_VALIDATIONS
+        .map((v, i) => { 
             // check description
             if (v.description === undefined || typeof v.description !== 'string') {
                 throw new TypeError(`The ${i}th Validation Rule should have a 'description' property of type 'string'.`);
@@ -383,16 +384,16 @@ export class OrgCheckDataFactory {
         });
         Object.freeze(this.#allValidations); 
 
-        this.#needDepencencies = [
+        this.#needDependencies = [ // START:ALL_NEED_DEPENDENCIES
             SFDC_ApexClass, SFDC_ApexTrigger, SFDC_Field, SFDC_CustomLabel, SFDC_Flow,
             SFDC_LightningAuraComponent, SFDC_LightningPage, SFDC_LightningWebComponent,
             SFDC_VisualForceComponent, SFDC_VisualForcePage
-        ];
+        ]; // END:ALL_NEED_DEPENDENCIES
         // check if '#needDepencencies' array contains only OrgCheckData instances
-        if (this.#needDepencencies === undefined || Array.isArray(this.#needDepencencies) === false || this.#needDepencencies.every((dc) => IS_CLASS_EXTENDS(dc, OrgCheckData) === false)) {
+        if (this.#needDependencies === undefined || Array.isArray(this.#needDependencies) === false || this.#needDependencies.every((dc) => IS_CLASS_EXTENDS(dc, OrgCheckData) === false)) {
             throw new TypeError(`The list of classes that needs Dependencies must be of type 'array' with only OrgCheckData items.`);
         }
-        Object.freeze(this.#needDepencencies); 
+        Object.freeze(this.#needDependencies); 
 
         this.#instances = new Map();
     }
@@ -413,7 +414,7 @@ export class OrgCheckDataFactory {
             this.#instances.set(dataClass, new OrgCheckDataFactory2(
                 dataClass, 
                 isDataClassExtendsData ? this.#allValidations.filter(v => v.applicable.includes(dataClass)) : [], 
-                isDataClassExtendsData ? this.#needDepencencies.includes(dataClass) : []
+                isDataClassExtendsData ? this.#needDependencies.includes(dataClass) : []
             ));
         }
         // Return the instance

force-app/main/default/lwc/orgcheckApp/api/core/orgcheck-api-sfconnectionmanager.js

@@ -276,9 +276,11 @@ export class OrgCheckSalesforceManager {
         let nbQueriesDone = 0, nbQueriesByPassed = 0, nbQueriesError = 0, nbQueryMore = 0, nbQueriesPending = queries.length;
         return Promise.all(queries.map((query) => {
             const conn = query.tooling === true ? this.#connection.tooling : this.#connection;
+            let queryMoreStartingId = '000000000000000000';
+            const uniqueFieldName = query.uniqueFieldName || 'Id';
             const sequential_query = (callback) => {
                 if (query.queryMore === false) {
-                    conn.query(`${query.string} LIMIT ${MAX_NOQUERYMORE_BATCH_SIZE} OFFSET ${nbQueryMore * MAX_NOQUERYMORE_BATCH_SIZE}`, { autoFetch: false }, callback);
+                    conn.query(`${query.string} AND ${uniqueFieldName} > '${queryMoreStartingId}' ORDER BY ${uniqueFieldName} LIMIT ${MAX_NOQUERYMORE_BATCH_SIZE}`, { autoFetch: false }, callback);
                 } else {
                     conn.query(query.string, { autoFetch: true }, callback);
                 }
@@ -306,6 +308,7 @@ export class OrgCheckSalesforceManager {
                                 nbQueriesPending--;
                                 resolve({ records: records });
                             } else {
+                                queryMoreStartingId = records[records.length-1][uniqueFieldName];
                                 nbQueryMore++;
                                 sequential_query(recursive_query);
                             }

force-app/main/default/lwc/orgcheckApp/api/dataset/orgcheck-api-dataset-apexclasses.js

@@ -21,7 +21,8 @@ export class OrgCheckDatasetApexClasses extends OrgCheckDataset {
                     'FROM ApexClass '+
                     'WHERE ManageableState IN (\'installedEditable\', \'unmanaged\') ',
             tooling: true,
-            queryMore: false
+            queryMore: false,
+            uniqueFieldName: 'Id', // unique field name (to be used by the custom QueryMore)
         }, {
             string: 'SELECT ApexClassOrTriggerId, ApexTestClassId '+
                     'FROM ApexCodeCoverage',

force-app/main/default/lwc/orgcheckApp/api/dataset/orgcheck-api-dataset-objects.js

@@ -18,7 +18,8 @@ export class OrgCheckDatasetObjects extends OrgCheckDataset {
 
             // Some information are not in the global describe, we need to append them with EntityDefinition soql query
             sfdcManager.soqlQuery([{ 
-                queryMore: false, // entityDef does not support calling QueryMore, we will get records with LIMIT/OFFSET
+                queryMore: false, // entityDef does not support calling QueryMore
+                uniqueFieldName: 'DurableId', // unique field name (to be used by the custom QueryMore)
                 string: 'SELECT DurableId, NamespacePrefix, DeveloperName, QualifiedApiName, '+
                             'ExternalSharingModel, InternalSharingModel '+
                         'FROM EntityDefinition '+