Merge pull request #64 from akretion/16.0-mig-sale-import

[16.0] migrate/refactor sale import

.pre-commit-config.yaml

@@ -7,9 +7,7 @@ exclude: |
   ^sale_channel_hook_sale_state/|
   ^sale_channel_hook_stock_variation/|
   ^sale_channel_product/|
-  ^sale_import_base/|
   ^sale_import_delivery_carrier/|
-  ^sale_import_rest/|
   # END NOT INSTALLABLE ADDONS
   # Files and folders generated by bots, to avoid loops
   ^setup/|/static/description/index\.html$|

queue_job_chunk/__init__.py

@@ -1,2 +1 @@
 from . import models
-from . import components

queue_job_chunk/__manifest__.py

@@ -8,7 +8,7 @@
     "website": "https://github.com/OCA/sale-channel",
     "license": "AGPL-3",
     "category": "Generic Modules",
-    "depends": ["queue_job", "component"],
+    "depends": ["queue_job"],
     "data": [
         "views/queue_job_chunk.xml",
         "security/security.xml",

queue_job_chunk/models/queue_job_chunk.py

@@ -1,21 +1,48 @@
 #  Copyright (c) Akretion 2020
 #  License AGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html)
 
+import json
 import traceback
 
 from psycopg2 import OperationalError
 
 from odoo import api, fields, models
 from odoo.service.model import PG_CONCURRENCY_ERRORS_TO_RETRY
 
+from odoo.addons.queue_job.exception import RetryableJobError
+
 # Use to bypass chunks entirely for easier debugging
 DEBUG_MODE = False
 
 
 class QueueJobChunk(models.Model):
     _name = "queue.job.chunk"
     _description = "Queue Job Chunk"
-    _inherit = "collection.base"
+    _order = "id desc"
+
+    processor = fields.Selection([])
+    data_str = fields.Text(string="Editable data")
+    state = fields.Selection(
+        [("pending", "Pending"), ("done", "Done"), ("fail", "Failed")],
+        default="pending",
+        readonly=True,
+    )
+    state_info = fields.Text("Additional state information", readonly=True)
+    model_name = fields.Char(readonly=True)
+    record_id = fields.Integer(readonly=True)
+    reference = fields.Reference(
+        selection="_selection_target_model",
+        compute="_compute_reference",
+        store=True,
+        readonly=True,
+    )
+    company_id = fields.Many2one(
+        "res.company",
+        compute="_compute_reference",
+        store=True,
+        readonly=True,
+    )
+    stack_trace = fields.Text(readonly=True)
 
     @api.model
     def _selection_target_model(self):
@@ -34,26 +61,6 @@ def _compute_reference(self):
             else:
                 rec.reference = False
 
-    # component fields
-    usage = fields.Char()
-    apply_on_model = fields.Char()
-
-    data_str = fields.Text(string="Editable data")
-    state = fields.Selection(
-        [("pending", "Pending"), ("done", "Done"), ("fail", "Failed")],
-        default="pending",
-    )
-    state_info = fields.Text("Additional state information")
-    model_name = fields.Char()
-    record_id = fields.Integer()
-    reference = fields.Reference(
-        selection="_selection_target_model",
-        compute=_compute_reference,
-        store=True,
-    )
-    company_id = fields.Many2one("res.company", compute=_compute_reference, store=True)
-    stack_trace = fields.Text()
-
     @api.model_create_multi
     def create(self, vals):
         result = super().create(vals)
@@ -70,38 +77,39 @@ def enqueue_job(self):
         else:
             return self.with_delay().process_chunk()
 
+    def _get_processor(self):
+        # return here whatever class you want
+        # it can be a pure python class, an odoo TransientModel ...
+        raise NotImplementedError
+
+    def _get_data(self):
+        return json.loads(self.data_str)
+
     def process_chunk(self):
         self.ensure_one()
-        usage = self.usage
-        apply_on = self.apply_on_model
-        with self.work_on(apply_on) as work:
+        try:
+            with self.env.cr.savepoint():
+                processor = self._get_processor()
+                result = processor.run()
+        except RetryableJobError:
+            raise
+        except Exception as e:
             if DEBUG_MODE:
-                with self.env.cr.savepoint():
-                    processor = work.component(usage=usage)
-                    result = processor.run()
-                    self.state_info = ""
-                    self.state = "done"
-                    return result
-            else:
-                try:
-                    with self.env.cr.savepoint():
-                        processor = work.component(usage=usage)
-                        result = processor.run()
-                except Exception as e:
-                    # TODO maybe it will be simplier to have a kind of inherits
-                    #  on queue.job to avoid a double error management
-                    # so a failling chunk will have a failling job
-                    if (
-                        isinstance(e, OperationalError)
-                        and e.pgcode in PG_CONCURRENCY_ERRORS_TO_RETRY
-                    ):
-                        # In that case we raise an error so queue_job
-                        # will do a RetryableJobError
-                        raise
-                    self.state = "fail"
-                    self.state_info = type(e).__name__ + str(e.args)
-                    self.stack_trace = traceback.format_exc()
-                    return False
-                self.state_info = ""
-                self.state = "done"
-                return result
+                raise
+            # TODO maybe it will be simplier to have a kind of inherits
+            #  on queue.job to avoid a double error management
+            # so a failling chunk will have a failling job
+            if (
+                isinstance(e, OperationalError)
+                and e.pgcode in PG_CONCURRENCY_ERRORS_TO_RETRY
+            ):
+                # In that case we raise an error so queue_job
+                # will do a RetryableJobError
+                raise
+            self.state = "fail"
+            self.state_info = type(e).__name__ + str(e.args)
+            self.stack_trace = traceback.format_exc()
+            return False
+        self.state_info = ""
+        self.state = "done"
+        return result

queue_job_chunk/tests/models.py

@@ -0,0 +1,44 @@
+# Copyright 2022 Akretion (https://www.akretion.com).
+# @author Sébastien BEAU <sebastien.beau@akretion.com>
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
+
+from odoo import fields, models
+
+
+# Exemple of pure python processor (inherit is not possible)
+class TestPythonPartnerProcessor:
+    def __init__(self, chunk):
+        super().__init__()
+        self.env = chunk.env
+        self.chunk = chunk
+
+    def run(self):
+        return self.env["res.partner"].create(self.chunk._get_data())
+
+
+# Exemple of odoo processor (inherit is possible)
+class TestOdooStateProcessor(models.TransientModel):
+    _name = "test.odoo.state.processor"
+    _description = "Chunk Processor State Create"
+
+    chunk_id = fields.Many2one("queue.job.chunk", "Chunk")
+
+    def run(self):
+        return self.env["res.country.state"].create(self.chunk_id._get_data())
+
+
+class QueueJobChunk(models.Model):
+    _inherit = "queue.job.chunk"
+
+    processor = fields.Selection(
+        selection_add=[
+            ("test_partner", "Test create Partner"),
+            ("test_state", "Test create Country State"),
+        ],
+    )
+
+    def _get_processor(self):
+        if self.processor == "test_partner":
+            return TestPythonPartnerProcessor(self)
+        elif self.processor == "test_state":
+            return self.env["test.odoo.state.processor"].new({"chunk_id": self.id})

queue_job_chunk/tests/test_queue_job_chunk.py

@@ -2,56 +2,60 @@
 #  License AGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html)
 
 
-from odoo.addons.component.tests.common import TransactionComponentCase
+from odoo_test_helper import FakeModelLoader
 
+from odoo.tests import TransactionCase
 
-class TestQueueJobChunk(TransactionComponentCase):
-    def setUp(self):
-        super().setUp()
-        self.env = self.env(
-            context=dict(self.env.context, test_queue_job_no_delay=True)
-        )
-        self.main_company = self.env.ref("base.main_company")
-        self.another_company_partner = self.env["res.partner"].create(
+
+class TestQueueJobChunk(TransactionCase, FakeModelLoader):
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+
+        cls.loader = FakeModelLoader(cls.env, cls.__module__)
+        cls.loader.backup_registry()
+        from .models import QueueJobChunk, TestOdooStateProcessor
+
+        cls.loader.update_registry((TestOdooStateProcessor, QueueJobChunk))
+
+        cls.env = cls.env(context=dict(cls.env.context, test_queue_job_no_delay=True))
+        cls.main_company = cls.env.ref("base.main_company")
+        cls.another_company_partner = cls.env["res.partner"].create(
             {"name": "Company2"}
         )
-        self.another_company = self.env["res.company"].create(
+        cls.another_company = cls.env["res.company"].create(
             {
-                "name": self.another_company_partner.name,
-                "partner_id": self.another_company_partner.id,
-                "currency_id": self.env.ref("base.main_company").currency_id.id,
+                "name": cls.another_company_partner.name,
+                "partner_id": cls.another_company_partner.id,
+                "currency_id": cls.env.ref("base.main_company").currency_id.id,
             }
         )
-        self.partner = self.env.ref("base.res_partner_3")
-        self.chunk_data_contact = [
+        cls.partner = cls.env.ref("base.res_partner_3")
+        cls.chunk_data_contact = [
             {
-                "apply_on_model": "res.partner",
                 "data_str": '{"name": "Steve Queue Job"}',
-                "usage": "basic_create",
+                "processor": "test_partner",
                 "model_name": "res.partner",
-                "record_id": self.partner.id,
+                "record_id": cls.partner.id,
             },
             {
-                "apply_on_model": "res.partner",
                 "data_str": '{"name": "Other"}',
-                "usage": "basic_create",
+                "processor": "test_partner",
                 "model_name": "res.partner",
-                "record_id": self.partner.id,
+                "record_id": cls.partner.id,
             },
         ]
-        self.chunk_data_bad = {
-            "apply_on_model": "res.partner",
+        cls.chunk_data_bad = {
             "data_str": "{''(;,),x*}",
-            "usage": "basic_create",
+            "processor": "test_partner",
             "model_name": "res.partner",
-            "record_id": self.partner.id,
+            "record_id": cls.partner.id,
         }
-        USA = self.env.ref("base.us")
-        self.chunk_data_state = {
-            "apply_on_model": "res.country.state",
+        USA = cls.env.ref("base.us")
+        cls.chunk_data_state = {
+            "processor": "test_state",
             "data_str": '{"name": "New Stateshire", "code": "NS", "country_id": %d}'
             % USA.id,
-            "usage": "basic_create",
             "model_name": "res.country",
             "record_id": USA.id,
         }