diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9afc6921..4d12a08c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,11 +1,9 @@ exclude: | (?x) # NOT INSTALLABLE ADDONS - ^pattern_import_export/| ^pattern_import_export_csv/| ^pattern_import_export_custom_header/| ^pattern_import_export_synchronize/| - ^pattern_import_export_xlsx/| # END NOT INSTALLABLE ADDONS # Files and folders generated by bots, to avoid loops ^setup/|/static/description/index\.html$| diff --git a/pattern_import_export/README.rst b/pattern_import_export/README.rst index 61b8e7bb..73a61b2b 100644 --- a/pattern_import_export/README.rst +++ b/pattern_import_export/README.rst @@ -2,10 +2,13 @@ Pattern Import Export ===================== -.. !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! !! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:3705d1bfe3c899c7c2037cad5305a0a6f080ddd8ca1d03bc732613a481115d2e + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! .. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png :target: https://odoo-community.org/page/development-status @@ -14,10 +17,10 @@ Pattern Import Export :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html :alt: License: AGPL-3 .. |badge3| image:: https://img.shields.io/badge/github-shopinvader%2Fpattern--import--export-lightgray.png?logo=github - :target: https://github.com/shopinvader/pattern-import-export/tree/14.0/pattern_import_export + :target: https://github.com/shopinvader/pattern-import-export/tree/16.0/pattern_import_export :alt: shopinvader/pattern-import-export -|badge1| |badge2| |badge3| +|badge1| |badge2| |badge3| Overview ~~~~~~~~ @@ -79,16 +82,29 @@ You have two options: * Open the tree view of any model and tick some record selection boxes (for this step, these don't matter, we only just want to show the sidebar). * In the sidebar, click on the "Import with Pattern" button -* Select the pattern that you used to generate the export, upload your file and click import. +* Select a pattern, upload your file and click import. * A "Pattern file" is created, and its job along with it. Depending on the success or failure of the job, you will receive a red/green notification on your window. You can check the details in the appropriate Import/Export menu. Or: * Access the Import wizard through the Import/Export menu -* Select the Pattern that you want to use +* Select a pattern * Click on the "Import" button + +Import syntax +------------- + +One of the strength of pattern_import_export module is the ability to +reference records by natural keys (business keys) instead of technical keys (xmlid or database id). + +One or more columns can be the natural key of the record to find and update or to create a new record. +Each column in the natural key has to be suffixed by "#key". + +One or more columns can be used as foreign keys can be accessed with "|" syntax. (for instance on partner: country_id|code ) + + Example ------- @@ -179,8 +195,8 @@ Bug Tracker Bugs are tracked on `GitHub Issues `_. In case of trouble, please check there if your issue has already been reported. -If you spotted it first, help us smashing it by providing a detailed and welcomed -`feedback `_. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. Do not contact contributors directly about support or help with technical issues. @@ -199,10 +215,11 @@ Contributors * Sébastien Beau * François Honoré (ACSONE SA/NV) * Kevin Khao +* Raphaël Reverdy Maintainers ~~~~~~~~~~~ -This module is part of the `shopinvader/pattern-import-export `_ project on GitHub. +This module is part of the `shopinvader/pattern-import-export `_ project on GitHub. You are welcome to contribute. diff --git a/pattern_import_export/__manifest__.py b/pattern_import_export/__manifest__.py index f004cf9b..a6a6b066 100644 --- a/pattern_import_export/__manifest__.py +++ b/pattern_import_export/__manifest__.py @@ -3,7 +3,7 @@ { "name": "Pattern Import Export", "summary": "Pattern for import or export", - "version": "14.0.2.5.1", + "version": "16.0.1.0.0", "category": "Extra Tools", "author": "Akretion", "website": "https://github.com/Shopinvader/pattern-import-export", @@ -30,5 +30,5 @@ "data/queue_job_function_data.xml", ], "demo": ["demo/demo.xml"], - "installable": False, + "installable": True, } diff --git a/pattern_import_export/demo/demo.xml b/pattern_import_export/demo/demo.xml index 5297c3ef..15c522cb 100644 --- a/pattern_import_export/demo/demo.xml +++ b/pattern_import_export/demo/demo.xml @@ -1,108 +1,107 @@ - - - - - US, FR, BE - [("code", "in", ["US", "FR", "BE"])] - res.country - - - European countries - [("code", "in", ["UK", "FR", "BE", "ES", "DE"])] - res.country - - - Ignore one - [("name", "!=", "Ignored company")] - res.company - + + + + US, FR, BE + [("code", "in", ["US", "FR", "BE"])] + res.country + + + + European countries + [("code", "in", ["UK", "FR", "BE", "ES", "DE"])] + res.country + + + + Ignore one + [("name", "!=", "Ignored company")] + res.company + + - - - Partner - res.partner - - - Users list - M2M - res.users - - - Partner with contact - res.partner - - - - - - - - - - + + + Partner + res.partner + + + Users list - M2M + res.users + + + Partner with contact + res.partner + + + + + + + + + + - - - id - - - - name - - - - street - - - - country_id/code - - - - - - category_id/name - - - - - id - - - - name - - - - company_ids/name - 1 - - - - - - id - - - - name - - - - child_ids - 3 - - - - - country_id/code - - - - - + + + id + + + + name + + + + street + + + + country_id/code + + + + + + category_id/name + + + + + id + + + + name + + + + company_ids/name + 1 + + + + + + id + + + + name + + + + child_ids + 3 + + + + + country_id/code + + + + diff --git a/pattern_import_export/migrations/12.0.4.0.0/post-migrate.py b/pattern_import_export/migrations/12.0.4.0.0/post-migrate.py index 49b2bfa8..9f5c4db3 100644 --- a/pattern_import_export/migrations/12.0.4.0.0/post-migrate.py +++ b/pattern_import_export/migrations/12.0.4.0.0/post-migrate.py @@ -16,12 +16,12 @@ def migrate(env, version): ("export_format", "NULL"), ]: if openupgrade.column_exists(env.cr, "ir_exports", field): - params[field] = '"{}"'.format(field) + params[field] = f'"{field}"' else: params[field] = default_value env.cr.execute( - """ + f""" INSERT INTO pattern_config ( use_description, pattern_file, @@ -34,17 +34,15 @@ def migrate(env, version): ) SELECT use_description, - {p[pattern_file]}, - {p[pattern_file_name]}, - {p[pattern_last_generation_date]}, - {p[export_format]}, - {p[partial_commit]}, - {p[flush_step]}, + {params["pattern_file"]}, + {params["pattern_file_name"]}, + {params["pattern_last_generation_date"]}, + {params["export_format"]}, + {params["partial_commit"]}, + {params["flush_step"]}, id FROM ir_exports WHERE is_pattern IS TRUE - """.format( - p=params - ) + """ ) env.cr.execute( diff --git a/pattern_import_export/models/base.py b/pattern_import_export/models/base.py index eaa60702..6f07477f 100644 --- a/pattern_import_export/models/base.py +++ b/pattern_import_export/models/base.py @@ -103,15 +103,22 @@ def _clean_identifier_key(self, res, ident_keys): res[key.replace(IDENTIFIER_SUFFIX, "")] = res.pop(key) def _convert_value_to_domain(self, field_name, value): + # fieldname may be None + # todo: rename field_name to prefix if isinstance(value, dict): domain = [] + subdom = [] for key, val in value.items(): if key == ".id": # .id is internal db id, so we rename it key = "id" - domain.append(("{}.{}".format(field_name, key), "=", val)) + # field_name may be None + # then key = value directly + dom_key = f"{field_name}.{key}" if field_name else key + subdom += self._convert_value_to_domain(dom_key, val) + domain = subdom else: - domain = [(field_name, "=", value)] + domain = [[field_name, "=", value]] return domain def _get_domain_from_identifier_key(self, res): @@ -120,9 +127,7 @@ def _get_domain_from_identifier_key(self, res): for key in list(res.keys()): if key.endswith(IDENTIFIER_SUFFIX): field_name = key.replace(IDENTIFIER_SUFFIX, "") - domain = expression.AND( - [domain, self._convert_value_to_domain(field_name, res[key])] - ) + domain += self._convert_value_to_domain(field_name, res[key]) ident_keys.append(key) return domain, ident_keys @@ -232,6 +237,12 @@ def _extract_records(self, fields_, data, log=lambda a: None, limit=FLOAT_INF): yield self._pattern_format2json(row), {"rows": {"from": idx, "to": idx}} # WARNING: complex code + # This feature has been inactivated for v16 + # it's not something easy to do + # and need to be carefully done + # + # it's about allowing to import working lines + # and reporting lines in error # As we are in an generator the following code is executed # after the "for id, xid, record, info in converted:" in model.py:1090 # the idea is to call the flush manually for the last line @@ -243,8 +254,9 @@ def _extract_records(self, fields_, data, log=lambda a: None, limit=FLOAT_INF): # in V15 we should propose a refactor of load method if data[-1][0] == idx: self._context["import_flush"]() - self._cr.execute("RELEASE SAVEPOINT model_load") - self._cr.execute("SAVEPOINT model_load") + # https://github.com/odoo/odoo/pull/76243 + # self._cr.execute("RELEASE SAVEPOINT model_load") + # self._cr.execute("SAVEPOINT model_load") else: yield from super()._extract_records(fields_, data, log=log, limit=limit) diff --git a/pattern_import_export/models/ir_actions.py b/pattern_import_export/models/ir_actions.py index c9d16815..64037d65 100644 --- a/pattern_import_export/models/ir_actions.py +++ b/pattern_import_export/models/ir_actions.py @@ -23,8 +23,11 @@ def get_bindings(self, model_name): # when we append the action in res["action"] it's added in the dict # and as the dict is mutuable the value is cached is updated # so we need to be careful to not add it again and again + if res == {}: + res["action"] = [] if self.env.user.has_group("pattern_import_export.group_pattern_user"): for xml_id in xml_ids: - if xml_id not in [act.get("xml_id") for act in res["action"]]: - res["action"].append(self.env.ref(xml_id).sudo().read()[0]) + patimpex = self.env.ref(xml_id) + if patimpex.id not in [act.get("id") for act in res["action"]]: + res["action"].append({"id": patimpex.id, "name": patimpex.name}) return res diff --git a/pattern_import_export/models/ir_exports_line.py b/pattern_import_export/models/ir_exports_line.py index 3ba735b8..3475b9bd 100644 --- a/pattern_import_export/models/ir_exports_line.py +++ b/pattern_import_export/models/ir_exports_line.py @@ -88,7 +88,7 @@ def _compute_required_fields(self): level += 1 hidden_fields.remove("add_select_tab") for idx in range(2, level + 1): - required.append("field{}_id".format(idx)) + required.append(f"field{idx}_id") if ftype in ["one2many", "many2many"]: required.append("number_occurence") if ftype in "one2many": @@ -162,7 +162,7 @@ def _compute_related_level_field(self): def _build_header(self, level, use_description): base_header = [] for idx in range(1, level + 1): - field = self["field{}_id".format(idx)] + field = self[f"field{idx}_id"] if use_description: base_header.append(field.field_description) else: @@ -191,7 +191,7 @@ def _get_header(self, use_description=False): header += IDENTIFIER_SUFFIX headers.append(header) else: - last_relation_field = record["field{}_id".format(record.level)] + last_relation_field = record[f"field{record.level}_id"] if last_relation_field.ttype == "many2one": headers.append( record._build_header(record.level + 1, use_description) @@ -213,7 +213,7 @@ def _get_header(self, use_description=False): ] ) else: - field = record["field{}_id".format(record.level + 1)] + field = record[f"field{record.level + 1}_id"] if use_description: field_name = field.field_description else: @@ -238,7 +238,7 @@ def _format_tab_records(self, permitted_records): def _get_tab_name(self): tab_filter = self.tab_filter_id if tab_filter: - name = "({}) {}".format(str(tab_filter.id), tab_filter.name) + name = f"({str(tab_filter.id)}) {tab_filter.name}" else: name = self.field1_id.field_description if len(name) > 31: diff --git a/pattern_import_export/models/ir_fields.py b/pattern_import_export/models/ir_fields.py index 4add32ca..971042bd 100644 --- a/pattern_import_export/models/ir_fields.py +++ b/pattern_import_export/models/ir_fields.py @@ -4,11 +4,9 @@ import ast -from odoo import _, api, models +from odoo import Command, _, api, models from odoo.osv import expression -from odoo.addons.base.models import ir_fields - from .common import IDENTIFIER_SUFFIX @@ -29,9 +27,7 @@ def fn_with_key_support(record, log): cleanned[field] = vals converted = fn(cleanned, log) for field in keyfields: - converted["{}{}".format(field, IDENTIFIER_SUFFIX)] = converted.pop( - field - ) + converted[f"{field}{IDENTIFIER_SUFFIX}"] = converted.pop(field) return converted return fn_with_key_support @@ -110,9 +106,9 @@ def _list_to_many2many(self, model, field, value): warnings.extend(ws) if self._context.get("update_many2many"): - return [ir_fields.LINK_TO(id) for id in ids], warnings + return [Command.link(id) for id in ids], warnings else: - return [ir_fields.REPLACE_WITH(ids)], warnings + return [Command.set(ids)], warnings @api.model def _str_to_many2many(self, model, field, value): @@ -127,12 +123,35 @@ def _str_to_many2many(self, model, field, value): def _str_to_many2one(self, model, field, value): if isinstance(value, dict): # odoo expect a list with one item - value = [value] + if len(value) == 1: + one_value = [value] + return super()._str_to_many2one(model, field, one_value) + else: + domain = model._convert_value_to_domain(None, value) + tosearch = field._related_comodel_name + record = self.env[tosearch].search(domain) + if len(record) > 1: + # TODO improve here + raise self._format_import_error( + ValueError, + _("%s Too many records found for %s in field '%s'"), + (_(record._description), domain, tosearch), + ) + if len(record) == 0: + raise self._format_import_error( + ValueError, + _("%s No matching record found for %s in field '%s'"), + (_(record._description), domain, tosearch), + ) + + # call core function to be sure not to miss something + an_id, donotcare, w2 = self.db_id_for(model, field, ".id", record.id) + return an_id, [] + w2 return super()._str_to_many2one(model, field, value) @api.model def _str_to_boolean(self, model, field, value): - if isinstance(value, (int, float)): + if isinstance(value, (int | float)): return bool(value), [] if isinstance(value, bool): return value, [] diff --git a/pattern_import_export/models/patch.py b/pattern_import_export/models/patch.py index 17f91b98..b1d3fc9b 100644 --- a/pattern_import_export/models/patch.py +++ b/pattern_import_export/models/patch.py @@ -22,8 +22,9 @@ def _convert_records(self, records, log=lambda a: None): :rtype: list((int|None, str|None, dict)) """ field_names = {name: field.string for name, field in self._fields.items()} - if self.env.lang: - field_names.update(self.env["ir.translation"].get_field_string(self._name)) + # if self.env.lang: + # field._get_stored_translations + # field_names.update(self.env["ir.translation"].get_field_string(self._name)) convert = self.env["ir.fields.converter"].for_model(self) diff --git a/pattern_import_export/models/pattern_chunk.py b/pattern_import_export/models/pattern_chunk.py index 23c0947b..c1664cb4 100644 --- a/pattern_import_export/models/pattern_chunk.py +++ b/pattern_import_export/models/pattern_chunk.py @@ -39,7 +39,9 @@ def run_import(self): pattern_config={ "model": model, "record_ids": [], - "purge_one2many": self.pattern_file_id.pattern_config_id.purge_one2many, + "purge_one2many": ( + self.pattern_file_id.pattern_config_id.purge_one2many + ), } ) .env[model] diff --git a/pattern_import_export/models/pattern_config.py b/pattern_import_export/models/pattern_config.py index 74d025f3..bb78cf20 100644 --- a/pattern_import_export/models/pattern_config.py +++ b/pattern_import_export/models/pattern_config.py @@ -71,7 +71,9 @@ def _compute_pattern_file_counts(self): for state in ("failed", "pending", "done"): field_name = "count_pattern_file_" + state count = len( - rec.pattern_file_ids.filtered(lambda r: r.state == state).ids + rec.pattern_file_ids.filtered( + lambda r, state=state: r.state == state + ).ids ) setattr(rec, field_name, count) @@ -113,9 +115,13 @@ def _get_output_headers(self): headers = [] if self.header_format == "description_and_tech": headers.append( - dict(zip(tech_header, self._get_header(use_description=True))) + dict( + zip( + tech_header, self._get_header(use_description=True), strict=True + ) + ) ) - headers.append(dict(zip(tech_header, tech_header))) + headers.append(dict(zip(tech_header, tech_header, strict=True))) return headers def _get_header(self, use_description=False): @@ -223,7 +229,7 @@ def _export_with_record(self, records): pattern_file_exports = self.env["pattern.file"] all_data = self._generate_with_records(records) if all_data and self.env.context.get("export_as_attachment", True): - for export, attachment_data in zip(self, all_data): + for export, attachment_data in zip(self, all_data, strict=True): pattern_file_exports |= export._create_pattern_file_export( attachment_data ) @@ -236,7 +242,7 @@ def _create_pattern_file_export(self, attachment_datas): @return: ir.attachment recordset """ self.ensure_one() - name = "{name}.{format}".format(name=self.name, format=self.export_format) + name = f"{self.name}.{self.export_format}" return self.env["pattern.file"].create( { "name": name, diff --git a/pattern_import_export/models/pattern_file.py b/pattern_import_export/models/pattern_file.py index ccbe4879..df6dd9a5 100644 --- a/pattern_import_export/models/pattern_file.py +++ b/pattern_import_export/models/pattern_file.py @@ -12,6 +12,7 @@ class PatternFile(models.Model): _name = "pattern.file" _inherits = {"ir.attachment": "attachment_id"} _description = "Attachment with pattern file metadata" + _order = "id desc" attachment_id = fields.Many2one("ir.attachment", required=True, ondelete="cascade") state = fields.Selection( @@ -61,9 +62,7 @@ def _notify_user(self): if self.state == "failed": self.env.user.notify_danger( message=_( - "{} job has failed. \nFor more details: {}".format( - import_or_export, details - ) + f"{import_or_export} job has failed. \nFor more details: {details}" ), sticky=True, ) @@ -125,8 +124,7 @@ def _parse_data(self): def _parse_data_json(self, data): items = json.loads(data.decode("utf-8")) - for idx, item in enumerate(items): - yield idx + 1, item + yield from enumerate(items, start=1) def _prepare_chunk(self, start_idx, stop_idx, data): return { @@ -170,6 +168,11 @@ def split_in_chunk(self): previous_idx = idx if items: self._create_chunk(start_idx, idx, items) + else: + # document has an header and no data lines + # valid document. So create a dummy chunk + # to have progression and status + self._create_chunk(-1, -1, []) except Exception as e: self.state = "failed" self.info = _("Failed to create the chunk: %s") % e diff --git a/pattern_import_export/readme/CONTRIBUTORS.rst b/pattern_import_export/readme/CONTRIBUTORS.rst index bd541f8a..f720d113 100644 --- a/pattern_import_export/readme/CONTRIBUTORS.rst +++ b/pattern_import_export/readme/CONTRIBUTORS.rst @@ -2,3 +2,4 @@ * Sébastien Beau * François Honoré (ACSONE SA/NV) * Kevin Khao +* Raphaël Reverdy diff --git a/pattern_import_export/readme/USAGE.rst b/pattern_import_export/readme/USAGE.rst index 1200e758..5f3880c8 100644 --- a/pattern_import_export/readme/USAGE.rst +++ b/pattern_import_export/readme/USAGE.rst @@ -23,16 +23,29 @@ You have two options: * Open the tree view of any model and tick some record selection boxes (for this step, these don't matter, we only just want to show the sidebar). * In the sidebar, click on the "Import with Pattern" button -* Select the pattern that you used to generate the export, upload your file and click import. +* Select a pattern, upload your file and click import. * A "Pattern file" is created, and its job along with it. Depending on the success or failure of the job, you will receive a red/green notification on your window. You can check the details in the appropriate Import/Export menu. Or: * Access the Import wizard through the Import/Export menu -* Select the Pattern that you want to use +* Select a pattern * Click on the "Import" button + +Import syntax +------------- + +One of the strength of pattern_import_export module is the ability to +reference records by natural keys (business keys) instead of technical keys (xmlid or database id). + +One or more columns can be the natural key of the record to find and update or to create a new record. +Each column in the natural key has to be suffixed by "#key". + +One or more columns can be used as foreign keys can be accessed with "|" syntax. (for instance on partner: country_id|code ) + + Example ------- diff --git a/pattern_import_export/static/description/index.html b/pattern_import_export/static/description/index.html index 634d6361..f972ef49 100644 --- a/pattern_import_export/static/description/index.html +++ b/pattern_import_export/static/description/index.html @@ -1,20 +1,19 @@ - - + Pattern Import Export