From 7401fd87095b5fbb0be4c605d7a0e0f4551ba8cf Mon Sep 17 00:00:00 2001 From: Maxime Aveline Date: Wed, 2 Nov 2022 17:52:21 +0100 Subject: [PATCH 01/16] AKCOMAG2001-276: Manage attribute column length behavior into helper --- Helper/Import/Entities.php | 144 ++++++++++++++++++++++++++++++++++--- 1 file changed, 134 insertions(+), 10 deletions(-) diff --git a/Helper/Import/Entities.php b/Helper/Import/Entities.php index 0fc340e6..a8e42ffe 100644 --- a/Helper/Import/Entities.php +++ b/Helper/Import/Entities.php @@ -2,7 +2,10 @@ namespace Akeneo\Connector\Helper\Import; +use Akeneo\Connector\Helper\Authenticator; use Akeneo\Connector\Helper\Config as ConfigHelper; +use Akeneo\Pim\ApiClient\AkeneoPimClientInterface; +use Akeneo\Pim\ApiClient\Pagination\ResourceCursor; use Exception; use Magento\Catalog\Api\Data\ProductAttributeInterface; use Magento\Catalog\Model\Product as BaseProductModel; @@ -45,6 +48,29 @@ class Entities * @var string IMPORT_CODE_PRODUCT */ const IMPORT_CODE_PRODUCT = 'product'; + /** + * @var mixed[] ATTRIBUTE_TYPES_LENGTH + */ + public const ATTRIBUTE_TYPES_LENGTH = [ + 'pim_catalog_identifier' => '255', + 'pim_catalog_text' => '255', + 'pim_catalog_textarea' => '65535', + 'pim_catalog_simpleselect' => null, + 'pim_catalog_multiselect' => null, + 'pim_catalog_boolean' => '3', + 'pim_catalog_date' => '20', + 'pim_catalog_number' => '100', + 'pim_catalog_metric' => '255', + 'pim_catalog_price_collection' => null, + 'pim_catalog_image' => null, + 'pim_catalog_file' => null, + 'pim_catalog_asset_collection' => '2M', + 'akeneo_reference_entity' => null, + 'akeneo_reference_entity_collection' => null, + 'pim_reference_data_simpleselect' => null, + 'pim_reference_data_multiselect' => null, + 'pim_catalog_table' => null, + ]; /** * This variable contains a ResourceConnection * @@ -57,6 +83,7 @@ class Entities * @var BaseProductModel $product */ protected $product; + protected Authenticator $authenticator; /** * This variable contains a ConfigHelper * @@ -83,26 +110,30 @@ class Entities * @var bool[] $rowIdExists */ protected $rowIdExists = []; + protected array $attributeLength = []; /** * Entities constructor * - * @param Context $context + * @param Context $context * @param ResourceConnection $connection - * @param DeploymentConfig $deploymentConfig - * @param ConfigHelper $configHelper - * @param BaseProductModel $product + * @param DeploymentConfig $deploymentConfig + * @param ConfigHelper $configHelper + * @param BaseProductModel $product + * @param Authenticator $authenticator */ public function __construct( ResourceConnection $connection, DeploymentConfig $deploymentConfig, BaseProductModel $product, - ConfigHelper $configHelper + ConfigHelper $configHelper, + Authenticator $authenticator ) { $this->connection = $connection->getConnection(); $this->deploymentConfig = $deploymentConfig; $this->configHelper = $configHelper; $this->product = $product; + $this->authenticator = $authenticator; } /** @@ -347,10 +378,11 @@ public function formatColumn($column) * * @param array $result * @param null|string $tableSuffix + * @param null|string $family * * @return bool */ - public function insertDataFromApi(array $result, $tableSuffix = null) + public function insertDataFromApi(array $result, ?string $tableSuffix = null, ?string $family = null) { if (empty($result)) { return false; @@ -377,7 +409,7 @@ public function insertDataFromApi(array $result, $tableSuffix = null) $key, [ 'type' => 'text', - 'length' => '2M', + 'length' => $this->getAttributeColumnLength($family, $key), // Get correct column length 'default' => null, 'COMMENT' => ' ' ] @@ -721,10 +753,11 @@ public function getColumnIdentifier($table, $identifier = 'entity_id') * @param string $tableName * @param string $source * @param string $target + * @param string|null $family * * @return \Akeneo\Connector\Helper\Import\Entities */ - public function copyColumn($tableName, $source, $target) + public function copyColumn(string $tableName, string $source, string $target, ?string $family = null) { /** @var AdapterInterface $connection */ $connection = $this->getConnection(); @@ -734,8 +767,8 @@ public function copyColumn($tableName, $source, $target) $tableName, $target, [ - 'type' => 'text', - 'length' => '2M', + 'type' => 'text', + 'length' => $this->getAttributeColumnLength($family, $target), // Get correct column length 'default' => '', 'COMMENT' => ' ' ] @@ -979,4 +1012,95 @@ public function getMysqlVersion(): string return $mysqlVersion['version']; } + + /** + * Get family attributes database recommended length + * + * @param string $familyCode + * + * @return mixed[] + */ + protected function getAttributesLength(string $familyCode): array + { + /** @var AkeneoPimClientInterface|false $akeneoClient */ + $akeneoClient = $this->authenticator->getAkeneoApiClient(); + + if (!empty($this->attributeLength) || !$akeneoClient) { + return $this->attributeLength; + } + + $attributeTypesLength = self::ATTRIBUTE_TYPES_LENGTH; + /** @var mixed[] $family */ + $family = $akeneoClient->getFamilyApi()->get($familyCode); + /** @var string[] $familyAttributesCode */ + $familyAttributesCode = $family['attributes'] ?? []; + /** @var string|int $paginationSize */ + $paginationSize = $this->configHelper->getPaginationSize(); + $searchAttributesResult = []; + $searchAttributesCode = []; + // Batch API calls to avoid too large request URI + foreach ($familyAttributesCode as $attributeCode) { + $searchAttributesCode[] = $attributeCode; + if (count($searchAttributesCode) === $paginationSize) { + $searchAttributesResult[] = $akeneoClient->getAttributeApi()->all($paginationSize, [ + 'search' => [ + 'code' => [ + [ + 'operator' => 'IN', + 'value' => $searchAttributesCode, + ], + ], + ], + ]); + $searchAttributesCode = []; + } + } + // Don't forget last page of attributes + if (count($searchAttributesCode) > 1) { + $searchAttributesResult[] = $akeneoClient->getAttributeApi()->all($paginationSize, [ + 'search' => [ + 'code' => [ + [ + 'operator' => 'IN', + 'value' => $searchAttributesCode, + ], + ], + ], + ]); + } + + /** @var ResourceCursor $familyAttributes */ + foreach ($searchAttributesResult as $familyAttributes) { + foreach ($familyAttributes as $attribute) { + if (!isset($attribute['code'], $attribute['type'])) { + continue; + } + $attributeCode = $attribute['code']; + $attributeType = $attribute['type']; + + $this->attributeLength[$attributeCode] = $attributeTypesLength[$attributeType]; + } + } + + return $this->attributeLength; + } + + /** + * Get attribute column length with family ATTRIBUTE_TYPES_LENGTH + * + * @param string|null $familyCode + * @param string $attributeCode + * + * @return string|null + */ + public function getAttributeColumnLength(?string $familyCode, string $attributeCode): ?string + { + if (!$familyCode) { + return null; + } + + $attributesLength = $this->getAttributesLength($familyCode); + + return $attributesLength[$attributeCode] ?? '2M'; // Add 2M by default to ensure "fake" reference entity attributes correct length + } } From 8c05db6febe436ac16d4649a57172bb6505ab09f Mon Sep 17 00:00:00 2001 From: Maxime Aveline Date: Wed, 2 Nov 2022 17:56:32 +0100 Subject: [PATCH 02/16] AKCOMAG2001-276: Add authenticator into children product helper --- Helper/Import/Product.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Helper/Import/Product.php b/Helper/Import/Product.php index b88e48c7..24f71619 100755 --- a/Helper/Import/Product.php +++ b/Helper/Import/Product.php @@ -2,6 +2,7 @@ namespace Akeneo\Connector\Helper\Import; +use Akeneo\Connector\Helper\Authenticator; use Akeneo\Connector\Helper\Config as ConfigHelper; use Magento\Catalog\Model\Product as BaseProductModel; use Magento\CatalogUrlRewrite\Model\ProductUrlPathGenerator; @@ -73,6 +74,7 @@ class Product extends Entities * @var ScopeConfigInterface $scopeConfig */ protected $scopeConfig; + protected Authenticator $authenticator; /** * Product constructor @@ -82,6 +84,7 @@ class Product extends Entities * @param BaseProductModel $product * @param ProductUrlPathGenerator $productUrlPathGenerator * @param ConfigHelper $configHelper + * @param Authenticator $authenticator * @param Json $jsonSerializer * @param ScopeConfigInterface $scopeConfig */ @@ -91,10 +94,11 @@ public function __construct( BaseProductModel $product, ProductUrlPathGenerator $productUrlPathGenerator, ConfigHelper $configHelper, + Authenticator $authenticator, Json $jsonSerializer, ScopeConfigInterface $scopeConfig ) { - parent::__construct($connection, $deploymentConfig, $product, $configHelper); + parent::__construct($connection, $deploymentConfig, $product, $configHelper, $authenticator); $this->jsonSerializer = $jsonSerializer; $this->productUrlPathGenerator = $productUrlPathGenerator; From 8a126cd88161d0bd6fdf03af4b12293374c23d51 Mon Sep 17 00:00:00 2001 From: Maxime Aveline Date: Wed, 2 Nov 2022 18:04:45 +0100 Subject: [PATCH 03/16] AKCOMAG2001-276: Handle product model helper column length behavior --- Helper/ProductModel.php | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Helper/ProductModel.php b/Helper/ProductModel.php index 578817e4..56555e79 100644 --- a/Helper/ProductModel.php +++ b/Helper/ProductModel.php @@ -164,11 +164,12 @@ public function createTable($akeneoClient, $filters) * Insert data into temporary table * * @param AkeneoPimClientInterface $akeneoClient - * @param string[] $filters + * @param string[] $filters + * * @param string|null $family * * @return void */ - public function insertData($akeneoClient, $filters) + public function insertData(AkeneoPimClientInterface $akeneoClient, array $filters, ?string $family = null) { /** @var mixed[] $messages */ $messages = []; @@ -345,7 +346,7 @@ public function insertData($akeneoClient, $filters) if (isset($productModel['code'])) { $productModel['identifier'] = $productModel['code']; } - $this->entitiesHelper->insertDataFromApi($productModel, 'product_model'); + $this->entitiesHelper->insertDataFromApi($productModel, 'product_model', $family); $index++; } } @@ -438,9 +439,11 @@ public function getMetricsSymbols($akeneoClient) /** * Add columns to product table * + * @param string|null $family + * * @return void */ - public function addColumns($code) + public function addColumns($code, ?string $family = null) { /** @var AdapterInterface $connection */ $connection = $this->entitiesHelper->getConnection(); @@ -462,12 +465,13 @@ public function addColumns($code) if (in_array($column, $except)) { continue; } + $columnName = $this->_columnName($column); $connection->addColumn( $tmpTable, - $this->_columnName($column), + $columnName, [ 'type' => 'text', - 'length' => '2M', + $this->entitiesHelper->getAttributeColumnLength($family, $columnName), // Get correct column length 'default' => '', 'COMMENT' => ' ' ] From 7f7241411f1caf3c2f0fa54666e2a544ecbcfac7 Mon Sep 17 00:00:00 2001 From: Maxime Aveline Date: Wed, 2 Nov 2022 18:10:39 +0100 Subject: [PATCH 04/16] AKCOMAG2001-276: Add family parameters into product job --- Job/Product.php | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/Job/Product.php b/Job/Product.php index 1911da2f..cceed857 100644 --- a/Job/Product.php +++ b/Job/Product.php @@ -494,8 +494,9 @@ public function insertData() $paginationSize = $this->configHelper->getPaginationSize(); /** @var int $index */ $index = 0; + $family = $this->getFamily(); /** @var mixed[] $filters */ - $filters = $this->getFilters($this->getFamily()); + $filters = $this->getFilters($family); /** @var mixed[] $metricsConcatSettings */ $metricsConcatSettings = $this->configHelper->getMetricsColumns(null, true); /** @var string[] $metricSymbols */ @@ -661,7 +662,8 @@ public function insertData() /** @var bool $result */ $result = $this->entitiesHelper->insertDataFromApi( $product, - $this->jobExecutor->getCurrentJob()->getCode() + $this->jobExecutor->getCurrentJob()->getCode(), + $family ); if (!$result) { @@ -729,19 +731,20 @@ public function insertData() */ public function productModelImport() { - if ($this->entitiesHelper->isFamilyGrouped($this->getFamily())) { + $family = $this->getFamily(); + if ($this->entitiesHelper->isFamilyGrouped($family)) { return; } /** @var string[] $messages */ $messages = []; /** @var mixed[] $filters */ - $filters = $this->getProductModelFilters($this->getFamily()); + $filters = $this->getProductModelFilters($family); if ($this->configHelper->isAdvancedLogActivated()) { $this->logger->debug(__('Product Model API call Filters : ') . print_r($filters, true)); } /** @var mixed[] $step */ - $step = $this->productModelHelper->createTable($this->akeneoClient, $filters); + $step = $this->productModelHelper->createTable($this->akeneoClient, $filters, $family); $messages[] = $step; if (array_keys(array_column($step, 'status'), false)) { $this->jobExecutor->displayMessages($messages, $this->logger); @@ -750,7 +753,7 @@ public function productModelImport() } /** @var mixed[] $stepInsertData */ - $step = $this->productModelHelper->insertData($this->akeneoClient, $filters); + $step = $this->productModelHelper->insertData($this->akeneoClient, $filters, $family); $messages[] = $step; if (array_keys(array_column($step, 'status'), false)) { $this->jobExecutor->displayMessages($messages, $this->logger); @@ -758,7 +761,7 @@ public function productModelImport() return; } // Add missing columns from product models in product tmp table - $this->productModelHelper->addColumns($this->jobExecutor->getCurrentJob()->getCode()); + $this->productModelHelper->addColumns($this->jobExecutor->getCurrentJob()->getCode(), $family); $this->jobExecutor->displayMessages($messages, $this->logger); } @@ -837,6 +840,7 @@ public function getMetricsSymbols() */ public function addRequiredData() { + $family = $this->getFamily(); /** @var AdapterInterface $connection */ $connection = $this->entitiesHelper->getConnection(); /** @var string $tmpTable */ @@ -846,7 +850,7 @@ public function addRequiredData() $edition = $this->configHelper->getEdition(); // If family is grouped, create grouped products if (($edition === Edition::SERENITY || $edition === Edition::GREATER_OR_FIVE || $edition === Edition::GROWTH) && $this->entitiesHelper->isFamilyGrouped( - $this->getFamily() + $family ) ) { $connection->addColumn( @@ -1020,7 +1024,7 @@ public function addRequiredData() /** @var string $magentoAttribute */ $magentoAttribute = $match['magento_attribute']; - $this->entitiesHelper->copyColumn($tmpTable, $pimAttribute, $magentoAttribute); + $this->entitiesHelper->copyColumn($tmpTable, $pimAttribute, $magentoAttribute, $family); /** * @var string $local @@ -1030,7 +1034,8 @@ public function addRequiredData() $this->entitiesHelper->copyColumn( $tmpTable, $pimAttribute . '-' . $local, - $magentoAttribute . '-' . $local + $magentoAttribute . '-' . $local, + $family ); } } @@ -1088,7 +1093,7 @@ public function createMetricsOptions() 'labels' => $labels, ]; - $this->entitiesHelper->insertDataFromApi($insertedData, $this->jobOption->getCode()); + $this->entitiesHelper->insertDataFromApi($insertedData, $this->jobOption->getCode(), $this->getFamily()); } } From 98885e0d886c1b5390c6dd07f4b3bd211b116524 Mon Sep 17 00:00:00 2001 From: Maxime Aveline Date: Wed, 2 Nov 2022 18:17:57 +0100 Subject: [PATCH 05/16] AKCOMAG2001-276: Remove useless family parameter --- Job/Product.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Job/Product.php b/Job/Product.php index cceed857..00294095 100644 --- a/Job/Product.php +++ b/Job/Product.php @@ -744,7 +744,7 @@ public function productModelImport() $this->logger->debug(__('Product Model API call Filters : ') . print_r($filters, true)); } /** @var mixed[] $step */ - $step = $this->productModelHelper->createTable($this->akeneoClient, $filters, $family); + $step = $this->productModelHelper->createTable($this->akeneoClient, $filters); $messages[] = $step; if (array_keys(array_column($step, 'status'), false)) { $this->jobExecutor->displayMessages($messages, $this->logger); From 805a33d5b9d832bd573c47c8a1fa7d747cf71609 Mon Sep 17 00:00:00 2001 From: Maxime Aveline Date: Thu, 3 Nov 2022 11:02:39 +0100 Subject: [PATCH 06/16] AKCOMAG2001-276: Use constants for attributes length --- Helper/Import/Entities.php | 54 ++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/Helper/Import/Entities.php b/Helper/Import/Entities.php index a8e42ffe..df6ad3c7 100644 --- a/Helper/Import/Entities.php +++ b/Helper/Import/Entities.php @@ -48,28 +48,42 @@ class Entities * @var string IMPORT_CODE_PRODUCT */ const IMPORT_CODE_PRODUCT = 'product'; + /** @var null DEFAULT_ATTRIBUTE_LENGTH */ + public const DEFAULT_ATTRIBUTE_LENGTH = null; + /** @var string NORMAL_TEXT_ATTRIBUTE_LENGTH */ + public const NORMAL_TEXT_ATTRIBUTE_LENGTH = '255'; + /** @var string TEXTAREA_ATTRIBUTE_LENGTH */ + public const TEXTAREA_ATTRIBUTE_LENGTH = '65535'; + /** @var string BOOLEAN_ATTRIBUTE_LENGTH */ + public const BOOLEAN_ATTRIBUTE_LENGTH = '3'; + /** @var string DATE_ATTRIBUTE_LENGTH */ + public const DATE_ATTRIBUTE_LENGTH = '20'; + /** @var string NUMBER_ATTRIBUTE_LENGTH */ + public const NUMBER_ATTRIBUTE_LENGTH = '100'; + /** @var string LARGE_ATTRIBUTE_LENGTH */ + public const LARGE_ATTRIBUTE_LENGTH = '2M'; /** * @var mixed[] ATTRIBUTE_TYPES_LENGTH */ public const ATTRIBUTE_TYPES_LENGTH = [ - 'pim_catalog_identifier' => '255', - 'pim_catalog_text' => '255', - 'pim_catalog_textarea' => '65535', - 'pim_catalog_simpleselect' => null, - 'pim_catalog_multiselect' => null, - 'pim_catalog_boolean' => '3', - 'pim_catalog_date' => '20', - 'pim_catalog_number' => '100', - 'pim_catalog_metric' => '255', - 'pim_catalog_price_collection' => null, - 'pim_catalog_image' => null, - 'pim_catalog_file' => null, - 'pim_catalog_asset_collection' => '2M', - 'akeneo_reference_entity' => null, - 'akeneo_reference_entity_collection' => null, - 'pim_reference_data_simpleselect' => null, - 'pim_reference_data_multiselect' => null, - 'pim_catalog_table' => null, + 'pim_catalog_identifier' => self::NORMAL_TEXT_ATTRIBUTE_LENGTH, + 'pim_catalog_text' => self::NORMAL_TEXT_ATTRIBUTE_LENGTH, + 'pim_catalog_textarea' => self::TEXTAREA_ATTRIBUTE_LENGTH, + 'pim_catalog_simpleselect' => self::DEFAULT_ATTRIBUTE_LENGTH, + 'pim_catalog_multiselect' => self::DEFAULT_ATTRIBUTE_LENGTH, + 'pim_catalog_boolean' => self::BOOLEAN_ATTRIBUTE_LENGTH, + 'pim_catalog_date' => self::DATE_ATTRIBUTE_LENGTH, + 'pim_catalog_number' => self::NUMBER_ATTRIBUTE_LENGTH, + 'pim_catalog_metric' => self::NORMAL_TEXT_ATTRIBUTE_LENGTH, + 'pim_catalog_price_collection' => self::DEFAULT_ATTRIBUTE_LENGTH, + 'pim_catalog_image' => self::DEFAULT_ATTRIBUTE_LENGTH, + 'pim_catalog_file' => self::DEFAULT_ATTRIBUTE_LENGTH, + 'pim_catalog_asset_collection' => self::LARGE_ATTRIBUTE_LENGTH, + 'akeneo_reference_entity' => self::DEFAULT_ATTRIBUTE_LENGTH, + 'akeneo_reference_entity_collection' => self::DEFAULT_ATTRIBUTE_LENGTH, + 'pim_reference_data_simpleselect' => self::DEFAULT_ATTRIBUTE_LENGTH, + 'pim_reference_data_multiselect' => self::DEFAULT_ATTRIBUTE_LENGTH, + 'pim_catalog_table' => self::DEFAULT_ATTRIBUTE_LENGTH, ]; /** * This variable contains a ResourceConnection @@ -1012,7 +1026,7 @@ public function getMysqlVersion(): string return $mysqlVersion['version']; } - + /** * Get family attributes database recommended length * @@ -1101,6 +1115,6 @@ public function getAttributeColumnLength(?string $familyCode, string $attributeC $attributesLength = $this->getAttributesLength($familyCode); - return $attributesLength[$attributeCode] ?? '2M'; // Add 2M by default to ensure "fake" reference entity attributes correct length + return $attributesLength[$attributeCode] ?? self::LARGE_ATTRIBUTE_LENGTH; // Add 2M by default to ensure "fake" reference entity attributes correct length } } From 954c672c35213c57823ec6a51421ce64a5807e35 Mon Sep 17 00:00:00 2001 From: Maxime Aveline Date: Thu, 3 Nov 2022 11:36:26 +0100 Subject: [PATCH 07/16] AKCOMAG2001-276: Replace google number by shorter one --- Helper/Import/Entities.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Helper/Import/Entities.php b/Helper/Import/Entities.php index df6ad3c7..3a42f0d8 100644 --- a/Helper/Import/Entities.php +++ b/Helper/Import/Entities.php @@ -59,7 +59,7 @@ class Entities /** @var string DATE_ATTRIBUTE_LENGTH */ public const DATE_ATTRIBUTE_LENGTH = '20'; /** @var string NUMBER_ATTRIBUTE_LENGTH */ - public const NUMBER_ATTRIBUTE_LENGTH = '100'; + public const NUMBER_ATTRIBUTE_LENGTH = '20'; /** @var string LARGE_ATTRIBUTE_LENGTH */ public const LARGE_ATTRIBUTE_LENGTH = '2M'; /** From 3f6aa6cefda7621acd204a3888d8224617acebea Mon Sep 17 00:00:00 2001 From: Maxime Aveline Date: Thu, 3 Nov 2022 14:11:54 +0100 Subject: [PATCH 08/16] AKCOMAG2001-276: Add boolean length as default --- Helper/Import/Entities.php | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Helper/Import/Entities.php b/Helper/Import/Entities.php index 3a42f0d8..531828ed 100644 --- a/Helper/Import/Entities.php +++ b/Helper/Import/Entities.php @@ -54,8 +54,6 @@ class Entities public const NORMAL_TEXT_ATTRIBUTE_LENGTH = '255'; /** @var string TEXTAREA_ATTRIBUTE_LENGTH */ public const TEXTAREA_ATTRIBUTE_LENGTH = '65535'; - /** @var string BOOLEAN_ATTRIBUTE_LENGTH */ - public const BOOLEAN_ATTRIBUTE_LENGTH = '3'; /** @var string DATE_ATTRIBUTE_LENGTH */ public const DATE_ATTRIBUTE_LENGTH = '20'; /** @var string NUMBER_ATTRIBUTE_LENGTH */ @@ -71,7 +69,7 @@ class Entities 'pim_catalog_textarea' => self::TEXTAREA_ATTRIBUTE_LENGTH, 'pim_catalog_simpleselect' => self::DEFAULT_ATTRIBUTE_LENGTH, 'pim_catalog_multiselect' => self::DEFAULT_ATTRIBUTE_LENGTH, - 'pim_catalog_boolean' => self::BOOLEAN_ATTRIBUTE_LENGTH, + 'pim_catalog_boolean' => self::DEFAULT_ATTRIBUTE_LENGTH, 'pim_catalog_date' => self::DATE_ATTRIBUTE_LENGTH, 'pim_catalog_number' => self::NUMBER_ATTRIBUTE_LENGTH, 'pim_catalog_metric' => self::NORMAL_TEXT_ATTRIBUTE_LENGTH, From f814323f6e6c4086026d4e568c870716d372b73e Mon Sep 17 00:00:00 2001 From: Maxime Aveline Date: Fri, 4 Nov 2022 18:39:27 +0100 Subject: [PATCH 09/16] AKCOMAG2001-276: Handle real attribute code with hyphen explode --- Helper/Import/Entities.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Helper/Import/Entities.php b/Helper/Import/Entities.php index 531828ed..a8ba3f34 100644 --- a/Helper/Import/Entities.php +++ b/Helper/Import/Entities.php @@ -1113,6 +1113,6 @@ public function getAttributeColumnLength(?string $familyCode, string $attributeC $attributesLength = $this->getAttributesLength($familyCode); - return $attributesLength[$attributeCode] ?? self::LARGE_ATTRIBUTE_LENGTH; // Add 2M by default to ensure "fake" reference entity attributes correct length + return $attributesLength[strtok($attributeCode, '-')] ?? self::LARGE_ATTRIBUTE_LENGTH; // Add 2M by default to ensure "fake" reference entity attributes correct length } } From e8d72f3c289fde5ad9cc08ef7353494a572ccc1b Mon Sep 17 00:00:00 2001 From: Maxime Aveline Date: Mon, 7 Nov 2022 09:34:25 +0100 Subject: [PATCH 10/16] AKCOMAG2001-276: Add correct attribute column length into createTmpTable method --- Helper/Import/Entities.php | 10 ++++++---- Helper/ProductModel.php | 7 ++++--- Job/Product.php | 36 +++++++++++++++++++----------------- 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/Helper/Import/Entities.php b/Helper/Import/Entities.php index a8ba3f34..f0b3d5f3 100644 --- a/Helper/Import/Entities.php +++ b/Helper/Import/Entities.php @@ -213,14 +213,15 @@ public function getTablePrefix() * * @param array $result * @param string $tableSuffix + * @param string|null $family * * @return $this */ - public function createTmpTableFromApi($result, $tableSuffix) + public function createTmpTableFromApi(array $result, string $tableSuffix, ?string $family = null) { /** @var array $columns */ $columns = $this->getColumnsFromResult($result); - $this->createTmpTable(array_keys($columns), $tableSuffix); + $this->createTmpTable(array_keys($columns), $tableSuffix, $family); return $this; } @@ -230,11 +231,12 @@ public function createTmpTableFromApi($result, $tableSuffix) * * @param array $fields * @param string $tableSuffix + * @param string|null $family * * @return $this * @throws \Zend_Db_Exception */ - public function createTmpTable($fields, $tableSuffix) + public function createTmpTable(array $fields, string $tableSuffix, ?string $family = null) { /* Delete table if exists */ $this->dropTable($tableSuffix); @@ -269,7 +271,7 @@ public function createTmpTable($fields, $tableSuffix) $table->addColumn( $column, Table::TYPE_TEXT, - '2M', + $this->getAttributeColumnLength($family, $column), [], $column ); diff --git a/Helper/ProductModel.php b/Helper/ProductModel.php index 56555e79..7d7081ab 100644 --- a/Helper/ProductModel.php +++ b/Helper/ProductModel.php @@ -122,11 +122,12 @@ public function __construct( * Description createTable function * * @param AkeneoPimClientInterface $akeneoClient - * @param string[] $filters + * @param string[] $filters + * @param string|null $family * * @return string[] */ - public function createTable($akeneoClient, $filters) + public function createTable(AkeneoPimClientInterface $akeneoClient, array $filters, ?string $family = null) { /** @var string[] $messages */ $messages = []; @@ -155,7 +156,7 @@ public function createTable($akeneoClient, $filters) } $productModel = reset($productModels); - $this->entitiesHelper->createTmpTableFromApi($productModel, 'product_model'); + $this->entitiesHelper->createTmpTableFromApi($productModel, 'product_model', $family); return $messages; } diff --git a/Job/Product.php b/Job/Product.php index 00294095..bd326542 100644 --- a/Job/Product.php +++ b/Job/Product.php @@ -389,7 +389,8 @@ public function createTable() } // Stop the import if the family is not imported - if ($this->getFamily()) { + $family = $this->getFamily(); + if ($family) { /** @var AdapterInterface $connection */ $connection = $this->entitiesHelper->getConnection(); /** @var string $connectorEntitiesTable */ @@ -398,13 +399,13 @@ public function createTable() $isFamilyImported = (bool)$connection->fetchOne( $connection->select() ->from($connectorEntitiesTable, ['code']) - ->where('code = ?', $this->getFamily()) + ->where('code = ?', $family) ->limit(1) ); if (!$isFamilyImported) { $this->jobExecutor->setAdditionalMessage( - __('The family %1 is not imported yet, please run Family import.', $this->getFamily()), + __('The family %1 is not imported yet, please run Family import.', $family), $this->logger ); $this->jobExecutor->afterRun(true); @@ -414,7 +415,7 @@ public function createTable() } /** @var mixed[] $filters */ - $filters = $this->getFilters($this->getFamily()); + $filters = $this->getFilters($family); if ($this->configHelper->isAdvancedLogActivated()) { $this->logger->debug(__('Product API call Filters : ') . print_r($filters, true)); } @@ -432,9 +433,9 @@ public function createTable() if (empty($products)) { // No product were found and we're in a grouped family, we don't import product models for it, so we stop the import - if ($this->entitiesHelper->isFamilyGrouped($this->getFamily())) { + if ($this->entitiesHelper->isFamilyGrouped($family)) { $this->jobExecutor->setAdditionalMessage( - __('No results from Akeneo for the family: %1', $this->getFamily()), + __('No results from Akeneo for the family: %1', $family), $this->logger )->afterRun(null, true); @@ -442,7 +443,7 @@ public function createTable() } /** @var mixed[] $modelFilters */ - $modelFilters = $this->getProductModelFilters($this->getFamily()); + $modelFilters = $this->getProductModelFilters($family); foreach ($modelFilters as $filter) { /** @var PageInterface $productModels */ $productModels = $this->akeneoClient->getProductModelApi()->listPerPage(1, false, $filter); @@ -456,17 +457,17 @@ public function createTable() if (empty($productModels)) { $this->jobExecutor->setAdditionalMessage( - __('No results from Akeneo for the family: %1', $this->getFamily()), + __('No results from Akeneo for the family: %1', $family), $this->logger )->afterRun(null, true); return; } $productModel = reset($productModels); - $this->entitiesHelper->createTmpTableFromApi($productModel, $this->jobExecutor->getCurrentJob()->getCode()); - $this->entitiesHelper->createTmpTableFromApi($productModel, 'product_model'); + $this->entitiesHelper->createTmpTableFromApi($productModel, $this->jobExecutor->getCurrentJob()->getCode(), $family); + $this->entitiesHelper->createTmpTableFromApi($productModel, 'product_model', $family); $this->jobExecutor->setAdditionalMessage( - __('No product found for family: %1 but product model found, process with import', $this->getFamily()), + __('No product found for family: %1 but product model found, process with import', $family), $this->logger ); @@ -475,10 +476,10 @@ public function createTable() $product = reset($products); // Make sure to delete product model table $this->entitiesHelper->dropTable('product_model'); - $this->entitiesHelper->createTmpTableFromApi($product, $this->jobExecutor->getCurrentJob()->getCode()); + $this->entitiesHelper->createTmpTableFromApi($product, $this->jobExecutor->getCurrentJob()->getCode(), $family); /** @var string $message */ - $message = __('Family imported in this batch: %1', $this->getFamily()); + $message = __('Family imported in this batch: %1', $family); $this->jobExecutor->setAdditionalMessage($message, $this->logger); } } @@ -744,7 +745,7 @@ public function productModelImport() $this->logger->debug(__('Product Model API call Filters : ') . print_r($filters, true)); } /** @var mixed[] $step */ - $step = $this->productModelHelper->createTable($this->akeneoClient, $filters); + $step = $this->productModelHelper->createTable($this->akeneoClient, $filters, $family); $messages[] = $step; if (array_keys(array_column($step, 'status'), false)) { $this->jobExecutor->displayMessages($messages, $this->logger); @@ -773,7 +774,8 @@ public function productModelImport() */ public function familyVariantImport() { - if ($this->entitiesHelper->isFamilyGrouped($this->getFamily())) { + $family = $this->getFamily(); + if ($this->entitiesHelper->isFamilyGrouped($family)) { return; } @@ -784,7 +786,7 @@ public function familyVariantImport() $messages = []; /** @var mixed[] $step */ - $step = $this->familyVariantHelper->createTable($this->akeneoClient, $this->getFamily()); + $step = $this->familyVariantHelper->createTable($this->akeneoClient, $family); $messages[] = $step; if (array_keys(array_column($step, 'status'), false)) { $this->jobExecutor->displayMessages($messages, $this->logger); @@ -792,7 +794,7 @@ public function familyVariantImport() return; } /** @var mixed[] $step */ - $step = $this->familyVariantHelper->insertData($this->akeneoClient, $this->getFamily()); + $step = $this->familyVariantHelper->insertData($this->akeneoClient, $family); $messages[] = $step; if (array_keys(array_column($step, 'status'), false)) { $this->jobExecutor->displayMessages($messages, $this->logger); From 296eb457e3f6590c6395574440dea210aeaccc3f Mon Sep 17 00:00:00 2001 From: Maxime Aveline Date: Mon, 7 Nov 2022 09:39:32 +0100 Subject: [PATCH 11/16] AKCOMAG2001-276: Add family lvl for attributeLength to fix double import issue --- Helper/Import/Entities.php | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Helper/Import/Entities.php b/Helper/Import/Entities.php index f0b3d5f3..a4766d46 100644 --- a/Helper/Import/Entities.php +++ b/Helper/Import/Entities.php @@ -1039,8 +1039,8 @@ protected function getAttributesLength(string $familyCode): array /** @var AkeneoPimClientInterface|false $akeneoClient */ $akeneoClient = $this->authenticator->getAkeneoApiClient(); - if (!empty($this->attributeLength) || !$akeneoClient) { - return $this->attributeLength; + if (!empty($this->attributeLength[$familyCode]) || !$akeneoClient) { + return $this->attributeLength[$familyCode]; } $attributeTypesLength = self::ATTRIBUTE_TYPES_LENGTH; @@ -1092,11 +1092,10 @@ protected function getAttributesLength(string $familyCode): array $attributeCode = $attribute['code']; $attributeType = $attribute['type']; - $this->attributeLength[$attributeCode] = $attributeTypesLength[$attributeType]; - } + $this->attributeLength[$familyCode][$attributeCode] = $attributeTypesLength[$attributeType]; } } - return $this->attributeLength; + return $this->attributeLength[$familyCode]; } /** From 52f89e2a449800c1eb5b021b1f9ef4399f73f844 Mon Sep 17 00:00:00 2001 From: Maxime Aveline Date: Mon, 7 Nov 2022 09:42:04 +0100 Subject: [PATCH 12/16] AKCOMAG2001-276: Use default length to avoid varchar attributes --- Helper/Import/Entities.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Helper/Import/Entities.php b/Helper/Import/Entities.php index a4766d46..84202a41 100644 --- a/Helper/Import/Entities.php +++ b/Helper/Import/Entities.php @@ -64,15 +64,15 @@ class Entities * @var mixed[] ATTRIBUTE_TYPES_LENGTH */ public const ATTRIBUTE_TYPES_LENGTH = [ - 'pim_catalog_identifier' => self::NORMAL_TEXT_ATTRIBUTE_LENGTH, - 'pim_catalog_text' => self::NORMAL_TEXT_ATTRIBUTE_LENGTH, - 'pim_catalog_textarea' => self::TEXTAREA_ATTRIBUTE_LENGTH, + 'pim_catalog_identifier' => self::DEFAULT_ATTRIBUTE_LENGTH, + 'pim_catalog_text' => self::DEFAULT_ATTRIBUTE_LENGTH, + 'pim_catalog_textarea' => self::DEFAULT_ATTRIBUTE_LENGTH, 'pim_catalog_simpleselect' => self::DEFAULT_ATTRIBUTE_LENGTH, 'pim_catalog_multiselect' => self::DEFAULT_ATTRIBUTE_LENGTH, 'pim_catalog_boolean' => self::DEFAULT_ATTRIBUTE_LENGTH, - 'pim_catalog_date' => self::DATE_ATTRIBUTE_LENGTH, - 'pim_catalog_number' => self::NUMBER_ATTRIBUTE_LENGTH, - 'pim_catalog_metric' => self::NORMAL_TEXT_ATTRIBUTE_LENGTH, + 'pim_catalog_date' => self::DEFAULT_ATTRIBUTE_LENGTH, + 'pim_catalog_number' => self::DEFAULT_ATTRIBUTE_LENGTH, + 'pim_catalog_metric' => self::DEFAULT_ATTRIBUTE_LENGTH, 'pim_catalog_price_collection' => self::DEFAULT_ATTRIBUTE_LENGTH, 'pim_catalog_image' => self::DEFAULT_ATTRIBUTE_LENGTH, 'pim_catalog_file' => self::DEFAULT_ATTRIBUTE_LENGTH, @@ -81,7 +81,7 @@ class Entities 'akeneo_reference_entity_collection' => self::DEFAULT_ATTRIBUTE_LENGTH, 'pim_reference_data_simpleselect' => self::DEFAULT_ATTRIBUTE_LENGTH, 'pim_reference_data_multiselect' => self::DEFAULT_ATTRIBUTE_LENGTH, - 'pim_catalog_table' => self::DEFAULT_ATTRIBUTE_LENGTH, + 'pim_catalog_table' => self::LARGE_ATTRIBUTE_LENGTH, ]; /** * This variable contains a ResourceConnection From 6b828daceca30576397c82dd19f3b0c9eca5b928 Mon Sep 17 00:00:00 2001 From: Maxime Aveline Date: Mon, 7 Nov 2022 09:43:47 +0100 Subject: [PATCH 13/16] AKCOMAG2001-276: Return null value for default attributes length --- Helper/Import/Entities.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Helper/Import/Entities.php b/Helper/Import/Entities.php index 84202a41..0fb17f3a 100644 --- a/Helper/Import/Entities.php +++ b/Helper/Import/Entities.php @@ -48,8 +48,8 @@ class Entities * @var string IMPORT_CODE_PRODUCT */ const IMPORT_CODE_PRODUCT = 'product'; - /** @var null DEFAULT_ATTRIBUTE_LENGTH */ - public const DEFAULT_ATTRIBUTE_LENGTH = null; + /** @var string DEFAULT_ATTRIBUTE_LENGTH */ + public const DEFAULT_ATTRIBUTE_LENGTH = 'default'; /** @var string NORMAL_TEXT_ATTRIBUTE_LENGTH */ public const NORMAL_TEXT_ATTRIBUTE_LENGTH = '255'; /** @var string TEXTAREA_ATTRIBUTE_LENGTH */ @@ -1113,7 +1113,7 @@ public function getAttributeColumnLength(?string $familyCode, string $attributeC } $attributesLength = $this->getAttributesLength($familyCode); - - return $attributesLength[strtok($attributeCode, '-')] ?? self::LARGE_ATTRIBUTE_LENGTH; // Add 2M by default to ensure "fake" reference entity attributes correct length + $attributeColumnLength = $attributesLength[strtok($attributeCode, '-')] ?? self::LARGE_ATTRIBUTE_LENGTH; // Add 2M by default to ensure "fake" reference entity attributes correct length + return $attributeColumnLength === self::DEFAULT_ATTRIBUTE_LENGTH ? null : $attributeColumnLength; // Return null value for default attributes length } } From 5aa1fe745a6503fc82982b1f7214bc316478a7f7 Mon Sep 17 00:00:00 2001 From: Maxime Aveline Date: Mon, 7 Nov 2022 17:38:32 +0100 Subject: [PATCH 14/16] AKCOMAG2001-276: Use textarea medium length --- Helper/Import/Entities.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Helper/Import/Entities.php b/Helper/Import/Entities.php index 0fb17f3a..e6c61f7d 100644 --- a/Helper/Import/Entities.php +++ b/Helper/Import/Entities.php @@ -66,7 +66,7 @@ class Entities public const ATTRIBUTE_TYPES_LENGTH = [ 'pim_catalog_identifier' => self::DEFAULT_ATTRIBUTE_LENGTH, 'pim_catalog_text' => self::DEFAULT_ATTRIBUTE_LENGTH, - 'pim_catalog_textarea' => self::DEFAULT_ATTRIBUTE_LENGTH, + 'pim_catalog_textarea' => self::TEXTAREA_ATTRIBUTE_LENGTH, 'pim_catalog_simpleselect' => self::DEFAULT_ATTRIBUTE_LENGTH, 'pim_catalog_multiselect' => self::DEFAULT_ATTRIBUTE_LENGTH, 'pim_catalog_boolean' => self::DEFAULT_ATTRIBUTE_LENGTH, From a8b5bdc37c68be09d33f02f81eebef980de5341d Mon Sep 17 00:00:00 2001 From: Maxime Aveline Date: Mon, 7 Nov 2022 17:47:08 +0100 Subject: [PATCH 15/16] AKCOMAG2001-276: Remove unused constants --- Helper/Import/Entities.php | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Helper/Import/Entities.php b/Helper/Import/Entities.php index e6c61f7d..58f85884 100644 --- a/Helper/Import/Entities.php +++ b/Helper/Import/Entities.php @@ -50,14 +50,8 @@ class Entities const IMPORT_CODE_PRODUCT = 'product'; /** @var string DEFAULT_ATTRIBUTE_LENGTH */ public const DEFAULT_ATTRIBUTE_LENGTH = 'default'; - /** @var string NORMAL_TEXT_ATTRIBUTE_LENGTH */ - public const NORMAL_TEXT_ATTRIBUTE_LENGTH = '255'; /** @var string TEXTAREA_ATTRIBUTE_LENGTH */ public const TEXTAREA_ATTRIBUTE_LENGTH = '65535'; - /** @var string DATE_ATTRIBUTE_LENGTH */ - public const DATE_ATTRIBUTE_LENGTH = '20'; - /** @var string NUMBER_ATTRIBUTE_LENGTH */ - public const NUMBER_ATTRIBUTE_LENGTH = '20'; /** @var string LARGE_ATTRIBUTE_LENGTH */ public const LARGE_ATTRIBUTE_LENGTH = '2M'; /** From ef365d65d80d83aa0ffd25ca339fa0fee3a4ff09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verri=C3=A8re?= Date: Tue, 8 Nov 2022 16:40:01 +0100 Subject: [PATCH 16/16] v103.2.0: Add version 103.2.0 change log and bump composer.json version --- CHANGELOG.md | 3 +++ composer.json | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e5088cc..1be770a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -377,3 +377,6 @@ ### Version 103.0.8 : * Fix URL rewrite generation when multiple url keys are duplicated + +### Version 103.2.0 : +* Improve column sizes per attribute type inside product temporary table in order to reduce the MYSQL table volume diff --git a/composer.json b/composer.json index bd32a778..00d5a3ee 100644 --- a/composer.json +++ b/composer.json @@ -13,7 +13,7 @@ "nyholm/psr7": "^1.5" }, "type": "magento2-module", - "version": "103.0.8", + "version": "103.2.0", "license": [ "OSL-3.0", "AFL-3.0"