Skip to content

Commit

Permalink
Merge branch 'release/v103.2.0' into 'master'
Browse files Browse the repository at this point in the history 
release/v103.2.0 into master

See merge request agence-dnd/marketplace/magento-2/external/magento2-connector-community!51
  • Loading branch information
Rémi V committed Nov 8, 2022
2 parents a8509c7 + ef365d6 commit d690519
Show file tree
Hide file tree
Showing 6 changed files with 202 additions and 52 deletions.
3 changes: 3 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -377,3 +377,6 @@

### Version 103.0.8 :
* Fix URL rewrite generation when multiple url keys are duplicated

### Version 103.2.0 :
* Improve column sizes per attribute type inside product temporary table in order to reduce the MYSQL table volume
159 changes: 145 additions & 14 deletions Helper/Import/Entities.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@

namespace Akeneo\Connector\Helper\Import;

use Akeneo\Connector\Helper\Authenticator;
use Akeneo\Connector\Helper\Config as ConfigHelper;
use Akeneo\Pim\ApiClient\AkeneoPimClientInterface;
use Akeneo\Pim\ApiClient\Pagination\ResourceCursor;
use Exception;
use Magento\Catalog\Api\Data\ProductAttributeInterface;
use Magento\Catalog\Model\Product as BaseProductModel;
Expand Down Expand Up @@ -45,6 +48,35 @@ class Entities
* @var string IMPORT_CODE_PRODUCT
*/
const IMPORT_CODE_PRODUCT = 'product';
/** @var string DEFAULT_ATTRIBUTE_LENGTH */
public const DEFAULT_ATTRIBUTE_LENGTH = 'default';
/** @var string TEXTAREA_ATTRIBUTE_LENGTH */
public const TEXTAREA_ATTRIBUTE_LENGTH = '65535';
/** @var string LARGE_ATTRIBUTE_LENGTH */
public const LARGE_ATTRIBUTE_LENGTH = '2M';
/**
* @var mixed[] ATTRIBUTE_TYPES_LENGTH
*/
public const ATTRIBUTE_TYPES_LENGTH = [
'pim_catalog_identifier' => self::DEFAULT_ATTRIBUTE_LENGTH,
'pim_catalog_text' => self::DEFAULT_ATTRIBUTE_LENGTH,
'pim_catalog_textarea' => self::TEXTAREA_ATTRIBUTE_LENGTH,
'pim_catalog_simpleselect' => self::DEFAULT_ATTRIBUTE_LENGTH,
'pim_catalog_multiselect' => self::DEFAULT_ATTRIBUTE_LENGTH,
'pim_catalog_boolean' => self::DEFAULT_ATTRIBUTE_LENGTH,
'pim_catalog_date' => self::DEFAULT_ATTRIBUTE_LENGTH,
'pim_catalog_number' => self::DEFAULT_ATTRIBUTE_LENGTH,
'pim_catalog_metric' => self::DEFAULT_ATTRIBUTE_LENGTH,
'pim_catalog_price_collection' => self::DEFAULT_ATTRIBUTE_LENGTH,
'pim_catalog_image' => self::DEFAULT_ATTRIBUTE_LENGTH,
'pim_catalog_file' => self::DEFAULT_ATTRIBUTE_LENGTH,
'pim_catalog_asset_collection' => self::LARGE_ATTRIBUTE_LENGTH,
'akeneo_reference_entity' => self::DEFAULT_ATTRIBUTE_LENGTH,
'akeneo_reference_entity_collection' => self::DEFAULT_ATTRIBUTE_LENGTH,
'pim_reference_data_simpleselect' => self::DEFAULT_ATTRIBUTE_LENGTH,
'pim_reference_data_multiselect' => self::DEFAULT_ATTRIBUTE_LENGTH,
'pim_catalog_table' => self::LARGE_ATTRIBUTE_LENGTH,
];
/**
* This variable contains a ResourceConnection
*
Expand All @@ -57,6 +89,7 @@ class Entities
* @var BaseProductModel $product
*/
protected $product;
protected Authenticator $authenticator;
/**
* This variable contains a ConfigHelper
*
Expand All @@ -83,26 +116,30 @@ class Entities
* @var bool[] $rowIdExists
*/
protected $rowIdExists = [];
protected array $attributeLength = [];

/**
* Entities constructor
*
* @param Context $context
* @param Context $context
* @param ResourceConnection $connection
* @param DeploymentConfig $deploymentConfig
* @param ConfigHelper $configHelper
* @param BaseProductModel $product
* @param DeploymentConfig $deploymentConfig
* @param ConfigHelper $configHelper
* @param BaseProductModel $product
* @param Authenticator $authenticator
*/
public function __construct(
ResourceConnection $connection,
DeploymentConfig $deploymentConfig,
BaseProductModel $product,
ConfigHelper $configHelper
ConfigHelper $configHelper,
Authenticator $authenticator
) {
$this->connection = $connection->getConnection();
$this->deploymentConfig = $deploymentConfig;
$this->configHelper = $configHelper;
$this->product = $product;
$this->authenticator = $authenticator;
}

/**
Expand Down Expand Up @@ -170,14 +207,15 @@ public function getTablePrefix()
*
* @param array $result
* @param string $tableSuffix
* @param string|null $family
*
* @return $this
*/
public function createTmpTableFromApi($result, $tableSuffix)
public function createTmpTableFromApi(array $result, string $tableSuffix, ?string $family = null)
{
/** @var array $columns */
$columns = $this->getColumnsFromResult($result);
$this->createTmpTable(array_keys($columns), $tableSuffix);
$this->createTmpTable(array_keys($columns), $tableSuffix, $family);

return $this;
}
Expand All @@ -187,11 +225,12 @@ public function createTmpTableFromApi($result, $tableSuffix)
*
* @param array $fields
* @param string $tableSuffix
* @param string|null $family
*
* @return $this
* @throws \Zend_Db_Exception
*/
public function createTmpTable($fields, $tableSuffix)
public function createTmpTable(array $fields, string $tableSuffix, ?string $family = null)
{
/* Delete table if exists */
$this->dropTable($tableSuffix);
Expand Down Expand Up @@ -226,7 +265,7 @@ public function createTmpTable($fields, $tableSuffix)
$table->addColumn(
$column,
Table::TYPE_TEXT,
'2M',
$this->getAttributeColumnLength($family, $column),
[],
$column
);
Expand Down Expand Up @@ -347,10 +386,11 @@ public function formatColumn($column)
*
* @param array $result
* @param null|string $tableSuffix
* @param null|string $family
*
* @return bool
*/
public function insertDataFromApi(array $result, $tableSuffix = null)
public function insertDataFromApi(array $result, ?string $tableSuffix = null, ?string $family = null)
{
if (empty($result)) {
return false;
Expand All @@ -377,7 +417,7 @@ public function insertDataFromApi(array $result, $tableSuffix = null)
$key,
[
'type' => 'text',
'length' => '2M',
'length' => $this->getAttributeColumnLength($family, $key), // Get correct column length
'default' => null,
'COMMENT' => ' '
]
Expand Down Expand Up @@ -721,10 +761,11 @@ public function getColumnIdentifier($table, $identifier = 'entity_id')
* @param string $tableName
* @param string $source
* @param string $target
* @param string|null $family
*
* @return \Akeneo\Connector\Helper\Import\Entities
*/
public function copyColumn($tableName, $source, $target)
public function copyColumn(string $tableName, string $source, string $target, ?string $family = null)
{
/** @var AdapterInterface $connection */
$connection = $this->getConnection();
Expand All @@ -734,8 +775,8 @@ public function copyColumn($tableName, $source, $target)
$tableName,
$target,
[
'type' => 'text',
'length' => '2M',
'type' => 'text',
'length' => $this->getAttributeColumnLength($family, $target), // Get correct column length
'default' => '',
'COMMENT' => ' '
]
Expand Down Expand Up @@ -979,4 +1020,94 @@ public function getMysqlVersion(): string

return $mysqlVersion['version'];
}

/**
* Get family attributes database recommended length
*
* @param string $familyCode
*
* @return mixed[]
*/
protected function getAttributesLength(string $familyCode): array
{
/** @var AkeneoPimClientInterface|false $akeneoClient */
$akeneoClient = $this->authenticator->getAkeneoApiClient();

if (!empty($this->attributeLength[$familyCode]) || !$akeneoClient) {
return $this->attributeLength[$familyCode];
}

$attributeTypesLength = self::ATTRIBUTE_TYPES_LENGTH;
/** @var mixed[] $family */
$family = $akeneoClient->getFamilyApi()->get($familyCode);
/** @var string[] $familyAttributesCode */
$familyAttributesCode = $family['attributes'] ?? [];
/** @var string|int $paginationSize */
$paginationSize = $this->configHelper->getPaginationSize();
$searchAttributesResult = [];
$searchAttributesCode = [];
// Batch API calls to avoid too large request URI
foreach ($familyAttributesCode as $attributeCode) {
$searchAttributesCode[] = $attributeCode;
if (count($searchAttributesCode) === $paginationSize) {
$searchAttributesResult[] = $akeneoClient->getAttributeApi()->all($paginationSize, [
'search' => [
'code' => [
[
'operator' => 'IN',
'value' => $searchAttributesCode,
],
],
],
]);
$searchAttributesCode = [];
}
}
// Don't forget last page of attributes
if (count($searchAttributesCode) > 1) {
$searchAttributesResult[] = $akeneoClient->getAttributeApi()->all($paginationSize, [
'search' => [
'code' => [
[
'operator' => 'IN',
'value' => $searchAttributesCode,
],
],
],
]);
}

/** @var ResourceCursor $familyAttributes */
foreach ($searchAttributesResult as $familyAttributes) {
foreach ($familyAttributes as $attribute) {
if (!isset($attribute['code'], $attribute['type'])) {
continue;
}
$attributeCode = $attribute['code'];
$attributeType = $attribute['type'];

$this->attributeLength[$familyCode][$attributeCode] = $attributeTypesLength[$attributeType]; }
}

return $this->attributeLength[$familyCode];
}

/**
* Get attribute column length with family ATTRIBUTE_TYPES_LENGTH
*
* @param string|null $familyCode
* @param string $attributeCode
*
* @return string|null
*/
public function getAttributeColumnLength(?string $familyCode, string $attributeCode): ?string
{
if (!$familyCode) {
return null;
}

$attributesLength = $this->getAttributesLength($familyCode);
$attributeColumnLength = $attributesLength[strtok($attributeCode, '-')] ?? self::LARGE_ATTRIBUTE_LENGTH; // Add 2M by default to ensure "fake" reference entity attributes correct length
return $attributeColumnLength === self::DEFAULT_ATTRIBUTE_LENGTH ? null : $attributeColumnLength; // Return null value for default attributes length
}
}
6 changes: 5 additions & 1 deletion Helper/Import/Product.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

namespace Akeneo\Connector\Helper\Import;

use Akeneo\Connector\Helper\Authenticator;
use Akeneo\Connector\Helper\Config as ConfigHelper;
use Magento\Catalog\Model\Product as BaseProductModel;
use Magento\CatalogUrlRewrite\Model\ProductUrlPathGenerator;
Expand Down Expand Up @@ -73,6 +74,7 @@ class Product extends Entities
* @var ScopeConfigInterface $scopeConfig
*/
protected $scopeConfig;
protected Authenticator $authenticator;

/**
* Product constructor
Expand All @@ -82,6 +84,7 @@ class Product extends Entities
* @param BaseProductModel $product
* @param ProductUrlPathGenerator $productUrlPathGenerator
* @param ConfigHelper $configHelper
* @param Authenticator $authenticator
* @param Json $jsonSerializer
* @param ScopeConfigInterface $scopeConfig
*/
Expand All @@ -91,10 +94,11 @@ public function __construct(
BaseProductModel $product,
ProductUrlPathGenerator $productUrlPathGenerator,
ConfigHelper $configHelper,
Authenticator $authenticator,
Json $jsonSerializer,
ScopeConfigInterface $scopeConfig
) {
parent::__construct($connection, $deploymentConfig, $product, $configHelper);
parent::__construct($connection, $deploymentConfig, $product, $configHelper, $authenticator);

$this->jsonSerializer = $jsonSerializer;
$this->productUrlPathGenerator = $productUrlPathGenerator;
Expand Down
23 changes: 14 additions & 9 deletions Helper/ProductModel.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,11 +122,12 @@ public function __construct(
* Description createTable function
*
* @param AkeneoPimClientInterface $akeneoClient
* @param string[] $filters
* @param string[] $filters
* @param string|null $family
*
* @return string[]
*/
public function createTable($akeneoClient, $filters)
public function createTable(AkeneoPimClientInterface $akeneoClient, array $filters, ?string $family = null)
{
/** @var string[] $messages */
$messages = [];
Expand Down Expand Up @@ -155,7 +156,7 @@ public function createTable($akeneoClient, $filters)
}

$productModel = reset($productModels);
$this->entitiesHelper->createTmpTableFromApi($productModel, 'product_model');
$this->entitiesHelper->createTmpTableFromApi($productModel, 'product_model', $family);

return $messages;
}
Expand All @@ -164,11 +165,12 @@ public function createTable($akeneoClient, $filters)
* Insert data into temporary table
*
* @param AkeneoPimClientInterface $akeneoClient
* @param string[] $filters
* @param string[] $filters
* * @param string|null $family
*
* @return void
*/
public function insertData($akeneoClient, $filters)
public function insertData(AkeneoPimClientInterface $akeneoClient, array $filters, ?string $family = null)
{
/** @var mixed[] $messages */
$messages = [];
Expand Down Expand Up @@ -345,7 +347,7 @@ public function insertData($akeneoClient, $filters)
if (isset($productModel['code'])) {
$productModel['identifier'] = $productModel['code'];
}
$this->entitiesHelper->insertDataFromApi($productModel, 'product_model');
$this->entitiesHelper->insertDataFromApi($productModel, 'product_model', $family);
$index++;
}
}
Expand Down Expand Up @@ -438,9 +440,11 @@ public function getMetricsSymbols($akeneoClient)
/**
* Add columns to product table
*
* @param string|null $family
*
* @return void
*/
public function addColumns($code)
public function addColumns($code, ?string $family = null)
{
/** @var AdapterInterface $connection */
$connection = $this->entitiesHelper->getConnection();
Expand All @@ -462,12 +466,13 @@ public function addColumns($code)
if (in_array($column, $except)) {
continue;
}
$columnName = $this->_columnName($column);
$connection->addColumn(
$tmpTable,
$this->_columnName($column),
$columnName,
[
'type' => 'text',
'length' => '2M',
$this->entitiesHelper->getAttributeColumnLength($family, $columnName), // Get correct column length
'default' => '',
'COMMENT' => ' '
]
Expand Down
Loading

0 comments on commit d690519

Please sign in to comment.