Merge pull request #124 from akeneo/release/100.3.3

Release/100.3.3
akeneo · Jan 20, 2020 · c89444b · c89444b
2 parents ce32073 + 9da975a
commit c89444b
Show file tree

Hide file tree

Showing 11 changed files with 159 additions and 47 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -55,4 +55,10 @@
 
 ### Version 100.3.2 :
 * Fix Object Manager usage
-* Fix category URL request missing "parent_id" select
+* Fix category URL request missing "parent_id" select
+
+### Version 100.3.3 :
+* Fix error on price attribute import
+* Fix category attribute set getter to prevent mixed id in case of data migration
+* Fix metric import when metric attribute code contains uppercase characters
+* Add product model batch size and request size to prevent MYSQL errors (https://help.akeneo.com/magento2-connector/v100/articles/05-configure-products.html#product-model-batch-size-and-product-model-update-length)
diff --git a/Helper/Config.php b/Helper/Config.php
@@ -62,6 +62,8 @@ class Config extends AbstractHelper
     const PRODUCT_ATTRIBUTE_MAPPING = 'akeneo_connector/product/attribute_mapping';
     const PRODUCT_WEBSITE_ATTRIBUTE = 'akeneo_connector/product/website_attribute';
     const PRODUCT_CONFIGURABLE_ATTRIBUTES = 'akeneo_connector/product/configurable_attributes';
+    const PRODUCT_PRODUCT_MODEL_BATCH_SIZE = 'akeneo_connector/product/product_model_batch_size';
+    const PRODUCT_PRODUCT_MODEL_UPDATE_LENGTH = 'akeneo_connector/product/product_model_update_length';
     const PRODUCT_TAX_CLASS = 'akeneo_connector/product/tax_class';
     const PRODUCT_URL_GENERATION_ENABLED = 'akeneo_connector/product/url_generation_enabled';
     const PRODUCT_MEDIA_ENABLED = 'akeneo_connector/product/media_enabled';
@@ -73,6 +75,18 @@ class Config extends AbstractHelper
      * @var int PAGINATION_SIZE_DEFAULT_VALUE
      */
     const PAGINATION_SIZE_DEFAULT_VALUE = 10;
+    /**
+     * @var int PRODUCT_PRODUCT_MODEL_BATCH_SIZE_DEFAULT_VALUE
+     */
+    const PRODUCT_PRODUCT_MODEL_BATCH_SIZE_DEFAULT_VALUE = 500;
+    /**
+     * @var int PRODUCT_PRODUCT_MODEL_LENGTH_DEFAULT_VALUE
+     */
+    const PRODUCT_PRODUCT_MODEL_UPDATE_LENGTH_DEFAULT_VALUE = 5000;
+    /**
+     * @var int PRODUCT_PRODUCT_MODEL_UPDATE_LENGTH_MINIMUM
+     */
+    const PRODUCT_PRODUCT_MODEL_UPDATE_LENGTH_MINIMUM = 1000;
     /**
      * This variable contains a Encryptor
      *
@@ -814,6 +828,38 @@ public function isUrlKeyMapped()
         return false;
     }
 
+    /**
+     * Retrieve product_model batch size
+     *
+     * @return int
+     */
+    public function getAdvancedPmBatchSize()
+    {
+        /** @var int $advancedPmBatchSize */
+        $advancedPmBatchSize = $this->scopeConfig->getValue(self::PRODUCT_PRODUCT_MODEL_BATCH_SIZE);
+        if (filter_var($advancedPmBatchSize, FILTER_VALIDATE_INT) === false) {
+            $advancedPmBatchSize = self::PRODUCT_PRODUCT_MODEL_BATCH_SIZE_DEFAULT_VALUE;
+        }
+
+        return $advancedPmBatchSize;
+    }
+
+    /**
+     * Retrieve product_model update length
+     *
+     * @return int
+     */
+    public function getAdvancedPmUpdateLength()
+    {
+        /** @var int $advancedPmUpdateLength */
+        $advancedPmUpdateLength = $this->scopeConfig->getValue(self::PRODUCT_PRODUCT_MODEL_UPDATE_LENGTH);
+        if ((filter_var($advancedPmUpdateLength, FILTER_VALIDATE_INT)) === false || ($advancedPmUpdateLength < self::PRODUCT_PRODUCT_MODEL_UPDATE_LENGTH_MINIMUM)) {
+            $advancedPmUpdateLength = self::PRODUCT_PRODUCT_MODEL_UPDATE_LENGTH_DEFAULT_VALUE;
+        }
+
+        return $advancedPmUpdateLength;
+    }
+
     /**
      * Get the attribute mapping, with lowered values
      *
@@ -834,4 +880,16 @@ public function getAttributeMapping()
 
         return $loweredMatches;
     }
+
+    /**
+     * Returns default attribute-set id for given entity
+     *
+     * @param string $entity
+     * @return int
+     * @throws \Magento\Framework\Exception\LocalizedException
+     */
+    public function getDefaultAttributeSetId($entity)
+    {
+        return $this->eavConfig->getEntityType($entity)->getDefaultAttributeSetId();
+    }
 }
diff --git a/Helper/Import/Product.php b/Helper/Import/Product.php
@@ -183,7 +183,7 @@ private function formatValues(array $values)
                 // Attribute is a price
                 /** @var array $price */
                 foreach ($attributeValue['data'] as $price) {
-                    if (!array_key_exists('currency', $price) || !array_key_exists('amount', $price)) {
+                    if (!is_array($price) || !array_key_exists('currency', $price) || !array_key_exists('amount', $price)) {
                         continue;
                     }
                     /** @var string $priceKey */

diff --git a/Job/Attribute.php b/Job/Attribute.php
@@ -177,8 +177,8 @@ public function insertData()
          */
         foreach ($attributes as $index => $attribute) {
             /** @var string $attributeCode */
-            $attributeCode     = strtolower($attribute['code']);
-            $attribute['code'] = $attributeCode;
+            $attributeCode     = $attribute['code'];
+            $attribute['code'] = strtolower($attributeCode);
 
             if ($attribute['type'] == 'pim_catalog_metric' && in_array(
                     $attributeCode,

diff --git a/Job/Category.php b/Job/Category.php
@@ -397,7 +397,7 @@ public function createEntities()
         /** @var array $values */
         $values = [
             'entity_id'        => '_entity_id',
-            'attribute_set_id' => new Expr($this->configHelper->getEntityTypeId(CategoryModel::ENTITY)),
+            'attribute_set_id' => new Expr($this->configHelper->getDefaultAttributeSetId(CategoryModel::ENTITY)),
             'parent_id'        => 'parent_id',
             'updated_at'       => new Expr('now()'),
             'path'             => 'path',

diff --git a/Job/Import.php b/Job/Import.php
@@ -602,6 +602,52 @@ public function checkLabelPerLocales(array $entity, array $lang, string $respons
         return $response;
     }
 
+    /**
+     * Slice InsertOnDuplicate into separate updates to prevent MySQL hitting row size max.
+     *
+     * @param string $table
+     * @param array $data
+     * @param array $fields
+     *
+     * @return void
+     */
+    public function sliceInsertOnDuplicate($table, array $data, array $fields = [])
+    {
+        /** @var AdapterInterface $connection */
+        $connection = $this->entitiesHelper->getConnection();
+        /** @var int $updateLength */
+        $updateLength = $this->configHelper->getAdvancedPmUpdateLength();
+        /** @var array $row */
+        foreach ($data as $row) {
+            // create empty row with primaryKey if not present
+            /** @var string $primaryKey */
+            $primaryKey = $row['code'];
+            if (!$connection->select()->from($table)->where('code = ?', $primaryKey)) {
+                $connection->insert($table, ['code = ?', $primaryKey]);
+            }
+            unset($row['code']);
+            // slice the data in separate updates
+            while (count($row)) {
+                /** @var int $sliceSize */
+                $sliceSize = 0;
+                /** @var array $slice */
+                $slice = [];
+                foreach ($row as $column => $value) {
+                    $sliceSize += strlen($column) + strlen($value);
+                    // Ignore "Update Length" on first column update to prevent
+                    // possible endless loop if a column is bigger.
+                    if (count($slice) && ($sliceSize >= $updateLength)) {
+                        break;
+                    }
+                    $slice[$column] = $value;
+                    unset($row[$column]);
+                }
+                $connection->update($table, $slice, ['code = ?' => $primaryKey]);
+            }
+        }
+        return;
+    }
+
     /**
      * Get Akeneo Client instance
      *

diff --git a/Job/Product.php b/Job/Product.php
@@ -28,13 +28,11 @@
 use Akeneo\Connector\Helper\Config as ConfigHelper;
 use Akeneo\Connector\Helper\Output as OutputHelper;
 use Akeneo\Connector\Helper\Store as StoreHelper;
-use Akeneo\Connector\Helper\Locales as LocalesHelper;
 use Akeneo\Connector\Helper\ProductFilters;
 use Akeneo\Connector\Helper\Serializer as JsonSerializer;
 use Akeneo\Connector\Helper\Import\Product as ProductImportHelper;
 use Akeneo\Connector\Job\Option as JobOption;
 use Akeneo\Connector\Model\Source\Attribute\Metrics as AttributeMetrics;
-use Magento\Backend\Model\Locale\Manager as LocaleManager;
 use Zend_Db_Expr as Expr;
 use Zend_Db_Statement_Pdo;
 
@@ -196,24 +194,12 @@ class Product extends Import
      * @var StoreHelper $storeHelper
      */
     protected $storeHelper;
-    /**
-     * This variable contains a LocalesHelper
-     *
-     * @var LocalesHelper $localesHelper
-     */
-    protected $localesHelper;
     /**
      * This variable contains a JobOption
      *
      * @var JobOption $jobOption
      */
     protected $jobOption;
-    /**
-     * This variable contains a LocaleManager
-     *
-     * @var LocaleManager $localeManager
-     */
-    protected $localeManager;
     /**
      * This variable contains an AttributeMetrics
      *
@@ -243,8 +229,6 @@ class Product extends Import
      * @param ProductUrlPathGenerator $productUrlPathGenerator
      * @param TypeListInterface       $cacheTypeList
      * @param StoreHelper             $storeHelper
-     * @param LocalesHelper           $localesHelper
-     * @param LocaleManager           $localeManager
      * @param AttributeMetrics        $attributeMetrics
      * @param StoreManagerInterface   $storeManager
      * @param array                   $data
@@ -263,9 +247,7 @@ public function __construct(
         ProductUrlPathGenerator $productUrlPathGenerator,
         TypeListInterface $cacheTypeList,
         StoreHelper $storeHelper,
-        LocalesHelper $localesHelper,
         JobOption $jobOption,
-        LocaleManager $localeManager,
         AttributeMetrics $attributeMetrics,
         StoreManagerInterface $storeManager,
         array $data = []
@@ -281,9 +263,7 @@ public function __construct(
         $this->product                 = $product;
         $this->cacheTypeList           = $cacheTypeList;
         $this->storeHelper             = $storeHelper;
-        $this->localesHelper           = $localesHelper;
         $this->jobOption               = $jobOption;
-        $this->localeManager           = $localeManager;
         $this->productUrlPathGenerator = $productUrlPathGenerator;
         $this->attributeMetrics        = $attributeMetrics;
         $this->storeManager            = $storeManager;
@@ -384,7 +364,7 @@ public function insertData()
                         /** @var string|false $symbol */
                         $symbol = array_key_exists($unit, $metricSymbols);
 
-                        if (!array_key_exists($unit, $metricSymbols)) {
+                        if (!$symbol) {
                             continue;
                         }
 
@@ -620,14 +600,13 @@ public function createMetricsOptions()
         $tmpTable = $this->entitiesHelper->getTableName($this->getCode());
         /** @var mixed[] $metricsVariantSettings */
         $metricsVariantSettings = $this->configHelper->getMetricsColumns(true);
-        /** @var string[] $metricsSymbols */
-        $metricsSymbols = $this->getMetricsSymbols();
-        /** @var string $adminLocale */
-        $adminLocale = $this->localeManager->getGeneralLocale();
+        /** @var string[] $locales */
+        $locales = $this->storeHelper->getMappedWebsitesStoreLangs();
 
         $this->jobOption->createTable();
 
         foreach ($metricsVariantSettings as $metricsVariantSetting) {
+            $metricsVariantSetting = strtolower($metricsVariantSetting);
             $columnExist = $connection->tableColumnExists($tmpTable, $metricsVariantSetting);
 
             if (!$columnExist) {
@@ -645,9 +624,7 @@ public function createMetricsOptions()
                 }
 
                 /** @var string[] $labels */
-                $labels = [];
-
-                $labels[$adminLocale] = $option;
+                $labels = array_fill_keys($locales, $option);
 
                 /** @var mixed[] $insertedData */
                 $insertedData = [

diff --git a/Job/ProductModel.php b/Job/ProductModel.php
@@ -25,11 +25,6 @@
  */
 class ProductModel extends Import
 {
-    /**
-     * @var int BATCH_SIZE
-     */
-    const BATCH_SIZE = 500;
-
     /**
      * This variable contains a string value
      *
@@ -200,6 +195,8 @@ public function updateData()
     {
         /** @var AdapterInterface $connection */
         $connection = $this->entitiesHelper->getConnection();
+        /** @var int $updateLength */
+        $batchSize = $this->configHelper->getAdvancedPmBatchSize();
         /** @var array $tmpTable */
         $tmpTable = $this->entitiesHelper->getTableName($this->getCode());
         /** @var array $variantTable */
@@ -248,14 +245,22 @@ public function updateData()
                 }
             }
             $i++;
-            if (count($values) > self::BATCH_SIZE) {
-                $connection->insertOnDuplicate($variantTable, $values, $keys);
+            if (count($values) > $batchSize) {
+                if (0 == $batchSize) {
+                    $this->sliceInsertOnDuplicate($variantTable, $values);
+                } else {
+                    $connection->insertOnDuplicate($variantTable, $values, $keys);
+                }
                 $values = [];
                 $i      = 0;
             }
         }
         if (count($values) > 0) {
-            $connection->insertOnDuplicate($variantTable, $values, $keys);
+            if (0 == $batchSize) {
+                $this->sliceInsertOnDuplicate($variantTable, $values);
+            } else {
+                $connection->insertOnDuplicate($variantTable, $values, $keys);
+            }
         }
     }
 

diff --git a/composer.json b/composer.json
@@ -11,7 +11,7 @@
     "php-http/guzzle6-adapter": "^1.1"
   },
   "type": "magento2-module",
-  "version": "100.3.2",
+  "version": "100.3.3",
   "license": [
     "OSL-3.0",
     "AFL-3.0"

diff --git a/etc/adminhtml/system.xml b/etc/adminhtml/system.xml
@@ -290,21 +290,39 @@ For example :<br/>
                         ]]>
                     </comment>
                 </field>
-                <field id="media_enabled" translate="label" type="select" sortOrder="50" showInDefault="1" showInWebsite="0" showInStore="0">
+                <field id="product_model_batch_size" translate="label comment" type="text" sortOrder="40" showInDefault="1" showInWebsite="0" showInStore="0">
+                    <label>Product Model Batch Size</label>
+                    <comment>
+                        <![CDATA[The number of product models handled per database update.</br>
+                        You can lower this value to prevent MYSQL errors from database limitations.<br />
+                        If set to 1, product models will be imported one by one.<br />
+                        If set to 0, "Product Model Update Length" value below will be used instead.<br />
+                        <u>Default value: 500</u>]]>
+                    </comment>
+                </field>
+                <field id="product_model_update_length" translate="label comment" type="text" sortOrder="50" showInDefault="1" showInWebsite="0" showInStore="0">
+                    <label>Product Model Update Length</label>
+                    <comment>
+                        <![CDATA[If the above "Product Model Batch Size" value is set to 0, the database command to insert product models will be sliced into separate statements no bigger than the given value.</br>
+                        This is an approximate value, and must be superior to 1000 characters.</br>
+                        <u>Default value: 5000</u>]]>
+                    </comment>
+                </field>
+                <field id="media_enabled" translate="label" type="select" sortOrder="60" showInDefault="1" showInWebsite="0" showInStore="0">
                     <label>Import Media Files</label>
                     <source_model>Magento\Config\Model\Config\Source\Yesno</source_model>
                 </field>
-                <field id="media_gallery" translate="label comment" type="text" sortOrder="60" showInDefault="1" showInWebsite="0" showInStore="0">
-                    <label>Akeneo Attributes for Image Gallery</label>
+                <field id="media_gallery" translate="label" type="text" sortOrder="70" showInDefault="1" showInWebsite="0" showInStore="0">
+                    <label>Akeneo Images Attributes</label>
                     <frontend_model>Akeneo\Connector\Block\Adminhtml\System\Config\Form\Field\Gallery</frontend_model>
                     <backend_model>Magento\Config\Model\Config\Backend\Serialized\ArraySerialized</backend_model>
                     <comment>Enter all PIM attributes containing product images. All images specified will be imported to the product image gallery.</comment>
                     <depends>
                         <field id="media_enabled">1</field>
                     </depends>
                 </field>
-                <field id="media_images" translate="label comment" type="text" sortOrder="70" showInDefault="1" showInWebsite="0" showInStore="0">
-                    <label>Product Image Association</label>
+                <field id="media_images" translate="label" type="text" sortOrder="80" showInDefault="1" showInWebsite="0" showInStore="0">
+                    <label>Product Images Mapping</label>
                     <frontend_model>Akeneo\Connector\Block\Adminhtml\System\Config\Form\Field\Image</frontend_model>
                     <backend_model>Magento\Config\Model\Config\Backend\Serialized\ArraySerialized</backend_model>
                     <comment>Map Magento image type for products. Associate the image type option to a PIM attribute specified in the Akeneo Attribute for Image Gallery field. After import, all specified options will be filled by their image if provided.</comment>

diff --git a/etc/config.xml b/etc/config.xml
@@ -28,6 +28,8 @@
             </products_filters>
             <product>
                 <url_generation_enabled>1</url_generation_enabled>
+                <product_model_batch_size>500</product_model_batch_size>
+                <product_model_update_length>5000</product_model_update_length>
             </product>
             <category>
                 <is_active>1</is_active>