From 6aca123dd21e1f8278f8e1dd86da334cb998c4c9 Mon Sep 17 00:00:00 2001 From: Andrew Stoltz Date: Mon, 22 Aug 2016 20:15:59 -0500 Subject: [PATCH] Fix performance issues with Formerly export --- controllers/Formerly_ExportController.php | 294 +++++++++++++--------- 1 file changed, 177 insertions(+), 117 deletions(-) mode change 100644 => 100755 controllers/Formerly_ExportController.php diff --git a/controllers/Formerly_ExportController.php b/controllers/Formerly_ExportController.php old mode 100644 new mode 100755 index 6294075..3ee4b26 --- a/controllers/Formerly_ExportController.php +++ b/controllers/Formerly_ExportController.php @@ -3,118 +3,181 @@ class Formerly_ExportController extends BaseController { - public function actionIndex() - { - $this->renderTemplate('formerly/export/_index', array( - 'forms' => craft()->formerly_forms->getAllForms() - )); - } - - - public function actionCsv() - { - $dataToProcess = true; - $first = true; - $offset = 0; - $blocksize = 500; - $maximumLoops = 1000; - $loops = 0; - - $formId = craft()->request->getPost('form'); - $form = craft()->formerly_forms->getFormById($formId); - - set_time_limit('1000'); - //ini_set('memory_limit', '1024M'); - header('Content-Type: application/octet-stream'); - header('Content-Disposition: attachment; filename="' . ($form->handle . '_submissions.csv')); - header('Content-Transfer-Encoding: binary'); - $stream = fopen('php://output', 'w'); - - while ($dataToProcess && $loops <= $maximumLoops) { - $dataToProcess = false; - $loops++; - - $criteria = craft()->elements->getCriteria('Formerly_Submission'); - $criteria->formId = $formId; - $criteria->limit = $blocksize; - $criteria->offset = $offset; - - if (isset($_POST['fromDate']) && - !empty($_POST['fromDate']['date']) && - isset($_POST['toDate']) && - !empty($_POST['toDate']['date']) - ) { - - $fromDate = craft()->request->getPost('fromDate'); - $fromDate = DateTime::createFromString($fromDate, craft()->timezone); - $toDate = craft()->request->getPost('toDate'); - $toDate = DateTime::createFromString($toDate, craft()->timezone); - //A bit of a hack, I can't work out how to do a betweendates or AND query. These fields are always the same anyway - $criteria->dateCreated = '>= ' . $fromDate->format(DateTime::MYSQL_DATETIME); - $criteria->dateUpdated = '<= ' . $toDate->format(DateTime::MYSQL_DATETIME); - } elseif (isset($_POST['fromDate']) && !empty($_POST['fromDate']['date'])) { - $fromDate = craft()->request->getPost('fromDate'); - $fromDate = DateTime::createFromString($fromDate, craft()->timezone); - $criteria->dateCreated = '>= ' . $fromDate->format(DateTime::MYSQL_DATETIME); - } else if (isset($_POST['toDate']) && !empty($_POST['toDate']['date'])) { - $toDate = craft()->request->getPost('toDate'); - $toDate = DateTime::createFromString($toDate, craft()->timezone); - $criteria->dateCreated = '<= ' . $toDate->format(DateTime::MYSQL_DATETIME); - } - - $criteria->order = 'dateCreated desc'; - - // Write column names first. - + /** + * Switch this to true if you require the onPopulateElement hook. + * + * This significantly increases memory and CPU consumption. + * + * @var bool + */ + private $enableHooks = false; + + public function actionIndex() + { + $this->renderTemplate('formerly/export/_index', array( + 'forms' => craft()->formerly_forms->getAllForms() + )); + } + + public function actionCsv() + { + $formId = craft()->request->getPost('form'); + $form = craft()->formerly_forms->getFormById($formId); + $formQuestions = $form->getQuestions(); + $fieldTypes = craft()->fields->getAllFieldTypes(); + + set_time_limit('1000'); + header('Content-Type: application/octet-stream'); + header('Content-Disposition: attachment; filename="' . ($form->handle . '_submissions.csv')); + header('Content-Transfer-Encoding: binary'); + $stream = fopen('php://output', 'w'); + + + $criteria = craft()->elements->getCriteria('Formerly_Submission'); + $criteria->formId = $formId; + + $query = craft()->elements->buildElementsQuery($criteria, $contentTable, $fieldColumns); + $query->limit(500000); + $query->order('dateCreated desc'); + + if (isset($_POST['fromDate']) && + !empty($_POST['fromDate']['date']) && + isset($_POST['toDate']) && + !empty($_POST['toDate']['date']) + ) { + + $fromDate = craft()->request->getPost('fromDate'); + $fromDate = DateTime::createFromString($fromDate, craft()->timezone); + $toDate = craft()->request->getPost('toDate'); + $toDate = DateTime::createFromString($toDate, craft()->timezone); + + $query->andWhere(DbHelper::parseDateParam('elements.dateCreated', '>= ' . $fromDate->format(DateTime::MYSQL_DATETIME), $query->params)); + $query->andWhere(DbHelper::parseDateParam('elements.dateCreated', '<= ' . $toDate->format(DateTime::MYSQL_DATETIME), $query->params)); + } elseif (isset($_POST['fromDate']) && !empty($_POST['fromDate']['date'])) { + $fromDate = craft()->request->getPost('fromDate'); + $fromDate = DateTime::createFromString($fromDate, craft()->timezone); + $query->andWhere(DbHelper::parseDateParam('elements.dateCreated', '>= ' . $fromDate->format(DateTime::MYSQL_DATETIME), $query->params)); + } else if (isset($_POST['toDate']) && !empty($_POST['toDate']['date'])) { + $toDate = craft()->request->getPost('toDate'); + $toDate = DateTime::createFromString($toDate, craft()->timezone); + $query->andWhere(DbHelper::parseDateParam('elements.dateCreated', '<= ' . $toDate->format(DateTime::MYSQL_DATETIME), $query->params)); + } + + // Write column names first. $first = true; + $queryResult = $query->query(); + $elementType = $criteria->getElementType(); + while (false !== ($result = $queryResult->read())) { + if ($this->enableHooks) { + // Make a copy to pass to the onPopulateElement event + $originalResult = array_merge($result); + } + + // Separate the content values from the main element attributes + $content = array( + 'id' => (isset($result['contentId']) ? $result['contentId'] : null), + 'elementId' => $result['id'], + 'locale' => $criteria->locale, + 'title' => (isset($result['title']) ? $result['title'] : null) + ); + + unset($result['title']); + + if ($fieldColumns) { + foreach ($fieldColumns as $column) { + // Account for results where multiple fields have the same handle, but from + // different columns e.g. two Matrix block types that each have a field with the + // same handle + + $colName = $column['column']; + $fieldHandle = $column['handle']; + + if (!isset($content[$fieldHandle]) || (empty($content[$fieldHandle]) && !empty($result[$colName]))) { + $content[$fieldHandle] = $result[$colName]; + } + + unset($result[$colName]); + } + } - foreach ($criteria->find() as $submission) { - $dataToProcess = true; - $row = array( - 'Id' => $submission->id, - 'Time' => $submission->dateCreated->format('d/m/Y H:i:s') - ); - - foreach($form->getQuestions() as $question) - { - if ($question->type != 'RawHTML') { - $columnName = str_replace($form->handle . '_', '', $question->handle); - $columnName = str_replace(Formerly_QuestionType::CustomListHandle, '', $columnName); - $columnName = str_replace(Formerly_QuestionType::RawHTMLHandle, '', $columnName); - $columnName = str_replace(Formerly_QuestionType::CustomHandle, '', $columnName); - $columnName = ucwords($columnName); - - $row[$columnName] = $submission->{$question->handle}; - $value = $submission->{$question->handle}; - if ($value instanceof MultiOptionsFieldData) { - $options = $value->getOptions(); - - $summary = array(); - if ($question->type == Formerly_QuestionType::CustomList) { - for ($j = 0; $j < count($value); ++$j) { - $v = $value[$j]; - if ($v->selected) { - $summary[] = $v->value; - } - } - } else { - for ($j = 0; $j < count($options); ++$j) { - $option = $options[$j]; - if ($option->selected) - $summary[] = $option->value; - } - } - $row[$columnName] = implode($summary, ', '); - } - elseif ($question->type == Formerly_QuestionType::MultilineText) { - $row[$columnName] = str_replace('
', "\n", $value); - } - else { - if ($question->type != Formerly_QuestionType::RawHTML) - $row[$columnName] = $value; - } - } - } + $result['locale'] = $criteria->locale; + + if ($this->enableHooks) { + $submission = $elementType->populateElementModel($result); + + // Was an element returned? + if (!$submission || !($submission instanceof BaseElementModel)) { + continue; + } + + $submission->setContent($content); + + // Fire an 'onPopulateElement' event + craft()->elements->onPopulateElement(new Event($this, array( + 'element' => $submission, + 'result' => $originalResult + ))); + } else { + $result['dateCreated'] = DateTime::createFromFormat(DateTime::MYSQL_DATETIME, $result['dateCreated']); + $submission = (object)array_merge($content, $result); + } + + $row = array( + 'Id' => $submission->id, + 'Time' => $submission->dateCreated->format('d/m/Y H:i:s') + ); + + foreach ($formQuestions as $question) { + if ($question->type == Formerly_QuestionType::RawHTML) { + continue; + } + + $columnName = str_replace(array( + $form->handle . '_', + Formerly_QuestionType::CustomListHandle, + Formerly_QuestionType::RawHTMLHandle, + Formerly_QuestionType::CustomHandle, + ), '', $question->handle); + $columnName = ucwords($columnName); + + $value = $submission->{$question->handle}; + if (!$this->enableHooks && isset($fieldTypes[$question->type])) { + $fieldType = clone $fieldTypes[$question->type]; + $fieldType->setSettings(array( + 'options' => $question->options, + )); + + if ($value && is_string($value) && mb_strpos('{[', $value[0]) !== false) { + // Presumably this is JSON. + $value = JsonHelper::decode($value); + } + + $value = $fieldType->prepValue($value); + } + + if ($value instanceof MultiOptionsFieldData) { + $summary = array(); + if ($question->type == Formerly_QuestionType::CustomList) { + for ($j = 0; $j < count($value); ++$j) { + $v = $value[$j]; + if ($v->selected) { + $summary[] = $v->value; + } + } + } else { + foreach ($value->getOptions() as $option) { + if ($option->selected) { + $summary[] = $option->value; + } + } + } + $row[$columnName] = implode($summary, ', '); + } elseif ($question->type == Formerly_QuestionType::MultilineText) { + $row[$columnName] = str_replace('
', "\n", $value); + } else { + $row[$columnName] = $value; + } + } if ($first) { fputcsv($stream, array_keys($row)); @@ -122,13 +185,10 @@ public function actionCsv() } fputcsv($stream, $row); + } - } - $first = true; - $offset += $blocksize; - - } fclose($stream); - } -} \ No newline at end of file + } + +}