diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c237056..20c51f31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ # Changelog +## Unreleased +### Add +-Alternative product export command based on a queue system + ## [v5.4.0] - 2026.08.05 ### Add - Add support for Atlas AI diff --git a/README.md b/README.md index 39c38c91..c0f92307 100644 --- a/README.md +++ b/README.md @@ -276,6 +276,15 @@ Command name: `factfinder:export [TYPE]`. You can add execution of this command - store - define a store, which the product data will be taken from - skip-ftp-upload - skips the ftp upload - skip-push-import - skips triggering import + +#### Running export with a worker + +Since version 5.5.0, we've introduced a new export flow based on the worker. +The biggest advantage of this export method is the reduced memory usage, which is especially helpful when you have a large or complex products catalog. +Currently, this command is only available from the CLI and can be executed by: + + php [MAGENTO_ROOT]/bin/magento factfinder:worker-export + ## Web Component Integration diff --git a/src/Console/Command/ExportBatch.php b/src/Console/Command/ExportBatch.php new file mode 100644 index 00000000..54c6b730 --- /dev/null +++ b/src/Console/Command/ExportBatch.php @@ -0,0 +1,80 @@ +setName('factfinder:export:batch') + ->setDescription('Internal worker command for exporting a batch of products') + ->setHidden(true); + + $this->addArgument('type', InputArgument::REQUIRED, 'Type of data to export'); + $this->addArgument('store', InputArgument::REQUIRED, 'Store ID'); + $this->addArgument('offset', InputArgument::REQUIRED, 'Offset'); + $this->addArgument('limit', InputArgument::REQUIRED, 'Limit'); + $this->addArgument('file_path', InputArgument::REQUIRED, 'Path to output file'); + + parent::configure(); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $this->state->setAreaCode('frontend'); + + $type = $input->getArgument('type'); + $storeId = (int) $input->getArgument('store'); + $offset = (int) $input->getArgument('offset'); + $limit = (int) $input->getArgument('limit'); + $filePath = $input->getArgument('file_path'); + + $processedCount = 0; + + $this->storeEmulation->runInStore($storeId, function () use ($type, $offset, $limit, $filePath, &$processedCount) { + $mode = ($offset === 0) ? 'w+' : 'a+'; + $stream = $this->streamFactory->create([ + 'filename' => $filePath, + 'mode' => $mode + ]); + + $generator = $this->feedGeneratorFactory->create($type); + $processedCount = $generator->generateBatch($stream, $offset, $limit); + }); + + $memoryUsageMB = memory_get_usage(true) / 1024 / 1024; + $peakMemoryMB = memory_get_peak_usage(true) / 1024 / 1024; + + $result = [ + 'count' => $processedCount, + 'memory' => round($memoryUsageMB, 2), + 'peak' => round($peakMemoryMB, 2), + ]; + + $output->write(json_encode($result)); + + return Command::SUCCESS; + } +} diff --git a/src/Console/Command/WorkerExport.php b/src/Console/Command/WorkerExport.php new file mode 100644 index 00000000..e895f3a3 --- /dev/null +++ b/src/Console/Command/WorkerExport.php @@ -0,0 +1,336 @@ +setName('factfinder:worker-export') + ->setDescription('Export feed data using a queue/batch mechanism to prevent memory issues'); + + $this->addArgument( + 'type', + InputArgument::OPTIONAL, + 'Type of data to be exported (default: product)', + self::PRODUCTS_EXPORT_TYPE + ); + $this->addOption('store', 's', InputOption::VALUE_OPTIONAL, 'Store ID or Store Code'); + $this->addOption('upload', 'u', InputOption::VALUE_NONE, 'Upload feed via FTP'); + $this->addOption('push-import', 'i', InputOption::VALUE_NONE, 'Push Import'); + + parent::configure(); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $this->state->setAreaCode('frontend'); + + [$storeIds, $upload, $pushImport] = $this->resolveExecutionParameters($input, $output); + + if (empty($storeIds)) { + $output->writeln('[ERROR] There is no integration enabled for any store.'); + return Command::FAILURE; + } + + $phpBinaryFinder = new PhpExecutableFinder(); + $phpBinary = $phpBinaryFinder->find() ?: 'php'; + $type = $input->getArgument('type') ?? self::PRODUCTS_EXPORT_TYPE; + + foreach ($storeIds as $storeId) { + $success = $this->exportForStore($storeId, $type, $upload, $pushImport, $output, $phpBinary); + if (!$success) { + return Command::FAILURE; + } + } + + return Command::SUCCESS; + } + + private function resolveExecutionParameters(InputInterface $input, OutputInterface $output): array + { + $storeIdInput = $input->getOption('store'); + $upload = (bool) $input->getOption('upload'); + $pushImport = (bool) $input->getOption('push-import'); + + if ($input->isInteractive()) { + $helper = $this->getHelper('question'); + + if (empty($storeIdInput)) { + $storeIdInput = $this->askStoreId($input, $output, $helper); + } + + if (!$upload) { + $upload = $this->askYesNoQuestion( + $input, + $output, + $helper, + 'Should upload feed to FTP after exporting?' + ); + } + + if (!$pushImport) { + $pushImport = $this->askYesNoQuestion( + $input, + $output, + $helper, + 'Should trigger Push Import after uploading?' + ); + } + } + + $storeIds = $this->getStoreIds($storeIdInput ? (int) $storeIdInput : 0); + + return [$storeIds, $upload, $pushImport]; + } + + private function askStoreId(InputInterface $input, OutputInterface $output, mixed $helper): ?string + { + $storeChoices = []; + foreach ($this->storeManager->getStores() as $store) { + if ($this->communicationConfig->isChannelEnabled((int) $store->getId())) { + $storeChoices[$store->getId()] = "{$store->getName()} (ID: {$store->getId()})"; + } + } + + if (empty($storeChoices)) { + return null; + } + + $question = new ChoiceQuestion('Select store ID:', $storeChoices); + $storeIdInput = $helper->ask($input, $output, $question); + + if (preg_match('/ID: (\d+)\)/', (string) $storeIdInput, $matches)) { + return $matches[1]; + } + + return (string) $storeIdInput; + } + + private function askYesNoQuestion( + InputInterface $input, + OutputInterface $output, + mixed $helper, + string $questionText + ): bool { + $question = new ChoiceQuestion("{$questionText} (default: no)", ['no', 'yes'], 0); + $answer = $helper->ask($input, $output, $question); + + return $answer === 'yes'; + } + + private function exportForStore( + int $storeId, + string $type, + bool $upload, + bool $pushImport, + OutputInterface $output, + string $phpBinary + ): bool { + $output->writeln(''); + $output->writeln('=================================================='); + $output->writeln(">>> STARTING EXPORT FOR STORE ID: {$storeId} <<<"); + $output->writeln('=================================================='); + + $channelId = $this->communicationConfig->getChannel($storeId); + $filename = $this->feedFileService->getFeedExportFilename($type, $channelId); + $relativePath = "factfinder/{$filename}"; + $absolutePath = $this->feedFileService->getExportPath($filename); + + $this->prepareExportFile($absolutePath, $output); + + if ($type === self::PRODUCTS_EXPORT_TYPE) { + $batchSuccess = $this->runProductBatchExport($storeId, $type, $relativePath, $output, $phpBinary); + if (!$batchSuccess) { + return false; + } + $output->writeln("[FILE CREATED] {$absolutePath}"); + } + + if (!$this->handleUpload($upload, $filename, $relativePath, $output)) { + return false; + } + + if (!$this->handlePushImport($pushImport, $storeId, $output)) { + return false; + } + + $output->writeln(" SUCCESS: Export process completed for Store {$storeId}\n"); + return true; + } + + private function prepareExportFile(string $absolutePath, OutputInterface $output): void + { + $dir = dirname($absolutePath); + if (!is_dir($dir)) { + mkdir($dir, 0755, true); + $output->writeln("[STEP 1] Created directory: {$dir}"); + } + + if (file_exists($absolutePath)) { + unlink($absolutePath); + $output->writeln("[STEP 1] Removed existing export file: {$absolutePath}"); + } + } + + private function runProductBatchExport( + int $storeId, + string $type, + string $relativePath, + OutputInterface $output, + string $phpBinary + ): bool { + $batchSize = 100; + $offset = 0; + $batchNumber = 1; + $totalCount = 0; + + $output->writeln("[STEP 2] Starting product batch processing (Batch Size: {$batchSize})..."); + + while (true) { + $output->write(sprintf(' -> Processing Batch #%d (Offset: %d)... ', $batchNumber, $offset)); + + $process = new Process([ + $phpBinary, + 'bin/magento', + 'factfinder:export:batch', + $type, + (string) $storeId, + (string) $offset, + (string) $batchSize, + $relativePath, + ]); + $process->setTimeout(600); + $process->run(); + + if (!$process->isSuccessful()) { + $output->writeln("\n[ERROR] Batch #{$batchNumber} failed at Offset {$offset}!"); + $output->writeln("{$process->getErrorOutput()}"); + return false; + } + + $result = $this->parseProcessOutput($process->getOutput()); + $count = $result['count'] ?? 0; + + if ($count === 0) { + $output->writeln('Done! No more items to process.'); + break; + } + + $totalCount += $count; + $output->writeln(sprintf( + 'OK (Exported: %d items | RAM: %.2f MB | Peak RAM: %.2f MB)', + $count, + $result['memory'] ?? 0, + $result['peak'] ?? 0 + )); + + $offset += $batchSize; + $batchNumber++; + } + + $output->writeln("[STEP 2 COMPLETED] Total exported items: {$totalCount}"); + return true; + } + + private function parseProcessOutput(string $rawOutput): array + { + preg_match('/\{.*\}/s', trim($rawOutput), $matches); + return json_decode($matches[0] ?? '{}', true) ?: []; + } + + private function handleUpload( + bool $upload, + string $filename, + string $relativePath, + OutputInterface $output + ): bool { + if (!$upload) { + $output->writeln('[STEP 3] FTP Upload skipped.'); + return true; + } + + $output->writeln("[STEP 3] Uploading file {$filename} to FTP server..."); + try { + $stream = $this->streamFactory->create([ + 'filename' => $relativePath, + 'mode' => 'r', + ]); + $this->ftpUploader->upload($filename, $stream); + $output->writeln('[STEP 3 COMPLETED] File successfully uploaded to FTP.'); + return true; + } catch (\Throwable $e) { + $output->writeln("[ERROR] FTP Upload failed: {$e->getMessage()}"); + return false; + } + } + + private function handlePushImport( + bool $pushImport, + int $storeId, + OutputInterface $output + ): bool { + if (!$pushImport) { + $output->writeln('[STEP 4] Push Import skipped.'); + return true; + } + + $output->writeln('[STEP 4] Triggering Push Import on FactFinder side...'); + try { + if ($this->pushImport->execute($storeId)) { + $output->writeln('[STEP 4 COMPLETED] Push Import triggered successfully.'); + return true; + } + $output->writeln('[STEP 4 FAILED] Push Import execution failed.'); + return false; + } catch (\Throwable $e) { + $output->writeln("[ERROR] Push Import failed: {$e->getMessage()}"); + return false; + } + } + + private function getStoreIds(int $storeId): array + { + $storeIds = array_map( + fn ($store) => (int) $store->getId(), + $storeId ? [$this->storeManager->getStore($storeId)] : $this->storeManager->getStores() + ); + + return array_filter($storeIds, [$this->communicationConfig, 'isChannelEnabled']); + } +} diff --git a/src/Model/Export/Catalog/DataProvider.php b/src/Model/Export/Catalog/DataProvider.php index c156b032..e96dd78b 100644 --- a/src/Model/Export/Catalog/DataProvider.php +++ b/src/Model/Export/Catalog/DataProvider.php @@ -31,6 +31,41 @@ public function getEntities(): iterable } } + /** + * @return ExportEntityInterface[] + */ + public function getEntitiesBatch(int $offset, int $limit): iterable + { + yield from []; // init generator + + $productsBatch = $this->getProductsSlice($offset, $limit); + + foreach ($productsBatch as $product) { + yield from $this->entitiesFrom($product)->getEntities(); + } + } + + private function getProductsSlice(int $offset, int $limit): iterable + { + if (method_exists($this->products, 'getBatch')) { + return $this->products->getBatch($offset, $limit); + } + + if (method_exists($this->products, 'setPageSize') && method_exists($this->products, 'setCurPage')) { + $pageNumber = (int) floor($offset / $limit) + 1; + $this->products->setPageSize($limit); + $this->products->setCurPage($pageNumber); + + return $this->products; + } + + $productsArray = is_array($this->products) + ? $this->products + : iterator_to_array($this->products, false); + + return array_slice($productsArray, $offset, $limit); + } + private function entitiesFrom(ProductInterface $product): DataProviderInterface { $type = $this->entityTypes[$product->getTypeId()] ?? $this->entityTypes[ProductType::DEFAULT_TYPE]; diff --git a/src/Model/Export/Catalog/ProductType/SimpleDataProvider.php b/src/Model/Export/Catalog/ProductType/SimpleDataProvider.php index 6f8fc8ad..24ff540f 100644 --- a/src/Model/Export/Catalog/ProductType/SimpleDataProvider.php +++ b/src/Model/Export/Catalog/ProductType/SimpleDataProvider.php @@ -29,6 +29,15 @@ public function getEntities(): iterable return [$this]; } + public function getEntitiesBatch(int $offset, int $limit): iterable + { + $entities = is_array($this->getEntities()) + ? $this->getEntities() + : iterator_to_array($this->getEntities(), false); + + return array_slice($entities, $offset, $limit); + } + public function getId(): int { return (int) $this->product->getId(); diff --git a/src/Model/Export/Feed.php b/src/Model/Export/Feed.php index 3196f60d..c7641461 100644 --- a/src/Model/Export/Feed.php +++ b/src/Model/Export/Feed.php @@ -30,6 +30,28 @@ public function generate(StreamInterface $stream): void $stream->finalize(); } + public function generateBatch( + StreamInterface $stream, + int $offset, + int $limit + ): int { + $columns = $this->getColumns($this->fields); + + if ($offset === 0) { + $stream->addEntity($columns); + } + + $processedCount = $this->exporter->exportEntitiesBatch( + $stream, + $this->dataProvider, + $columns, + $offset, + $limit + ); + + return (int) $processedCount; + } + private function getColumns(array $fields): array { return array_values(array_unique([...$this->columns, ...array_map([$this, 'getFieldName'], $fields)])); diff --git a/src/Model/Exporter.php b/src/Model/Exporter.php index e89f5fd9..32c488b5 100644 --- a/src/Model/Exporter.php +++ b/src/Model/Exporter.php @@ -23,8 +23,31 @@ public function exportEntities(StreamInterface $stream, DataProviderInterface $d } } + public function exportEntitiesBatch( + StreamInterface $stream, + DataProviderInterface $dataProvider, + array $columns, + int $offset, + int $limit + ): int { + $emptyRecord = array_combine($columns, array_fill(0, count($columns), '')); + $processedCount = 0; + + $entities = $dataProvider->getEntitiesBatch($offset, $limit); + + foreach ($entities as $entity) { + $stream->addEntity($this->prepareRow($entity->toArray(), $emptyRecord)); + $processedCount++; + } + + return $processedCount; + } + private function prepareRow(array $entityData, array $emptyRecord): array { - return array_map([$this->filter, 'filterValue'], [...$emptyRecord, ...array_intersect_key($entityData, $emptyRecord)]); + return array_map( + [$this->filter, 'filterValue'], + [...$emptyRecord, ...array_intersect_key($entityData, $emptyRecord)] + ); } } diff --git a/src/Model/Stream/Csv.php b/src/Model/Stream/Csv.php index be04b847..d2e8105e 100644 --- a/src/Model/Stream/Csv.php +++ b/src/Model/Stream/Csv.php @@ -17,7 +17,8 @@ class Csv implements StreamInterface public function __construct( private readonly Filesystem $filesystem, - private readonly string $filename = 'factfinder/export.csv' + private readonly string $filename = 'factfinder/export.csv', + private readonly string $mode = 'w+' ) { } @@ -40,7 +41,7 @@ private function getStream(): WriteInterface { if (!isset($this->stream)) { $directory = $this->filesystem->getDirectoryWrite(DirectoryList::VAR_DIR); - $this->stream = $directory->openFile($directory->getAbsolutePath($this->filename), 'w+'); + $this->stream = $directory->openFile($directory->getAbsolutePath($this->filename), $this->mode); $this->stream->lock(); } diff --git a/src/etc/di.xml b/src/etc/di.xml index 99d91f66..89cfa9a8 100644 --- a/src/etc/di.xml +++ b/src/etc/di.xml @@ -230,6 +230,16 @@ Omikron\Factfinder\Console\Command\Export + + + Omikron\Factfinder\Console\Command\ExportBatch + + + + + Omikron\Factfinder\Console\Command\WorkerExport + + diff --git a/src/view/frontend/templates/ff/searchbox.phtml b/src/view/frontend/templates/ff/searchbox.phtml index fc16b18a..5a93e4e7 100644 --- a/src/view/frontend/templates/ff/searchbox.phtml +++ b/src/view/frontend/templates/ff/searchbox.phtml @@ -1,7 +1,7 @@ helper('Omikron\Factfinder\Helper\ConfigurationHelper')->getConfig('factfinder/components/popular_searches'); +$isPopularSearchesEnabled = $block->helper('Omikron\Factfinder\Helper\ConfigurationHelper')->getConfig('factfinder/components/popular_searches'); ?>