https://www.spryker.com logo
Join Slack
Powered by
Hi, I have memory problem on jenkins during queue ...
# help
q
Hi, I have memory problem on jenkins during queue consuming. Any idea how to resolve it? I have active 5 processes. Memory 500MB. I'm trying to process 2k abstract products. I'm thinking about decreasing QueueConfig::_DEFAULT_THRESHOLD_ but I am not sure if it is a good idea. I am also not sure which value should I set. Any better sollutions?
a
It hardly depends on the content of your messages, but 500M memory is too less in my opinion for spryker.
w
Pretty much depends on the product structure, how many stores, how big is the data per product (concrete/abstract). You can either try to use smaller batch sizes to fit one batch into memory or find the root of cause of the high memory consumption by using profiling tools like the Xdebug profiling or tideways/blackfire. We found a ton of issue already in our project, most of them are not that obvious with the spryker demo data, but become more exposed once you have big data sets and a complex product structure. One example: During search product publishing the whole category tree is loaded in memory. This works fine if you have a small category tree and the purpose is to find all parent categories for the product. If you publish big batches, this is more efficient then doing a ton of DB requests, but with 500MB of memory you probably will not be able to process big batches, then this becomes a bad tradeoff and selecting all parent category nodes from the DB is not only more efficient memory wise, but also significantly faster. We solved it as following:
👀 1
Copy code
<?php

/**
 * This file is part of the Spryker Commerce OS.
 * For full license information, please view the LICENSE file that was distributed with this source code.
 */

namespace Pyz\Zed\ProductCategorySearch\Business\Expander;

use Generated\Shared\Transfer\LocaleTransfer;
use Generated\Shared\Transfer\ProductPageSearchTransfer;
use Generated\Shared\Transfer\StoreTransfer;
use Pyz\Zed\ProductCategorySearch\Persistence\ProductCategorySearchRepositoryInterface;
use Spryker\Zed\ProductCategorySearch\Business\Builder\ProductCategoryTreeBuilderInterface;
use Spryker\Zed\ProductCategorySearch\Business\Expander\ProductPageDataExpander as SprykerProductPageDataExpander;

class ProductPageDataExpander extends SprykerProductPageDataExpander
{
    protected ProductCategorySearchRepositoryInterface $productCategorySearchRepository;

    /**
     * @param \Spryker\Zed\ProductCategorySearch\Business\Builder\ProductCategoryTreeBuilderInterface $productCategoryTreeBuilder
     * @param \Pyz\Zed\ProductCategorySearch\Persistence\ProductCategorySearchRepositoryInterface $productCategorySearchRepository
     */
    public function __construct(
        ProductCategoryTreeBuilderInterface $productCategoryTreeBuilder,
        ProductCategorySearchRepositoryInterface $productCategorySearchRepository
    ) {
        parent::__construct($productCategoryTreeBuilder);
        $this->productCategorySearchRepository = $productCategorySearchRepository;
    }

    /**
     * @param \Generated\Shared\Transfer\ProductPageSearchTransfer $productAbstractPageSearchTransfer
     * @param \Generated\Shared\Transfer\LocaleTransfer $localeTransfer
     * @param \Generated\Shared\Transfer\StoreTransfer $storeTransfer
     *
     * @return array<int>
     */
    protected function getAllParentCategoryNodeIds(
        ProductPageSearchTransfer $productAbstractPageSearchTransfer,
        LocaleTransfer $localeTransfer,
        StoreTransfer $storeTransfer
    ): array {
        return $this->productCategorySearchRepository->getParentCategoryNodeIds($storeTransfer, $productAbstractPageSearchTransfer->getCategoryNodeIds());
    }

    /**
     * @param int $idCategoryNode
     * @param \Generated\Shared\Transfer\LocaleTransfer $localeTransfer
     * @param \Generated\Shared\Transfer\StoreTransfer $storeTransfer
     *
     * @return array<int>
     */
    protected function getCategoryNodeParentIds(int $idCategoryNode, LocaleTransfer $localeTransfer, StoreTransfer $storeTransfer): array
    {
        return $this->productCategorySearchRepository->getParentCategoryNodeIds($storeTransfer, [$idCategoryNode]);
    }
}
Copy code
<?php

/**
 * This file is part of the Spryker Commerce OS.
 * For full license information, please view the LICENSE file that was distributed with this source code.
 */

namespace Pyz\Zed\ProductCategorySearch\Persistence;

use Generated\Shared\Transfer\StoreTransfer;
use Orm\Zed\Category\Persistence\Map\SpyCategoryClosureTableTableMap;
use Orm\Zed\Category\Persistence\Map\SpyCategoryNodeTableMap;
use Orm\Zed\Category\Persistence\SpyCategoryClosureTableQuery;
use Propel\Runtime\ActiveQuery\Criteria;
use Propel\Runtime\Formatter\SimpleArrayFormatter;
use Spryker\Zed\ProductCategorySearch\Persistence\ProductCategorySearchRepository as SprykerProductCategorySearchRepository;

/**
 * @method \Spryker\Zed\ProductCategorySearch\Persistence\ProductCategorySearchPersistenceFactory getFactory()
 */
class ProductCategorySearchRepository extends SprykerProductCategorySearchRepository implements ProductCategorySearchRepositoryInterface
{
    /**
     * @param array<int> $productAbstractIds
     * @param \Generated\Shared\Transfer\StoreTransfer $storeTransfer
     *
     * @return array<array<\Orm\Zed\ProductCategory\Persistence\SpyProductCategory>>
     */
    public function getMappedProductCategoriesByIdProductAbstractAndStoreFilteredByStore(
        array $productAbstractIds,
        StoreTransfer $storeTransfer
    ): array {
        /** @var \Orm\Zed\ProductCategory\Persistence\SpyProductCategoryQuery $productCategoryQuery */
        $productCategoryQuery = $this->getFactory()
            ->getProductCategoryPropelQuery()
            ->filterByFkProductAbstract_In($productAbstractIds)
            ->useSpyCategoryQuery()
            ->useNodeQuery()
            ->withColumn(SpyCategoryNodeTableMap::COL_ID_CATEGORY_NODE, static::COLUMN_ID_CATEGORY_NODE)
            ->endUse()
            ->endUse();

        $productCategoryQuery
            ->useSpyCategoryQuery(null, Criteria::LEFT_JOIN)
            ->useSpyCategoryStoreQuery(null, Criteria::LEFT_JOIN)
            ->leftJoinSpyStore()
            ->useSpyStoreQuery()
            ->filterByName($storeTransfer->getNameOrFail())
            ->endUse()
            ->endUse()
            ->endUse();

        return $this->getFactory()
            ->createProductCategoryMapper()
            ->mapProductCategoryEntitiesByIdProductAbstractAndStore($productCategoryQuery->find(), []);
    }

    /**
     * @param \Generated\Shared\Transfer\StoreTransfer $storeTransfer
     * @param array<int> $categoryNodeIds
     *
     * @return array<int>
     */
    public function getParentCategoryNodeIds(StoreTransfer $storeTransfer, array $categoryNodeIds): array
    {
        $query = SpyCategoryClosureTableQuery::create();
        $query->distinct();
        $query->select([
            SpyCategoryClosureTableTableMap::COL_FK_CATEGORY_NODE,
        ]);
        $query->useNodeQuery()
            ->useCategoryQuery()
            ->useSpyCategoryStoreQuery()
            ->useSpyStoreQuery()
            ->filterByName($storeTransfer->getNameOrFail())
            ->endUse()
            ->endUse()
            ->endUse()
            ->endUse();
        $query->filterByFkCategoryNodeDescendant_In($categoryNodeIds);
        $query->setFormatter(SimpleArrayFormatter::class);
        $parentCategoryNodeIds = $query->find()->getData();

        return array_unique(array_values($parentCategoryNodeIds));
    }
}
Open in Slack
PreviousNext
Powered by