first commit
This commit is contained in:
Sven Wappler
111
Classes/Domain/Index/Classification/Classification.php
Normal file
111
Classes/Domain/Index/Classification/Classification.php
Normal file
@@ -0,0 +1,111 @@
|
||||
<?php declare(strict_types = 1);
|
||||
namespace WapplerSystems\Meilisearch\Domain\Index\Classification;
|
||||
|
||||
/***************************************************************
|
||||
* Copyright notice
|
||||
*
|
||||
* (c) 2010-2017 dkd Internet Service GmbH <solr-support@dkd.de>
|
||||
* All rights reserved
|
||||
*
|
||||
* This script is part of the TYPO3 project. The TYPO3 project is
|
||||
* free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* The GNU General Public License can be found at
|
||||
* http://www.gnu.org/copyleft/gpl.html.
|
||||
*
|
||||
* This script is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* This copyright notice MUST APPEAR in all copies of the script!
|
||||
***************************************************************/
|
||||
|
||||
|
||||
/**
|
||||
* Class Classification
|
||||
*/
|
||||
class Classification {
|
||||
|
||||
/**
|
||||
* Array of regular expressions
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $matchPatterns = [];
|
||||
|
||||
/**
|
||||
* Array of regular expressions
|
||||
* @var array
|
||||
*/
|
||||
protected $unMatchPatterns = [];
|
||||
|
||||
/**
|
||||
* @var string
|
||||
*/
|
||||
protected $mappedClass = '';
|
||||
|
||||
/**
|
||||
* Classification constructor.
|
||||
* @param array $matchPatterns
|
||||
* @param array $unMatchPatterns
|
||||
* @param string $mappedClass
|
||||
*/
|
||||
public function __construct(array $matchPatterns = [], array $unMatchPatterns = [],string $mappedClass = '')
|
||||
{
|
||||
$this->matchPatterns = $matchPatterns;
|
||||
$this->unMatchPatterns = $unMatchPatterns;
|
||||
$this->mappedClass = $mappedClass;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array
|
||||
*/
|
||||
public function getUnMatchPatterns(): array
|
||||
{
|
||||
return $this->unMatchPatterns;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $unMatchPatterns
|
||||
*/
|
||||
public function setUnMatchPatterns(array $unMatchPatterns)
|
||||
{
|
||||
$this->unMatchPatterns = $unMatchPatterns;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array
|
||||
*/
|
||||
public function getMatchPatterns(): array
|
||||
{
|
||||
return $this->matchPatterns;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $matchPatterns
|
||||
*/
|
||||
public function setMatchPatterns(array $matchPatterns)
|
||||
{
|
||||
$this->matchPatterns = $matchPatterns;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return string
|
||||
*/
|
||||
public function getMappedClass(): string
|
||||
{
|
||||
return $this->mappedClass;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $mappedClass
|
||||
*/
|
||||
public function setMappedClass(string $mappedClass)
|
||||
{
|
||||
$this->mappedClass = $mappedClass;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
<?php declare(strict_types = 1);
|
||||
namespace WapplerSystems\Meilisearch\Domain\Index\Classification;
|
||||
|
||||
/***************************************************************
|
||||
* Copyright notice
|
||||
*
|
||||
* (c) 2010-2017 dkd Internet Service GmbH <solr-support@dkd.de>
|
||||
* All rights reserved
|
||||
*
|
||||
* This script is part of the TYPO3 project. The TYPO3 project is
|
||||
* free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* The GNU General Public License can be found at
|
||||
* http://www.gnu.org/copyleft/gpl.html.
|
||||
*
|
||||
* This script is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* This copyright notice MUST APPEAR in all copies of the script!
|
||||
***************************************************************/
|
||||
|
||||
/**
|
||||
* Class ClassificationService
|
||||
*/
|
||||
class ClassificationService {
|
||||
|
||||
/**
|
||||
* @param string $stringToMatch
|
||||
* @param Classification[] $classifications
|
||||
* @return array
|
||||
*/
|
||||
public function getMatchingClassNames(string $stringToMatch, $classifications) : array
|
||||
{
|
||||
$matchingClassification = [];
|
||||
foreach ($classifications as $classification) {
|
||||
$matchingClassification = $this->applyMatchPatterns($stringToMatch, $classification, $matchingClassification);
|
||||
$matchingClassification = $this->applyUnMatchPatterns($stringToMatch, $classification, $matchingClassification);
|
||||
}
|
||||
|
||||
return array_values($matchingClassification);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $stringToMatch
|
||||
* @param Classification $classification
|
||||
* @param $matchingClassification
|
||||
* @return array
|
||||
*/
|
||||
protected function applyMatchPatterns(string $stringToMatch, $classification, $matchingClassification): array
|
||||
{
|
||||
/** @var $classification Classification */
|
||||
foreach ($classification->getMatchPatterns() as $matchPattern) {
|
||||
if (preg_match_all('~' . $matchPattern . '~ims', $stringToMatch) > 0) {
|
||||
$matchingClassification[] = $classification->getMappedClass();
|
||||
// if we found one match, we do not need to check the other patterns
|
||||
break;
|
||||
}
|
||||
}
|
||||
return array_unique($matchingClassification);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $stringToMatch
|
||||
* @param Classification $classification
|
||||
* @param $matchingClassification
|
||||
* @param $messages
|
||||
* @return array
|
||||
*/
|
||||
protected function applyUnMatchPatterns(string $stringToMatch, $classification, $matchingClassification): array
|
||||
{
|
||||
foreach ($classification->getUnMatchPatterns() as $unMatchPattern) {
|
||||
if (preg_match_all('~' . $unMatchPattern . '~ims', $stringToMatch) > 0) {
|
||||
// if we found one match, we do not need to check the other patterns
|
||||
$position = array_search($classification->getMappedClass(), $matchingClassification);
|
||||
if ($position !== false) {
|
||||
unset($matchingClassification[$position]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $matchingClassification;
|
||||
}
|
||||
}
|
||||
318
Classes/Domain/Index/IndexService.php
Normal file
318
Classes/Domain/Index/IndexService.php
Normal file
@@ -0,0 +1,318 @@
|
||||
<?php
|
||||
|
||||
namespace WapplerSystems\Meilisearch\Domain\Index;
|
||||
|
||||
/***************************************************************
|
||||
* Copyright notice
|
||||
*
|
||||
* (c) 2015-2016 Timo Hund <timo.hund@dkd.de>
|
||||
* All rights reserved
|
||||
*
|
||||
* This script is part of the TYPO3 project. The TYPO3 project is
|
||||
* free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* The GNU General Public License can be found at
|
||||
* http://www.gnu.org/copyleft/gpl.html.
|
||||
*
|
||||
* This script is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* This copyright notice MUST APPEAR in all copies of the script!
|
||||
***************************************************************/
|
||||
|
||||
use WapplerSystems\Meilisearch\ConnectionManager;
|
||||
use WapplerSystems\Meilisearch\IndexQueue\Indexer;
|
||||
use WapplerSystems\Meilisearch\IndexQueue\Item;
|
||||
use WapplerSystems\Meilisearch\IndexQueue\Queue;
|
||||
use WapplerSystems\Meilisearch\Domain\Site\Site;
|
||||
use WapplerSystems\Meilisearch\System\Configuration\TypoScriptConfiguration;
|
||||
use WapplerSystems\Meilisearch\System\Logging\SolrLogManager;
|
||||
use WapplerSystems\Meilisearch\Task\IndexQueueWorkerTask;
|
||||
use Solarium\Exception\HttpException;
|
||||
use TYPO3\CMS\Backend\Utility\BackendUtility;
|
||||
use TYPO3\CMS\Core\Utility\GeneralUtility;
|
||||
use TYPO3\CMS\Extbase\SignalSlot\Dispatcher;
|
||||
|
||||
/**
|
||||
* Service to perform indexing operations
|
||||
*
|
||||
* @author Timo Hund <timo.schmidt@dkd.de>
|
||||
*/
|
||||
class IndexService
|
||||
{
|
||||
/**
|
||||
* @var TypoScriptConfiguration
|
||||
*/
|
||||
protected $configuration;
|
||||
|
||||
/**
|
||||
* @var Site
|
||||
*/
|
||||
protected $site;
|
||||
|
||||
/**
|
||||
* @var IndexQueueWorkerTask
|
||||
*/
|
||||
protected $contextTask;
|
||||
|
||||
/**
|
||||
* @var Queue
|
||||
*/
|
||||
protected $indexQueue;
|
||||
|
||||
/**
|
||||
* @var Dispatcher
|
||||
*/
|
||||
protected $signalSlotDispatcher;
|
||||
|
||||
/**
|
||||
* @var \WapplerSystems\Meilisearch\System\Logging\SolrLogManager
|
||||
*/
|
||||
protected $logger = null;
|
||||
|
||||
/**
|
||||
* IndexService constructor.
|
||||
* @param Site $site
|
||||
* @param Queue|null $queue
|
||||
* @param Dispatcher|null $dispatcher
|
||||
* @param SolrLogManager|null $solrLogManager
|
||||
*/
|
||||
public function __construct(Site $site, Queue $queue = null, Dispatcher $dispatcher = null, SolrLogManager $solrLogManager = null)
|
||||
{
|
||||
$this->site = $site;
|
||||
$this->indexQueue = $queue ?? GeneralUtility::makeInstance(Queue::class);
|
||||
$this->signalSlotDispatcher = $dispatcher ?? GeneralUtility::makeInstance(Dispatcher::class);
|
||||
$this->logger = $solrLogManager ?? GeneralUtility::makeInstance(SolrLogManager::class, /** @scrutinizer ignore-type */ __CLASS__);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param \WapplerSystems\Meilisearch\Task\IndexQueueWorkerTask $contextTask
|
||||
*/
|
||||
public function setContextTask($contextTask)
|
||||
{
|
||||
$this->contextTask = $contextTask;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return \WapplerSystems\Meilisearch\Task\IndexQueueWorkerTask
|
||||
*/
|
||||
public function getContextTask()
|
||||
{
|
||||
return $this->contextTask;
|
||||
}
|
||||
|
||||
/**
|
||||
* Indexes items from the Index Queue.
|
||||
*
|
||||
* @param int $limit
|
||||
* @return bool
|
||||
*/
|
||||
public function indexItems($limit)
|
||||
{
|
||||
$errors = 0;
|
||||
$indexRunId = uniqid();
|
||||
$configurationToUse = $this->site->getSolrConfiguration();
|
||||
$enableCommitsSetting = $configurationToUse->getEnableCommits();
|
||||
|
||||
// get items to index
|
||||
$itemsToIndex = $this->indexQueue->getItemsToIndex($this->site, $limit);
|
||||
|
||||
$this->emitSignal('beforeIndexItems', [$itemsToIndex, $this->getContextTask(), $indexRunId]);
|
||||
|
||||
foreach ($itemsToIndex as $itemToIndex) {
|
||||
try {
|
||||
// try indexing
|
||||
$this->emitSignal('beforeIndexItem', [$itemToIndex, $this->getContextTask(), $indexRunId]);
|
||||
$this->indexItem($itemToIndex, $configurationToUse);
|
||||
$this->emitSignal('afterIndexItem', [$itemToIndex, $this->getContextTask(), $indexRunId]);
|
||||
} catch (\Exception $e) {
|
||||
$errors++;
|
||||
$this->indexQueue->markItemAsFailed($itemToIndex, $e->getCode() . ': ' . $e->__toString());
|
||||
$this->generateIndexingErrorLog($itemToIndex, $e);
|
||||
}
|
||||
}
|
||||
|
||||
$this->emitSignal('afterIndexItems', [$itemsToIndex, $this->getContextTask(), $indexRunId]);
|
||||
|
||||
if ($enableCommitsSetting && count($itemsToIndex) > 0) {
|
||||
$solrServers = GeneralUtility::makeInstance(ConnectionManager::class)->getConnectionsBySite($this->site);
|
||||
foreach ($solrServers as $solrServer) {
|
||||
try {
|
||||
$solrServer->getWriteService()->commit(false, false, false);
|
||||
} catch (HttpException $e) {
|
||||
$errors++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ($errors === 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates a message in the error log when an error occured.
|
||||
*
|
||||
* @param Item $itemToIndex
|
||||
* @param \Exception $e
|
||||
*/
|
||||
protected function generateIndexingErrorLog(Item $itemToIndex, \Exception $e)
|
||||
{
|
||||
$message = 'Failed indexing Index Queue item ' . $itemToIndex->getIndexQueueUid();
|
||||
$data = ['code' => $e->getCode(), 'message' => $e->getMessage(), 'trace' => $e->getTraceAsString(), 'item' => (array)$itemToIndex];
|
||||
|
||||
$this->logger->log(
|
||||
SolrLogManager::ERROR,
|
||||
$message,
|
||||
$data
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds an emits a singal for the IndexService.
|
||||
*
|
||||
* @param string $name
|
||||
* @param array $arguments
|
||||
* @return mixed
|
||||
*/
|
||||
protected function emitSignal($name, $arguments)
|
||||
{
|
||||
return $this->signalSlotDispatcher->dispatch(__CLASS__, $name, $arguments);
|
||||
}
|
||||
|
||||
/**
|
||||
* Indexes an item from the Index Queue.
|
||||
*
|
||||
* @param Item $item An index queue item to index
|
||||
* @param TypoScriptConfiguration $configuration
|
||||
* @return bool TRUE if the item was successfully indexed, FALSE otherwise
|
||||
*/
|
||||
protected function indexItem(Item $item, TypoScriptConfiguration $configuration)
|
||||
{
|
||||
$indexer = $this->getIndexerByItem($item->getIndexingConfigurationName(), $configuration);
|
||||
|
||||
// Remember original http host value
|
||||
$originalHttpHost = isset($_SERVER['HTTP_HOST']) ? $_SERVER['HTTP_HOST'] : null;
|
||||
|
||||
$itemChangedDate = $item->getChanged();
|
||||
$itemChangedDateAfterIndex = 0;
|
||||
|
||||
try {
|
||||
$this->initializeHttpServerEnvironment($item);
|
||||
$itemIndexed = $indexer->index($item);
|
||||
|
||||
// update IQ item so that the IQ can determine what's been indexed already
|
||||
if ($itemIndexed) {
|
||||
$this->indexQueue->updateIndexTimeByItem($item);
|
||||
$itemChangedDateAfterIndex = $item->getChanged();
|
||||
}
|
||||
|
||||
if ($itemChangedDateAfterIndex > $itemChangedDate && $itemChangedDateAfterIndex > time()) {
|
||||
$this->indexQueue->setForcedChangeTimeByItem($item, $itemChangedDateAfterIndex);
|
||||
}
|
||||
} catch (\Exception $e) {
|
||||
$this->restoreOriginalHttpHost($originalHttpHost);
|
||||
throw $e;
|
||||
}
|
||||
|
||||
$this->restoreOriginalHttpHost($originalHttpHost);
|
||||
|
||||
return $itemIndexed;
|
||||
}
|
||||
|
||||
/**
|
||||
* A factory method to get an indexer depending on an item's configuration.
|
||||
*
|
||||
* By default all items are indexed using the default indexer
|
||||
* (WapplerSystems\Meilisearch\IndexQueue\Indexer) coming with EXT:meilisearch. Pages by default are
|
||||
* configured to be indexed through a dedicated indexer
|
||||
* (WapplerSystems\Meilisearch\IndexQueue\PageIndexer). In all other cases a dedicated indexer
|
||||
* can be specified through TypoScript if needed.
|
||||
*
|
||||
* @param string $indexingConfigurationName Indexing configuration name.
|
||||
* @param TypoScriptConfiguration $configuration
|
||||
* @return Indexer
|
||||
*/
|
||||
protected function getIndexerByItem($indexingConfigurationName, TypoScriptConfiguration $configuration)
|
||||
{
|
||||
$indexerClass = $configuration->getIndexQueueIndexerByConfigurationName($indexingConfigurationName);
|
||||
$indexerConfiguration = $configuration->getIndexQueueIndexerConfigurationByConfigurationName($indexingConfigurationName);
|
||||
|
||||
$indexer = GeneralUtility::makeInstance($indexerClass, /** @scrutinizer ignore-type */ $indexerConfiguration);
|
||||
if (!($indexer instanceof Indexer)) {
|
||||
throw new \RuntimeException(
|
||||
'The indexer class "' . $indexerClass . '" for indexing configuration "' . $indexingConfigurationName . '" is not a valid indexer. Must be a subclass of WapplerSystems\Meilisearch\IndexQueue\Indexer.',
|
||||
1260463206
|
||||
);
|
||||
}
|
||||
|
||||
return $indexer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the indexing progress.
|
||||
*
|
||||
* @return float Indexing progress as a two decimal precision float. f.e. 44.87
|
||||
*/
|
||||
public function getProgress()
|
||||
{
|
||||
return $this->indexQueue->getStatisticsBySite($this->site)->getSuccessPercentage();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the amount of failed queue items for the current site.
|
||||
*
|
||||
* @return int
|
||||
*/
|
||||
public function getFailCount()
|
||||
{
|
||||
return $this->indexQueue->getStatisticsBySite($this->site)->getFailedCount();
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes the $_SERVER['HTTP_HOST'] environment variable in CLI
|
||||
* environments dependent on the Index Queue item's root page.
|
||||
*
|
||||
* When the Index Queue Worker task is executed by a cron job there is no
|
||||
* HTTP_HOST since we are in a CLI environment. RealURL needs the host
|
||||
* information to generate a proper URL though. Using the Index Queue item's
|
||||
* root page information we can determine the correct host although being
|
||||
* in a CLI environment.
|
||||
*
|
||||
* @param Item $item Index Queue item to use to determine the host.
|
||||
* @param
|
||||
*/
|
||||
protected function initializeHttpServerEnvironment(Item $item)
|
||||
{
|
||||
static $hosts = [];
|
||||
$rootpageId = $item->getRootPageUid();
|
||||
$hostFound = !empty($hosts[$rootpageId]);
|
||||
|
||||
if (!$hostFound) {
|
||||
$hosts[$rootpageId] = $item->getSite()->getDomain();
|
||||
}
|
||||
|
||||
$_SERVER['HTTP_HOST'] = $hosts[$rootpageId];
|
||||
|
||||
// needed since TYPO3 7.5
|
||||
GeneralUtility::flushInternalRuntimeCaches();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string|null $originalHttpHost
|
||||
*/
|
||||
protected function restoreOriginalHttpHost($originalHttpHost)
|
||||
{
|
||||
if (!is_null($originalHttpHost)) {
|
||||
$_SERVER['HTTP_HOST'] = $originalHttpHost;
|
||||
} else {
|
||||
unset($_SERVER['HTTP_HOST']);
|
||||
}
|
||||
|
||||
// needed since TYPO3 7.5
|
||||
GeneralUtility::flushInternalRuntimeCaches();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
<?php declare(strict_types = 1);
|
||||
|
||||
namespace WapplerSystems\Meilisearch\Domain\Index\PageIndexer\Helper\UriBuilder;
|
||||
|
||||
/***************************************************************
|
||||
* Copyright notice
|
||||
*
|
||||
* (c) 2019 Timo Hund <timo.hund@dkd.de>
|
||||
* All rights reserved
|
||||
*
|
||||
* This script is part of the TYPO3 project. The TYPO3 project is
|
||||
* free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* The GNU General Public License can be found at
|
||||
* http://www.gnu.org/copyleft/gpl.html.
|
||||
* A copy is found in the textfile GPL.txt and important notices to the license
|
||||
* from the author is found in LICENSE.txt distributed with these scripts.
|
||||
*
|
||||
*
|
||||
* This script is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* This copyright notice MUST APPEAR in all copies of the script!
|
||||
***************************************************************/
|
||||
|
||||
use WapplerSystems\Meilisearch\IndexQueue\Item;
|
||||
use WapplerSystems\Meilisearch\IndexQueue\PageIndexerDataUrlModifier;
|
||||
use WapplerSystems\Meilisearch\System\Logging\SolrLogManager;
|
||||
use WapplerSystems\Meilisearch\System\Url\UrlHelper;
|
||||
use TYPO3\CMS\Core\Utility\GeneralUtility;
|
||||
|
||||
/**
|
||||
* Implementations of this class are able to build an indexing url for solr page indexing.
|
||||
*/
|
||||
abstract class AbstractUriStrategy
|
||||
{
|
||||
/**
|
||||
* @var SolrLogManager|null|object
|
||||
*/
|
||||
protected $logger;
|
||||
|
||||
/**
|
||||
* AbstractUriStrategy constructor.
|
||||
* @param SolrLogManager|null $logger
|
||||
*/
|
||||
public function __construct(SolrLogManager $logger = null)
|
||||
{
|
||||
$this->logger = $logger ?? GeneralUtility::makeInstance(SolrLogManager::class, /** @scrutinizer ignore-type */ __CLASS__);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param UrlHelper $urlHelper
|
||||
* @param array $overrideConfiguration
|
||||
* @return UrlHelper
|
||||
*/
|
||||
protected function applyTypoScriptOverridesOnIndexingUrl(UrlHelper $urlHelper, array $overrideConfiguration = []): UrlHelper
|
||||
{
|
||||
// check whether we should use ssl / https
|
||||
if (!empty($overrideConfiguration['scheme'])) {
|
||||
$urlHelper->setScheme($overrideConfiguration['scheme']);
|
||||
}
|
||||
|
||||
// overwriting the host
|
||||
if (!empty($overrideConfiguration['host'])) {
|
||||
$urlHelper->setHost($overrideConfiguration['host']);
|
||||
}
|
||||
|
||||
// overwriting the port
|
||||
if (!empty($overrideConfiguration['port'])) {
|
||||
$urlHelper->setPort($overrideConfiguration['port']);
|
||||
}
|
||||
|
||||
// setting a path if TYPO3 is installed in a sub directory
|
||||
if (!empty($overrideConfiguration['path'])) {
|
||||
$urlHelper->setPath($overrideConfiguration['path']);
|
||||
}
|
||||
|
||||
return $urlHelper;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param Item $item
|
||||
* @param int $language
|
||||
* @param string $mountPointParameter
|
||||
* @param array $options
|
||||
* @return string
|
||||
*/
|
||||
public function getPageIndexingUriFromPageItemAndLanguageId(Item $item, int $language = 0, string $mountPointParameter = '', $options = []): string
|
||||
{
|
||||
$pageIndexUri = $this->buildPageIndexingUriFromPageItemAndLanguageId($item, $language, $mountPointParameter);
|
||||
$urlHelper = GeneralUtility::makeInstance(UrlHelper::class, $pageIndexUri);
|
||||
$overrideConfiguration = is_array($options['frontendDataHelper.']) ? $options['frontendDataHelper.'] : [];
|
||||
$urlHelper = $this->applyTypoScriptOverridesOnIndexingUrl($urlHelper, $overrideConfiguration);
|
||||
$dataUrl = $urlHelper->getUrl();
|
||||
|
||||
if (!GeneralUtility::isValidUrl($dataUrl)) {
|
||||
$this->logger->log(
|
||||
SolrLogManager::ERROR,
|
||||
'Could not create a valid URL to get frontend data while trying to index a page.',
|
||||
[
|
||||
'item' => (array)$item,
|
||||
'constructed URL' => $dataUrl,
|
||||
'scheme' => $urlHelper->getScheme(),
|
||||
'host' => $urlHelper->getHost(),
|
||||
'path' => $urlHelper->getPath(),
|
||||
'page ID' => $item->getRecordUid(),
|
||||
'indexer options' => $options
|
||||
]
|
||||
);
|
||||
|
||||
throw new \RuntimeException(
|
||||
'Could not create a valid URL to get frontend data while trying to index a page. Created URL: ' . $dataUrl,
|
||||
1311080805
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
return $this->applyDataUrlModifier($item, $language, $dataUrl, $urlHelper);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param Item $item
|
||||
* @param int $language
|
||||
* @param string $mountPointParameter
|
||||
* @return mixed
|
||||
*/
|
||||
abstract protected function buildPageIndexingUriFromPageItemAndLanguageId(Item $item, int $language = 0, string $mountPointParameter = '');
|
||||
|
||||
/**
|
||||
* @param Item $item
|
||||
* @param int $language
|
||||
* @param string $dataUrl
|
||||
* @param UrlHelper $urlHelper
|
||||
* @return string
|
||||
*/
|
||||
protected function applyDataUrlModifier(Item $item, int $language, $dataUrl, UrlHelper $urlHelper):string
|
||||
{
|
||||
if (empty($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['IndexQueuePageIndexer']['dataUrlModifier'])) {
|
||||
return $dataUrl;
|
||||
}
|
||||
|
||||
$dataUrlModifier = GeneralUtility::makeInstance($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['IndexQueuePageIndexer']['dataUrlModifier']);
|
||||
if (!$dataUrlModifier instanceof PageIndexerDataUrlModifier) {
|
||||
throw new \RuntimeException($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['IndexQueuePageIndexer']['dataUrlModifier'] . ' is not an implementation of WapplerSystems\Meilisearch\IndexQueue\PageIndexerDataUrlModifier', 1290523345);
|
||||
}
|
||||
|
||||
return $dataUrlModifier->modifyDataUrl($dataUrl,
|
||||
[
|
||||
'item' => $item, 'scheme' => $urlHelper->getScheme(), 'host' => $urlHelper->getHost(),
|
||||
'path' => $urlHelper->getPath(), 'pageId' => $item->getRecordUid(), 'language' => $language
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
<?php declare(strict_types = 1);
|
||||
|
||||
namespace WapplerSystems\Meilisearch\Domain\Index\PageIndexer\Helper\UriBuilder;
|
||||
|
||||
/***************************************************************
|
||||
* Copyright notice
|
||||
*
|
||||
* (c) 2019 Timo Hund <timo.hund@dkd.de>
|
||||
* All rights reserved
|
||||
*
|
||||
* This script is part of the TYPO3 project. The TYPO3 project is
|
||||
* free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* The GNU General Public License can be found at
|
||||
* http://www.gnu.org/copyleft/gpl.html.
|
||||
* A copy is found in the textfile GPL.txt and important notices to the license
|
||||
* from the author is found in LICENSE.txt distributed with these scripts.
|
||||
*
|
||||
*
|
||||
* This script is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* This copyright notice MUST APPEAR in all copies of the script!
|
||||
***************************************************************/
|
||||
|
||||
use WapplerSystems\Meilisearch\IndexQueue\Item;
|
||||
use WapplerSystems\Meilisearch\System\Logging\SolrLogManager;
|
||||
use TYPO3\CMS\Core\Exception\SiteNotFoundException;
|
||||
use TYPO3\CMS\Core\Routing\InvalidRouteArgumentsException;
|
||||
use TYPO3\CMS\Core\Site\SiteFinder;
|
||||
use TYPO3\CMS\Core\Utility\GeneralUtility;
|
||||
|
||||
/**
|
||||
* This class is used to build the indexing url for a TYPO3 site that is managed with the TYPO3 site management.
|
||||
*
|
||||
* These sites have the pageId and language information encoded in the speaking url.
|
||||
*/
|
||||
class TYPO3SiteStrategy extends AbstractUriStrategy
|
||||
{
|
||||
/**
|
||||
* @var SiteFinder
|
||||
*/
|
||||
protected $siteFinder = null;
|
||||
|
||||
/**
|
||||
* TYPO3SiteStrategy constructor.
|
||||
* @param SolrLogManager|null $logger
|
||||
* @param SiteFinder|null $siteFinder
|
||||
*/
|
||||
public function __construct(SolrLogManager $logger = null, SiteFinder $siteFinder = null)
|
||||
{
|
||||
parent::__construct($logger);
|
||||
$this->siteFinder = $siteFinder ?? GeneralUtility::makeInstance(SiteFinder::class);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param Item $item
|
||||
* @param int $language
|
||||
* @param string $mountPointParameter
|
||||
* @return string
|
||||
* @throws SiteNotFoundException
|
||||
* @throws InvalidRouteArgumentsException
|
||||
*/
|
||||
protected function buildPageIndexingUriFromPageItemAndLanguageId(Item $item, int $language = 0, string $mountPointParameter = '')
|
||||
{
|
||||
$site = $this->siteFinder->getSiteByPageId((int)$item->getRecordUid());
|
||||
$parameters = [];
|
||||
|
||||
if ($language > 0) {
|
||||
$parameters['_language'] = $language;
|
||||
};
|
||||
|
||||
if ($mountPointParameter !== '') {
|
||||
$parameters['MP'] = $mountPointParameter;
|
||||
}
|
||||
|
||||
$pageIndexUri = (string)$site->getRouter()->generateUri($item->getRecord(), $parameters);
|
||||
return $pageIndexUri;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
<?php declare(strict_types = 1);
|
||||
|
||||
namespace WapplerSystems\Meilisearch\Domain\Index\PageIndexer\Helper;
|
||||
|
||||
/***************************************************************
|
||||
* Copyright notice
|
||||
*
|
||||
* (c) 2019 Timo Hund <timo.hund@dkd.de>
|
||||
* All rights reserved
|
||||
*
|
||||
* This script is part of the TYPO3 project. The TYPO3 project is
|
||||
* free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* The GNU General Public License can be found at
|
||||
* http://www.gnu.org/copyleft/gpl.html.
|
||||
* A copy is found in the textfile GPL.txt and important notices to the license
|
||||
* from the author is found in LICENSE.txt distributed with these scripts.
|
||||
*
|
||||
*
|
||||
* This script is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* This copyright notice MUST APPEAR in all copies of the script!
|
||||
***************************************************************/
|
||||
|
||||
use WapplerSystems\Meilisearch\Domain\Index\PageIndexer\Helper\UriBuilder\AbstractUriStrategy;
|
||||
use WapplerSystems\Meilisearch\Domain\Index\PageIndexer\Helper\UriBuilder\TYPO3SiteStrategy;
|
||||
use WapplerSystems\Meilisearch\System\Util\SiteUtility;
|
||||
use TYPO3\CMS\Core\Utility\GeneralUtility;
|
||||
|
||||
/**
|
||||
* This class is responsible to retrieve an "UriStrategy" the can build uri's for the site where the
|
||||
* passed page belongs to.
|
||||
*
|
||||
* This can be:
|
||||
* * A TYPO3 site managed with site management
|
||||
* * A TYPO3 site without site management where the url is build by EXT:meilisearch with L and id param and information from the domain
|
||||
* record or solr specific configuration.
|
||||
*/
|
||||
class UriStrategyFactory
|
||||
{
|
||||
/**
|
||||
* @param integer $pageId
|
||||
* @oaram array $overrideConfiguration
|
||||
* @return AbstractUriStrategy
|
||||
* @throws \Exception
|
||||
*/
|
||||
public function getForPageId(int $pageId): AbstractUriStrategy
|
||||
{
|
||||
if (!SiteUtility::getIsSiteManagedSite($pageId)) {
|
||||
throw new \Exception('Site of page with uid ' . $pageId . ' is not a TYPO3 managed site');
|
||||
}
|
||||
|
||||
return GeneralUtility::makeInstance(TYPO3SiteStrategy::class);
|
||||
}
|
||||
}
|
||||
175
Classes/Domain/Index/Queue/GarbageRemover/AbstractStrategy.php
Normal file
175
Classes/Domain/Index/Queue/GarbageRemover/AbstractStrategy.php
Normal file
@@ -0,0 +1,175 @@
|
||||
<?php
|
||||
|
||||
namespace WapplerSystems\Meilisearch\Domain\Index\Queue\GarbageRemover;
|
||||
|
||||
/***************************************************************
|
||||
* Copyright notice
|
||||
*
|
||||
* (c) 2018 - Timo Hund <timo.hund@dkd.de>
|
||||
* All rights reserved
|
||||
*
|
||||
* This script is part of the TYPO3 project. The TYPO3 project is
|
||||
* free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* The GNU General Public License can be found at
|
||||
* http://www.gnu.org/copyleft/gpl.html.
|
||||
*
|
||||
* This script is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* This copyright notice MUST APPEAR in all copies of the script!
|
||||
***************************************************************/
|
||||
|
||||
use WapplerSystems\Meilisearch\ConnectionManager;
|
||||
use WapplerSystems\Meilisearch\GarbageCollectorPostProcessor;
|
||||
use WapplerSystems\Meilisearch\IndexQueue\Queue;
|
||||
use WapplerSystems\Meilisearch\System\Solr\SolrConnection;
|
||||
use TYPO3\CMS\Core\Utility\GeneralUtility;
|
||||
|
||||
/**
|
||||
* An implementation ob a garbage remover strategy is responsible to remove all garbage from the index queue and
|
||||
* the solr server for a certain table and uid combination.
|
||||
*/
|
||||
abstract class AbstractStrategy
|
||||
{
|
||||
|
||||
/**
|
||||
* @var Queue
|
||||
*/
|
||||
protected $queue;
|
||||
|
||||
/**
|
||||
* @var ConnectionManager
|
||||
*/
|
||||
protected $connectionManager;
|
||||
|
||||
/**
|
||||
* AbstractStrategy constructor.
|
||||
* @param Queue|null $queue
|
||||
* @param ConnectionManager|null $connectionManager
|
||||
*/
|
||||
public function __construct(Queue $queue = null, ConnectionManager $connectionManager = null)
|
||||
{
|
||||
$this->queue = $queue ?? GeneralUtility::makeInstance(Queue::class);
|
||||
$this->connectionManager = $connectionManager ?? GeneralUtility::makeInstance(ConnectionManager::class);
|
||||
}
|
||||
|
||||
/**
|
||||
* Call's the removal of the strategy and afterwards the garbagecollector post processing hook.
|
||||
*
|
||||
* @param string $table
|
||||
* @param int $uid
|
||||
* @return mixed
|
||||
*/
|
||||
public function removeGarbageOf($table, $uid)
|
||||
{
|
||||
$this->removeGarbageOfByStrategy($table, $uid);
|
||||
$this->callPostProcessGarbageCollectorHook($table, $uid);
|
||||
}
|
||||
|
||||
/**
|
||||
* A implementation of the GarbageCollection strategy is responsible to remove the garbage from
|
||||
* the indexqueue and from the solr server.
|
||||
*
|
||||
* @param string $table
|
||||
* @param int $uid
|
||||
* @return mixed
|
||||
*/
|
||||
abstract protected function removeGarbageOfByStrategy($table, $uid);
|
||||
|
||||
/**
|
||||
* Deletes a document from solr and from the index queue.
|
||||
*
|
||||
* @param string $table
|
||||
* @param integer $uid
|
||||
*/
|
||||
protected function deleteInSolrAndRemoveFromIndexQueue($table, $uid)
|
||||
{
|
||||
$this->deleteIndexDocuments($table, $uid);
|
||||
$this->queue->deleteItem($table, $uid);
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes a document from solr and updates the item in the index queue (e.g. on page content updates).
|
||||
*
|
||||
* @param string $table
|
||||
* @param integer $uid
|
||||
*/
|
||||
protected function deleteInSolrAndUpdateIndexQueue($table, $uid)
|
||||
{
|
||||
$this->deleteIndexDocuments($table, $uid);
|
||||
$this->queue->updateItem($table, $uid);
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes index documents for a given record identification.
|
||||
*
|
||||
* @param string $table The record's table name.
|
||||
* @param int $uid The record's uid.
|
||||
*/
|
||||
protected function deleteIndexDocuments($table, $uid, $language = 0)
|
||||
{
|
||||
// record can be indexed for multiple sites
|
||||
$indexQueueItems = $this->queue->getItems($table, $uid);
|
||||
foreach ($indexQueueItems as $indexQueueItem) {
|
||||
$site = $indexQueueItem->getSite();
|
||||
$enableCommitsSetting = $site->getSolrConfiguration()->getEnableCommits();
|
||||
$siteHash = $site->getSiteHash();
|
||||
// a site can have multiple connections (cores / languages)
|
||||
$solrConnections = $this->connectionManager->getConnectionsBySite($site);
|
||||
if ($language > 0) {
|
||||
$solrConnections = [$language => $solrConnections[$language]];
|
||||
}
|
||||
$this->deleteRecordInAllSolrConnections($table, $uid, $solrConnections, $siteHash, $enableCommitsSetting);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes the record in all solr connections from that site.
|
||||
*
|
||||
* @param string $table
|
||||
* @param int $uid
|
||||
* @param SolrConnection[] $solrConnections
|
||||
* @param string $siteHash
|
||||
* @param boolean $enableCommitsSetting
|
||||
*/
|
||||
protected function deleteRecordInAllSolrConnections($table, $uid, $solrConnections, $siteHash, $enableCommitsSetting)
|
||||
{
|
||||
foreach ($solrConnections as $solr) {
|
||||
$solr->getWriteService()->deleteByQuery('type:' . $table . ' AND uid:' . (int)$uid . ' AND siteHash:' . $siteHash);
|
||||
if ($enableCommitsSetting) {
|
||||
$solr->getWriteService()->commit(false, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calls the registered post processing hooks after the garbageCollection.
|
||||
*
|
||||
* @param string $table
|
||||
* @param int $uid
|
||||
*/
|
||||
protected function callPostProcessGarbageCollectorHook($table, $uid)
|
||||
{
|
||||
if (!is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['postProcessGarbageCollector'])) {
|
||||
return;
|
||||
}
|
||||
|
||||
foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['postProcessGarbageCollector'] as $classReference) {
|
||||
$garbageCollectorPostProcessor = GeneralUtility::makeInstance($classReference);
|
||||
|
||||
if ($garbageCollectorPostProcessor instanceof GarbageCollectorPostProcessor) {
|
||||
$garbageCollectorPostProcessor->postProcessGarbageCollector($table, $uid);
|
||||
} else {
|
||||
$message = get_class($garbageCollectorPostProcessor) . ' must implement interface ' .
|
||||
GarbageCollectorPostProcessor::class;
|
||||
throw new \UnexpectedValueException($message, 1345807460);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
80
Classes/Domain/Index/Queue/GarbageRemover/PageStrategy.php
Normal file
80
Classes/Domain/Index/Queue/GarbageRemover/PageStrategy.php
Normal file
@@ -0,0 +1,80 @@
|
||||
<?php
|
||||
|
||||
namespace WapplerSystems\Meilisearch\Domain\Index\Queue\GarbageRemover;
|
||||
|
||||
/***************************************************************
|
||||
* Copyright notice
|
||||
*
|
||||
* (c) 2018 - Timo Hund <timo.hund@dkd.de>
|
||||
* All rights reserved
|
||||
*
|
||||
* This script is part of the TYPO3 project. The TYPO3 project is
|
||||
* free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* The GNU General Public License can be found at
|
||||
* http://www.gnu.org/copyleft/gpl.html.
|
||||
*
|
||||
* This script is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* This copyright notice MUST APPEAR in all copies of the script!
|
||||
***************************************************************/
|
||||
|
||||
use TYPO3\CMS\Backend\Utility\BackendUtility;
|
||||
|
||||
|
||||
/**
|
||||
* Class PageStrategy
|
||||
*/
|
||||
class PageStrategy extends AbstractStrategy {
|
||||
|
||||
/**
|
||||
* @param string $table
|
||||
* @param int $uid
|
||||
* @return mixed
|
||||
*/
|
||||
protected function removeGarbageOfByStrategy($table, $uid)
|
||||
{
|
||||
if ($table === 'tt_content') {
|
||||
$this->collectPageGarbageByContentChange($uid);
|
||||
return;
|
||||
}
|
||||
|
||||
if ($table === 'pages') {
|
||||
$this->collectPageGarbageByPageChange($uid);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines the relevant page id for an content element update. Deletes the page from solr and requeues the
|
||||
* page for a reindex.
|
||||
*
|
||||
* @param int $ttContentUid
|
||||
*/
|
||||
protected function collectPageGarbageByContentChange($ttContentUid)
|
||||
{
|
||||
$contentElement = BackendUtility::getRecord('tt_content', $ttContentUid, 'uid, pid', '', false);
|
||||
$this->deleteInSolrAndUpdateIndexQueue('pages', $contentElement['pid']);
|
||||
}
|
||||
|
||||
/**
|
||||
* When a page was changed it is removed from the index and index queue.
|
||||
*
|
||||
* @param int $uid
|
||||
*/
|
||||
protected function collectPageGarbageByPageChange($uid)
|
||||
{
|
||||
$pageOverlay = BackendUtility::getRecord('pages', $uid, 'l10n_parent, sys_language_uid', '', false);
|
||||
if (!empty($pageOverlay['l10n_parent']) && intval($pageOverlay['l10n_parent']) !== 0) {
|
||||
$this->deleteIndexDocuments('pages', (int)$pageOverlay['l10n_parent'], (int)$pageOverlay['sys_language_uid']);
|
||||
} else {
|
||||
$this->deleteInSolrAndRemoveFromIndexQueue('pages', $uid);
|
||||
}
|
||||
}
|
||||
}
|
||||
43
Classes/Domain/Index/Queue/GarbageRemover/RecordStrategy.php
Normal file
43
Classes/Domain/Index/Queue/GarbageRemover/RecordStrategy.php
Normal file
@@ -0,0 +1,43 @@
|
||||
<?php
|
||||
|
||||
namespace WapplerSystems\Meilisearch\Domain\Index\Queue\GarbageRemover;
|
||||
|
||||
/***************************************************************
|
||||
* Copyright notice
|
||||
*
|
||||
* (c) 2018 - Timo Hund <timo.hund@dkd.de>
|
||||
* All rights reserved
|
||||
*
|
||||
* This script is part of the TYPO3 project. The TYPO3 project is
|
||||
* free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* The GNU General Public License can be found at
|
||||
* http://www.gnu.org/copyleft/gpl.html.
|
||||
*
|
||||
* This script is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* This copyright notice MUST APPEAR in all copies of the script!
|
||||
***************************************************************/
|
||||
/**
|
||||
* Class RecordStrategy
|
||||
*/
|
||||
class RecordStrategy extends AbstractStrategy {
|
||||
|
||||
/**
|
||||
* Removes the garbage of a record.
|
||||
*
|
||||
* @param string $table
|
||||
* @param int $uid
|
||||
* @return mixed
|
||||
*/
|
||||
protected function removeGarbageOfByStrategy($table, $uid)
|
||||
{
|
||||
$this->deleteInSolrAndRemoveFromIndexQueue($table, $uid);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
<?php
|
||||
|
||||
namespace WapplerSystems\Meilisearch\Domain\Index\Queue\GarbageRemover;
|
||||
|
||||
/***************************************************************
|
||||
* Copyright notice
|
||||
*
|
||||
* (c) 2018 - Timo Hund <timo.hund@dkd.de>
|
||||
* All rights reserved
|
||||
*
|
||||
* This script is part of the TYPO3 project. The TYPO3 project is
|
||||
* free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* The GNU General Public License can be found at
|
||||
* http://www.gnu.org/copyleft/gpl.html.
|
||||
*
|
||||
* This script is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* This copyright notice MUST APPEAR in all copies of the script!
|
||||
***************************************************************/
|
||||
|
||||
/**
|
||||
* Class StrategyFactory
|
||||
*/
|
||||
class StrategyFactory {
|
||||
|
||||
/**
|
||||
* @param string $table
|
||||
* @return PageStrategy|RecordStrategy
|
||||
*/
|
||||
public static function getByTable($table)
|
||||
{
|
||||
$isPageRelated = in_array($table, ['tt_content','pages']);
|
||||
return $isPageRelated ? new PageStrategy() : new RecordStrategy();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
<?php declare(strict_types = 1);
|
||||
namespace WapplerSystems\Meilisearch\Domain\Index\Queue;
|
||||
|
||||
/***************************************************************
|
||||
* Copyright notice
|
||||
*
|
||||
* (c) 2010-2017 dkd Internet Service GmbH <solr-support@dkd.de>
|
||||
* All rights reserved
|
||||
*
|
||||
* This script is part of the TYPO3 project. The TYPO3 project is
|
||||
* free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* The GNU General Public License can be found at
|
||||
* http://www.gnu.org/copyleft/gpl.html.
|
||||
*
|
||||
* This script is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* This copyright notice MUST APPEAR in all copies of the script!
|
||||
***************************************************************/
|
||||
|
||||
use WapplerSystems\Meilisearch\System\Records\AbstractRepository;
|
||||
|
||||
/**
|
||||
* Class IndexQueueIndexingPropertyRepository
|
||||
* Handles all CRUD operations to tx_meilisearch_indexqueue_indexing_property table
|
||||
*/
|
||||
class IndexQueueIndexingPropertyRepository extends AbstractRepository
|
||||
{
|
||||
/**
|
||||
* @var string
|
||||
*/
|
||||
protected $table = 'tx_meilisearch_indexqueue_indexing_property';
|
||||
|
||||
/**
|
||||
* Removes existing indexing properties.
|
||||
*
|
||||
* @param int $rootPid
|
||||
* @param int $indexQueueUid
|
||||
* @return int
|
||||
*/
|
||||
public function removeByRootPidAndIndexQueueUid(int $rootPid, int $indexQueueUid) : int
|
||||
{
|
||||
$queryBuider = $this->getQueryBuilder();
|
||||
return $queryBuider
|
||||
->delete($this->table)
|
||||
->where(
|
||||
$queryBuider->expr()->eq('root', $queryBuider->createNamedParameter($rootPid, \PDO::PARAM_INT)),
|
||||
$queryBuider->expr()->eq('item_id', $queryBuider->createNamedParameter($indexQueueUid, \PDO::PARAM_INT))
|
||||
)->execute();
|
||||
}
|
||||
|
||||
/**
|
||||
* Inserts a list of given properties
|
||||
*
|
||||
* @param array $properties assoc array with column names as key
|
||||
* @return int
|
||||
*/
|
||||
public function bulkInsert(array $properties) : int
|
||||
{
|
||||
return $this->getQueryBuilder()->getConnection()->bulkInsert($this->table, $properties, ['root', 'item_id', 'property_key', 'property_value']);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches a list of properties related to index queue item
|
||||
*
|
||||
* @param int $indexQueueUid
|
||||
* @return array list of records for searched index queue item
|
||||
*/
|
||||
public function findAllByIndexQueueUid(int $indexQueueUid) : array
|
||||
{
|
||||
$queryBuider = $this->getQueryBuilder();
|
||||
return $queryBuider
|
||||
->select('property_key', 'property_value')
|
||||
->from($this->table)
|
||||
->where(
|
||||
$queryBuider->expr()->eq('item_id', $queryBuider->createNamedParameter($indexQueueUid, \PDO::PARAM_INT))
|
||||
)
|
||||
->execute()->fetchAll();
|
||||
}
|
||||
}
|
||||
162
Classes/Domain/Index/Queue/QueueInitializationService.php
Normal file
162
Classes/Domain/Index/Queue/QueueInitializationService.php
Normal file
@@ -0,0 +1,162 @@
|
||||
<?php declare(strict_types = 1);
|
||||
namespace WapplerSystems\Meilisearch\Domain\Index\Queue;
|
||||
|
||||
/***************************************************************
|
||||
* Copyright notice
|
||||
*
|
||||
* (c) 2010-2017 dkd Internet Service GmbH <solr-support@dkd.de>
|
||||
* All rights reserved
|
||||
*
|
||||
* This script is part of the TYPO3 project. The TYPO3 project is
|
||||
* free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* The GNU General Public License can be found at
|
||||
* http://www.gnu.org/copyleft/gpl.html.
|
||||
*
|
||||
* This script is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* This copyright notice MUST APPEAR in all copies of the script!
|
||||
***************************************************************/
|
||||
|
||||
use WapplerSystems\Meilisearch\IndexQueue\InitializationPostProcessor;
|
||||
use WapplerSystems\Meilisearch\IndexQueue\Queue;
|
||||
use WapplerSystems\Meilisearch\Domain\Site\Site;
|
||||
use TYPO3\CMS\Core\Utility\GeneralUtility;
|
||||
|
||||
/**
|
||||
* The queue initialization service is responsible to run the initialization of the index queue for a combination of sites
|
||||
* and index queue configurations.
|
||||
*
|
||||
* @author Timo Hund <timo.hund@dkd.de>
|
||||
* @author Ingo Renner <ingo.renner@dkd.de>
|
||||
*/
|
||||
class QueueInitializationService {
|
||||
|
||||
/**
|
||||
* @var Queue
|
||||
*/
|
||||
protected $queue;
|
||||
|
||||
/**
|
||||
* QueueInitializationService constructor.
|
||||
*/
|
||||
public function __construct(Queue $queue)
|
||||
{
|
||||
$this->queue = $queue;
|
||||
}
|
||||
|
||||
/**
|
||||
* Truncate and rebuild the tx_meilisearch_indexqueue_item table. This is the most
|
||||
* complete way to force reindexing, or to build the Index Queue for the
|
||||
* first time. The Index Queue initialization is site-specific.
|
||||
*
|
||||
* @param Site $site The site to initialize
|
||||
* @param string $indexingConfigurationName Name of a specific indexing configuration, when * is passed any is used
|
||||
* @return array An array of booleans, each representing whether the
|
||||
* initialization for an indexing configuration was successful
|
||||
*/
|
||||
public function initializeBySiteAndIndexConfiguration(Site $site, $indexingConfigurationName = '*'): array
|
||||
{
|
||||
return $this->initializeBySiteAndIndexConfigurations($site, [$indexingConfigurationName]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Truncates and rebuilds the tx_meilisearch_indexqueue_item table for a set of sites and a set of index configurations.
|
||||
*
|
||||
* @param array $sites The array of sites to initialize
|
||||
* @param array $indexingConfigurationNames the array of index configurations to initialize.
|
||||
* @return array
|
||||
*/
|
||||
public function initializeBySitesAndConfigurations(array $sites, array $indexingConfigurationNames = ['*']): array
|
||||
{
|
||||
$initializationStatesBySiteId = [];
|
||||
foreach($sites as $site) {
|
||||
/** @var Site $site */
|
||||
$initializationResult = $this->initializeBySiteAndIndexConfigurations($site, $indexingConfigurationNames);
|
||||
$initializationStatesBySiteId[$site->getRootPageId()] = $initializationResult;
|
||||
}
|
||||
|
||||
return $initializationStatesBySiteId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes a set index configurations for a given site.
|
||||
*
|
||||
* @param Site $site
|
||||
* @param array $indexingConfigurationNames if one of the names is a * (wildcard) all configurations are used,
|
||||
* @return array
|
||||
*/
|
||||
public function initializeBySiteAndIndexConfigurations(Site $site, array $indexingConfigurationNames): array
|
||||
{
|
||||
$initializationStatus = [];
|
||||
|
||||
$hasWildcardConfiguration = in_array('*', $indexingConfigurationNames);
|
||||
$indexingConfigurationNames = $hasWildcardConfiguration ? $site->getSolrConfiguration()->getEnabledIndexQueueConfigurationNames() : $indexingConfigurationNames;
|
||||
foreach ($indexingConfigurationNames as $indexingConfigurationName) {
|
||||
$initializationStatus[$indexingConfigurationName] = $this->applyInitialization($site, (string)$indexingConfigurationName);
|
||||
}
|
||||
|
||||
if (!is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['postProcessIndexQueueInitialization'])) {
|
||||
return $initializationStatus;
|
||||
}
|
||||
|
||||
foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['postProcessIndexQueueInitialization'] as $classReference) {
|
||||
$indexQueueInitializationPostProcessor = GeneralUtility::makeInstance($classReference);
|
||||
if ($indexQueueInitializationPostProcessor instanceof InitializationPostProcessor) {
|
||||
$indexQueueInitializationPostProcessor->postProcessIndexQueueInitialization($site, $indexingConfigurationNames, $initializationStatus);
|
||||
} else {
|
||||
throw new \UnexpectedValueException(get_class($indexQueueInitializationPostProcessor) . ' must implement interface ' . InitializationPostProcessor::class, 1345815561);
|
||||
}
|
||||
}
|
||||
|
||||
return $initializationStatus;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes the Index Queue for a specific indexing configuration.
|
||||
*
|
||||
* @param Site $site The site to initialize
|
||||
* @param string $indexingConfigurationName name of a specific
|
||||
* indexing configuration
|
||||
* @return bool TRUE if the initialization was successful, FALSE otherwise
|
||||
*/
|
||||
protected function applyInitialization(Site $site, $indexingConfigurationName): bool
|
||||
{
|
||||
// clear queue
|
||||
$this->queue->deleteItemsBySite($site, $indexingConfigurationName);
|
||||
|
||||
$solrConfiguration = $site->getSolrConfiguration();
|
||||
$tableToIndex = $solrConfiguration->getIndexQueueTableNameOrFallbackToConfigurationName($indexingConfigurationName);
|
||||
$initializerClass = $solrConfiguration->getIndexQueueInitializerClassByConfigurationName($indexingConfigurationName);
|
||||
$indexConfiguration = $solrConfiguration->getIndexQueueConfigurationByName($indexingConfigurationName);
|
||||
|
||||
return $this->executeInitializer($site, $indexingConfigurationName, $initializerClass, $tableToIndex, $indexConfiguration);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param Site $site
|
||||
* @param string $indexingConfigurationName
|
||||
* @param string $initializerClass
|
||||
* @param string $tableToIndex
|
||||
* @param array $indexConfiguration
|
||||
* @return bool
|
||||
*/
|
||||
protected function executeInitializer(Site $site, $indexingConfigurationName, $initializerClass, $tableToIndex, $indexConfiguration): bool
|
||||
{
|
||||
$initializer = GeneralUtility::makeInstance($initializerClass);
|
||||
/** @var $initializer \WapplerSystems\Meilisearch\IndexQueue\Initializer\AbstractInitializer */
|
||||
$initializer->setSite($site);
|
||||
$initializer->setType($tableToIndex);
|
||||
$initializer->setIndexingConfigurationName($indexingConfigurationName);
|
||||
$initializer->setIndexingConfiguration($indexConfiguration);
|
||||
|
||||
return $initializer->initialize();
|
||||
}
|
||||
|
||||
}
|
||||
902
Classes/Domain/Index/Queue/QueueItemRepository.php
Normal file
902
Classes/Domain/Index/Queue/QueueItemRepository.php
Normal file
@@ -0,0 +1,902 @@
|
||||
<?php declare(strict_types = 1);
|
||||
namespace WapplerSystems\Meilisearch\Domain\Index\Queue;
|
||||
|
||||
/***************************************************************
|
||||
* Copyright notice
|
||||
*
|
||||
* (c) 2010-2017 dkd Internet Service GmbH <solr-support@dkd.de>
|
||||
* All rights reserved
|
||||
*
|
||||
* This script is part of the TYPO3 project. The TYPO3 project is
|
||||
* free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* The GNU General Public License can be found at
|
||||
* http://www.gnu.org/copyleft/gpl.html.
|
||||
*
|
||||
* This script is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* This copyright notice MUST APPEAR in all copies of the script!
|
||||
***************************************************************/
|
||||
|
||||
use WapplerSystems\Meilisearch\IndexQueue\Item;
|
||||
use WapplerSystems\Meilisearch\Domain\Site\Site;
|
||||
use WapplerSystems\Meilisearch\System\Logging\SolrLogManager;
|
||||
use WapplerSystems\Meilisearch\System\Records\AbstractRepository;
|
||||
use Doctrine\DBAL\DBALException;
|
||||
use TYPO3\CMS\Core\Database\ConnectionPool;
|
||||
use TYPO3\CMS\Core\Database\Query\Expression\CompositeExpression;
|
||||
use TYPO3\CMS\Core\Database\Query\QueryBuilder;
|
||||
use TYPO3\CMS\Core\Utility\GeneralUtility;
|
||||
|
||||
/**
|
||||
* Class QueueItemRepository
|
||||
* Handles all CRUD operations to tx_meilisearch_indexqueue_item table
|
||||
*
|
||||
*/
|
||||
class QueueItemRepository extends AbstractRepository
|
||||
{
|
||||
/**
|
||||
* @var string
|
||||
*/
|
||||
protected $table = 'tx_meilisearch_indexqueue_item';
|
||||
|
||||
/**
|
||||
* @var SolrLogManager
|
||||
*/
|
||||
protected $logger;
|
||||
|
||||
/**
|
||||
* QueueItemRepository constructor.
|
||||
*
|
||||
* @param SolrLogManager|null $logManager
|
||||
*/
|
||||
public function __construct(SolrLogManager $logManager = null)
|
||||
{
|
||||
$this->logger = $logManager ?? GeneralUtility::makeInstance(SolrLogManager::class, /** @scrutinizer ignore-type */ __CLASS__);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches the last indexed row
|
||||
*
|
||||
* @param int $rootPageId The root page uid for which to get the last indexed row
|
||||
* @return array
|
||||
*/
|
||||
public function findLastIndexedRow(int $rootPageId) : array
|
||||
{
|
||||
$queryBuilder = $this->getQueryBuilder();
|
||||
$row = $queryBuilder
|
||||
->select('uid', 'indexed')
|
||||
->from($this->table)
|
||||
->where($queryBuilder->expr()->eq('root', $rootPageId))
|
||||
->andWhere($queryBuilder->expr()->neq('indexed', 0))
|
||||
->orderBy('indexed', 'DESC')
|
||||
->setMaxResults(1)
|
||||
->execute()->fetchAll();
|
||||
|
||||
return $row;
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds indexing errors for the current site
|
||||
*
|
||||
* @param Site $site
|
||||
* @return array Error items for the current site's Index Queue
|
||||
*/
|
||||
public function findErrorsBySite(Site $site) : array
|
||||
{
|
||||
$queryBuilder = $this->getQueryBuilder();
|
||||
$errors = $queryBuilder
|
||||
->select('uid', 'item_type', 'item_uid', 'errors')
|
||||
->from($this->table)
|
||||
->andWhere(
|
||||
$queryBuilder->expr()->notLike('errors', $queryBuilder->createNamedParameter('')),
|
||||
$queryBuilder->expr()->eq('root', $site->getRootPageId())
|
||||
)
|
||||
->execute()->fetchAll();
|
||||
|
||||
return $errors;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resets all the errors for all index queue items.
|
||||
*
|
||||
* @return int affected rows
|
||||
*/
|
||||
public function flushAllErrors() : int
|
||||
{
|
||||
$queryBuilder = $this->getQueryBuilder();
|
||||
$affectedRows = $this->getPreparedFlushErrorQuery($queryBuilder)->execute();
|
||||
return $affectedRows;
|
||||
}
|
||||
|
||||
/**
|
||||
* Flushes the errors for a single site.
|
||||
*
|
||||
* @param Site $site
|
||||
* @return int
|
||||
*/
|
||||
public function flushErrorsBySite(Site $site) : int
|
||||
{
|
||||
$queryBuilder = $this->getQueryBuilder();
|
||||
$affectedRows = $this->getPreparedFlushErrorQuery($queryBuilder)
|
||||
->andWhere(
|
||||
$queryBuilder->expr()->eq('root', (int)$site->getRootPageId())
|
||||
)
|
||||
->execute();
|
||||
return $affectedRows;
|
||||
}
|
||||
|
||||
/**
|
||||
* Flushes the error for a single item.
|
||||
*
|
||||
* @param Item $item
|
||||
* @return int affected rows
|
||||
*/
|
||||
public function flushErrorByItem(Item $item) : int
|
||||
{
|
||||
$queryBuilder = $this->getQueryBuilder();
|
||||
$affectedRows = $this->getPreparedFlushErrorQuery($queryBuilder)
|
||||
->andWhere(
|
||||
$queryBuilder->expr()->eq('uid', $item->getIndexQueueUid())
|
||||
)
|
||||
->execute();
|
||||
return $affectedRows;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes the QueryBuilder with a query the resets the error field for items that have an error.
|
||||
*
|
||||
* @return QueryBuilder
|
||||
*/
|
||||
private function getPreparedFlushErrorQuery(QueryBuilder $queryBuilder)
|
||||
{
|
||||
return $queryBuilder
|
||||
->update($this->table)
|
||||
->set('errors', '')
|
||||
->where(
|
||||
$queryBuilder->expr()->notLike('errors', $queryBuilder->createNamedParameter(''))
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates an existing queue entry by $itemType $itemUid and $rootPageId.
|
||||
*
|
||||
* @param string $itemType The item's type, usually a table name.
|
||||
* @param int $itemUid The item's uid, usually an integer uid, could be a
|
||||
* different value for non-database-record types.
|
||||
* @param string $indexingConfiguration The name of the related indexConfiguration
|
||||
* @param int $rootPageId The uid of the rootPage
|
||||
* @param int $changedTime The forced change time that should be used for updating
|
||||
* @return int affected rows
|
||||
*/
|
||||
public function updateExistingItemByItemTypeAndItemUidAndRootPageId(string $itemType, int $itemUid, int $rootPageId, int $changedTime, string $indexingConfiguration = '') : int
|
||||
{
|
||||
$queryBuilder = $this->getQueryBuilder();
|
||||
$queryBuilder
|
||||
->update($this->table)
|
||||
->set('changed', $changedTime)
|
||||
->andWhere(
|
||||
$queryBuilder->expr()->eq('item_type', $queryBuilder->createNamedParameter($itemType)),
|
||||
$queryBuilder->expr()->eq('item_uid', $itemUid),
|
||||
$queryBuilder->expr()->eq('root', $rootPageId)
|
||||
);
|
||||
|
||||
if (!empty($indexingConfiguration)) {
|
||||
$queryBuilder->set('indexing_configuration', $indexingConfiguration);
|
||||
}
|
||||
|
||||
return $queryBuilder->execute();
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds an item to the index queue.
|
||||
*
|
||||
* Not meant for public use.
|
||||
*
|
||||
* @param string $itemType The item's type, usually a table name.
|
||||
* @param int $itemUid The item's uid, usually an integer uid, could be a different value for non-database-record types.
|
||||
* @param int $rootPageId
|
||||
* @param int $changedTime
|
||||
* @param string $indexingConfiguration The item's indexing configuration to use. Optional, overwrites existing / determined configuration.
|
||||
* @return int the number of inserted rows, which is typically 1
|
||||
*/
|
||||
public function add(string $itemType, int $itemUid, int $rootPageId, int $changedTime, string $indexingConfiguration) : int
|
||||
{
|
||||
$queryBuilder = $this->getQueryBuilder();
|
||||
return $queryBuilder
|
||||
->insert($this->table)
|
||||
->values([
|
||||
'root' => $rootPageId,
|
||||
'item_type' => $itemType,
|
||||
'item_uid' => $itemUid,
|
||||
'changed' => $changedTime,
|
||||
'errors' => '',
|
||||
'indexing_configuration' => $indexingConfiguration
|
||||
])
|
||||
->execute();
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves the count of items that match certain filters. Each filter is passed as parts of the where claus combined with AND.
|
||||
*
|
||||
* @param array $sites
|
||||
* @param array $indexQueueConfigurationNames
|
||||
* @param array $itemTypes
|
||||
* @param array $itemUids
|
||||
* @param array $uids
|
||||
* @return int
|
||||
*/
|
||||
public function countItems(array $sites = [], array $indexQueueConfigurationNames = [], array $itemTypes = [], array $itemUids = [], array $uids = []): int
|
||||
{
|
||||
$rootPageIds = Site::getRootPageIdsFromSites($sites);
|
||||
$indexQueueConfigurationList = implode(",", $indexQueueConfigurationNames);
|
||||
$itemTypeList = implode(",", $itemTypes);
|
||||
$itemUids = array_map("intval", $itemUids);
|
||||
$uids = array_map("intval", $uids);
|
||||
|
||||
$queryBuilderForCountingItems = $this->getQueryBuilder();
|
||||
$queryBuilderForCountingItems->count('uid')->from($this->table);
|
||||
$queryBuilderForCountingItems = $this->addItemWhereClauses($queryBuilderForCountingItems, $rootPageIds, $indexQueueConfigurationList, $itemTypeList, $itemUids, $uids);
|
||||
|
||||
return (int)$queryBuilderForCountingItems->execute()->fetchColumn(0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the most recent changed time of a page's content elements
|
||||
*
|
||||
* @param int $pageUid
|
||||
* @return int|null Timestamp of the most recent content element change or null if nothing is found.
|
||||
*/
|
||||
public function getPageItemChangedTimeByPageUid(int $pageUid)
|
||||
{
|
||||
$queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('tt_content');
|
||||
$queryBuilder->getRestrictions()->removeAll();
|
||||
$pageContentLastChangedTime = $queryBuilder
|
||||
->add('select', $queryBuilder->expr()->max('tstamp', 'changed_time'))
|
||||
->from('tt_content')
|
||||
->where(
|
||||
$queryBuilder->expr()->eq('pid', $pageUid)
|
||||
)
|
||||
->execute()->fetch();
|
||||
|
||||
return $pageContentLastChangedTime['changed_time'];
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the most recent changed time for an item taking into account
|
||||
* localized records.
|
||||
*
|
||||
* @param string $itemType The item's type, usually a table name.
|
||||
* @param int $itemUid The item's uid
|
||||
* @return int Timestamp of the most recent content element change
|
||||
*/
|
||||
public function getLocalizableItemChangedTime(string $itemType, int $itemUid) : int
|
||||
{
|
||||
$localizedChangedTime = 0;
|
||||
|
||||
if (isset($GLOBALS['TCA'][$itemType]['ctrl']['transOrigPointerField'])) {
|
||||
// table is localizable
|
||||
$translationOriginalPointerField = $GLOBALS['TCA'][$itemType]['ctrl']['transOrigPointerField'];
|
||||
|
||||
$queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable($itemType);
|
||||
$queryBuilder->getRestrictions()->removeAll();
|
||||
$localizedChangedTime = $queryBuilder
|
||||
->add('select', $queryBuilder->expr()->max('tstamp', 'changed_time'))
|
||||
->from($itemType)
|
||||
->orWhere(
|
||||
$queryBuilder->expr()->eq('uid', $itemUid),
|
||||
$queryBuilder->expr()->eq($translationOriginalPointerField, $itemUid)
|
||||
)->execute()->fetchColumn(0);
|
||||
}
|
||||
|
||||
return (int)$localizedChangedTime;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns prepared QueryBuilder for contains* methods in this repository
|
||||
*
|
||||
* @param string $itemType
|
||||
* @param int $itemUid
|
||||
* @return QueryBuilder
|
||||
*/
|
||||
protected function getQueryBuilderForContainsMethods(string $itemType, int $itemUid) : QueryBuilder
|
||||
{
|
||||
$queryBuilder = $this->getQueryBuilder();
|
||||
return $queryBuilder->count('uid')->from($this->table)
|
||||
->andWhere(
|
||||
$queryBuilder->expr()->eq('item_type', $queryBuilder->createNamedParameter($itemType)),
|
||||
$queryBuilder->expr()->eq('item_uid', $itemUid)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks whether the Index Queue contains a specific item.
|
||||
*
|
||||
* @param string $itemType The item's type, usually a table name.
|
||||
* @param int $itemUid The item's uid
|
||||
* @return bool TRUE if the item is found in the queue, FALSE otherwise
|
||||
*/
|
||||
public function containsItem(string $itemType, int $itemUid) : bool
|
||||
{
|
||||
return (bool)$this->getQueryBuilderForContainsMethods($itemType, $itemUid)->execute()->fetchColumn(0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks whether the Index Queue contains a specific item.
|
||||
*
|
||||
* @param string $itemType The item's type, usually a table name.
|
||||
* @param int $itemUid The item's uid
|
||||
* @param integer $rootPageId
|
||||
* @return bool TRUE if the item is found in the queue, FALSE otherwise
|
||||
*/
|
||||
public function containsItemWithRootPageId(string $itemType, int $itemUid, int $rootPageId) : bool
|
||||
{
|
||||
$queryBuilder = $this->getQueryBuilderForContainsMethods($itemType, $itemUid);
|
||||
return (bool)$queryBuilder
|
||||
->andWhere($queryBuilder->expr()->eq('root', $rootPageId))
|
||||
->execute()->fetchColumn(0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks whether the Index Queue contains a specific item that has been
|
||||
* marked as indexed.
|
||||
*
|
||||
* @param string $itemType The item's type, usually a table name.
|
||||
* @param int $itemUid The item's uid
|
||||
* @return bool TRUE if the item is found in the queue and marked as
|
||||
* indexed, FALSE otherwise
|
||||
*/
|
||||
public function containsIndexedItem(string $itemType, int $itemUid) : bool
|
||||
{
|
||||
$queryBuilder = $this->getQueryBuilderForContainsMethods($itemType, $itemUid);
|
||||
return (bool)$queryBuilder
|
||||
->andWhere($queryBuilder->expr()->gt('indexed', 0))
|
||||
->execute()->fetchColumn(0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes an item from the Index Queue.
|
||||
*
|
||||
* @param string $itemType The type of the item to remove, usually a table name.
|
||||
* @param int $itemUid The uid of the item to remove
|
||||
*/
|
||||
public function deleteItem(string $itemType, int $itemUid = null)
|
||||
{
|
||||
$itemUids = empty($itemUid) ? [] : [$itemUid];
|
||||
$this->deleteItems([], [], [$itemType], $itemUids);
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes all items of a certain type from the Index Queue.
|
||||
*
|
||||
* @param string $itemType The type of items to remove, usually a table name.
|
||||
*/
|
||||
public function deleteItemsByType(string $itemType)
|
||||
{
|
||||
$this->deleteItem($itemType);
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes all items of a certain site from the Index Queue. Accepts an
|
||||
* optional parameter to limit the deleted items by indexing configuration.
|
||||
*
|
||||
* @param Site $site The site to remove items for.
|
||||
* @param string $indexingConfigurationName Name of a specific indexing configuration
|
||||
* @throws \Exception
|
||||
*/
|
||||
public function deleteItemsBySite(Site $site, string $indexingConfigurationName = '')
|
||||
{
|
||||
$indexingConfigurationNames = empty($indexingConfigurationName) ? [] : [$indexingConfigurationName];
|
||||
$this->deleteItems([$site], $indexingConfigurationNames);
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes items in the index queue filtered by the passed arguments.
|
||||
*
|
||||
* @param array $sites
|
||||
* @param array $indexQueueConfigurationNames
|
||||
* @param array $itemTypes
|
||||
* @param array $itemUids
|
||||
* @param array $uids
|
||||
* @throws \Exception
|
||||
*/
|
||||
public function deleteItems(array $sites = [], array $indexQueueConfigurationNames = [], array $itemTypes = [], array $itemUids = [], array $uids = [])
|
||||
{
|
||||
$rootPageIds = Site::getRootPageIdsFromSites($sites);
|
||||
$indexQueueConfigurationList = implode(",", $indexQueueConfigurationNames);
|
||||
$itemTypeList = implode(",", $itemTypes);
|
||||
$itemUids = array_map("intval", $itemUids);
|
||||
$uids = array_map("intval", $uids);
|
||||
|
||||
$queryBuilderForDeletingItems = $this->getQueryBuilder();
|
||||
$queryBuilderForDeletingItems->delete($this->table);
|
||||
$queryBuilderForDeletingItems = $this->addItemWhereClauses($queryBuilderForDeletingItems, $rootPageIds, $indexQueueConfigurationList, $itemTypeList, $itemUids, $uids);
|
||||
|
||||
$queryBuilderForDeletingProperties = $this->buildQueryForPropertyDeletion($queryBuilderForDeletingItems, $rootPageIds, $indexQueueConfigurationList, $itemTypeList, $itemUids, $uids);
|
||||
|
||||
$queryBuilderForDeletingItems->getConnection()->beginTransaction();
|
||||
try {
|
||||
$queryBuilderForDeletingItems->execute();
|
||||
$queryBuilderForDeletingProperties->execute();
|
||||
|
||||
$queryBuilderForDeletingItems->getConnection()->commit();
|
||||
} catch (\Exception $e) {
|
||||
$queryBuilderForDeletingItems->getConnection()->rollback();
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes the query builder to delete items in the index queue filtered by the passed arguments.
|
||||
*
|
||||
* @param array $rootPageIds filter on a set of rootPageUids.
|
||||
* @param string $indexQueueConfigurationList
|
||||
* @param string $itemTypeList
|
||||
* @param array $itemUids filter on a set of item uids
|
||||
* @param array $uids filter on a set of queue item uids
|
||||
* @return QueryBuilder
|
||||
*/
|
||||
private function addItemWhereClauses(QueryBuilder $queryBuilderForDeletingItems, array $rootPageIds, string $indexQueueConfigurationList, string $itemTypeList, array $itemUids, array $uids): QueryBuilder
|
||||
{
|
||||
|
||||
if (!empty($rootPageIds)) {
|
||||
$queryBuilderForDeletingItems->andWhere($queryBuilderForDeletingItems->expr()->in('root', $rootPageIds));
|
||||
};
|
||||
|
||||
if (!empty($indexQueueConfigurationList)) {
|
||||
$queryBuilderForDeletingItems->andWhere($queryBuilderForDeletingItems->expr()->in('indexing_configuration', $queryBuilderForDeletingItems->createNamedParameter($indexQueueConfigurationList)));
|
||||
}
|
||||
|
||||
if (!empty($itemTypeList)) {
|
||||
$queryBuilderForDeletingItems->andWhere($queryBuilderForDeletingItems->expr()->in('item_type', $queryBuilderForDeletingItems->createNamedParameter($itemTypeList)));
|
||||
}
|
||||
|
||||
if (!empty($itemUids)) {
|
||||
$queryBuilderForDeletingItems->andWhere($queryBuilderForDeletingItems->expr()->in('item_uid', $itemUids));
|
||||
}
|
||||
|
||||
if (!empty($uids)) {
|
||||
$queryBuilderForDeletingItems->andWhere($queryBuilderForDeletingItems->expr()->in('uid', $uids));
|
||||
}
|
||||
|
||||
return $queryBuilderForDeletingItems;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes a query builder to delete the indexing properties of an item by the passed conditions.
|
||||
*
|
||||
* @param QueryBuilder $queryBuilderForDeletingItems
|
||||
* @param array $rootPageIds
|
||||
* @param string $indexQueueConfigurationList
|
||||
* @param string $itemTypeList
|
||||
* @param array $itemUids
|
||||
* @param array $uids
|
||||
* @return QueryBuilder
|
||||
*/
|
||||
private function buildQueryForPropertyDeletion(QueryBuilder $queryBuilderForDeletingItems, array $rootPageIds, string $indexQueueConfigurationList, string $itemTypeList, array $itemUids, array $uids): QueryBuilder
|
||||
{
|
||||
$queryBuilderForSelectingProperties = $queryBuilderForDeletingItems->getConnection()->createQueryBuilder();
|
||||
$queryBuilderForSelectingProperties->select('items.uid')->from('tx_meilisearch_indexqueue_indexing_property', 'properties')->innerJoin(
|
||||
'properties',
|
||||
$this->table,
|
||||
'items',
|
||||
(string)$queryBuilderForSelectingProperties->expr()->andX(
|
||||
$queryBuilderForSelectingProperties->expr()->eq('items.uid', $queryBuilderForSelectingProperties->quoteIdentifier('properties.item_id')),
|
||||
empty($rootPageIds) ? '' : $queryBuilderForSelectingProperties->expr()->in('items.root', $rootPageIds),
|
||||
empty($indexQueueConfigurationList) ? '' : $queryBuilderForSelectingProperties->expr()->in('items.indexing_configuration', $queryBuilderForSelectingProperties->createNamedParameter($indexQueueConfigurationList)),
|
||||
empty($itemTypeList) ? '' : $queryBuilderForSelectingProperties->expr()->in('items.item_type', $queryBuilderForSelectingProperties->createNamedParameter($itemTypeList)),
|
||||
empty($itemUids) ? '' : $queryBuilderForSelectingProperties->expr()->in('items.item_uid', $itemUids),
|
||||
empty($uids) ? '' : $queryBuilderForSelectingProperties->expr()->in('items.uid', $uids)
|
||||
)
|
||||
);
|
||||
$propertyEntriesToDelete = implode(',', array_column($queryBuilderForSelectingProperties->execute()->fetchAll(), 'uid'));
|
||||
|
||||
$queryBuilderForDeletingProperties = $queryBuilderForDeletingItems->getConnection()->createQueryBuilder();
|
||||
|
||||
// make sure executing the propety deletion query doesn't fail if there are no properties to delete
|
||||
if (empty($propertyEntriesToDelete)) {
|
||||
$propertyEntriesToDelete = '0';
|
||||
}
|
||||
|
||||
$queryBuilderForDeletingProperties->delete('tx_meilisearch_indexqueue_indexing_property')->where(
|
||||
$queryBuilderForDeletingProperties->expr()->in('item_id', $propertyEntriesToDelete)
|
||||
);
|
||||
|
||||
return $queryBuilderForDeletingProperties;
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes all items from the Index Queue.
|
||||
*
|
||||
* @return int The number of affected rows. For a truncate this is unreliable as theres no meaningful information.
|
||||
*/
|
||||
public function deleteAllItems()
|
||||
{
|
||||
return $this->getQueryBuilder()->getConnection()->truncate($this->table);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets a single Index Queue item by its uid.
|
||||
*
|
||||
* @param int $uid Index Queue item uid
|
||||
* @return Item|null The request Index Queue item or NULL if no item with $itemId was found
|
||||
*/
|
||||
public function findItemByUid(int $uid)
|
||||
{
|
||||
$queryBuilder = $this->getQueryBuilder();
|
||||
$indexQueueItemRecord = $queryBuilder
|
||||
->select('*')
|
||||
->from($this->table)
|
||||
->where($queryBuilder->expr()->eq('uid', $uid))
|
||||
->execute()->fetch();
|
||||
|
||||
if (!isset($indexQueueItemRecord['uid'])) {
|
||||
return null;
|
||||
}
|
||||
|
||||
/** @var Item $item*/
|
||||
$item = GeneralUtility::makeInstance(Item::class, /** @scrutinizer ignore-type */ $indexQueueItemRecord);
|
||||
return $item;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets Index Queue items by type and uid.
|
||||
*
|
||||
* @param string $itemType item type, usually the table name
|
||||
* @param int $itemUid item uid
|
||||
* @return Item[] An array of items matching $itemType and $itemUid
|
||||
*/
|
||||
public function findItemsByItemTypeAndItemUid(string $itemType, int $itemUid) : array
|
||||
{
|
||||
$queryBuilder = $this->getQueryBuilder();
|
||||
$compositeExpression = $queryBuilder->expr()->andX(
|
||||
$queryBuilder->expr()->eq('item_type', $queryBuilder->getConnection()->quote($itemType, \PDO::PARAM_STR)),
|
||||
$queryBuilder->expr()->eq('item_uid', $itemUid)
|
||||
);
|
||||
return $this->getItemsByCompositeExpression($compositeExpression, $queryBuilder);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a collection of items by CompositeExpression.
|
||||
* D
|
||||
*
|
||||
* @param CompositeExpression|null $expression Optional expression to filter records.
|
||||
* @param QueryBuilder|null $queryBuilder QueryBuilder to use
|
||||
* @return array
|
||||
*/
|
||||
protected function getItemsByCompositeExpression(CompositeExpression $expression = null, QueryBuilder $queryBuilder = null) : array
|
||||
{
|
||||
if (!$queryBuilder instanceof QueryBuilder) {
|
||||
$queryBuilder = $this->getQueryBuilder();
|
||||
}
|
||||
|
||||
$queryBuilder->select('*')->from($this->table);
|
||||
if (isset($expression)) {
|
||||
$queryBuilder->where($expression);
|
||||
}
|
||||
|
||||
$indexQueueItemRecords = $queryBuilder->execute()->fetchAll();
|
||||
return $this->getIndexQueueItemObjectsFromRecords($indexQueueItemRecords);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns all items in the queue.
|
||||
*
|
||||
* @return Item[] all Items from Queue without restrictions
|
||||
*/
|
||||
public function findAll() : array
|
||||
{
|
||||
$queryBuilder = $this->getQueryBuilder();
|
||||
$allRecords = $queryBuilder
|
||||
->select('*')
|
||||
->from($this->table)
|
||||
->execute()->fetchAll();
|
||||
return $this->getIndexQueueItemObjectsFromRecords($allRecords);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets $limit number of items to index for a particular $site.
|
||||
*
|
||||
* @param Site $site TYPO3 site
|
||||
* @param int $limit Number of items to get from the queue
|
||||
* @return Item[] Items to index to the given solr server
|
||||
*/
|
||||
public function findItemsToIndex(Site $site, int $limit = 50) : array
|
||||
{
|
||||
$queryBuilder = $this->getQueryBuilder();
|
||||
// determine which items to index with this run
|
||||
$indexQueueItemRecords = $queryBuilder
|
||||
->select('*')
|
||||
->from($this->table)
|
||||
->andWhere(
|
||||
$queryBuilder->expr()->eq('root', $site->getRootPageId()),
|
||||
$queryBuilder->expr()->gt('changed', 'indexed'),
|
||||
$queryBuilder->expr()->lte('changed', time()),
|
||||
$queryBuilder->expr()->eq('errors', $queryBuilder->createNamedParameter(''))
|
||||
)
|
||||
->orderBy('indexing_priority', 'DESC')
|
||||
->addOrderBy('changed', 'DESC')
|
||||
->addOrderBy('uid', 'DESC')
|
||||
->setMaxResults($limit)
|
||||
->execute()->fetchAll();
|
||||
|
||||
return $this->getIndexQueueItemObjectsFromRecords($indexQueueItemRecords);
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves the count of items that match certain filters. Each filter is passed as parts of the where claus combined with AND.
|
||||
*
|
||||
* @param array $sites
|
||||
* @param array $indexQueueConfigurationNames
|
||||
* @param array $itemTypes
|
||||
* @param array $itemUids
|
||||
* @param array $uids
|
||||
* @param int $start
|
||||
* @param int $limit
|
||||
* @return array
|
||||
*/
|
||||
public function findItems(array $sites = [], array $indexQueueConfigurationNames = [], array $itemTypes = [], array $itemUids = [], array $uids = [], $start = 0, $limit = 50): array
|
||||
{
|
||||
$rootPageIds = Site::getRootPageIdsFromSites($sites);
|
||||
$indexQueueConfigurationList = implode(",", $indexQueueConfigurationNames);
|
||||
$itemTypeList = implode(",", $itemTypes);
|
||||
$itemUids = array_map("intval", $itemUids);
|
||||
$uids = array_map("intval", $uids);
|
||||
$itemQueryBuilder = $this->getQueryBuilder()->select('*')->from($this->table);
|
||||
$itemQueryBuilder = $this->addItemWhereClauses($itemQueryBuilder, $rootPageIds, $indexQueueConfigurationList, $itemTypeList, $itemUids, $uids);
|
||||
$itemRecords = $itemQueryBuilder->setFirstResult($start)->setMaxResults($limit)->execute()->fetchAll();
|
||||
return $this->getIndexQueueItemObjectsFromRecords($itemRecords);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates an array of WapplerSystems\Meilisearch\IndexQueue\Item objects from an array of
|
||||
* index queue records.
|
||||
*
|
||||
* @param array $indexQueueItemRecords Array of plain index queue records
|
||||
* @return array Array of WapplerSystems\Meilisearch\IndexQueue\Item objects
|
||||
*/
|
||||
protected function getIndexQueueItemObjectsFromRecords(array $indexQueueItemRecords) : array
|
||||
{
|
||||
$tableRecords = $this->getAllQueueItemRecordsByUidsGroupedByTable($indexQueueItemRecords);
|
||||
return $this->getQueueItemObjectsByRecords($indexQueueItemRecords, $tableRecords);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the records for suitable item type.
|
||||
*
|
||||
* @param array $indexQueueItemRecords
|
||||
* @return array
|
||||
*/
|
||||
protected function getAllQueueItemRecordsByUidsGroupedByTable(array $indexQueueItemRecords) : array
|
||||
{
|
||||
$tableUids = [];
|
||||
$tableRecords = [];
|
||||
// grouping records by table
|
||||
foreach ($indexQueueItemRecords as $indexQueueItemRecord) {
|
||||
$tableUids[$indexQueueItemRecord['item_type']][] = $indexQueueItemRecord['item_uid'];
|
||||
}
|
||||
|
||||
// fetching records by table, saves us a lot of single queries
|
||||
foreach ($tableUids as $table => $uids) {
|
||||
$uidList = implode(',', $uids);
|
||||
|
||||
$queryBuilderForRecordTable = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable($table);
|
||||
$queryBuilderForRecordTable->getRestrictions()->removeAll();
|
||||
$resultsFromRecordTable = $queryBuilderForRecordTable
|
||||
->select('*')
|
||||
->from($table)
|
||||
->where($queryBuilderForRecordTable->expr()->in('uid', $uidList))
|
||||
->execute();
|
||||
$records = [];
|
||||
while ($record = $resultsFromRecordTable->fetch()) {
|
||||
$records[$record['uid']] = $record;
|
||||
}
|
||||
|
||||
$tableRecords[$table] = $records;
|
||||
$this->hookPostProcessFetchRecordsForIndexQueueItem($table, $uids, $tableRecords);
|
||||
}
|
||||
|
||||
return $tableRecords;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calls defined in postProcessFetchRecordsForIndexQueueItem hook method.
|
||||
*
|
||||
* @param string $table
|
||||
* @param array $uids
|
||||
* @param array $tableRecords
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
protected function hookPostProcessFetchRecordsForIndexQueueItem(string $table, array $uids, array &$tableRecords)
|
||||
{
|
||||
if (!is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['postProcessFetchRecordsForIndexQueueItem'])) {
|
||||
return;
|
||||
}
|
||||
$params = ['table' => $table, 'uids' => $uids, 'tableRecords' => &$tableRecords];
|
||||
foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['postProcessFetchRecordsForIndexQueueItem'] as $reference) {
|
||||
GeneralUtility::callUserFunction($reference, $params, $this);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Instantiates a list of Item objects from database records.
|
||||
*
|
||||
* @param array $indexQueueItemRecords records from database
|
||||
* @param array $tableRecords
|
||||
* @return array
|
||||
*/
|
||||
protected function getQueueItemObjectsByRecords(array $indexQueueItemRecords, array $tableRecords) : array
|
||||
{
|
||||
$indexQueueItems = [];
|
||||
foreach ($indexQueueItemRecords as $indexQueueItemRecord) {
|
||||
if (isset($tableRecords[$indexQueueItemRecord['item_type']][$indexQueueItemRecord['item_uid']])) {
|
||||
$indexQueueItems[] = GeneralUtility::makeInstance(
|
||||
Item::class,
|
||||
/** @scrutinizer ignore-type */ $indexQueueItemRecord,
|
||||
/** @scrutinizer ignore-type */ $tableRecords[$indexQueueItemRecord['item_type']][$indexQueueItemRecord['item_uid']]
|
||||
);
|
||||
} else {
|
||||
$this->logger->log(
|
||||
SolrLogManager::ERROR,
|
||||
'Record missing for Index Queue item. Item removed.',
|
||||
[
|
||||
$indexQueueItemRecord
|
||||
]
|
||||
);
|
||||
$this->deleteItem($indexQueueItemRecord['item_type'],
|
||||
$indexQueueItemRecord['item_uid']);
|
||||
}
|
||||
}
|
||||
|
||||
return $indexQueueItems;
|
||||
}
|
||||
|
||||
/**
|
||||
* Marks an item as failed and causes the indexer to skip the item in the
|
||||
* next run.
|
||||
*
|
||||
* @param int|Item $item Either the item's Index Queue uid or the complete item
|
||||
* @param string $errorMessage Error message
|
||||
* @return int affected rows
|
||||
*/
|
||||
public function markItemAsFailed($item, string $errorMessage = ''): int
|
||||
{
|
||||
$itemUid = ($item instanceof Item) ? $item->getIndexQueueUid() : (int)$item;
|
||||
$errorMessage = empty($errorMessage) ? '1' : $errorMessage;
|
||||
|
||||
$queryBuilder = $this->getQueryBuilder();
|
||||
return (int)$queryBuilder
|
||||
->update($this->table)
|
||||
->set('errors', $errorMessage)
|
||||
->where($queryBuilder->expr()->eq('uid', $itemUid))
|
||||
->execute();
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the timestamp of when an item last has been indexed.
|
||||
*
|
||||
* @param Item $item
|
||||
* @return int affected rows
|
||||
*/
|
||||
public function updateIndexTimeByItem(Item $item) : int
|
||||
{
|
||||
$queryBuilder = $this->getQueryBuilder();
|
||||
return (int)$queryBuilder
|
||||
->update($this->table)
|
||||
->set('indexed', time())
|
||||
->where($queryBuilder->expr()->eq('uid', $item->getIndexQueueUid()))
|
||||
->execute();
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the change timestamp of an item.
|
||||
*
|
||||
* @param Item $item
|
||||
* @param int $changedTime
|
||||
* @return int affected rows
|
||||
*/
|
||||
public function updateChangedTimeByItem(Item $item, int $changedTime) : int
|
||||
{
|
||||
$queryBuilder = $this->getQueryBuilder();
|
||||
return (int)$queryBuilder
|
||||
->update($this->table)
|
||||
->set('changed', $changedTime)
|
||||
->where($queryBuilder->expr()->eq('uid', $item->getIndexQueueUid()))
|
||||
->execute();
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes Queue by given sql
|
||||
*
|
||||
* Note: Do not use platform specific functions!
|
||||
*
|
||||
* @param string $sqlStatement Native SQL statement
|
||||
* @return int The number of affected rows.
|
||||
* @internal
|
||||
* @throws DBALException
|
||||
*/
|
||||
public function initializeByNativeSQLStatement(string $sqlStatement) : int
|
||||
{
|
||||
return $this->getQueryBuilder()->getConnection()->exec($sqlStatement);
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves an array of pageIds from mountPoints that allready have a queue entry.
|
||||
*
|
||||
* @param string $identifier identifier of the mount point
|
||||
* @return array pageIds from mountPoints that allready have a queue entry
|
||||
*/
|
||||
public function findPageIdsOfExistingMountPagesByMountIdentifier(string $identifier) : array
|
||||
{
|
||||
$queryBuilder = $this->getQueryBuilder();
|
||||
$resultSet = $queryBuilder
|
||||
->select('item_uid')
|
||||
->add('select', $queryBuilder->expr()->count('*', 'queueItemCount'), true)
|
||||
->from($this->table)
|
||||
->where(
|
||||
$queryBuilder->expr()->eq('item_type', $queryBuilder->createNamedParameter('pages')),
|
||||
$queryBuilder->expr()->eq('pages_mountidentifier', $queryBuilder->createNamedParameter($identifier))
|
||||
)
|
||||
->groupBy('item_uid')
|
||||
->execute();
|
||||
|
||||
$mountedPagesIdsWithQueueItems = [];
|
||||
while ($record = $resultSet->fetch()) {
|
||||
if ($record['queueItemCount'] > 0) {
|
||||
$mountedPagesIdsWithQueueItems[] = $record['item_uid'];
|
||||
}
|
||||
}
|
||||
|
||||
return $mountedPagesIdsWithQueueItems;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves an array of items for mount destinations mathed by root page ID, Mount Identifier and a list of mounted page IDs.
|
||||
*
|
||||
* @param int $rootPid
|
||||
* @param string $identifier identifier of the mount point
|
||||
* @param array $mountedPids An array of mounted page IDs
|
||||
* @return array
|
||||
*/
|
||||
public function findAllIndexQueueItemsByRootPidAndMountIdentifierAndMountedPids(int $rootPid, string $identifier, array $mountedPids) : array
|
||||
{
|
||||
$queryBuilder = $this->getQueryBuilder();
|
||||
return $queryBuilder
|
||||
->select('*')
|
||||
->from($this->table)
|
||||
->where(
|
||||
$queryBuilder->expr()->eq('root', $queryBuilder->createNamedParameter($rootPid, \PDO::PARAM_INT)),
|
||||
$queryBuilder->expr()->eq('item_type', $queryBuilder->createNamedParameter('pages')),
|
||||
$queryBuilder->expr()->in('item_uid', $mountedPids),
|
||||
$queryBuilder->expr()->eq('has_indexing_properties', $queryBuilder->createNamedParameter(1, \PDO::PARAM_INT)),
|
||||
$queryBuilder->expr()->eq('pages_mountidentifier', $queryBuilder->createNamedParameter($identifier))
|
||||
)
|
||||
->execute()->fetchAll();
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates has_indexing_properties field for given Item
|
||||
*
|
||||
* @param int $itemUid
|
||||
* @param bool $hasIndexingPropertiesFlag
|
||||
* @return int number of affected rows, 1 on success
|
||||
*/
|
||||
public function updateHasIndexingPropertiesFlagByItemUid(int $itemUid, bool $hasIndexingPropertiesFlag): int
|
||||
{
|
||||
$queryBuilder = $this->getQueryBuilder();
|
||||
|
||||
return $queryBuilder
|
||||
->update($this->table)
|
||||
->where($queryBuilder->expr()->eq('uid', $queryBuilder->createNamedParameter($itemUid, \PDO::PARAM_INT)))
|
||||
->set('has_indexing_properties', $queryBuilder->createNamedParameter($hasIndexingPropertiesFlag, \PDO::PARAM_INT), false)
|
||||
->execute();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
<?php
|
||||
|
||||
namespace WapplerSystems\Meilisearch\Domain\Index\Queue\RecordMonitor\Helper;
|
||||
|
||||
/***************************************************************
|
||||
* Copyright notice
|
||||
*
|
||||
* (c) 2017 - Thomas Hohn <tho@systime.dk>
|
||||
* All rights reserved
|
||||
*
|
||||
* This script is part of the TYPO3 project. The TYPO3 project is
|
||||
* free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* The GNU General Public License can be found at
|
||||
* http://www.gnu.org/copyleft/gpl.html.
|
||||
*
|
||||
* This script is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* This copyright notice MUST APPEAR in all copies of the script!
|
||||
***************************************************************/
|
||||
|
||||
use WapplerSystems\Meilisearch\System\Cache\TwoLevelCache;
|
||||
use WapplerSystems\Meilisearch\System\Configuration\TypoScriptConfiguration;
|
||||
use TYPO3\CMS\Backend\Utility\BackendUtility;
|
||||
use TYPO3\CMS\Core\Utility\GeneralUtility;
|
||||
|
||||
/**
|
||||
* Extracted logic from the AbstractDataHandlerListener in order to
|
||||
* handle ConfigurationAwareRecords
|
||||
*
|
||||
* @author Thomas Hohn Hund <tho@systime.dk>
|
||||
*/
|
||||
class ConfigurationAwareRecordService
|
||||
{
|
||||
/**
|
||||
* Retrieves the name of the Index Queue Configuration for a record.
|
||||
*
|
||||
* @param string $recordTable Table to read from
|
||||
* @param int $recordUid Id of the record
|
||||
* @param TypoScriptConfiguration $solrConfiguration
|
||||
* @return string|null Name of indexing configuration
|
||||
*/
|
||||
public function getIndexingConfigurationName($recordTable, $recordUid, TypoScriptConfiguration $solrConfiguration)
|
||||
{
|
||||
$name = null;
|
||||
$indexingConfigurations = $solrConfiguration->getEnabledIndexQueueConfigurationNames();
|
||||
foreach ($indexingConfigurations as $indexingConfigurationName) {
|
||||
if (!$solrConfiguration->getIndexQueueConfigurationIsEnabled($indexingConfigurationName)) {
|
||||
// ignore disabled indexing configurations
|
||||
continue;
|
||||
}
|
||||
|
||||
$record = $this->getRecordIfIndexConfigurationIsValid($recordTable, $recordUid,
|
||||
$indexingConfigurationName, $solrConfiguration);
|
||||
if (!empty($record)) {
|
||||
$name = $indexingConfigurationName;
|
||||
// FIXME currently returns after the first configuration match
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return $name;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves a record, taking into account the additionalWhereClauses of the
|
||||
* Indexing Queue configurations.
|
||||
*
|
||||
* @param string $recordTable Table to read from
|
||||
* @param int $recordUid Id of the record
|
||||
* @param TypoScriptConfiguration $solrConfiguration
|
||||
* @return array Record if found, otherwise empty array
|
||||
*/
|
||||
public function getRecord($recordTable, $recordUid, TypoScriptConfiguration $solrConfiguration)
|
||||
{
|
||||
$record = [];
|
||||
$indexingConfigurations = $solrConfiguration->getEnabledIndexQueueConfigurationNames();
|
||||
foreach ($indexingConfigurations as $indexingConfigurationName) {
|
||||
$record = $this->getRecordIfIndexConfigurationIsValid($recordTable, $recordUid,
|
||||
$indexingConfigurationName, $solrConfiguration);
|
||||
if (!empty($record)) {
|
||||
// if we found a record which matches the conditions, we can continue
|
||||
break;
|
||||
}
|
||||
}
|
||||
return $record;
|
||||
}
|
||||
|
||||
/**
|
||||
* This method return the record array if the table is valid for this indexingConfiguration.
|
||||
* Otherwise an empty array will be returned.
|
||||
*
|
||||
* @param string $recordTable
|
||||
* @param integer $recordUid
|
||||
* @param string $indexingConfigurationName
|
||||
* @param TypoScriptConfiguration $solrConfiguration
|
||||
* @return array
|
||||
*/
|
||||
protected function getRecordIfIndexConfigurationIsValid($recordTable, $recordUid, $indexingConfigurationName, TypoScriptConfiguration $solrConfiguration)
|
||||
{
|
||||
if (!$this->isValidTableForIndexConfigurationName($recordTable, $indexingConfigurationName, $solrConfiguration)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$recordWhereClause = $solrConfiguration->getIndexQueueAdditionalWhereClauseByConfigurationName($indexingConfigurationName);
|
||||
|
||||
$row = $this->getRecordForIndexConfigurationIsValid($recordTable, $recordUid, $recordWhereClause);
|
||||
|
||||
return $row;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the row need by getRecordIfIndexConfigurationIsValid either directly from database
|
||||
* or from cache
|
||||
*
|
||||
* @param string $recordTable
|
||||
* @param integer $recordUid
|
||||
* @param string $recordWhereClause
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
protected function getRecordForIndexConfigurationIsValid($recordTable, $recordUid, $recordWhereClause)
|
||||
{
|
||||
$cache = GeneralUtility::makeInstance(TwoLevelCache::class, /** @scrutinizer ignore-type */ 'cache_runtime');
|
||||
$cacheId = md5('ConfigurationAwareRecordService' . ':' . 'getRecordIfIndexConfigurationIsValid' . ':' . $recordTable . ':' . $recordUid . ':' . $recordWhereClause);
|
||||
|
||||
$row = $cache->get($cacheId);
|
||||
if (!empty($row)) {
|
||||
return $row;
|
||||
}
|
||||
|
||||
$row = (array)BackendUtility::getRecord($recordTable, $recordUid, '*', $recordWhereClause);
|
||||
$cache->set($cacheId, $row);
|
||||
|
||||
return $row;
|
||||
}
|
||||
|
||||
/**
|
||||
* This method is used to check if a table is an allowed table for an index configuration.
|
||||
*
|
||||
* @param string $recordTable
|
||||
* @param string $indexingConfigurationName
|
||||
* @param TypoScriptConfiguration $solrConfiguration
|
||||
* @return boolean
|
||||
*/
|
||||
protected function isValidTableForIndexConfigurationName($recordTable, $indexingConfigurationName, TypoScriptConfiguration $solrConfiguration)
|
||||
{
|
||||
$tableToIndex = $solrConfiguration->getIndexQueueTableNameOrFallbackToConfigurationName($indexingConfigurationName);
|
||||
|
||||
$isMatchingTable = ($tableToIndex === $recordTable);
|
||||
|
||||
if ($isMatchingTable) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* This method retrieves the parent pages record when the parent record is accessible
|
||||
* through the recordWhereClause
|
||||
*
|
||||
* @param int $recordUid
|
||||
* @param string $parentWhereClause
|
||||
* @return array
|
||||
*/
|
||||
protected function getPageOverlayRecordIfParentIsAccessible($recordUid, $parentWhereClause)
|
||||
{
|
||||
$overlayRecord = (array)BackendUtility::getRecord('pages', $recordUid, '*');
|
||||
$overlayParentId = $overlayRecord['l10n_parent'];
|
||||
|
||||
$pageRecord = (array)BackendUtility::getRecord('pages', $overlayParentId, '*', $parentWhereClause);
|
||||
|
||||
if (empty($pageRecord)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return $overlayRecord;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
<?php
|
||||
|
||||
namespace WapplerSystems\Meilisearch\Domain\Index\Queue\RecordMonitor\Helper;
|
||||
|
||||
/***************************************************************
|
||||
* Copyright notice
|
||||
*
|
||||
* (c) 2015-2016 Timo Hund <timo.hund@dkd.de>
|
||||
* All rights reserved
|
||||
*
|
||||
* This script is part of the TYPO3 project. The TYPO3 project is
|
||||
* free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* The GNU General Public License can be found at
|
||||
* http://www.gnu.org/copyleft/gpl.html.
|
||||
*
|
||||
* This script is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* This copyright notice MUST APPEAR in all copies of the script!
|
||||
***************************************************************/
|
||||
|
||||
use WapplerSystems\Meilisearch\Domain\Site\SiteRepository;
|
||||
use WapplerSystems\Meilisearch\IndexQueue\Initializer\Page;
|
||||
use WapplerSystems\Meilisearch\System\Page\Rootline;
|
||||
use WapplerSystems\Meilisearch\System\Records\Pages\PagesRepository;
|
||||
use TYPO3\CMS\Core\Utility\GeneralUtility;
|
||||
use TYPO3\CMS\Core\Utility\RootlineUtility;
|
||||
use TYPO3\CMS\Frontend\Page\PageRepository;
|
||||
|
||||
/**
|
||||
* Extracted logic from the RecordMonitor to trigger mount page updates.
|
||||
*
|
||||
* @author Timo Hund <timo.hund@dkd.de>
|
||||
*/
|
||||
class MountPagesUpdater
|
||||
{
|
||||
|
||||
/**
|
||||
* @var PagesRepository
|
||||
*/
|
||||
protected $pagesRepository;
|
||||
|
||||
/**
|
||||
* MountPagesUpdater constructor.
|
||||
*
|
||||
* @param PagesRepository|null $pagesRepository
|
||||
*/
|
||||
public function __construct(PagesRepository $pagesRepository = null) {
|
||||
$this->pagesRepository = $pagesRepository ?? GeneralUtility::makeInstance(PagesRepository::class);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles updates of the Index Queue in case a newly created or changed
|
||||
* page is part of a tree that is mounted into a another site.
|
||||
*
|
||||
* @param int $pageId Page Id (uid).
|
||||
*/
|
||||
public function update($pageId)
|
||||
{
|
||||
// get the root line of the page, every parent page could be a Mount Page source
|
||||
$rootlineUtility = GeneralUtility::makeInstance(RootlineUtility::class, $pageId);
|
||||
try {
|
||||
$rootLineArray = $rootlineUtility->get();
|
||||
} catch (\RuntimeException $e) {
|
||||
$rootLineArray = [];
|
||||
}
|
||||
|
||||
$currentPage = array_shift($rootLineArray);
|
||||
$currentPageUid = (int)$currentPage['uid'];
|
||||
|
||||
if (empty($rootLineArray) && $currentPageUid === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
/** @var $rootLine Rootline */
|
||||
$rootLine = GeneralUtility::makeInstance(Rootline::class, /** @scrutinizer ignore-type */ $rootLineArray);
|
||||
$rootLineParentPageIds = array_map('intval', $rootLine->getParentPageIds());
|
||||
$destinationMountProperties = $this->pagesRepository->findMountPointPropertiesByPageIdOrByRootLineParentPageIds($currentPageUid, $rootLineParentPageIds);
|
||||
|
||||
if (empty($destinationMountProperties)) {
|
||||
return;
|
||||
}
|
||||
|
||||
foreach ($destinationMountProperties as $destinationMount) {
|
||||
$this->addPageToMountingSiteIndexQueue($pageId, $destinationMount);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a page to the Index Queue of a site mounting the page.
|
||||
*
|
||||
* @param int $mountedPageId ID (uid) of the mounted page.
|
||||
* @param array $mountProperties Array of mount point properties mountPageSource, mountPageDestination, and mountPageOverlayed
|
||||
*/
|
||||
protected function addPageToMountingSiteIndexQueue($mountedPageId, array $mountProperties)
|
||||
{
|
||||
$siteRepository = GeneralUtility::makeInstance(SiteRepository::class);
|
||||
$mountingSite = $siteRepository->getSiteByPageId($mountProperties['mountPageDestination']);
|
||||
|
||||
/** @var $pageInitializer Page */
|
||||
$pageInitializer = GeneralUtility::makeInstance(Page::class);
|
||||
$pageInitializer->setSite($mountingSite);
|
||||
|
||||
$pageInitializer->initializeMountedPage($mountProperties, $mountedPageId);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,315 @@
|
||||
<?php
|
||||
|
||||
namespace WapplerSystems\Meilisearch\Domain\Index\Queue\RecordMonitor\Helper;
|
||||
|
||||
/***************************************************************
|
||||
* Copyright notice
|
||||
*
|
||||
* (c) 2017- Timo Hund <timo.hund@dkd.de>
|
||||
* All rights reserved
|
||||
*
|
||||
* This script is part of the TYPO3 project. The TYPO3 project is
|
||||
* free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* The GNU General Public License can be found at
|
||||
* http://www.gnu.org/copyleft/gpl.html.
|
||||
*
|
||||
* This script is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* This copyright notice MUST APPEAR in all copies of the script!
|
||||
***************************************************************/
|
||||
|
||||
use WapplerSystems\Meilisearch\Domain\Site\SiteRepository;
|
||||
use WapplerSystems\Meilisearch\Domain\Site\Site;
|
||||
use WapplerSystems\Meilisearch\System\Cache\TwoLevelCache;
|
||||
use WapplerSystems\Meilisearch\System\Configuration\ExtensionConfiguration;
|
||||
use WapplerSystems\Meilisearch\System\Page\Rootline;
|
||||
use TYPO3\CMS\Backend\Utility\BackendUtility;
|
||||
use TYPO3\CMS\Core\SingletonInterface;
|
||||
use TYPO3\CMS\Core\Utility\GeneralUtility;
|
||||
use TYPO3\CMS\Core\Utility\RootlineUtility;
|
||||
use TYPO3\CMS\Frontend\Page\PageRepository;
|
||||
|
||||
/**
|
||||
* RootPageResolver.
|
||||
*
|
||||
* Responsibility: The RootPageResolver is responsible to determine all relevant site root page id's
|
||||
* for a certain records, by table and uid.
|
||||
*
|
||||
* @author Timo Hund <timo.hund@dkd.de>
|
||||
*/
|
||||
class RootPageResolver implements SingletonInterface
|
||||
{
|
||||
|
||||
/**
|
||||
* @var ConfigurationAwareRecordService
|
||||
*/
|
||||
protected $recordService;
|
||||
|
||||
/**
|
||||
* @var TwoLevelCache
|
||||
*/
|
||||
protected $runtimeCache;
|
||||
|
||||
/**
|
||||
* @var ExtensionConfiguration
|
||||
*/
|
||||
protected $extensionConfiguration;
|
||||
|
||||
/**
|
||||
* RootPageResolver constructor.
|
||||
* @param ConfigurationAwareRecordService|null $recordService
|
||||
* @param TwoLevelCache|null $twoLevelCache
|
||||
*/
|
||||
public function __construct(ConfigurationAwareRecordService $recordService = null, TwoLevelCache $twoLevelCache = null)
|
||||
{
|
||||
$this->recordService = $recordService ?? GeneralUtility::makeInstance(ConfigurationAwareRecordService::class);
|
||||
$this->runtimeCache = $twoLevelCache ?? GeneralUtility::makeInstance(TwoLevelCache::class, /** @scrutinizer ignore-type */ 'cache_runtime');
|
||||
$this->extensionConfiguration = GeneralUtility::makeInstance(ExtensionConfiguration::class);
|
||||
}
|
||||
|
||||
/**
|
||||
* This method determines the responsible site roots for a record by getting the rootPage of the record and checking
|
||||
* if the pid is references in another site with additionalPageIds and returning those rootPageIds as well.
|
||||
* The result is cached by the caching framework.
|
||||
*
|
||||
* @param string $table
|
||||
* @param int $uid
|
||||
* @return array
|
||||
*/
|
||||
public function getResponsibleRootPageIds($table, $uid)
|
||||
{
|
||||
$cacheId = 'RootPageResolver' . '_' . 'getResponsibleRootPageIds' . '_' . $table . '_' . $uid;
|
||||
$methodResult = $this->runtimeCache->get($cacheId);
|
||||
if (!empty($methodResult)) {
|
||||
return $methodResult;
|
||||
}
|
||||
|
||||
$methodResult = $this->buildResponsibleRootPageIds($table, $uid);
|
||||
$this->runtimeCache->set($cacheId, $methodResult);
|
||||
|
||||
return $methodResult;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the passed pageId is a root page.
|
||||
*
|
||||
* @param int $pageId Page ID
|
||||
* @return bool TRUE if the page is marked as root page, FALSE otherwise
|
||||
*/
|
||||
public function getIsRootPageId($pageId)
|
||||
{
|
||||
// Page 0 can never be a root page
|
||||
if ($pageId === 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Page -1 is a workspace thing
|
||||
if ($pageId === -1) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$cacheId = 'RootPageResolver' . '_' . 'getIsRootPageId' . '_' . $pageId;
|
||||
$isSiteRoot = $this->runtimeCache->get($cacheId);
|
||||
|
||||
if (!empty($isSiteRoot)) {
|
||||
return $isSiteRoot;
|
||||
}
|
||||
|
||||
$page = $this->getPageRecordByPageId($pageId);
|
||||
if (empty($page)) {
|
||||
throw new \InvalidArgumentException(
|
||||
'The page for the given page ID \'' . $pageId
|
||||
. '\' could not be found in the database and can therefore not be used as site root page.',
|
||||
1487171426
|
||||
);
|
||||
}
|
||||
|
||||
$isSiteRoot = Site::isRootPage($page);
|
||||
$this->runtimeCache->set($cacheId, $isSiteRoot);
|
||||
|
||||
return $isSiteRoot;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param $pageId
|
||||
* @param string $fieldList
|
||||
* @return array
|
||||
*/
|
||||
protected function getPageRecordByPageId($pageId, $fieldList = 'is_siteroot')
|
||||
{
|
||||
return (array)BackendUtility::getRecord('pages', $pageId, $fieldList);
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines the rootpage ID for a given page.
|
||||
*
|
||||
* @param int $pageId A page ID somewhere in a tree.
|
||||
* @param bool $forceFallback Force the explicit detection and do not use the current frontend root line
|
||||
* @param string $mountPointIdentifier
|
||||
* @return int The page's tree branch's root page ID
|
||||
*/
|
||||
public function getRootPageId($pageId = 0, $forceFallback = false, $mountPointIdentifier = '')
|
||||
{
|
||||
/** @var Rootline $rootLine */
|
||||
$rootLine = GeneralUtility::makeInstance(Rootline::class);
|
||||
$rootPageId = intval($pageId) ?: intval($GLOBALS['TSFE']->id);
|
||||
|
||||
// frontend
|
||||
if (!empty($GLOBALS['TSFE']->rootLine)) {
|
||||
$rootLine->setRootLineArray($GLOBALS['TSFE']->rootLine);
|
||||
}
|
||||
|
||||
// fallback, backend
|
||||
if ($pageId != 0 && ($forceFallback || !$rootLine->getHasRootPage())) {
|
||||
$rootlineUtility = GeneralUtility::makeInstance(RootlineUtility::class, $pageId, $mountPointIdentifier);
|
||||
try {
|
||||
$rootLineArray = $rootlineUtility->get();
|
||||
} catch (\RuntimeException $e) {
|
||||
$rootLineArray = [];
|
||||
}
|
||||
$rootLine->setRootLineArray($rootLineArray);
|
||||
}
|
||||
|
||||
$rootPageFromRootLine = $rootLine->getRootPageId();
|
||||
|
||||
return $rootPageFromRootLine === 0 ? $rootPageId : $rootPageFromRootLine;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* This method determines the responsible site roots for a record by getting the rootPage of the record and checking
|
||||
* if the pid is references in another site with additionalPageIds and returning those rootPageIds as well.
|
||||
*
|
||||
* @param string $table
|
||||
* @param integer $uid
|
||||
* @return array
|
||||
*/
|
||||
protected function buildResponsibleRootPageIds($table, $uid)
|
||||
{
|
||||
$rootPages = [];
|
||||
$rootPageId = $this->getRootPageIdByTableAndUid($table, $uid);
|
||||
if ($this->getIsRootPageId($rootPageId)) {
|
||||
$rootPages[] = $rootPageId;
|
||||
}
|
||||
if ($this->extensionConfiguration->getIsUseConfigurationTrackRecordsOutsideSiteroot()) {
|
||||
$recordPageId = $this->getRecordPageId($table, $uid);
|
||||
if ($recordPageId === 0) {
|
||||
return $rootPages;
|
||||
}
|
||||
$alternativeSiteRoots = $this->getAlternativeSiteRootPagesIds($table, $uid, $recordPageId);
|
||||
$rootPages = array_merge($rootPages, $alternativeSiteRoots);
|
||||
}
|
||||
|
||||
return $rootPages;
|
||||
}
|
||||
|
||||
/**
|
||||
* This method checks if the record is a pages record or another one and determines the rootPageId from the records
|
||||
* rootline.
|
||||
*
|
||||
* @param string $table
|
||||
* @param int $uid
|
||||
* @return int
|
||||
*/
|
||||
protected function getRootPageIdByTableAndUid($table, $uid)
|
||||
{
|
||||
if ($table === 'pages') {
|
||||
$rootPageId = $this->getRootPageId($uid);
|
||||
return $rootPageId;
|
||||
} else {
|
||||
$recordPageId = $this->getRecordPageId($table, $uid);
|
||||
$rootPageId = $this->getRootPageId($recordPageId, true);
|
||||
return $rootPageId;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the pageId of the record or 0 when no valid record was given.
|
||||
*
|
||||
* @param string $table
|
||||
* @param integer $uid
|
||||
* @return mixed
|
||||
*/
|
||||
protected function getRecordPageId($table, $uid)
|
||||
{
|
||||
$record = BackendUtility::getRecord($table, $uid, 'pid');
|
||||
return !empty($record['pid']) ? (int)$record['pid'] : 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* When no root page can be determined we check if the pageIdOf the record is configured as additionalPageId in the index
|
||||
* configuration of another site, if so we return the rootPageId of this site.
|
||||
* The result is cached by the caching framework.
|
||||
*
|
||||
* @param string $table
|
||||
* @param int $uid
|
||||
* @param int $recordPageId
|
||||
* @return array
|
||||
*/
|
||||
public function getAlternativeSiteRootPagesIds($table, $uid, $recordPageId)
|
||||
{
|
||||
$siteRootsByObservedPageIds = $this->getSiteRootsByObservedPageIds($table, $uid);
|
||||
if (!isset($siteRootsByObservedPageIds[$recordPageId])) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return $siteRootsByObservedPageIds[$recordPageId];
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves an optimized array structure we the monitored pageId as key and the relevant site rootIds as value.
|
||||
*
|
||||
* @param string $table
|
||||
* @param integer $uid
|
||||
* @return array
|
||||
*/
|
||||
protected function getSiteRootsByObservedPageIds($table, $uid)
|
||||
{
|
||||
$cacheId = 'RootPageResolver' . '_' . 'getSiteRootsByObservedPageIds' . '_' . $table . '_' . $uid;
|
||||
$methodResult = $this->runtimeCache->get($cacheId);
|
||||
if (!empty($methodResult)) {
|
||||
return $methodResult;
|
||||
}
|
||||
|
||||
$methodResult = $this->buildSiteRootsByObservedPageIds($table, $uid);
|
||||
$this->runtimeCache->set($cacheId, $methodResult);
|
||||
|
||||
return $methodResult;
|
||||
}
|
||||
|
||||
/**
|
||||
* This methods build an array with observer page id as key and rootPageIds as values to determine which root pages
|
||||
* are responsible for this record by referencing the pageId in additionalPageIds configuration.
|
||||
*
|
||||
* @param string $table
|
||||
* @param integer $uid
|
||||
* @return array
|
||||
*/
|
||||
protected function buildSiteRootsByObservedPageIds($table, $uid)
|
||||
{
|
||||
$siteRootByObservedPageIds = [];
|
||||
$siteRepository = GeneralUtility::makeInstance(SiteRepository::class);
|
||||
$allSites = $siteRepository->getAvailableSites();
|
||||
|
||||
foreach ($allSites as $site) {
|
||||
$solrConfiguration = $site->getSolrConfiguration();
|
||||
$indexingConfigurationName = $this->recordService->getIndexingConfigurationName($table, $uid, $solrConfiguration);
|
||||
if ($indexingConfigurationName === null) {
|
||||
continue;
|
||||
}
|
||||
$observedPageIdsOfSiteRoot = $solrConfiguration->getIndexQueueAdditionalPageIdsByConfigurationName($indexingConfigurationName);
|
||||
foreach ($observedPageIdsOfSiteRoot as $observedPageIdOfSiteRoot) {
|
||||
$siteRootByObservedPageIds[$observedPageIdOfSiteRoot][] = $site->getRootPageId();
|
||||
}
|
||||
}
|
||||
|
||||
return $siteRootByObservedPageIds;
|
||||
}
|
||||
}
|
||||
138
Classes/Domain/Index/Queue/Statistic/QueueStatistic.php
Normal file
138
Classes/Domain/Index/Queue/Statistic/QueueStatistic.php
Normal file
@@ -0,0 +1,138 @@
|
||||
<?php
|
||||
|
||||
namespace WapplerSystems\Meilisearch\Domain\Index\Queue\Statistic;
|
||||
|
||||
/***************************************************************
|
||||
* Copyright notice
|
||||
*
|
||||
* (c) 2017 Timo Hund <timo.hund@dkd.de>
|
||||
* All rights reserved
|
||||
*
|
||||
* This script is part of the TYPO3 project. The TYPO3 project is
|
||||
* free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* The GNU General Public License can be found at
|
||||
* http://www.gnu.org/copyleft/gpl.html.
|
||||
*
|
||||
* This script is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* This copyright notice MUST APPEAR in all copies of the script!
|
||||
***************************************************************/
|
||||
|
||||
class QueueStatistic
|
||||
{
|
||||
|
||||
/**
|
||||
* @var int
|
||||
*/
|
||||
protected $failedCount = 0;
|
||||
|
||||
/**
|
||||
* @var int
|
||||
*/
|
||||
protected $pendingCount = 0;
|
||||
|
||||
/**
|
||||
* @var int
|
||||
*/
|
||||
protected $successCount = 0;
|
||||
|
||||
/**
|
||||
* @param int $failedCount
|
||||
*/
|
||||
public function setFailedCount($failedCount)
|
||||
{
|
||||
$this->failedCount = $failedCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return int
|
||||
*/
|
||||
public function getFailedCount()
|
||||
{
|
||||
return $this->failedCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return float|int
|
||||
*/
|
||||
public function getFailedPercentage()
|
||||
{
|
||||
return $this->getPercentage($this->getFailedCount());
|
||||
}
|
||||
|
||||
/**
|
||||
* @param int $pendingCount
|
||||
*/
|
||||
public function setPendingCount($pendingCount)
|
||||
{
|
||||
$this->pendingCount = $pendingCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return int
|
||||
*/
|
||||
public function getPendingCount()
|
||||
{
|
||||
return $this->pendingCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return float|int
|
||||
*/
|
||||
public function getPendingPercentage()
|
||||
{
|
||||
return $this->getPercentage($this->getPendingCount());
|
||||
}
|
||||
|
||||
/**
|
||||
* @param int $successCount
|
||||
*/
|
||||
public function setSuccessCount($successCount)
|
||||
{
|
||||
$this->successCount = $successCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return int
|
||||
*/
|
||||
public function getSuccessCount()
|
||||
{
|
||||
return $this->successCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return float|int
|
||||
*/
|
||||
public function getSuccessPercentage()
|
||||
{
|
||||
return $this->getPercentage($this->getSuccessCount());
|
||||
}
|
||||
|
||||
/**
|
||||
* @return int
|
||||
*/
|
||||
public function getTotalCount()
|
||||
{
|
||||
return $this->pendingCount + $this->failedCount + $this->successCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param integer $count
|
||||
* @return float
|
||||
*/
|
||||
protected function getPercentage($count)
|
||||
{
|
||||
$total = $this->getTotalCount();
|
||||
if ($total === 0) {
|
||||
return 0.0;
|
||||
}
|
||||
return (float)round((100 / $total) * $count, 2);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
<?php declare(strict_types = 1);
|
||||
namespace WapplerSystems\Meilisearch\Domain\Index\Queue\Statistic;
|
||||
|
||||
/***************************************************************
|
||||
* Copyright notice
|
||||
*
|
||||
* (c) 2010-2017 dkd Internet Service GmbH <solr-support@dkd.de>
|
||||
* All rights reserved
|
||||
*
|
||||
* This script is part of the TYPO3 project. The TYPO3 project is
|
||||
* free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* The GNU General Public License can be found at
|
||||
* http://www.gnu.org/copyleft/gpl.html.
|
||||
*
|
||||
* This script is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* This copyright notice MUST APPEAR in all copies of the script!
|
||||
***************************************************************/
|
||||
|
||||
use WapplerSystems\Meilisearch\System\Records\AbstractRepository;
|
||||
use TYPO3\CMS\Core\Utility\GeneralUtility;
|
||||
use TYPO3\CMS\Extbase\Persistence\Generic\Exception\UnsupportedMethodException;
|
||||
|
||||
/**
|
||||
* Class QueueStatisticsRepository
|
||||
*/
|
||||
class QueueStatisticsRepository extends AbstractRepository
|
||||
{
|
||||
/**
|
||||
* @var string
|
||||
*/
|
||||
protected $table = 'tx_meilisearch_indexqueue_item';
|
||||
|
||||
/**
|
||||
* Extracts the number of pending, indexed and erroneous items from the
|
||||
* Index Queue.
|
||||
*
|
||||
* @param int $rootPid
|
||||
* @param string $indexingConfigurationName
|
||||
*
|
||||
* @return QueueStatistic
|
||||
*/
|
||||
public function findOneByRootPidAndOptionalIndexingConfigurationName(int $rootPid, $indexingConfigurationName = null): QueueStatistic
|
||||
{
|
||||
$queryBuilder = $this->getQueryBuilder();
|
||||
$queryBuilder
|
||||
->add('select', vsprintf('(%s < %s) AS %s', [
|
||||
$queryBuilder->quoteIdentifier('indexed'),
|
||||
$queryBuilder->quoteIdentifier('changed'),
|
||||
$queryBuilder->quoteIdentifier('pending')
|
||||
]), true)
|
||||
->add('select', vsprintf('(%s) AS %s', [
|
||||
$queryBuilder->expr()->notLike('errors', $queryBuilder->createNamedParameter('')),
|
||||
$queryBuilder->quoteIdentifier('failed')
|
||||
]), true)
|
||||
->add('select', $queryBuilder->expr()->count('*', 'count'), true)
|
||||
->from($this->table)
|
||||
->where($queryBuilder->expr()->eq('root', $queryBuilder->createNamedParameter($rootPid, \PDO::PARAM_INT)))
|
||||
->groupBy('pending', 'failed');
|
||||
|
||||
if (!empty($indexingConfigurationName)) {
|
||||
$queryBuilder->andWhere($queryBuilder->expr()->eq('indexing_configuration', $queryBuilder->createNamedParameter($indexingConfigurationName)));
|
||||
}
|
||||
|
||||
return $this->buildQueueStatisticFromResultSet($queryBuilder->execute()->fetchAll());
|
||||
}
|
||||
|
||||
/**
|
||||
* Instantiates and fills QueueStatistic with values
|
||||
*
|
||||
* @param array $indexQueueStatisticResultSet
|
||||
* @return QueueStatistic
|
||||
*/
|
||||
protected function buildQueueStatisticFromResultSet(array $indexQueueStatisticResultSet): QueueStatistic
|
||||
{
|
||||
/* @var $statistic QueueStatistic */
|
||||
$statistic = GeneralUtility::makeInstance(QueueStatistic::class);
|
||||
foreach ($indexQueueStatisticResultSet as $row) {
|
||||
if ($row['failed'] == 1) {
|
||||
$statistic->setFailedCount((int)$row['count']);
|
||||
} elseif ($row['pending'] == 1) {
|
||||
$statistic->setPendingCount((int)$row['count']);
|
||||
} else {
|
||||
$statistic->setSuccessCount((int)$row['count']);
|
||||
}
|
||||
}
|
||||
|
||||
return $statistic;
|
||||
}
|
||||
|
||||
/**
|
||||
* Don't use this method.
|
||||
*
|
||||
* @return int
|
||||
* @throws UnsupportedMethodException
|
||||
*/
|
||||
public function count(): int
|
||||
{
|
||||
throw new UnsupportedMethodException('Can not count the Index Queue Statistics.', 1504694750);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user