WapplerSystems/meilisearch
6
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

first commit

Browse Source
This commit is contained in:
Sven Wappler
2021-04-17 21:20:54 +02:00
parent c93ec9492a
commit cadcc8edb4
406 changed files with 4917 additions and 5157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
Classes/Domain/Index/Queue/GarbageRemover/AbstractStrategy.php
View File

@@ -28,12 +28,12 @@ namespace WapplerSystems\Meilisearch\Domain\Index\Queue\GarbageRemover;
use WapplerSystems\Meilisearch\ConnectionManager;
use WapplerSystems\Meilisearch\GarbageCollectorPostProcessor;
use WapplerSystems\Meilisearch\IndexQueue\Queue;
use WapplerSystems\Meilisearch\System\Solr\SolrConnection;
use WapplerSystems\Meilisearch\System\Meilisearch\MeilisearchConnection;
use TYPO3\CMS\Core\Utility\GeneralUtility;
/**
* An implementation ob a garbage remover strategy is responsible to remove all garbage from the index queue and
* the solr server for a certain table and uid combination.
* the meilisearch server for a certain table and uid combination.
*/
abstract class AbstractStrategy
{
@@ -74,7 +74,7 @@ abstract class AbstractStrategy
/**
* A implementation of the GarbageCollection strategy is responsible to remove the garbage from
* the indexqueue and from the solr server.
* the indexqueue and from the meilisearch server.
*
* @param string $table
* @param int $uid
@@ -83,24 +83,24 @@ abstract class AbstractStrategy
abstract protected function removeGarbageOfByStrategy($table, $uid);
/**
* Deletes a document from solr and from the index queue.
* Deletes a document from meilisearch and from the index queue.
*
* @param string $table
* @param integer $uid
*/
protected function deleteInSolrAndRemoveFromIndexQueue($table, $uid)
protected function deleteInMeilisearchAndRemoveFromIndexQueue($table, $uid)
{
$this->deleteIndexDocuments($table, $uid);
$this->queue->deleteItem($table, $uid);
}
/**
* Deletes a document from solr and updates the item in the index queue (e.g. on page content updates).
* Deletes a document from meilisearch and updates the item in the index queue (e.g. on page content updates).
*
* @param string $table
* @param integer $uid
*/
protected function deleteInSolrAndUpdateIndexQueue($table, $uid)
protected function deleteInMeilisearchAndUpdateIndexQueue($table, $uid)
{
$this->deleteIndexDocuments($table, $uid);
$this->queue->updateItem($table, $uid);
@@ -118,32 +118,32 @@ abstract class AbstractStrategy
$indexQueueItems = $this->queue->getItems($table, $uid);
foreach ($indexQueueItems as $indexQueueItem) {
$site = $indexQueueItem->getSite();
$enableCommitsSetting = $site->getSolrConfiguration()->getEnableCommits();
$enableCommitsSetting = $site->getMeilisearchConfiguration()->getEnableCommits();
$siteHash = $site->getSiteHash();
// a site can have multiple connections (cores / languages)
$solrConnections = $this->connectionManager->getConnectionsBySite($site);
$meilisearchConnections = $this->connectionManager->getConnectionsBySite($site);
if ($language > 0) {
$solrConnections = [$language => $solrConnections[$language]];
$meilisearchConnections = [$language => $meilisearchConnections[$language]];
}
$this->deleteRecordInAllSolrConnections($table, $uid, $solrConnections, $siteHash, $enableCommitsSetting);
$this->deleteRecordInAllMeilisearchConnections($table, $uid, $meilisearchConnections, $siteHash, $enableCommitsSetting);
}
}
/**
* Deletes the record in all solr connections from that site.
* Deletes the record in all meilisearch connections from that site.
*
* @param string $table
* @param int $uid
* @param SolrConnection[] $solrConnections
* @param MeilisearchConnection[] $meilisearchConnections
* @param string $siteHash
* @param boolean $enableCommitsSetting
*/
protected function deleteRecordInAllSolrConnections($table, $uid, $solrConnections, $siteHash, $enableCommitsSetting)
protected function deleteRecordInAllMeilisearchConnections($table, $uid, $meilisearchConnections, $siteHash, $enableCommitsSetting)
{
foreach ($solrConnections as $solr) {
$solr->getWriteService()->deleteByQuery('type:' . $table . ' AND uid:' . (int)$uid . ' AND siteHash:' . $siteHash);
foreach ($meilisearchConnections as $meilisearch) {
$meilisearch->getWriteService()->deleteByQuery('type:' . $table . ' AND uid:' . (int)$uid . ' AND siteHash:' . $siteHash);
if ($enableCommitsSetting) {
$solr->getWriteService()->commit(false, false);
$meilisearch->getWriteService()->commit(false, false);
}
}
}
@@ -156,11 +156,11 @@ abstract class AbstractStrategy
*/
protected function callPostProcessGarbageCollectorHook($table, $uid)
{
if (!is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['postProcessGarbageCollector'])) {
if (!is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['meilisearch']['postProcessGarbageCollector'])) {
return;
}
foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['postProcessGarbageCollector'] as $classReference) {
foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['meilisearch']['postProcessGarbageCollector'] as $classReference) {
$garbageCollectorPostProcessor = GeneralUtility::makeInstance($classReference);
if ($garbageCollectorPostProcessor instanceof GarbageCollectorPostProcessor) {

6
Classes/Domain/Index/Queue/GarbageRemover/PageStrategy.php
View File

@@ -52,7 +52,7 @@ class PageStrategy extends AbstractStrategy {
}
/**
* Determines the relevant page id for an content element update. Deletes the page from solr and requeues the
* Determines the relevant page id for an content element update. Deletes the page from meilisearch and requeues the
* page for a reindex.
*
* @param int $ttContentUid
@@ -60,7 +60,7 @@ class PageStrategy extends AbstractStrategy {
protected function collectPageGarbageByContentChange($ttContentUid)
{
$contentElement = BackendUtility::getRecord('tt_content', $ttContentUid, 'uid, pid', '', false);
$this->deleteInSolrAndUpdateIndexQueue('pages', $contentElement['pid']);
$this->deleteInMeilisearchAndUpdateIndexQueue('pages', $contentElement['pid']);
}
/**
@@ -74,7 +74,7 @@ class PageStrategy extends AbstractStrategy {
if (!empty($pageOverlay['l10n_parent']) && intval($pageOverlay['l10n_parent']) !== 0) {
$this->deleteIndexDocuments('pages', (int)$pageOverlay['l10n_parent'], (int)$pageOverlay['sys_language_uid']);
} else {
$this->deleteInSolrAndRemoveFromIndexQueue('pages', $uid);
$this->deleteInMeilisearchAndRemoveFromIndexQueue('pages', $uid);
}
}
}

2
Classes/Domain/Index/Queue/GarbageRemover/RecordStrategy.php
View File

@@ -38,6 +38,6 @@ class RecordStrategy extends AbstractStrategy {
*/
protected function removeGarbageOfByStrategy($table, $uid)
{
$this->deleteInSolrAndRemoveFromIndexQueue($table, $uid);
$this->deleteInMeilisearchAndRemoveFromIndexQueue($table, $uid);
}
}

2
Classes/Domain/Index/Queue/IndexQueueIndexingPropertyRepository.php
View File

@@ -4,7 +4,7 @@ namespace WapplerSystems\Meilisearch\Domain\Index\Queue;
/***************************************************************
* Copyright notice
*
* (c) 2010-2017 dkd Internet Service GmbH <solr-support@dkd.de>
* (c) 2010-2017 dkd Internet Service GmbH <meilisearch-support@dkd.de>
* All rights reserved
*
* This script is part of the TYPO3 project. The TYPO3 project is

16
Classes/Domain/Index/Queue/QueueInitializationService.php
View File

@@ -4,7 +4,7 @@ namespace WapplerSystems\Meilisearch\Domain\Index\Queue;
/***************************************************************
* Copyright notice
*
* (c) 2010-2017 dkd Internet Service GmbH <solr-support@dkd.de>
* (c) 2010-2017 dkd Internet Service GmbH <meilisearch-support@dkd.de>
* All rights reserved
*
* This script is part of the TYPO3 project. The TYPO3 project is
@@ -97,16 +97,16 @@ class QueueInitializationService {
$initializationStatus = [];
$hasWildcardConfiguration = in_array('*', $indexingConfigurationNames);
$indexingConfigurationNames = $hasWildcardConfiguration ? $site->getSolrConfiguration()->getEnabledIndexQueueConfigurationNames() : $indexingConfigurationNames;
$indexingConfigurationNames = $hasWildcardConfiguration ? $site->getMeilisearchConfiguration()->getEnabledIndexQueueConfigurationNames() : $indexingConfigurationNames;
foreach ($indexingConfigurationNames as $indexingConfigurationName) {
$initializationStatus[$indexingConfigurationName] = $this->applyInitialization($site, (string)$indexingConfigurationName);
}
if (!is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['postProcessIndexQueueInitialization'])) {
if (!is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['meilisearch']['postProcessIndexQueueInitialization'])) {
return $initializationStatus;
}
foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['postProcessIndexQueueInitialization'] as $classReference) {
foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['meilisearch']['postProcessIndexQueueInitialization'] as $classReference) {
$indexQueueInitializationPostProcessor = GeneralUtility::makeInstance($classReference);
if ($indexQueueInitializationPostProcessor instanceof InitializationPostProcessor) {
$indexQueueInitializationPostProcessor->postProcessIndexQueueInitialization($site, $indexingConfigurationNames, $initializationStatus);
@@ -131,10 +131,10 @@ class QueueInitializationService {
// clear queue
$this->queue->deleteItemsBySite($site, $indexingConfigurationName);
$solrConfiguration = $site->getSolrConfiguration();
$tableToIndex = $solrConfiguration->getIndexQueueTableNameOrFallbackToConfigurationName($indexingConfigurationName);
$initializerClass = $solrConfiguration->getIndexQueueInitializerClassByConfigurationName($indexingConfigurationName);
$indexConfiguration = $solrConfiguration->getIndexQueueConfigurationByName($indexingConfigurationName);
$meilisearchConfiguration = $site->getMeilisearchConfiguration();
$tableToIndex = $meilisearchConfiguration->getIndexQueueTableNameOrFallbackToConfigurationName($indexingConfigurationName);
$initializerClass = $meilisearchConfiguration->getIndexQueueInitializerClassByConfigurationName($indexingConfigurationName);
$indexConfiguration = $meilisearchConfiguration->getIndexQueueConfigurationByName($indexingConfigurationName);
return $this->executeInitializer($site, $indexingConfigurationName, $initializerClass, $tableToIndex, $indexConfiguration);
}

20
Classes/Domain/Index/Queue/QueueItemRepository.php
View File

@@ -4,7 +4,7 @@ namespace WapplerSystems\Meilisearch\Domain\Index\Queue;
/***************************************************************
* Copyright notice
*
* (c) 2010-2017 dkd Internet Service GmbH <solr-support@dkd.de>
* (c) 2010-2017 dkd Internet Service GmbH <meilisearch-support@dkd.de>
* All rights reserved
*
* This script is part of the TYPO3 project. The TYPO3 project is
@@ -26,7 +26,7 @@ namespace WapplerSystems\Meilisearch\Domain\Index\Queue;
use WapplerSystems\Meilisearch\IndexQueue\Item;
use WapplerSystems\Meilisearch\Domain\Site\Site;
use WapplerSystems\Meilisearch\System\Logging\SolrLogManager;
use WapplerSystems\Meilisearch\System\Logging\MeilisearchLogManager;
use WapplerSystems\Meilisearch\System\Records\AbstractRepository;
use Doctrine\DBAL\DBALException;
use TYPO3\CMS\Core\Database\ConnectionPool;
@@ -47,18 +47,18 @@ class QueueItemRepository extends AbstractRepository
protected $table = 'tx_meilisearch_indexqueue_item';
/**
* @var SolrLogManager
* @var MeilisearchLogManager
*/
protected $logger;
/**
* QueueItemRepository constructor.
*
* @param SolrLogManager|null $logManager
* @param MeilisearchLogManager|null $logManager
*/
public function __construct(SolrLogManager $logManager = null)
public function __construct(MeilisearchLogManager $logManager = null)
{
$this->logger = $logManager ?? GeneralUtility::makeInstance(SolrLogManager::class, /** @scrutinizer ignore-type */ __CLASS__);
$this->logger = $logManager ?? GeneralUtility::makeInstance(MeilisearchLogManager::class, /** @scrutinizer ignore-type */ __CLASS__);
}
/**
@@ -606,7 +606,7 @@ class QueueItemRepository extends AbstractRepository
*
* @param Site $site TYPO3 site
* @param int $limit Number of items to get from the queue
* @return Item[] Items to index to the given solr server
* @return Item[] Items to index to the given meilisearch server
*/
public function findItemsToIndex(Site $site, int $limit = 50) : array
{
@@ -717,11 +717,11 @@ class QueueItemRepository extends AbstractRepository
*/
protected function hookPostProcessFetchRecordsForIndexQueueItem(string $table, array $uids, array &$tableRecords)
{
if (!is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['postProcessFetchRecordsForIndexQueueItem'])) {
if (!is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['meilisearch']['postProcessFetchRecordsForIndexQueueItem'])) {
return;
}
$params = ['table' => $table, 'uids' => $uids, 'tableRecords' => &$tableRecords];
foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['postProcessFetchRecordsForIndexQueueItem'] as $reference) {
foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['meilisearch']['postProcessFetchRecordsForIndexQueueItem'] as $reference) {
GeneralUtility::callUserFunction($reference, $params, $this);
}
}
@@ -745,7 +745,7 @@ class QueueItemRepository extends AbstractRepository
);
} else {
$this->logger->log(
SolrLogManager::ERROR,
MeilisearchLogManager::ERROR,
'Record missing for Index Queue item. Item removed.',
[
$indexQueueItemRecord

32
Classes/Domain/Index/Queue/RecordMonitor/Helper/ConfigurationAwareRecordService.php
View File

@@ -43,21 +43,21 @@ class ConfigurationAwareRecordService
*
* @param string $recordTable Table to read from
* @param int $recordUid Id of the record
* @param TypoScriptConfiguration $solrConfiguration
* @param TypoScriptConfiguration $meilisearchConfiguration
* @return string|null Name of indexing configuration
*/
public function getIndexingConfigurationName($recordTable, $recordUid, TypoScriptConfiguration $solrConfiguration)
public function getIndexingConfigurationName($recordTable, $recordUid, TypoScriptConfiguration $meilisearchConfiguration)
{
$name = null;
$indexingConfigurations = $solrConfiguration->getEnabledIndexQueueConfigurationNames();
$indexingConfigurations = $meilisearchConfiguration->getEnabledIndexQueueConfigurationNames();
foreach ($indexingConfigurations as $indexingConfigurationName) {
if (!$solrConfiguration->getIndexQueueConfigurationIsEnabled($indexingConfigurationName)) {
if (!$meilisearchConfiguration->getIndexQueueConfigurationIsEnabled($indexingConfigurationName)) {
// ignore disabled indexing configurations
continue;
}
$record = $this->getRecordIfIndexConfigurationIsValid($recordTable, $recordUid,
$indexingConfigurationName, $solrConfiguration);
$indexingConfigurationName, $meilisearchConfiguration);
if (!empty($record)) {
$name = $indexingConfigurationName;
// FIXME currently returns after the first configuration match
@@ -74,16 +74,16 @@ class ConfigurationAwareRecordService
*
* @param string $recordTable Table to read from
* @param int $recordUid Id of the record
* @param TypoScriptConfiguration $solrConfiguration
* @param TypoScriptConfiguration $meilisearchConfiguration
* @return array Record if found, otherwise empty array
*/
public function getRecord($recordTable, $recordUid, TypoScriptConfiguration $solrConfiguration)
public function getRecord($recordTable, $recordUid, TypoScriptConfiguration $meilisearchConfiguration)
{
$record = [];
$indexingConfigurations = $solrConfiguration->getEnabledIndexQueueConfigurationNames();
$indexingConfigurations = $meilisearchConfiguration->getEnabledIndexQueueConfigurationNames();
foreach ($indexingConfigurations as $indexingConfigurationName) {
$record = $this->getRecordIfIndexConfigurationIsValid($recordTable, $recordUid,
$indexingConfigurationName, $solrConfiguration);
$indexingConfigurationName, $meilisearchConfiguration);
if (!empty($record)) {
// if we found a record which matches the conditions, we can continue
break;
@@ -99,16 +99,16 @@ class ConfigurationAwareRecordService
* @param string $recordTable
* @param integer $recordUid
* @param string $indexingConfigurationName
* @param TypoScriptConfiguration $solrConfiguration
* @param TypoScriptConfiguration $meilisearchConfiguration
* @return array
*/
protected function getRecordIfIndexConfigurationIsValid($recordTable, $recordUid, $indexingConfigurationName, TypoScriptConfiguration $solrConfiguration)
protected function getRecordIfIndexConfigurationIsValid($recordTable, $recordUid, $indexingConfigurationName, TypoScriptConfiguration $meilisearchConfiguration)
{
if (!$this->isValidTableForIndexConfigurationName($recordTable, $indexingConfigurationName, $solrConfiguration)) {
if (!$this->isValidTableForIndexConfigurationName($recordTable, $indexingConfigurationName, $meilisearchConfiguration)) {
return [];
}
$recordWhereClause = $solrConfiguration->getIndexQueueAdditionalWhereClauseByConfigurationName($indexingConfigurationName);
$recordWhereClause = $meilisearchConfiguration->getIndexQueueAdditionalWhereClauseByConfigurationName($indexingConfigurationName);
$row = $this->getRecordForIndexConfigurationIsValid($recordTable, $recordUid, $recordWhereClause);
@@ -146,12 +146,12 @@ class ConfigurationAwareRecordService
*
* @param string $recordTable
* @param string $indexingConfigurationName
* @param TypoScriptConfiguration $solrConfiguration
* @param TypoScriptConfiguration $meilisearchConfiguration
* @return boolean
*/
protected function isValidTableForIndexConfigurationName($recordTable, $indexingConfigurationName, TypoScriptConfiguration $solrConfiguration)
protected function isValidTableForIndexConfigurationName($recordTable, $indexingConfigurationName, TypoScriptConfiguration $meilisearchConfiguration)
{
$tableToIndex = $solrConfiguration->getIndexQueueTableNameOrFallbackToConfigurationName($indexingConfigurationName);
$tableToIndex = $meilisearchConfiguration->getIndexQueueTableNameOrFallbackToConfigurationName($indexingConfigurationName);
$isMatchingTable = ($tableToIndex === $recordTable);

6
Classes/Domain/Index/Queue/RecordMonitor/Helper/RootPageResolver.php
View File

@@ -299,12 +299,12 @@ class RootPageResolver implements SingletonInterface
$allSites = $siteRepository->getAvailableSites();
foreach ($allSites as $site) {
$solrConfiguration = $site->getSolrConfiguration();
$indexingConfigurationName = $this->recordService->getIndexingConfigurationName($table, $uid, $solrConfiguration);
$meilisearchConfiguration = $site->getMeilisearchConfiguration();
$indexingConfigurationName = $this->recordService->getIndexingConfigurationName($table, $uid, $meilisearchConfiguration);
if ($indexingConfigurationName === null) {
continue;
}
$observedPageIdsOfSiteRoot = $solrConfiguration->getIndexQueueAdditionalPageIdsByConfigurationName($indexingConfigurationName);
$observedPageIdsOfSiteRoot = $meilisearchConfiguration->getIndexQueueAdditionalPageIdsByConfigurationName($indexingConfigurationName);
foreach ($observedPageIdsOfSiteRoot as $observedPageIdOfSiteRoot) {
$siteRootByObservedPageIds[$observedPageIdOfSiteRoot][] = $site->getRootPageId();
}

2
Classes/Domain/Index/Queue/Statistic/QueueStatisticsRepository.php
View File

@@ -4,7 +4,7 @@ namespace WapplerSystems\Meilisearch\Domain\Index\Queue\Statistic;
/***************************************************************
* Copyright notice
*
* (c) 2010-2017 dkd Internet Service GmbH <solr-support@dkd.de>
* (c) 2010-2017 dkd Internet Service GmbH <meilisearch-support@dkd.de>
* All rights reserved
*
* This script is part of the TYPO3 project. The TYPO3 project is