WapplerSystems/meilisearch
6
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

first commit

Browse Source
This commit is contained in:
Sven Wappler
2021-04-17 21:20:54 +02:00
parent c93ec9492a
commit cadcc8edb4
406 changed files with 4917 additions and 5157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

114
Classes/IndexQueue/Indexer.php
View File

@@ -25,16 +25,16 @@ namespace WapplerSystems\Meilisearch\IndexQueue;
***************************************************************/
use WapplerSystems\Meilisearch\ConnectionManager;
use WapplerSystems\Meilisearch\Domain\Search\ApacheSolrDocument\Builder;
use WapplerSystems\Meilisearch\Domain\Search\ApacheMeilisearchDocument\Builder;
use WapplerSystems\Meilisearch\FieldProcessor\Service;
use WapplerSystems\Meilisearch\FrontendEnvironment;
use WapplerSystems\Meilisearch\NoSolrConnectionFoundException;
use WapplerSystems\Meilisearch\NoMeilisearchConnectionFoundException;
use WapplerSystems\Meilisearch\Domain\Site\Site;
use WapplerSystems\Meilisearch\System\Logging\SolrLogManager;
use WapplerSystems\Meilisearch\System\Logging\MeilisearchLogManager;
use WapplerSystems\Meilisearch\System\Records\Pages\PagesRepository;
use WapplerSystems\Meilisearch\System\Solr\Document\Document;
use WapplerSystems\Meilisearch\System\Solr\ResponseAdapter;
use WapplerSystems\Meilisearch\System\Solr\SolrConnection;
use WapplerSystems\Meilisearch\System\Meilisearch\Document\Document;
use WapplerSystems\Meilisearch\System\Meilisearch\ResponseAdapter;
use WapplerSystems\Meilisearch\System\Meilisearch\MeilisearchConnection;
use Exception;
use RuntimeException;
use Solarium\Exception\HttpException;
@@ -61,11 +61,11 @@ class Indexer extends AbstractIndexer
# TODO change to singular $document instead of plural $documents
/**
* A Solr service instance to interact with the Solr server
* A Meilisearch service instance to interact with the Meilisearch server
*
* @var SolrConnection
* @var MeilisearchConnection
*/
protected $solr;
protected $meilisearch;
/**
* @var ConnectionManager
@@ -87,7 +87,7 @@ class Indexer extends AbstractIndexer
protected $loggingEnabled = false;
/**
* @var SolrLogManager
* @var MeilisearchLogManager
*/
protected $logger = null;
@@ -112,7 +112,7 @@ class Indexer extends AbstractIndexer
* @param array $options array of indexer options
* @param PagesRepository|null $pagesRepository
* @param Builder|null $documentBuilder
* @param SolrLogManager|null $logger
* @param MeilisearchLogManager|null $logger
* @param ConnectionManager|null $connectionManager
* @param FrontendEnvironment|null $frontendEnvironment
*/
@@ -120,7 +120,7 @@ class Indexer extends AbstractIndexer
array $options = [],
PagesRepository $pagesRepository = null,
Builder $documentBuilder = null,
SolrLogManager $logger = null,
MeilisearchLogManager $logger = null,
ConnectionManager $connectionManager = null,
FrontendEnvironment $frontendEnvironment = null
)
@@ -128,7 +128,7 @@ class Indexer extends AbstractIndexer
$this->options = $options;
$this->pagesRepository = $pagesRepository ?? GeneralUtility::makeInstance(PagesRepository::class);
$this->documentBuilder = $documentBuilder ?? GeneralUtility::makeInstance(Builder::class);
$this->logger = $logger ?? GeneralUtility::makeInstance(SolrLogManager::class, /** @scrutinizer ignore-type */ __CLASS__);
$this->logger = $logger ?? GeneralUtility::makeInstance(MeilisearchLogManager::class, /** @scrutinizer ignore-type */ __CLASS__);
$this->connectionManager = $connectionManager ?? GeneralUtility::makeInstance(ConnectionManager::class);
$this->frontendEnvironment = $frontendEnvironment ?? GeneralUtility::makeInstance(FrontendEnvironment::class);
}
@@ -146,9 +146,9 @@ class Indexer extends AbstractIndexer
$this->type = $item->getType();
$this->setLogging($item);
$solrConnections = $this->getSolrConnectionsByItem($item);
foreach ($solrConnections as $systemLanguageUid => $solrConnection) {
$this->solr = $solrConnection;
$meilisearchConnections = $this->getMeilisearchConnectionsByItem($item);
foreach ($meilisearchConnections as $systemLanguageUid => $meilisearchConnection) {
$this->meilisearch = $meilisearchConnection;
if (!$this->indexItem($item, $systemLanguageUid)) {
/*
@@ -167,7 +167,7 @@ class Indexer extends AbstractIndexer
}
/**
* Creates a single Solr Document for an item in a specific language.
* Creates a single Meilisearch Document for an item in a specific language.
*
* @param Item $item An index queue item to index.
* @param int $language The language to use.
@@ -195,7 +195,7 @@ class Indexer extends AbstractIndexer
$documents = $this->preAddModifyDocuments($item, $language, $documents);
try {
$response = $this->solr->getWriteService()->addDocuments($documents);
$response = $this->meilisearch->getWriteService()->addDocuments($documents);
if ($response->getHttpStatus() == 200) {
$itemIndexed = true;
}
@@ -224,7 +224,7 @@ class Indexer extends AbstractIndexer
$itemRecord = $this->getItemRecordOverlayed($item, $language);
if (!is_null($itemRecord)) {
$itemRecord['__solr_index_language'] = $language;
$itemRecord['__meilisearch_index_language'] = $language;
}
return $itemRecord;
@@ -288,8 +288,8 @@ class Indexer extends AbstractIndexer
{
try {
$pageId = $this->getPageIdOfItem($item);
$solrConfiguration = $this->frontendEnvironment->getSolrConfigurationFromPageId($pageId, $language);
return $solrConfiguration->getIndexQueueFieldsConfigurationByConfigurationName($indexConfigurationName, []);
$meilisearchConfiguration = $this->frontendEnvironment->getMeilisearchConfigurationFromPageId($pageId, $language);
return $meilisearchConfiguration->getIndexQueueFieldsConfigurationByConfigurationName($indexConfigurationName, []);
} catch (Exception $e) {
return [];
}
@@ -317,9 +317,9 @@ class Indexer extends AbstractIndexer
*/
protected function getFieldConfigurationFromItemRootPage(Item $item, $language, $indexConfigurationName)
{
$solrConfiguration = $this->frontendEnvironment->getSolrConfigurationFromPageId($item->getRootPageUid(), $language);
$meilisearchConfiguration = $this->frontendEnvironment->getMeilisearchConfigurationFromPageId($item->getRootPageUid(), $language);
return $solrConfiguration->getIndexQueueFieldsConfigurationByConfigurationName($indexConfigurationName, []);
return $meilisearchConfiguration->getIndexQueueFieldsConfigurationByConfigurationName($indexConfigurationName, []);
}
/**
@@ -343,12 +343,12 @@ class Indexer extends AbstractIndexer
}
/**
* Converts an item array (record) to a Solr document by mapping the
* record's fields onto Solr document fields as configured in TypoScript.
* Converts an item array (record) to a Meilisearch document by mapping the
* record's fields onto Meilisearch document fields as configured in TypoScript.
*
* @param Item $item An index queue item
* @param int $language Language Id
* @return Document|null The Solr document converted from the record
* @return Document|null The Meilisearch document converted from the record
* @throws SiteNotFoundException
* @throws ServiceUnavailableException
* @throws ImmediateResponseException
@@ -373,11 +373,11 @@ class Indexer extends AbstractIndexer
}
/**
* Creates a Solr document with the basic / core fields set already.
* Creates a Meilisearch document with the basic / core fields set already.
*
* @param Item $item The item to index
* @param array $itemRecord The record to use to build the base document
* @return Document A basic Solr document
* @return Document A basic Meilisearch document
*/
protected function getBaseDocument(Item $item, array $itemRecord)
{
@@ -417,14 +417,14 @@ class Indexer extends AbstractIndexer
* manipulating fields as defined in the field's configuration.
*
* @param Item $item An index queue item
* @param array $documents An array of \WapplerSystems\Meilisearch\System\Solr\Document\Document objects to manipulate.
* @param array $documents An array of \WapplerSystems\Meilisearch\System\Meilisearch\Document\Document objects to manipulate.
* @return Document[] array Array of manipulated Document objects.
*/
protected function processDocuments(Item $item, array $documents)
{
// needs to respect the TS settings for the page the item is on, conditions may apply
$solrConfiguration = $this->frontendEnvironment->getSolrConfigurationFromPageId($item->getRootPageUid());
$fieldProcessingInstructions = $solrConfiguration->getIndexFieldProcessingInstructionsConfiguration();
$meilisearchConfiguration = $this->frontendEnvironment->getMeilisearchConfigurationFromPageId($item->getRootPageUid());
$fieldProcessingInstructions = $meilisearchConfiguration->getIndexFieldProcessingInstructionsConfiguration();
// same as in the FE indexer
if (is_array($fieldProcessingInstructions)) {
@@ -448,8 +448,8 @@ class Indexer extends AbstractIndexer
{
$documents = [];
if (is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['IndexQueueIndexer']['indexItemAddDocuments'])) {
foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['IndexQueueIndexer']['indexItemAddDocuments'] as $classReference) {
if (is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['meilisearch']['IndexQueueIndexer']['indexItemAddDocuments'])) {
foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['meilisearch']['IndexQueueIndexer']['indexItemAddDocuments'] as $classReference) {
if (!class_exists($classReference)) {
throw new \InvalidArgumentException('Class does not exits' . $classReference, 1490363487);
}
@@ -474,7 +474,7 @@ class Indexer extends AbstractIndexer
/**
* Provides a hook to manipulate documents right before they get added to
* the Solr index.
* the Meilisearch index.
*
* @param Item $item The item currently being indexed.
* @param int $language The language uid of the documents
@@ -483,8 +483,8 @@ class Indexer extends AbstractIndexer
*/
protected function preAddModifyDocuments(Item $item, $language, array $documents)
{
if (is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['IndexQueueIndexer']['preAddModifyDocuments'])) {
foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['IndexQueueIndexer']['preAddModifyDocuments'] as $classReference) {
if (is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['meilisearch']['IndexQueueIndexer']['preAddModifyDocuments'])) {
foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['meilisearch']['IndexQueueIndexer']['preAddModifyDocuments'] as $classReference) {
$documentsModifier = GeneralUtility::makeInstance($classReference);
if ($documentsModifier instanceof PageIndexerDocumentsModifier) {
@@ -507,17 +507,17 @@ class Indexer extends AbstractIndexer
// Initialization
/**
* Gets the Solr connections applicable for an item.
* Gets the Meilisearch connections applicable for an item.
*
* The connections include the default connection and connections to be used
* for translations of an item.
*
* @param Item $item An index queue item
* @return array An array of WapplerSystems\Meilisearch\System\Solr\SolrConnection connections, the array's keys are the sys_language_uid of the language of the connection
* @return array An array of WapplerSystems\Meilisearch\System\Meilisearch\MeilisearchConnection connections, the array's keys are the sys_language_uid of the language of the connection
*/
protected function getSolrConnectionsByItem(Item $item)
protected function getMeilisearchConnectionsByItem(Item $item)
{
$solrConnections = [];
$meilisearchConnections = [];
$rootPageId = $item->getRootPageUid();
if ($item->getType() === 'pages') {
@@ -526,12 +526,12 @@ class Indexer extends AbstractIndexer
$pageId = $item->getRecordPageId();
}
// Solr configurations possible for this item
// Meilisearch configurations possible for this item
$site = $item->getSite();
$solrConfigurationsBySite = $site->getAllSolrConnectionConfigurations();
$meilisearchConfigurationsBySite = $site->getAllMeilisearchConnectionConfigurations();
$siteLanguages = [];
foreach ($solrConfigurationsBySite as $solrConfiguration) {
$siteLanguages[] = $solrConfiguration['language'];
foreach ($meilisearchConfigurationsBySite as $meilisearchConfiguration) {
$siteLanguages[] = $meilisearchConfiguration['language'];
}
$defaultLanguageUid = $this->getDefaultLanguageUid($item, $site->getRootPage(), $siteLanguages);
@@ -541,13 +541,13 @@ class Indexer extends AbstractIndexer
$translationConnections = $this->getConnectionsForIndexableLanguages($translationOverlays);
if ($defaultLanguageUid == 0) {
$solrConnections[0] = $defaultConnection;
$meilisearchConnections[0] = $defaultConnection;
}
foreach ($translationConnections as $systemLanguageUid => $solrConnection) {
$solrConnections[$systemLanguageUid] = $solrConnection;
foreach ($translationConnections as $systemLanguageUid => $meilisearchConnection) {
$meilisearchConnections[$systemLanguageUid] = $meilisearchConnection;
}
return $solrConnections;
return $meilisearchConnections;
}
/**
@@ -641,7 +641,7 @@ class Indexer extends AbstractIndexer
* these connections.
*
* @param array $translationOverlays An array of translation overlays to check for configured connections.
* @return array An array of WapplerSystems\Meilisearch\System\Solr\SolrConnection connections.
* @return array An array of WapplerSystems\Meilisearch\System\Meilisearch\MeilisearchConnection connections.
*/
protected function getConnectionsForIndexableLanguages(array $translationOverlays)
{
@@ -654,7 +654,7 @@ class Indexer extends AbstractIndexer
try {
$connection = $this->connectionManager->getConnectionByPageId($pageId, $languageId);
$connections[$languageId] = $connection;
} catch (NoSolrConnectionFoundException $e) {
} catch (NoMeilisearchConnectionFoundException $e) {
// ignore the exception as we seek only those connections
// actually available
}
@@ -666,7 +666,7 @@ class Indexer extends AbstractIndexer
// Utility methods
// FIXME extract log() and setLogging() to WapplerSystems\Meilisearch\IndexQueue\AbstractIndexer
// FIXME extract an interface Tx_Solr_IndexQueue_ItemInterface
// FIXME extract an interface Tx_Meilisearch_IndexQueue_ItemInterface
/**
* Enables logging dependent on the configuration of the item's site
@@ -676,8 +676,8 @@ class Indexer extends AbstractIndexer
*/
protected function setLogging(Item $item)
{
$solrConfiguration = $this->frontendEnvironment->getSolrConfigurationFromPageId($item->getRootPageUid());
$this->loggingEnabled = $solrConfiguration->getLoggingIndexingQueueOperationsByConfigurationNameWithFallBack(
$meilisearchConfiguration = $this->frontendEnvironment->getMeilisearchConfigurationFromPageId($item->getRootPageUid());
$this->loggingEnabled = $meilisearchConfiguration->getLoggingIndexingQueueOperationsByConfigurationNameWithFallBack(
$item->getIndexingConfigurationName()
);
}
@@ -686,8 +686,8 @@ class Indexer extends AbstractIndexer
* Logs the item and what document was created from it
*
* @param Item $item The item that is being indexed.
* @param array $itemDocuments An array of Solr documents created from the item's data
* @param ResponseAdapter $response The Solr response for the particular index document
* @param array $itemDocuments An array of Meilisearch documents created from the item's data
* @param ResponseAdapter $response The Meilisearch response for the particular index document
*/
protected function log(Item $item, array $itemDocuments, ResponseAdapter $response)
{
@@ -706,10 +706,10 @@ class Indexer extends AbstractIndexer
$logData = ['item' => (array)$item, 'documents' => $documents, 'response' => (array)$response];
if ($response->getHttpStatus() == 200) {
$severity = SolrLogManager::NOTICE;
$severity = MeilisearchLogManager::NOTICE;
$message .= 'Success';
} else {
$severity = SolrLogManager::ERROR;
$severity = MeilisearchLogManager::ERROR;
$message .= 'Failure';
$logData['status'] = $response->getHttpStatus();