first commit

Classes/IndexQueue/AbstractIndexer.php

@@ -0,0 +1,336 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2012-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+use WapplerSystems\Meilisearch\ContentObject\Classification;
+use WapplerSystems\Meilisearch\ContentObject\Multivalue;
+use WapplerSystems\Meilisearch\ContentObject\Relation;
+use WapplerSystems\Meilisearch\System\Solr\Document\Document;
+use TYPO3\CMS\Core\Core\Environment;
+use TYPO3\CMS\Core\TypoScript\Parser\TypoScriptParser;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Frontend\ContentObject\ContentObjectRenderer;
+
+/**
+ * An abstract indexer class to collect a few common methods shared with other
+ * indexers.
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+abstract class AbstractIndexer
+{
+
+    /**
+     * Holds the type of the data to be indexed, usually that is the table name.
+     *
+     * @var string
+     */
+    protected $type = '';
+
+    /**
+     * Holds field names that are denied to overwrite in thy indexing configuration.
+     *
+     * @var array
+     */
+    protected static $unAllowedOverrideFields = ['type'];
+
+    /**
+     * @param string $solrFieldName
+     * @return bool
+     */
+    public static function isAllowedToOverrideField($solrFieldName)
+    {
+        return !in_array($solrFieldName, static::$unAllowedOverrideFields);
+    }
+
+    /**
+     * Adds fields to the document as defined in $indexingConfiguration
+     *
+     * @param Document $document base document to add fields to
+     * @param array $indexingConfiguration Indexing configuration / mapping
+     * @param array $data Record data
+     * @return Document Modified document with added fields
+     */
+    protected function addDocumentFieldsFromTyposcript(Document $document, array $indexingConfiguration, array $data) {
+        $data = static::addVirtualContentFieldToRecord($document, $data);
+
+        // mapping of record fields => solr document fields, resolving cObj
+        foreach ($indexingConfiguration as $solrFieldName => $recordFieldName) {
+            if (is_array($recordFieldName)) {
+                // configuration for a content object, skipping
+                continue;
+            }
+
+            if (!static::isAllowedToOverrideField($solrFieldName)) {
+                throw new InvalidFieldNameException(
+                    'Must not overwrite field .' . $solrFieldName,
+                    1435441863
+                );
+            }
+
+            $fieldValue = $this->resolveFieldValue($indexingConfiguration, $solrFieldName, $data);
+
+            if (is_array($fieldValue)) {
+                // multi value
+                $document->setField($solrFieldName, $fieldValue);
+            } else {
+                if ($fieldValue !== '' && $fieldValue !== null) {
+                    $document->setField($solrFieldName, $fieldValue);
+                }
+            }
+        }
+
+        return $document;
+    }
+
+
+    /**
+     * Add's the content of the field 'content' from the solr document as virtual field __solr_content in the record,
+     * to have it available in typoscript.
+     *
+     * @param Document $document
+     * @param array $data
+     * @return array
+     */
+    public static function addVirtualContentFieldToRecord(Document $document, array $data): array
+    {
+        if (isset($document['content'])) {
+            $data['__solr_content'] = $document['content'];
+            return $data;
+        }
+        return $data;
+    }
+
+    /**
+     * Resolves a field to its value depending on its configuration.
+     *
+     * This enables you to configure the indexer to put the item/record through
+     * cObj processing if wanted/needed. Otherwise the plain item/record value
+     * is taken.
+     *
+     * @param array $indexingConfiguration Indexing configuration as defined in plugin.tx_meilisearch_index.queue.[indexingConfigurationName].fields
+     * @param string $solrFieldName A Solr field name that is configured in the indexing configuration
+     * @param array $data A record or item's data
+     * @return string The resolved string value to be indexed
+     */
+    protected function resolveFieldValue(
+        array $indexingConfiguration,
+        $solrFieldName,
+        array $data
+    ) {
+        $contentObject = GeneralUtility::makeInstance(ContentObjectRenderer::class);
+
+        if (isset($indexingConfiguration[$solrFieldName . '.'])) {
+            // configuration found => need to resolve a cObj
+
+            // need to change directory to make IMAGE content objects work in BE context
+            // see http://blog.netzelf.de/lang/de/tipps-und-tricks/tslib_cobj-image-im-backend
+            $backupWorkingDirectory = getcwd();
+            chdir(Environment::getPublicPath() . '/');
+
+            $contentObject->start($data, $this->type);
+            $fieldValue = $contentObject->cObjGetSingle(
+                $indexingConfiguration[$solrFieldName],
+                $indexingConfiguration[$solrFieldName . '.']
+            );
+
+            chdir($backupWorkingDirectory);
+
+            if ($this->isSerializedValue($indexingConfiguration,
+                $solrFieldName)
+            ) {
+                $fieldValue = unserialize($fieldValue);
+            }
+        } elseif (substr($indexingConfiguration[$solrFieldName], 0,
+                1) === '<'
+        ) {
+            $referencedTsPath = trim(substr($indexingConfiguration[$solrFieldName],
+                1));
+            $typoScriptParser = GeneralUtility::makeInstance(TypoScriptParser::class);
+            // $name and $conf is loaded with the referenced values.
+            list($name, $conf) = $typoScriptParser->getVal($referencedTsPath,
+                $GLOBALS['TSFE']->tmpl->setup);
+
+            // need to change directory to make IMAGE content objects work in BE context
+            // see http://blog.netzelf.de/lang/de/tipps-und-tricks/tslib_cobj-image-im-backend
+            $backupWorkingDirectory = getcwd();
+            chdir(Environment::getPublicPath() . '/');
+
+            $contentObject->start($data, $this->type);
+            $fieldValue = $contentObject->cObjGetSingle($name, $conf);
+
+            chdir($backupWorkingDirectory);
+
+            if ($this->isSerializedValue($indexingConfiguration,
+                $solrFieldName)
+            ) {
+                $fieldValue = unserialize($fieldValue);
+            }
+        } else {
+            $fieldValue = $data[$indexingConfiguration[$solrFieldName]];
+        }
+
+        // detect and correct type for dynamic fields
+
+        // find last underscore, substr from there, cut off last character (S/M)
+        $fieldType = substr($solrFieldName, strrpos($solrFieldName, '_') + 1,
+            -1);
+        if (is_array($fieldValue)) {
+            foreach ($fieldValue as $key => $value) {
+                $fieldValue[$key] = $this->ensureFieldValueType($value,
+                    $fieldType);
+            }
+        } else {
+            $fieldValue = $this->ensureFieldValueType($fieldValue, $fieldType);
+        }
+
+        return $fieldValue;
+    }
+
+    // Utility methods
+
+    /**
+     * Uses a field's configuration to detect whether its value returned by a
+     * content object is expected to be serialized and thus needs to be
+     * unserialized.
+     *
+     * @param array $indexingConfiguration Current item's indexing configuration
+     * @param string $solrFieldName Current field being indexed
+     * @return bool TRUE if the value is expected to be serialized, FALSE otherwise
+     */
+    public static function isSerializedValue(array $indexingConfiguration, $solrFieldName)
+    {
+        $isSerialized = static::isSerializedResultFromRegisteredHook($indexingConfiguration, $solrFieldName);
+        if ($isSerialized === true) {
+            return $isSerialized;
+        }
+
+        $isSerialized = static::isSerializedResultFromCustomContentElement($indexingConfiguration, $solrFieldName);
+        return $isSerialized;
+    }
+
+    /**
+     * Checks if the response comes from a custom content element that returns a serialized value.
+     *
+     * @param array $indexingConfiguration
+     * @param string $solrFieldName
+     * @return bool
+     */
+    protected static function isSerializedResultFromCustomContentElement(array $indexingConfiguration, $solrFieldName): bool
+    {
+        $isSerialized = false;
+
+        // SOLR_CLASSIFICATION - always returns serialized array
+        if ($indexingConfiguration[$solrFieldName] == Classification::CONTENT_OBJECT_NAME) {
+            $isSerialized = true;
+        }
+
+        // SOLR_MULTIVALUE - always returns serialized array
+        if ($indexingConfiguration[$solrFieldName] == Multivalue::CONTENT_OBJECT_NAME) {
+            $isSerialized = true;
+        }
+
+        // SOLR_RELATION - returns serialized array if multiValue option is set
+        if ($indexingConfiguration[$solrFieldName] == Relation::CONTENT_OBJECT_NAME && !empty($indexingConfiguration[$solrFieldName . '.']['multiValue'])) {
+            $isSerialized = true;
+        }
+
+        return $isSerialized;
+    }
+
+    /**
+     * Checks registered hooks if a SerializedValueDetector detects a serialized response.
+     *
+     * @param array $indexingConfiguration
+     * @param string $solrFieldName
+     * @return bool
+     */
+    protected static function isSerializedResultFromRegisteredHook(array $indexingConfiguration, $solrFieldName)
+    {
+        if (!is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['detectSerializedValue'])) {
+            return false;
+        }
+
+        foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['detectSerializedValue'] as $classReference) {
+            $serializedValueDetector = GeneralUtility::makeInstance($classReference);
+            if (!$serializedValueDetector instanceof SerializedValueDetector) {
+                $message = get_class($serializedValueDetector) . ' must implement interface ' . SerializedValueDetector::class;
+                throw new \UnexpectedValueException($message, 1404471741);
+            }
+
+            $isSerialized = (boolean)$serializedValueDetector->isSerializedValue($indexingConfiguration, $solrFieldName);
+            if ($isSerialized) {
+                return true;
+            }
+        }
+    }
+
+    /**
+     * Makes sure a field's value matches a (dynamic) field's type.
+     *
+     * @param mixed $value Value to be added to a document
+     * @param string $fieldType The dynamic field's type
+     * @return mixed Returns the value in the correct format for the field type
+     */
+    protected function ensureFieldValueType($value, $fieldType)
+    {
+        switch ($fieldType) {
+            case 'int':
+            case 'tInt':
+                $value = intval($value);
+                break;
+
+            case 'float':
+            case 'tFloat':
+                $value = floatval($value);
+                break;
+
+            // long and double do not exist in PHP
+            // simply make sure it somehow looks like a number
+            // <insert PHP rant here>
+            case 'long':
+            case 'tLong':
+                // remove anything that's not a number or negative/minus sign
+                $value = preg_replace('/[^0-9\\-]/', '', $value);
+                if (trim($value) === '') {
+                    $value = 0;
+                }
+                break;
+            case 'double':
+            case 'tDouble':
+            case 'tDouble4':
+                // as long as it's numeric we'll take it, int or float doesn't matter
+                if (!is_numeric($value)) {
+                    $value = 0;
+                }
+                break;
+
+            default:
+                // assume things are correct for non-dynamic fields
+        }
+
+        return $value;
+    }
+}

Classes/IndexQueue/AdditionalIndexQueueItemIndexer.php

@@ -0,0 +1,49 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2012-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *  A copy is found in the textfile GPL.txt and important notices to the license
+ *  from the author is found in LICENSE.txt distributed with these scripts.
+ *
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+use WapplerSystems\Meilisearch\System\Solr\Document\Document;
+
+/**
+ * Interface that defines the method an indexer must implement to provide
+ * additional documents to index for an item being indexed by the Index Queue.
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+interface AdditionalIndexQueueItemIndexer
+{
+
+    /**
+     * Provides additional documents that should be indexed together with an Index Queue item.
+     *
+     * @param Item $item The item currently being indexed.
+     * @param int $language The language uid of the documents
+     * @param Document $itemDocument The original item document.
+     * @return Document[] array An array of additional Document objects
+     */
+    public function getAdditionalItemDocuments(Item $item, $language, Document $itemDocument);
+}

Classes/IndexQueue/Exception/DocumentPreparationException.php

@@ -0,0 +1,33 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue\Exception;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2010-2017 dkd Internet Service GmbH <solr-eb-support@dkd.de>
+ *
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+/**
+ * Exception that is thrown on TYPO3 side in indexing process.
+ */
+class DocumentPreparationException extends IndexingException
+{
+}

Classes/IndexQueue/Exception/IndexingException.php

@@ -0,0 +1,34 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue\Exception;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2010-2017 dkd Internet Service GmbH <solr-eb-support@dkd.de>
+ *
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+/**
+ * Exception that is thrown on indexing process. Does not matter on which side, TYPO3 or Apache.
+ * This exception should be used for any errors on indexing process.
+ */
+class IndexingException extends \Exception
+{
+}

Classes/IndexQueue/FrontendHelper/AbstractFrontendHelper.php

@@ -0,0 +1,123 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue\FrontendHelper;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2010-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+use WapplerSystems\Meilisearch\IndexQueue\PageIndexerRequest;
+use WapplerSystems\Meilisearch\IndexQueue\PageIndexerResponse;
+use WapplerSystems\Meilisearch\System\Logging\SolrLogManager;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController;
+
+/**
+ * Index Queue page indexer frontend helper base class implementing common
+ * functionality.
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+abstract class AbstractFrontendHelper implements FrontendHelper
+{
+
+    /**
+     * Index Queue page indexer request.
+     *
+     * @var PageIndexerRequest
+     */
+    protected $request;
+
+    /**
+     * Index Queue page indexer response.
+     *
+     * @var PageIndexerResponse
+     */
+    protected $response;
+
+    /**
+     * The action a frontend helper executes.
+     */
+    protected $action = null;
+
+    /**
+     * @var SolrLogManager
+     */
+    protected $logger = null;
+
+    /**
+     * Disables the frontend output for index queue requests.
+     *
+     * @param array $parameters Parameters from frontend
+     */
+    public function disableFrontendOutput(&$parameters)
+    {
+        $parameters['enableOutput'] = false;
+    }
+
+    /**
+     * Disables caching for page generation to get reliable results.
+     *
+     * @param array $parameters Parameters from frontend
+     * @param TypoScriptFrontendController $parentObject TSFE object
+     */
+    public function disableCaching(
+        /** @noinspection PhpUnusedParameterInspection */
+        &$parameters,
+        $parentObject
+    ) {
+        $parentObject->no_cache = true;
+    }
+
+    /**
+     * Starts the execution of a frontend helper.
+     *
+     * @param PageIndexerRequest $request Page indexer request
+     * @param PageIndexerResponse $response Page indexer response
+     */
+    public function processRequest(
+        PageIndexerRequest $request,
+        PageIndexerResponse $response
+    ) {
+        $this->request = $request;
+        $this->response = $response;
+        $this->logger = GeneralUtility::makeInstance(SolrLogManager::class, /** @scrutinizer ignore-type */ __CLASS__);
+
+        if ($request->getParameter('loggingEnabled')) {
+            $this->logger->log(
+                SolrLogManager::INFO,
+                'Page indexer request received',
+                [
+                    'request' => (array)$request,
+                ]
+            );
+        }
+    }
+
+    /**
+     * Deactivates a frontend helper by unregistering from hooks and releasing
+     * resources.
+     */
+    public function deactivate()
+    {
+        $this->response->addActionResult($this->action, $this->getData());
+    }
+}

Classes/IndexQueue/FrontendHelper/AuthorizationService.php

@@ -0,0 +1,126 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue\FrontendHelper;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2011-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+use WapplerSystems\Meilisearch\Access\Rootline;
+use WapplerSystems\Meilisearch\IndexQueue\PageIndexerRequestHandler;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Core\Authentication\AbstractAuthenticationService;
+
+/**
+ * Authentication service to authorize the Index Queue page indexer to access
+ * protected pages.
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+class AuthorizationService extends AbstractAuthenticationService
+{
+
+    /**
+     * User used when authenticating the page indexer for protected pages,
+     * to allow the indexer to access and protected content. May also allow to
+     * identify requests by the page indexer.
+     *
+     * @var string
+     */
+    const SOLR_INDEXER_USERNAME = '__SolrIndexerUser__';
+
+    /**
+     * Gets a fake frontend user record to allow access to protected pages.
+     *
+     * @return array An array representing a frontend user.
+     */
+    public function getUser()
+    {
+        return [
+            'uid' => 0,
+            'username' => self::SOLR_INDEXER_USERNAME,
+            'authenticated' => true
+        ];
+    }
+
+    /**
+     * Authenticates the page indexer frontend user to grant it access to
+     * protected pages and page content.
+     *
+     * Returns 200 which automatically grants access for the current fake page
+     * indexer user. A status of >= 200 also tells TYPO3 that it doesn't need to
+     * conduct other services that might be registered for "their opinion"
+     * whether a user is authenticated.
+     *
+     * @see \TYPO3\CMS\Core\Authentication\AbstractUserAuthentication::checkAuthentication()
+     * @param array $user Array of user data
+     * @return int Returns 200 to grant access for the page indexer.
+     */
+    public function authUser($user)
+    {
+        // shouldn't happen, but in case we get a regular user we just
+        // pass it on to another (regular) auth service
+        $authenticationLevel = 100;
+
+        if ($user['username'] == self::SOLR_INDEXER_USERNAME) {
+            $authenticationLevel = 200;
+        }
+
+        return $authenticationLevel;
+    }
+
+    /**
+     * Creates user group records so that the page indexer is granted access to
+     * protected pages.
+     *
+     * @param array $user Data of user.
+     * @param array $knownGroups Group data array of already known groups. This is handy if you want select other related groups. Keys in this array are unique IDs of those groups.
+     * @return mixed Groups array, keys = uid which must be unique
+     */
+    public function getGroups(
+        $user,
+        /** @noinspection PhpUnusedParameterInspection */
+        $knownGroups
+    ) {
+        $groupData = [];
+
+            /** @var $requestHandler PageIndexerRequestHandler */
+        $requestHandler = GeneralUtility::makeInstance(PageIndexerRequestHandler::class);
+        $accessRootline = $requestHandler->getRequest()->getParameter('accessRootline');
+
+        if ($user['username'] == self::SOLR_INDEXER_USERNAME && !empty($accessRootline)) {
+            $accessRootline = GeneralUtility::makeInstance(Rootline::class, /** @scrutinizer ignore-type */ $accessRootline);
+            $groups = $accessRootline->getGroups();
+
+            foreach ($groups as $groupId) {
+                // faking a user group record
+                $groupData[] = [
+                    'uid' => $groupId,
+                    'pid' => 0,
+                    'title' => '__SolrIndexerGroup__',
+                    'TSconfig' => ''
+                ];
+            }
+        }
+
+        return $groupData;
+    }
+}

Classes/IndexQueue/FrontendHelper/Dispatcher.php

@@ -0,0 +1,89 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue\FrontendHelper;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2010-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+use WapplerSystems\Meilisearch\IndexQueue\PageIndexerRequest;
+use WapplerSystems\Meilisearch\IndexQueue\PageIndexerResponse;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+/**
+ * Dispatches the actions requested to the matching frontend helpers.
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+class Dispatcher
+{
+
+    /**
+     * Frontend helper manager.
+     *
+     * @var Manager
+     */
+    protected $frontendHelperManager;
+
+    /**
+     * Constructor
+     *
+     */
+    public function __construct()
+    {
+        $this->frontendHelperManager = GeneralUtility::makeInstance(Manager::class);
+    }
+
+    /**
+     * Takes the request's actions and hands them of to the according frontend
+     * helpers.
+     *
+     * @param PageIndexerRequest $request The request to dispatch
+     * @param PageIndexerResponse $response The request's response
+     */
+    public function dispatch(
+        PageIndexerRequest $request,
+        PageIndexerResponse $response
+    ) {
+        $actions = $request->getActions();
+
+        foreach ($actions as $action) {
+            $frontendHelper = $this->frontendHelperManager->resolveAction($action);
+            $frontendHelper->activate();
+            $frontendHelper->processRequest($request, $response);
+        }
+    }
+
+    /**
+     * Sends a shutdown signal to all activated frontend helpers.
+     *
+     * @return void
+     */
+    public function shutdown()
+    {
+        $frontendHelpers = $this->frontendHelperManager->getActivatedFrontendHelpers();
+
+        foreach ($frontendHelpers as $frontendHelper) {
+            /** @var FrontendHelper $frontendHelper */
+            $frontendHelper->deactivate();
+        }
+    }
+}

Classes/IndexQueue/FrontendHelper/FrontendHelper.php

@@ -0,0 +1,70 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue\FrontendHelper;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2010-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *  A copy is found in the textfile GPL.txt and important notices to the license
+ *  from the author is found in LICENSE.txt distributed with these scripts.
+ *
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+use WapplerSystems\Meilisearch\IndexQueue\PageIndexerRequest;
+use WapplerSystems\Meilisearch\IndexQueue\PageIndexerResponse;
+
+/**
+ * Index Queue Frontend Helper interface.
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+interface FrontendHelper
+{
+
+    /**
+     * Activates a frontend helper by registering for hooks and other
+     * resources required by the frontend helper to work.
+     */
+    public function activate();
+
+    /**
+     * Deactivates a frontend helper by unregistering from hooks and releasing
+     * resources.
+     */
+    public function deactivate();
+
+    /**
+     * Starts the execution of a frontend helper.
+     *
+     * @param PageIndexerRequest $request Page indexer request
+     * @param PageIndexerResponse $response Page indexer response
+     */
+    public function processRequest(
+        PageIndexerRequest $request,
+        PageIndexerResponse $response
+    );
+
+    /**
+     * Returns the collected data.
+     *
+     * @return array Collected data.
+     */
+    public function getData();
+}

Classes/IndexQueue/FrontendHelper/Manager.php

@@ -0,0 +1,98 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue\FrontendHelper;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2010-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+use RuntimeException;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+/**
+ * Index Queue Page Indexer frontend helper manager.
+ *
+ * Manages frontend helpers and creates instances.
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+class Manager
+{
+
+    /**
+     * Frontend helper descriptions.
+     *
+     * @var array
+     */
+    protected static $frontendHelperRegistry = [];
+
+    /**
+     * Instances of activated frontend helpers.
+     *
+     * @var array
+     */
+    protected $activatedFrontendHelpers = [];
+
+    /**
+     * Registers a frontend helper class for a certain action.
+     *
+     * @param string $action Action to register.
+     * @param string $class Class to register for an action.
+     */
+    public static function registerFrontendHelper($action, $class)
+    {
+        self::$frontendHelperRegistry[$action] = $class;
+    }
+
+    /**
+     * Tries to find a frontend helper for a given action. If found, creates an
+     * instance of the helper.
+     *
+     * @param string $action The action to get a frontend helper for.
+     * @return FrontendHelper Index Queue page indexer frontend helper
+     * @throws RuntimeException if the class registered for an action is not an implementation of WapplerSystems\Meilisearch\IndexQueue\FrontendHelper\FrontendHelper
+     */
+    public function resolveAction($action)
+    {
+        if (!array_key_exists($action, self::$frontendHelperRegistry)) {
+            return null;
+        }
+
+        $frontendHelper = GeneralUtility::makeInstance(self::$frontendHelperRegistry[$action]);
+        if (!$frontendHelper instanceof FrontendHelper) {
+            $message = self::$frontendHelperRegistry[$action] . ' is not an implementation of WapplerSystems\Meilisearch\IndexQueue\FrontendHelper\FrontendHelper';
+            throw new RuntimeException($message, 1292497896);
+        }
+
+        $this->activatedFrontendHelpers[$action] = $frontendHelper;
+        return $frontendHelper;
+    }
+
+    /**
+     * Gets an array with references to activated frontend helpers.
+     *
+     * @return array Array of references to activated frontend helpers.
+     */
+    public function getActivatedFrontendHelpers()
+    {
+        return $this->activatedFrontendHelpers;
+    }
+}

Classes/IndexQueue/FrontendHelper/PageFieldMappingIndexer.php

@@ -0,0 +1,164 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue\FrontendHelper;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2011-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+// TODO use/extend WapplerSystems\Meilisearch\IndexQueue\AbstractIndexer
+use WapplerSystems\Meilisearch\IndexQueue\AbstractIndexer;
+use WapplerSystems\Meilisearch\IndexQueue\InvalidFieldNameException;
+use WapplerSystems\Meilisearch\SubstitutePageIndexer;
+use WapplerSystems\Meilisearch\System\Configuration\TypoScriptConfiguration;
+use WapplerSystems\Meilisearch\System\Solr\Document\Document;
+use WapplerSystems\Meilisearch\Util;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Frontend\ContentObject\ContentObjectRenderer;
+
+/**
+ * Indexer to add / overwrite page document fields as defined in
+ * plugin.tx_meilisearch.index.queue.pages.fields.
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+class PageFieldMappingIndexer implements SubstitutePageIndexer
+{
+    /**
+     * @var \WapplerSystems\Meilisearch\System\Configuration\TypoScriptConfiguration
+     */
+    protected $configuration;
+
+    /**
+     * @var string
+     */
+    protected $pageIndexingConfigurationName = 'pages';
+
+    /**
+     * @param TypoScriptConfiguration $configuration
+     */
+    public function __construct(TypoScriptConfiguration $configuration = null)
+    {
+        $this->configuration = $configuration == null ? Util::getSolrConfiguration() : $configuration;
+    }
+
+    /**
+     * @param string $pageIndexingConfigurationName
+     */
+    public function setPageIndexingConfigurationName($pageIndexingConfigurationName)
+    {
+        $this->pageIndexingConfigurationName = $pageIndexingConfigurationName;
+    }
+
+    /**
+     * Returns a substitute document for the currently being indexed page.
+     *
+     * Uses the original document and adds fields as defined in
+     * plugin.tx_meilisearch.index.queue.pages.fields.
+     *
+     * @param Document $pageDocument The original page document.
+     * @return Document A Apache Solr Document object that replace the default page document
+     */
+    public function getPageDocument(Document $pageDocument)
+    {
+        $substitutePageDocument = clone $pageDocument;
+
+
+        $mappedFields = $this->getMappedFields($pageDocument);
+        foreach ($mappedFields as $fieldName => $fieldValue) {
+            if (isset($substitutePageDocument->{$fieldName})) {
+                // reset = overwrite, especially important to not make fields
+                // multi valued where they may not accept multiple values
+                unset($substitutePageDocument->{$fieldName});
+            }
+
+            // add new field / overwrite field if it was set before
+            if ($fieldValue !== '' && $fieldValue !== null) {
+                $substitutePageDocument->setField($fieldName, $fieldValue);
+            }
+        }
+
+        return $substitutePageDocument;
+    }
+
+    /**
+     * Gets the mapped fields as an array mapping field names to values.
+     *
+     * @throws InvalidFieldNameException
+     * @param Document $pageDocument The original page document.
+     * @return array An array mapping field names to their values.
+     */
+    protected function getMappedFields(Document $pageDocument)
+    {
+        $fields = [];
+
+        $mappedFieldNames = $this->configuration->getIndexQueueMappedFieldsByConfigurationName($this->pageIndexingConfigurationName);
+
+        foreach ($mappedFieldNames as $mappedFieldName) {
+            if (!AbstractIndexer::isAllowedToOverrideField($mappedFieldName)) {
+                throw new InvalidFieldNameException(
+                    'Must not overwrite field "type".',
+                    1435441863
+                );
+            }
+            $fields[$mappedFieldName] = $this->resolveFieldValue($mappedFieldName, $pageDocument);
+        }
+
+        return $fields;
+    }
+
+    /**
+     * Resolves a field mapping to its value depending on its configuration.
+     *
+     * Allows to put the page record through cObj processing if wanted / needed.
+     * Otherwise the plain page record field value is used.
+     *
+     * @param string $solrFieldName The Solr field name to resolve the value from the item's record
+     * @return string The resolved string value to be indexed
+     */
+    protected function resolveFieldValue($solrFieldName, Document $pageDocument)
+    {
+        $pageRecord = $GLOBALS['TSFE']->page;
+
+        $pageIndexingConfiguration = $this->configuration->getIndexQueueFieldsConfigurationByConfigurationName($this->pageIndexingConfigurationName);
+
+        if (isset($pageIndexingConfiguration[$solrFieldName . '.'])) {
+            $pageRecord = AbstractIndexer::addVirtualContentFieldToRecord($pageDocument, $pageRecord);
+
+            // configuration found => need to resolve a cObj
+            $contentObject = GeneralUtility::makeInstance(ContentObjectRenderer::class);
+            $contentObject->start($pageRecord, 'pages');
+
+            $fieldValue = $contentObject->cObjGetSingle(
+                $pageIndexingConfiguration[$solrFieldName],
+                $pageIndexingConfiguration[$solrFieldName . '.']
+            );
+
+            if (AbstractIndexer::isSerializedValue($pageIndexingConfiguration, $solrFieldName)) {
+                $fieldValue = unserialize($fieldValue);
+            }
+        } else {
+            $fieldValue = $pageRecord[$pageIndexingConfiguration[$solrFieldName]];
+        }
+
+        return $fieldValue;
+    }
+}

Classes/IndexQueue/FrontendHelper/PageIndexer.php

@@ -0,0 +1,364 @@
+<?php
+
+namespace WapplerSystems\Meilisearch\IndexQueue\FrontendHelper;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2011-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+use WapplerSystems\Meilisearch\Access\Rootline;
+use WapplerSystems\Meilisearch\ConnectionManager;
+use WapplerSystems\Meilisearch\IndexQueue\Item;
+use WapplerSystems\Meilisearch\IndexQueue\Queue;
+use WapplerSystems\Meilisearch\NoSolrConnectionFoundException;
+use WapplerSystems\Meilisearch\System\Logging\SolrLogManager;
+use WapplerSystems\Meilisearch\System\Solr\SolrConnection;
+use WapplerSystems\Meilisearch\Typo3PageIndexer;
+use WapplerSystems\Meilisearch\Util;
+use Exception;
+use TYPO3\CMS\Core\Log\Logger;
+use TYPO3\CMS\Core\SingletonInterface;
+use TYPO3\CMS\Core\Utility\ExtensionManagementUtility;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Frontend\ContentObject\ContentObjectRenderer;
+use TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController;
+use UnexpectedValueException;
+
+/**
+ * Index Queue Page Indexer frontend helper to ask the frontend page indexer to
+ * index the page.
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+class PageIndexer extends AbstractFrontendHelper implements SingletonInterface
+{
+
+    /**
+     * This frontend helper's executed action.
+     *
+     * @var string
+     */
+    protected $action = 'indexPage';
+
+    /**
+     * the page currently being indexed.
+     *
+     * @var TypoScriptFrontendController
+     */
+    protected $page;
+
+    /**
+     * Response data
+     *
+     * @var array
+     */
+    protected $responseData = [];
+
+    /**
+     * @var Logger
+     */
+    protected $logger = null;
+
+    /**
+     * Activates a frontend helper by registering for hooks and other
+     * resources required by the frontend helper to work.
+     *
+     * @noinspection PhpUnused
+     */
+    public function activate()
+    {
+        $pageIndexingHookRegistration = PageIndexer::class;
+
+        $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['tslib/class.tslib_fe.php']['pageIndexing'][__CLASS__] = $pageIndexingHookRegistration;
+
+        // indexes fields defined in plugin.tx_meilisearch.index.queue.pages.fields
+        $GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['Indexer']['indexPageSubstitutePageDocument'][PageFieldMappingIndexer::class] = PageFieldMappingIndexer::class;
+
+        $this->registerAuthorizationService();
+    }
+
+    /**
+     * Returns the status of whether a page was indexed.
+     *
+     * @return array Page indexed status.
+     * @noinspection PhpUnused
+     */
+    public function getData()
+    {
+        return $this->responseData;
+    }
+
+    #
+    # Indexer authorisation for access restricted pages / content
+    #
+
+    /**
+     * Fakes a logged in user to retrieve access restricted content.
+     *
+     * @return void
+     * @noinspection PhpUnused
+     */
+    public function authorizeFrontendUser()
+    {
+        $accessRootline = $this->getAccessRootline();
+        $stringAccessRootline = (string)$accessRootline;
+
+        if (empty($stringAccessRootline)) {
+            return;
+        }
+
+        if (!is_array($GLOBALS['TSFE']->fe_user->user)) {
+            $GLOBALS['TSFE']->fe_user->user = [];
+        }
+
+        $groups = $accessRootline->getGroups();
+        $groupList = implode(',', $groups);
+
+        $GLOBALS['TSFE']->fe_user->user['username'] = AuthorizationService::SOLR_INDEXER_USERNAME;
+        $GLOBALS['TSFE']->fe_user->user['usergroup'] = $groupList;
+
+        $this->responseData['authorization'] = [
+            'username' => $GLOBALS['TSFE']->fe_user->user['username'],
+            'usergroups' => $GLOBALS['TSFE']->fe_user->user['usergroup']
+        ];
+    }
+
+    /**
+     * Gets the access rootline as defined by the request.
+     *
+     * @return Rootline The access rootline to use for indexing.
+     */
+    protected function getAccessRootline()
+    {
+        $stringAccessRootline = '';
+
+        if ($this->request->getParameter('accessRootline')) {
+            $stringAccessRootline = $this->request->getParameter('accessRootline');
+        }
+
+        /** @noinspection PhpIncompatibleReturnTypeInspection */
+        return GeneralUtility::makeInstance(Rootline::class, /** @scrutinizer ignore-type */ $stringAccessRootline);
+    }
+
+    /**
+     * Registers an authentication service to authorize / grant the indexer to
+     * access protected pages.
+     *
+     * @return void
+     */
+    protected function registerAuthorizationService()
+    {
+        $overrulingPriority = $this->getHighestAuthenticationServicePriority() + 1;
+
+        ExtensionManagementUtility::addService(
+            'solr', // extension key
+            'auth', // service type
+            AuthorizationService::class,
+            // service key
+            [// service meta data
+                'title' => 'Solr Indexer Authorization',
+                'description' => 'Authorizes the Solr Index Queue indexer to access protected pages.',
+
+                'subtype' => 'getUserFE,authUserFE,getGroupsFE',
+
+                'available' => true,
+                'priority' => $overrulingPriority,
+                'quality' => 100,
+
+                'os' => '',
+                'exec' => '',
+
+                'classFile' => ExtensionManagementUtility::extPath('solr') . 'Classes/IndexQueue/FrontendHelper/AuthorizationService.php',
+                'className' => AuthorizationService::class,
+            ]
+        );
+    }
+
+    /**
+     * Determines the highest priority of all registered authentication
+     * services.
+     *
+     * @return int Highest priority of all registered authentication service
+     */
+    protected function getHighestAuthenticationServicePriority()
+    {
+        $highestPriority = 0;
+
+        if (is_array($GLOBALS['T3_SERVICES']['auth'])) {
+            foreach ($GLOBALS['T3_SERVICES']['auth'] as $service) {
+                if ($service['priority'] > $highestPriority) {
+                    $highestPriority = $service['priority'];
+                }
+            }
+        }
+
+        return $highestPriority;
+    }
+
+    #
+    # Indexing
+    #
+
+    /**
+     * Generates the current page's URL.
+     *
+     * Uses the provided GET parameters, page id and language id.
+     *
+     * @return string URL of the current page.
+     */
+    protected function generatePageUrl()
+    {
+        if ($this->request->getParameter('overridePageUrl')) {
+            return $this->request->getParameter('overridePageUrl');
+        }
+
+            /** @var $contentObject ContentObjectRenderer */
+        $contentObject = GeneralUtility::makeInstance(ContentObjectRenderer::class);
+
+        $typolinkConfiguration = [
+            'parameter' => intval($this->page->id),
+            'linkAccessRestrictedPages' => '1'
+        ];
+
+        $language = GeneralUtility::_GET('L');
+        if (!empty($language)) {
+            $typolinkConfiguration['additionalParams'] = '&L=' . $language;
+        }
+
+        $url = $contentObject->typoLink_URL($typolinkConfiguration);
+
+        // clean up
+        if ($url == '') {
+            $url = '/';
+        }
+
+        return $url;
+    }
+
+    /**
+     * Handles the indexing of the page content during post processing of a
+     * generated page.
+     *
+     * @param TypoScriptFrontendController $page TypoScript frontend
+     * @noinspection PhpUnused
+     */
+    public function hook_indexContent(TypoScriptFrontendController $page)
+    {
+        $this->logger = GeneralUtility::makeInstance(SolrLogManager::class, /** @scrutinizer ignore-type */ __CLASS__);
+
+        $this->page = $page;
+        $configuration = Util::getSolrConfiguration();
+
+        $logPageIndexed = $configuration->getLoggingIndexingPageIndexed();
+        if (!$this->page->config['config']['index_enable']) {
+            if ($logPageIndexed) {
+                $this->logger->log(
+                    SolrLogManager::ERROR,
+                    'Indexing is disabled. Set config.index_enable = 1 .'
+                );
+            }
+            return;
+        }
+
+        try {
+            $indexQueueItem = $this->getIndexQueueItem();
+            if (is_null($indexQueueItem)) {
+                throw new UnexpectedValueException('Can not get index queue item', 1482162337);
+            }
+
+            $solrConnection = $this->getSolrConnection($indexQueueItem);
+
+            /** @var $indexer Typo3PageIndexer */
+            $indexer = GeneralUtility::makeInstance(Typo3PageIndexer::class, /** @scrutinizer ignore-type */ $page);
+            $indexer->setSolrConnection($solrConnection);
+            $indexer->setPageAccessRootline($this->getAccessRootline());
+            $indexer->setPageUrl($this->generatePageUrl());
+            $indexer->setMountPointParameter($GLOBALS['TSFE']->MP);
+            $indexer->setIndexQueueItem($indexQueueItem);
+
+            $this->responseData['pageIndexed'] = (int)$indexer->indexPage();
+            $this->responseData['originalPageDocument'] = (array)$indexer->getPageSolrDocument();
+            $this->responseData['solrConnection'] = [
+                'rootPage' => $indexQueueItem->getRootPageUid(),
+                'sys_language_uid' => Util::getLanguageUid(),
+                'solr' => (string)$solrConnection->getNode('write')
+            ];
+
+            $documentsSentToSolr = $indexer->getDocumentsSentToSolr();
+            foreach ($documentsSentToSolr as $document) {
+                $this->responseData['documentsSentToSolr'][] = (array)$document;
+            }
+        } catch (Exception $e) {
+            if ($configuration->getLoggingExceptions()) {
+                $this->logger->log(
+                    SolrLogManager::ERROR,
+                    'Exception while trying to index page ' . $page->id,
+                    [
+                        $e->__toString()
+                    ]
+                );
+            }
+        }
+
+        if ($logPageIndexed) {
+            $success = $this->responseData['pageIndexed'] ? 'Success' : 'Failed';
+            $severity = $this->responseData['pageIndexed'] ? SolrLogManager::NOTICE : SolrLogManager::ERROR;
+
+            $this->logger->log(
+                $severity,
+                'Page indexed: ' . $success,
+                $this->responseData
+            );
+        }
+    }
+
+    /**
+     * Gets the solr connection to use for indexing the page based on the
+     * Index Queue item's properties.
+     *
+     * @param Item $indexQueueItem
+     * @return SolrConnection Solr server connection
+     * @throws NoSolrConnectionFoundException
+     */
+    protected function getSolrConnection(Item $indexQueueItem)
+    {
+        /** @var $connectionManager ConnectionManager */
+        $connectionManager = GeneralUtility::makeInstance(ConnectionManager::class);
+
+        return $connectionManager->getConnectionByRootPageId(
+            $indexQueueItem->getRootPageUid(),
+            Util::getLanguageUid()
+        );
+    }
+
+    /**
+     * This method retrieves the item from the index queue, that is indexed in this request.
+     *
+     * @return Item
+     */
+    protected function getIndexQueueItem()
+    {
+        /** @var $indexQueue Queue */
+        $indexQueue = GeneralUtility::makeInstance(Queue::class);
+        return $indexQueue->getItem($this->request->getParameter('item'));
+    }
+}

Classes/IndexQueue/FrontendHelper/UserGroupDetector.php

@@ -0,0 +1,253 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue\FrontendHelper;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2010-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+use WapplerSystems\Meilisearch\System\Logging\SolrLogManager;
+use TYPO3\CMS\Core\SingletonInterface;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Frontend\ContentObject\ContentObjectPostInitHookInterface;
+use TYPO3\CMS\Frontend\ContentObject\ContentObjectRenderer;
+use TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController;
+use TYPO3\CMS\Frontend\Page\PageRepository;
+use TYPO3\CMS\Frontend\Page\PageRepositoryGetPageHookInterface;
+use TYPO3\CMS\Frontend\Page\PageRepositoryGetPageOverlayHookInterface;
+
+/**
+ * The UserGroupDetector is responsible to identify the fe_group references on records that are visible on the page (not the page itself).
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+class UserGroupDetector extends AbstractFrontendHelper implements
+    SingletonInterface,
+    ContentObjectPostInitHookInterface,
+    PageRepositoryGetPageHookInterface,
+    PageRepositoryGetPageOverlayHookInterface
+{
+
+    /**
+     * This frontend helper's executed action.
+     */
+    protected $action = 'findUserGroups';
+
+    /**
+     * Holds the original, unmodified TCA during user group detection
+     *
+     * @var array
+     */
+    protected $originalTca = null;
+
+    /**
+     * Collects the usergroups used on a page.
+     *
+     * @var array
+     */
+    protected $frontendGroups = [];
+
+    /**
+     * @var \WapplerSystems\Meilisearch\System\Logging\SolrLogManager
+     */
+    protected $logger = null;
+
+    // activation
+
+    /**
+     * Activates a frontend helper by registering for hooks and other
+     * resources required by the frontend helper to work.
+     */
+    public function activate()
+    {
+        // register hooks
+        $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['tslib/class.tslib_fe.php']['isOutputting'][__CLASS__] = UserGroupDetector::class . '->disableFrontendOutput';
+        // disable TSFE cache for TYPO3 v9
+        $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['tslib/class.tslib_fe.php']['tslib_fe-PostProc'][__CLASS__] = UserGroupDetector::class . '->disableCaching';
+        $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['tslib/class.tslib_fe.php']['configArrayPostProc'][__CLASS__] = UserGroupDetector::class . '->deactivateTcaFrontendGroupEnableFields';
+        $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['tslib/class.tslib_fe.php']['hook_checkEnableFields'][__CLASS__] = UserGroupDetector::class . '->checkEnableFields';
+
+        $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['t3lib/class.t3lib_page.php']['getPage'][__CLASS__] = UserGroupDetector::class;
+        $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['t3lib/class.t3lib_page.php']['getPageOverlay'][__CLASS__] = UserGroupDetector::class;
+
+        $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['tslib/class.tslib_content.php']['postInit'][__CLASS__] = UserGroupDetector::class;
+    }
+
+    /**
+     * Disables the group access check by resetting the fe_group field in the given page table row.
+     * Will be called by the hook in the TypoScriptFrontendController in the checkEnableFields() method.
+     *
+     * @param array $parameters
+     * @param TypoScriptFrontendController $tsfe
+     * @see \TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController::checkEnableFields()
+     */
+    public function checkEnableFields(
+        $parameters,
+        /** @noinspection PhpUnusedParameterInspection */
+        $tsfe
+    ) {
+        $parameters['row']['fe_group'] = '';
+    }
+
+    /**
+     * Deactivates the frontend user group fields in TCA so that no access
+     * restrictions apply during page rendering.
+     *
+     * @param array $parameters Parameters from frontend
+     * @param TypoScriptFrontendController $parentObject TSFE object
+     */
+    public function deactivateTcaFrontendGroupEnableFields(
+        /** @noinspection PhpUnusedParameterInspection */
+        &$parameters,
+        /** @noinspection PhpUnusedParameterInspection */
+        $parentObject
+    ) {
+        $this->originalTca = $GLOBALS['TCA'];
+
+        foreach ($GLOBALS['TCA'] as $tableName => $tableConfiguration) {
+            if (isset($GLOBALS['TCA'][$tableName]['ctrl']['enablecolumns']['fe_group'])) {
+                unset($GLOBALS['TCA'][$tableName]['ctrl']['enablecolumns']['fe_group']);
+            }
+        }
+    }
+
+    // manipulation
+
+    /**
+     * Modifies the database query parameters so that access checks for pages
+     * are not performed any longer.
+     *
+     * @param int $uid The page ID
+     * @param bool $disableGroupAccessCheck If set, the check for group access is disabled. VERY rarely used
+     * @param PageRepository $parentObject parent \TYPO3\CMS\Frontend\Page\PageRepository object
+     */
+    public function getPage_preProcess(
+        &$uid,
+        &$disableGroupAccessCheck,
+        PageRepository $parentObject
+    ) {
+        $disableGroupAccessCheck = true;
+        $parentObject->where_groupAccess = ''; // just to be on the safe side
+    }
+
+    /**
+     * Modifies page records so that when checking for access through fe groups
+     * no groups or extendToSubpages flag is found and thus access is granted.
+     *
+     * @param array $pageRecord Page record
+     * @param int $languageUid Overlay language ID
+     * @param PageRepository $parentObject Parent \TYPO3\CMS\Frontend\Page\PageRepository object
+     */
+    public function getPageOverlay_preProcess(
+        &$pageRecord,
+        &$languageUid,
+        PageRepository $parentObject
+    ) {
+        if (is_array($pageRecord)) {
+            $pageRecord['fe_group'] = '';
+            $pageRecord['extendToSubpages'] = '0';
+        }
+    }
+
+    // execution
+
+    /**
+     * Hook for post processing the initialization of ContentObjectRenderer
+     *
+     * @param ContentObjectRenderer $parentObject parent content object
+     */
+    public function postProcessContentObjectInitialization(ContentObjectRenderer &$parentObject)
+    {
+        if (!empty($parentObject->currentRecord)) {
+            list($table) = explode(':', $parentObject->currentRecord);
+
+            if (!empty($table) && $table != 'pages') {
+                $this->findFrontendGroups($parentObject->data, $table);
+            }
+        }
+    }
+
+    /**
+     * Tracks user groups access restriction applied to records.
+     *
+     * @param array $record A record as an array of fieldname => fieldvalue mappings
+     * @param string $table Table name the record belongs to
+     */
+    protected function findFrontendGroups($record, $table)
+    {
+        if ($this->originalTca[$table]['ctrl']['enablecolumns']['fe_group']) {
+            $frontendGroups = $record[$this->originalTca[$table]['ctrl']['enablecolumns']['fe_group']];
+
+            if (empty($frontendGroups)) {
+                // default = public access
+                $frontendGroups = 0;
+            } else {
+                if ($this->request->getParameter('loggingEnabled')) {
+                    $this->logger->log(
+                        SolrLogManager::INFO,
+                        'Access restriction found',
+                        [
+                            'groups' => $frontendGroups,
+                            'record' => $record,
+                            'record type' => $table,
+                        ]
+                    );
+                }
+            }
+
+            $this->frontendGroups[] = $frontendGroups;
+        }
+    }
+
+    /**
+     * Returns an array of user groups that have been tracked during page
+     * rendering.
+     *
+     * @return array Array of user group IDs
+     */
+    protected function getFrontendGroups()
+    {
+        $frontendGroupsList = implode(',', $this->frontendGroups);
+        $frontendGroups = GeneralUtility::trimExplode(',', $frontendGroupsList,
+            true);
+
+        // clean up: filter double groups
+        $frontendGroups = array_unique($frontendGroups);
+        $frontendGroups = array_values($frontendGroups);
+
+        if (empty($frontendGroups)) {
+            // most likely an empty page with no content elements => public
+            $frontendGroups[] = '0';
+        }
+
+        return $frontendGroups;
+    }
+
+    /**
+     * Returns the user groups found.
+     *
+     * @return array Array of user groups.
+     */
+    public function getData()
+    {
+        return $this->getFrontendGroups();
+    }
+}

Classes/IndexQueue/Indexer.php

@@ -0,0 +1,721 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2009-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+use WapplerSystems\Meilisearch\ConnectionManager;
+use WapplerSystems\Meilisearch\Domain\Search\ApacheSolrDocument\Builder;
+use WapplerSystems\Meilisearch\FieldProcessor\Service;
+use WapplerSystems\Meilisearch\FrontendEnvironment;
+use WapplerSystems\Meilisearch\NoSolrConnectionFoundException;
+use WapplerSystems\Meilisearch\Domain\Site\Site;
+use WapplerSystems\Meilisearch\System\Logging\SolrLogManager;
+use WapplerSystems\Meilisearch\System\Records\Pages\PagesRepository;
+use WapplerSystems\Meilisearch\System\Solr\Document\Document;
+use WapplerSystems\Meilisearch\System\Solr\ResponseAdapter;
+use WapplerSystems\Meilisearch\System\Solr\SolrConnection;
+use Exception;
+use RuntimeException;
+use Solarium\Exception\HttpException;
+use TYPO3\CMS\Core\Context\LanguageAspectFactory;
+use TYPO3\CMS\Core\Error\Http\ServiceUnavailableException;
+use TYPO3\CMS\Core\Exception\SiteNotFoundException;
+use TYPO3\CMS\Core\Http\ImmediateResponseException;
+use TYPO3\CMS\Core\Site\SiteFinder;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Core\Utility\RootlineUtility;
+use TYPO3\CMS\Frontend\Page\PageRepository;
+
+/**
+ * A general purpose indexer to be used for indexing of any kind of regular
+ * records like tt_news, tt_address, and so on.
+ * Specialized indexers can extend this class to handle advanced stuff like
+ * category resolution in tt_news or file indexing.
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+class Indexer extends AbstractIndexer
+{
+
+    # TODO change to singular $document instead of plural $documents
+
+    /**
+     * A Solr service instance to interact with the Solr server
+     *
+     * @var SolrConnection
+     */
+    protected $solr;
+
+    /**
+     * @var ConnectionManager
+     */
+    protected $connectionManager;
+
+    /**
+     * Holds options for a specific indexer
+     *
+     * @var array
+     */
+    protected $options = [];
+
+    /**
+     * To log or not to log... #Shakespeare
+     *
+     * @var bool
+     */
+    protected $loggingEnabled = false;
+
+    /**
+     * @var SolrLogManager
+     */
+    protected $logger = null;
+
+    /**
+     * @var PagesRepository
+     */
+    protected $pagesRepository;
+
+    /**
+     * @var Builder
+     */
+    protected $documentBuilder;
+
+    /**
+     * @var FrontendEnvironment
+     */
+    protected $frontendEnvironment = null;
+
+    /**
+     * Constructor
+     *
+     * @param array $options array of indexer options
+     * @param PagesRepository|null $pagesRepository
+     * @param Builder|null $documentBuilder
+     * @param SolrLogManager|null $logger
+     * @param ConnectionManager|null $connectionManager
+     * @param FrontendEnvironment|null $frontendEnvironment
+     */
+    public function __construct(
+        array $options = [],
+        PagesRepository $pagesRepository = null,
+        Builder $documentBuilder = null,
+        SolrLogManager $logger = null,
+        ConnectionManager $connectionManager = null,
+        FrontendEnvironment $frontendEnvironment = null
+    )
+    {
+        $this->options = $options;
+        $this->pagesRepository = $pagesRepository ?? GeneralUtility::makeInstance(PagesRepository::class);
+        $this->documentBuilder = $documentBuilder ?? GeneralUtility::makeInstance(Builder::class);
+        $this->logger = $logger ?? GeneralUtility::makeInstance(SolrLogManager::class, /** @scrutinizer ignore-type */ __CLASS__);
+        $this->connectionManager = $connectionManager ?? GeneralUtility::makeInstance(ConnectionManager::class);
+        $this->frontendEnvironment = $frontendEnvironment ?? GeneralUtility::makeInstance(FrontendEnvironment::class);
+    }
+
+    /**
+     * Indexes an item from the indexing queue.
+     *
+     * @param Item $item An index queue item
+     * @return bool returns true when indexed, false when not
+     */
+    public function index(Item $item)
+    {
+        $indexed = true;
+
+        $this->type = $item->getType();
+        $this->setLogging($item);
+
+        $solrConnections = $this->getSolrConnectionsByItem($item);
+        foreach ($solrConnections as $systemLanguageUid => $solrConnection) {
+            $this->solr = $solrConnection;
+
+            if (!$this->indexItem($item, $systemLanguageUid)) {
+                /*
+                 * A single language voting for "not indexed" should make the whole
+                 * item count as being not indexed, even if all other languages are
+                 * indexed.
+                 * If there is no translation for a single language, this item counts
+                 * as TRUE since it's not an error which that should make the item
+                 * being reindexed during another index run.
+                 */
+                $indexed = false;
+            }
+        }
+
+        return $indexed;
+    }
+
+    /**
+     * Creates a single Solr Document for an item in a specific language.
+     *
+     * @param Item $item An index queue item to index.
+     * @param int $language The language to use.
+     * @return bool TRUE if item was indexed successfully, FALSE on failure
+     */
+    protected function indexItem(Item $item, $language = 0)
+    {
+        $itemIndexed = false;
+        $documents = [];
+
+        $itemDocument = $this->itemToDocument($item, $language);
+        if (is_null($itemDocument)) {
+            /*
+             * If there is no itemDocument, this means there was no translation
+             * for this record. This should not stop the current item to count as
+             * being valid because not-indexing not-translated items is perfectly
+             * fine.
+             */
+            return true;
+        }
+
+        $documents[] = $itemDocument;
+        $documents = array_merge($documents, $this->getAdditionalDocuments($item, $language, $itemDocument));
+        $documents = $this->processDocuments($item, $documents);
+        $documents = $this->preAddModifyDocuments($item, $language, $documents);
+
+        try {
+            $response = $this->solr->getWriteService()->addDocuments($documents);
+            if ($response->getHttpStatus() == 200) {
+                $itemIndexed = true;
+            }
+        } catch (HttpException $e) {
+            $response = new ResponseAdapter($e->getBody(), $httpStatus = 500, $e->getStatusMessage());
+        }
+
+        $this->log($item, $documents, $response);
+
+        return $itemIndexed;
+    }
+
+    /**
+     * Gets the full item record.
+     *
+     * This general record indexer simply gets the record from the item. Other
+     * more specialized indexers may provide more data for their specific item
+     * types.
+     *
+     * @param Item $item The item to be indexed
+     * @param int $language Language Id (sys_language.uid)
+     * @return array|NULL The full record with fields of data to be used for indexing or NULL to prevent an item from being indexed
+     */
+    protected function getFullItemRecord(Item $item, $language = 0)
+    {
+        $itemRecord = $this->getItemRecordOverlayed($item, $language);
+
+        if (!is_null($itemRecord)) {
+            $itemRecord['__solr_index_language'] = $language;
+        }
+
+        return $itemRecord;
+    }
+
+    /**
+     * Returns the overlayed item record.
+     *
+     * @param Item $item
+     * @param int $language
+     * @return array|mixed|null
+     */
+    protected function getItemRecordOverlayed(Item $item, int $language): ?array
+    {
+        $itemRecord = $item->getRecord();
+
+        if ($language > 0) {
+            $page = GeneralUtility::makeInstance(PageRepository::class);
+            $itemRecord = $page->getLanguageOverlay($item->getType(), $itemRecord);
+        }
+
+        if (!$itemRecord) {
+            $itemRecord = null;
+        }
+
+        return $itemRecord;
+    }
+
+    /**
+     * Gets the configuration how to process an item's fields for indexing.
+     *
+     * @param Item $item An index queue item
+     * @param int $language Language ID
+     * @throws RuntimeException
+     * @return array Configuration array from TypoScript
+     */
+    protected function getItemTypeConfiguration(Item $item, int $language = 0): array
+    {
+        $indexConfigurationName = $item->getIndexingConfigurationName();
+        $fields = $this->getFieldConfigurationFromItemRecordPage($item, $language, $indexConfigurationName);
+        if (!$this->isRootPageIdPartOfRootLine($item) || count($fields) === 0) {
+            $fields = $this->getFieldConfigurationFromItemRootPage($item, $language, $indexConfigurationName);
+            if (count($fields) === 0) {
+                throw new RuntimeException('The item indexing configuration "' . $item->getIndexingConfigurationName() .
+                    '" on root page uid ' . $item->getRootPageUid() . ' could not be found!', 1455530112);
+            }
+        }
+
+        return $fields;
+    }
+
+    /**
+     * The method retrieves the field configuration of the items record page id (pid).
+     *
+     * @param Item $item
+     * @param integer $language
+     * @param string $indexConfigurationName
+     * @return array
+     */
+    protected function getFieldConfigurationFromItemRecordPage(Item $item, $language, $indexConfigurationName): array
+    {
+        try {
+            $pageId = $this->getPageIdOfItem($item);
+            $solrConfiguration = $this->frontendEnvironment->getSolrConfigurationFromPageId($pageId, $language);
+            return $solrConfiguration->getIndexQueueFieldsConfigurationByConfigurationName($indexConfigurationName, []);
+        } catch (Exception $e) {
+            return [];
+        }
+    }
+
+    /**
+     * @param Item $item
+     * @return int
+     */
+    protected function getPageIdOfItem(Item $item): int
+    {
+        if ($item->getType() === 'pages') {
+            return (int)$item->getRecordUid();
+        }
+        return (int)$item->getRecordPageId();
+    }
+
+    /**
+     * The method returns the field configuration of the items root page id (uid of the related root page).
+     *
+     * @param Item $item
+     * @param integer $language
+     * @param string $indexConfigurationName
+     * @return array
+     */
+    protected function getFieldConfigurationFromItemRootPage(Item $item, $language, $indexConfigurationName)
+    {
+        $solrConfiguration = $this->frontendEnvironment->getSolrConfigurationFromPageId($item->getRootPageUid(), $language);
+
+        return $solrConfiguration->getIndexQueueFieldsConfigurationByConfigurationName($indexConfigurationName, []);
+    }
+
+    /**
+     * In case of additionalStoragePid config recordPageId can be outside of siteroot.
+     * In that case we should not read TS config of foreign siteroot.
+     *
+     * @param Item $item
+     * @return bool
+     */
+    protected function isRootPageIdPartOfRootLine(Item $item): bool
+    {
+        $rootPageId = (int)$item->getRootPageUid();
+        $buildRootlineWithPid = $this->getPageIdOfItem($item);
+        $rootlineUtility = GeneralUtility::makeInstance(RootlineUtility::class, $buildRootlineWithPid);
+        $rootline = $rootlineUtility->get();
+
+        $pageInRootline = array_filter($rootline, function($page) use ($rootPageId) {
+            return (int)$page['uid'] === $rootPageId;
+        });
+        return !empty($pageInRootline);
+    }
+
+    /**
+     * Converts an item array (record) to a Solr document by mapping the
+     * record's fields onto Solr document fields as configured in TypoScript.
+     *
+     * @param Item $item An index queue item
+     * @param int $language Language Id
+     * @return Document|null The Solr document converted from the record
+     * @throws SiteNotFoundException
+     * @throws ServiceUnavailableException
+     * @throws ImmediateResponseException
+     */
+    protected function itemToDocument(Item $item, $language = 0): ?Document
+    {
+        $document = null;
+        if ($item->getType() === 'pages') {
+            $this->frontendEnvironment->initializeTsfe($item->getRecordUid(), $language);
+        } else {
+            $this->frontendEnvironment->initializeTsfe($item->getRootPageUid(), $language);
+        }
+
+        $itemRecord = $this->getFullItemRecord($item, $language);
+        if (!is_null($itemRecord)) {
+            $itemIndexingConfiguration = $this->getItemTypeConfiguration($item, $language);
+            $document = $this->getBaseDocument($item, $itemRecord);
+            $document = $this->addDocumentFieldsFromTyposcript($document, $itemIndexingConfiguration, $itemRecord);
+        }
+
+        return $document;
+    }
+
+    /**
+     * Creates a Solr document with the basic / core fields set already.
+     *
+     * @param Item $item The item to index
+     * @param array $itemRecord The record to use to build the base document
+     * @return Document A basic Solr document
+     */
+    protected function getBaseDocument(Item $item, array $itemRecord)
+    {
+        $type = $item->getType();
+        $rootPageUid = $item->getRootPageUid();
+        $accessRootLine = $this->getAccessRootline($item);
+        return $this->documentBuilder->fromRecord($itemRecord, $type, $rootPageUid, $accessRootLine);
+    }
+
+    /**
+     * Generates an Access Rootline for an item.
+     *
+     * @param Item $item Index Queue item to index.
+     * @return string The Access Rootline for the item
+     */
+    protected function getAccessRootline(Item $item)
+    {
+        $accessRestriction = '0';
+        $itemRecord = $item->getRecord();
+
+        // TODO support access restrictions set on storage page
+
+        if (isset($GLOBALS['TCA'][$item->getType()]['ctrl']['enablecolumns']['fe_group'])) {
+            $accessRestriction = $itemRecord[$GLOBALS['TCA'][$item->getType()]['ctrl']['enablecolumns']['fe_group']];
+
+            if (empty($accessRestriction)) {
+                // public
+                $accessRestriction = '0';
+            }
+        }
+
+        return 'r:' . $accessRestriction;
+    }
+
+    /**
+     * Sends the documents to the field processing service which takes care of
+     * manipulating fields as defined in the field's configuration.
+     *
+     * @param Item $item An index queue item
+     * @param array $documents An array of \WapplerSystems\Meilisearch\System\Solr\Document\Document objects to manipulate.
+     * @return Document[] array Array of manipulated Document objects.
+     */
+    protected function processDocuments(Item $item, array $documents)
+    {
+        // needs to respect the TS settings for the page the item is on, conditions may apply
+        $solrConfiguration = $this->frontendEnvironment->getSolrConfigurationFromPageId($item->getRootPageUid());
+        $fieldProcessingInstructions = $solrConfiguration->getIndexFieldProcessingInstructionsConfiguration();
+
+        // same as in the FE indexer
+        if (is_array($fieldProcessingInstructions)) {
+            $service = GeneralUtility::makeInstance(Service::class);
+            $service->processDocuments($documents, $fieldProcessingInstructions);
+        }
+
+        return $documents;
+    }
+
+    /**
+     * Allows third party extensions to provide additional documents which
+     * should be indexed for the current item.
+     *
+     * @param Item $item The item currently being indexed.
+     * @param int $language The language uid currently being indexed.
+     * @param Document $itemDocument The document representing the item for the given language.
+     * @return Document[] array An array of additional Document objects to index.
+     */
+    protected function getAdditionalDocuments(Item $item, $language, Document $itemDocument)
+    {
+        $documents = [];
+
+        if (is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['IndexQueueIndexer']['indexItemAddDocuments'])) {
+            foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['IndexQueueIndexer']['indexItemAddDocuments'] as $classReference) {
+                if (!class_exists($classReference)) {
+                    throw new \InvalidArgumentException('Class does not exits' . $classReference, 1490363487);
+                }
+                $additionalIndexer = GeneralUtility::makeInstance($classReference);
+                if ($additionalIndexer instanceof AdditionalIndexQueueItemIndexer) {
+                    $additionalDocuments = $additionalIndexer->getAdditionalItemDocuments($item, $language, $itemDocument);
+
+                    if (is_array($additionalDocuments)) {
+                        $documents = array_merge($documents,
+                            $additionalDocuments);
+                    }
+                } else {
+                    throw new \UnexpectedValueException(
+                        get_class($additionalIndexer) . ' must implement interface ' . AdditionalIndexQueueItemIndexer::class,
+                        1326284551
+                    );
+                }
+            }
+        }
+        return $documents;
+    }
+
+    /**
+     * Provides a hook to manipulate documents right before they get added to
+     * the Solr index.
+     *
+     * @param Item $item The item currently being indexed.
+     * @param int $language The language uid of the documents
+     * @param array $documents An array of documents to be indexed
+     * @return array An array of modified documents
+     */
+    protected function preAddModifyDocuments(Item $item, $language, array $documents)
+    {
+        if (is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['IndexQueueIndexer']['preAddModifyDocuments'])) {
+            foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['IndexQueueIndexer']['preAddModifyDocuments'] as $classReference) {
+                $documentsModifier = GeneralUtility::makeInstance($classReference);
+
+                if ($documentsModifier instanceof PageIndexerDocumentsModifier) {
+                    $documents = $documentsModifier->modifyDocuments($item, $language, $documents);
+                } else {
+                    throw new RuntimeException(
+                        'The class "' . get_class($documentsModifier)
+                        . '" registered as document modifier in hook
+							preAddModifyDocuments must implement interface
+							WapplerSystems\Meilisearch\IndexQueue\PageIndexerDocumentsModifier',
+                        1309522677
+                    );
+                }
+            }
+        }
+
+        return $documents;
+    }
+
+    // Initialization
+
+    /**
+     * Gets the Solr connections applicable for an item.
+     *
+     * The connections include the default connection and connections to be used
+     * for translations of an item.
+     *
+     * @param Item $item An index queue item
+     * @return array An array of WapplerSystems\Meilisearch\System\Solr\SolrConnection connections, the array's keys are the sys_language_uid of the language of the connection
+     */
+    protected function getSolrConnectionsByItem(Item $item)
+    {
+        $solrConnections = [];
+
+        $rootPageId = $item->getRootPageUid();
+        if ($item->getType() === 'pages') {
+            $pageId = $item->getRecordUid();
+        } else {
+            $pageId = $item->getRecordPageId();
+        }
+
+        // Solr configurations possible for this item
+        $site = $item->getSite();
+        $solrConfigurationsBySite = $site->getAllSolrConnectionConfigurations();
+        $siteLanguages = [];
+        foreach ($solrConfigurationsBySite as $solrConfiguration) {
+            $siteLanguages[] = $solrConfiguration['language'];
+        }
+
+        $defaultLanguageUid = $this->getDefaultLanguageUid($item, $site->getRootPage(), $siteLanguages);
+        $translationOverlays = $this->getTranslationOverlaysWithConfiguredSite((int)$pageId, $site, (array)$siteLanguages);
+
+        $defaultConnection = $this->connectionManager->getConnectionByPageId($rootPageId, $defaultLanguageUid, $item->getMountPointIdentifier());
+        $translationConnections = $this->getConnectionsForIndexableLanguages($translationOverlays);
+
+        if ($defaultLanguageUid == 0) {
+            $solrConnections[0] = $defaultConnection;
+        }
+
+        foreach ($translationConnections as $systemLanguageUid => $solrConnection) {
+            $solrConnections[$systemLanguageUid] = $solrConnection;
+        }
+        return $solrConnections;
+    }
+
+    /**
+     * @param int $pageId
+     * @param Site $site
+     * @param array $siteLanguages
+     * @return array
+     */
+    protected function getTranslationOverlaysWithConfiguredSite(int $pageId, Site $site, array $siteLanguages): array
+    {
+        $translationOverlays = $this->pagesRepository->findTranslationOverlaysByPageId($pageId);
+        $translatedLanguages = [];
+        foreach ($translationOverlays as $key => $translationOverlay) {
+            if (!in_array($translationOverlay['sys_language_uid'], $siteLanguages)) {
+                unset($translationOverlays[$key]);
+            } else {
+                $translatedLanguages[] = (int)$translationOverlay['sys_language_uid'];
+            }
+        }
+
+        if (count($translationOverlays) + 1 !== count($siteLanguages)) {
+            // not all Languages are translated
+            // add Language Fallback
+            foreach ($siteLanguages as $languageId) {
+                if ($languageId !== 0 && !in_array((int)$languageId, $translatedLanguages, true)) {
+                    $fallbackLanguageIds = $this->getFallbackOrder($site, (int)$languageId, (int)$pageId);
+                    foreach ($fallbackLanguageIds as $fallbackLanguageId) {
+                        if ($fallbackLanguageId === 0 || in_array((int)$fallbackLanguageId, $translatedLanguages, true)) {
+                            $translationOverlay = [
+                                'pid' => $pageId,
+                                'sys_language_uid' => $languageId,
+                                'l10n_parent' => $pageId
+                            ];
+                            $translationOverlays[] = $translationOverlay;
+                            continue 2;
+                        }
+                    }
+                }
+            }
+        }
+        return $translationOverlays;
+    }
+
+    /**
+     * @param Site $site
+     * @param int $languageId
+     * @param int $pageId
+     * @return array
+     */
+    protected function getFallbackOrder(Site $site,  int $languageId, int $pageId): array
+    {
+        $fallbackChain = [];
+        $siteFinder = GeneralUtility::makeInstance(SiteFinder::class);
+        try {
+            $site = $siteFinder->getSiteByRootPageId($site->getRootPageId());
+            $languageAspect = LanguageAspectFactory::createFromSiteLanguage($site->getLanguageById($languageId));
+            $fallbackChain = $languageAspect->getFallbackChain();
+        } catch (SiteNotFoundException $e) {
+
+        }
+        return $fallbackChain;
+    }
+
+
+    /**
+     * @param Item $item An index queue item
+     * @param array $rootPage
+     * @param array $siteLanguages
+     *
+     * @return int
+     * @throws RuntimeException
+     */
+    protected function getDefaultLanguageUid(Item $item, array $rootPage, array $siteLanguages)
+    {
+        $defaultLanguageUid = 0;
+        if (($rootPage['l18n_cfg'] & 1) == 1 && count($siteLanguages) == 1 && $siteLanguages[min(array_keys($siteLanguages))] > 0) {
+            $defaultLanguageUid = $siteLanguages[min(array_keys($siteLanguages))];
+        } elseif (($rootPage['l18n_cfg'] & 1) == 1 && count($siteLanguages) > 1) {
+            unset($siteLanguages[array_search('0', $siteLanguages)]);
+            $defaultLanguageUid = $siteLanguages[min(array_keys($siteLanguages))];
+        } elseif (($rootPage['l18n_cfg'] & 1) == 1 && count($siteLanguages) == 1) {
+            $message = 'Root page ' . (int)$item->getRootPageUid() . ' is set to hide default translation, but no other language is configured!';
+            throw new RuntimeException($message);
+        }
+
+        return $defaultLanguageUid;
+    }
+
+    /**
+     * Checks for which languages connections have been configured and returns
+     * these connections.
+     *
+     * @param array $translationOverlays An array of translation overlays to check for configured connections.
+     * @return array An array of WapplerSystems\Meilisearch\System\Solr\SolrConnection connections.
+     */
+    protected function getConnectionsForIndexableLanguages(array $translationOverlays)
+    {
+        $connections = [];
+
+        foreach ($translationOverlays as $translationOverlay) {
+            $pageId = $translationOverlay['l10n_parent'];
+            $languageId = $translationOverlay['sys_language_uid'];
+
+            try {
+                $connection = $this->connectionManager->getConnectionByPageId($pageId, $languageId);
+                $connections[$languageId] = $connection;
+            } catch (NoSolrConnectionFoundException $e) {
+                // ignore the exception as we seek only those connections
+                // actually available
+            }
+        }
+
+        return $connections;
+    }
+
+    // Utility methods
+
+    // FIXME extract log() and setLogging() to WapplerSystems\Meilisearch\IndexQueue\AbstractIndexer
+    // FIXME extract an interface Tx_Solr_IndexQueue_ItemInterface
+
+    /**
+     * Enables logging dependent on the configuration of the item's site
+     *
+     * @param Item $item An item being indexed
+     * @return    void
+     */
+    protected function setLogging(Item $item)
+    {
+        $solrConfiguration = $this->frontendEnvironment->getSolrConfigurationFromPageId($item->getRootPageUid());
+        $this->loggingEnabled = $solrConfiguration->getLoggingIndexingQueueOperationsByConfigurationNameWithFallBack(
+            $item->getIndexingConfigurationName()
+        );
+    }
+
+    /**
+     * Logs the item and what document was created from it
+     *
+     * @param Item $item The item that is being indexed.
+     * @param array $itemDocuments An array of Solr documents created from the item's data
+     * @param ResponseAdapter $response The Solr response for the particular index document
+     */
+    protected function log(Item $item, array $itemDocuments, ResponseAdapter $response)
+    {
+        if (!$this->loggingEnabled) {
+            return;
+        }
+
+        $message = 'Index Queue indexing ' . $item->getType() . ':' . $item->getRecordUid() . ' - ';
+
+        // preparing data
+        $documents = [];
+        foreach ($itemDocuments as $document) {
+            $documents[] = (array)$document;
+        }
+
+        $logData = ['item' => (array)$item, 'documents' => $documents, 'response' => (array)$response];
+
+        if ($response->getHttpStatus() == 200) {
+            $severity = SolrLogManager::NOTICE;
+            $message .= 'Success';
+        } else {
+            $severity = SolrLogManager::ERROR;
+            $message .= 'Failure';
+
+            $logData['status'] = $response->getHttpStatus();
+            $logData['status message'] = $response->getHttpStatusMessage();
+        }
+
+        $this->logger->log($severity, $message, $logData);
+    }
+}

Classes/IndexQueue/InitializationPostProcessor.php

@@ -0,0 +1,52 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2012-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *  A copy is found in the textfile GPL.txt and important notices to the license
+ *  from the author is found in LICENSE.txt distributed with these scripts.
+ *
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+use WapplerSystems\Meilisearch\Domain\Site\Site;
+
+/**
+ * Interface to post process initialization of the Index Queue.
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+interface InitializationPostProcessor
+{
+
+    /**
+     * Post process Index Queue initialization
+     *
+     * @param Site $site The site to initialize
+     * @param array $indexingConfigurations Initialized indexing configurations
+     * @param array $initializationStatus Results of Index Queue initializations
+     */
+    public function postProcessIndexQueueInitialization(
+        Site $site,
+        array $indexingConfigurations,
+        array $initializationStatus
+    );
+}

Classes/IndexQueue/Initializer/AbstractInitializer.php

@@ -0,0 +1,405 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue\Initializer;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2011-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *  A copy is found in the textfile GPL.txt and important notices to the license
+ *  from the author is found in LICENSE.txt distributed with these scripts.
+ *
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+use WapplerSystems\Meilisearch\Domain\Index\Queue\QueueItemRepository;
+use WapplerSystems\Meilisearch\Domain\Site\Site;
+use WapplerSystems\Meilisearch\System\Logging\SolrLogManager;
+use Doctrine\DBAL\DBALException;
+use TYPO3\CMS\Backend\Utility\BackendUtility;
+use TYPO3\CMS\Core\Database\ConnectionPool;
+use TYPO3\CMS\Core\Messaging\FlashMessageService;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+/**
+ * Abstract Index Queue initializer with implementation  of methods for common
+ * needs during Index Queue initialization.
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+abstract class AbstractInitializer implements IndexQueueInitializer
+{
+
+    /**
+     * Site to initialize
+     *
+     * @var Site
+     */
+    protected $site;
+
+    /**
+     * The type of items this initializer is handling.
+     *
+     * @var string
+     */
+    protected $type;
+
+    /**
+     * Index Queue configuration.
+     *
+     * @var array
+     */
+    protected $indexingConfiguration;
+
+    /**
+     * Indexing configuration name.
+     *
+     * @var string
+     */
+    protected $indexingConfigurationName;
+
+    /**
+     * Flash message queue
+     *
+     * @var \TYPO3\CMS\Core\Messaging\FlashMessageQueue
+     */
+    protected $flashMessageQueue;
+
+    /**
+     * @var \WapplerSystems\Meilisearch\System\Logging\SolrLogManager
+     */
+    protected $logger = null;
+
+    /**
+     * @var QueueItemRepository
+     */
+    protected $queueItemRepository;
+
+    /**
+     * Constructor, prepares the flash message queue
+     * @param QueueItemRepository|null $queueItemRepository
+     */
+    public function __construct(QueueItemRepository $queueItemRepository = null)
+    {
+        $this->logger = GeneralUtility::makeInstance(SolrLogManager::class, /** @scrutinizer ignore-type */ __CLASS__);
+        $flashMessageService = GeneralUtility::makeInstance(FlashMessageService::class);
+        $this->flashMessageQueue = $flashMessageService->getMessageQueueByIdentifier('solr.queue.initializer');
+        $this->queueItemRepository = $queueItemRepository ?? GeneralUtility::makeInstance(QueueItemRepository::class);
+    }
+
+    /**
+     * Sets the site for the initializer.
+     *
+     * @param Site $site The site to initialize Index Queue items for.
+     */
+    public function setSite(Site $site)
+    {
+        $this->site = $site;
+    }
+
+    /**
+     * Set the type (usually a Db table name) of items to initialize.
+     *
+     * @param string $type Type to initialize.
+     */
+    public function setType($type)
+    {
+        $this->type = $type;
+    }
+
+    /**
+     * Sets the configuration for how to index a type of items.
+     *
+     * @param array $indexingConfiguration Indexing configuration from TypoScript
+     */
+    public function setIndexingConfiguration(array $indexingConfiguration)
+    {
+        $this->indexingConfiguration = $indexingConfiguration;
+    }
+
+    /**
+     * Sets the name of the indexing configuration to initialize.
+     *
+     * @param string $indexingConfigurationName Indexing configuration name
+     */
+    public function setIndexingConfigurationName($indexingConfigurationName)
+    {
+        $this->indexingConfigurationName = (string)$indexingConfigurationName;
+    }
+
+    /**
+     * Initializes Index Queue items for a certain site and indexing
+     * configuration.
+     *
+     * @return bool TRUE if initialization was successful, FALSE on error.
+     */
+    public function initialize()
+    {
+        /** @var ConnectionPool $connectionPool */
+        $connectionPool = GeneralUtility::makeInstance(ConnectionPool::class);
+
+        $fetchItemsQuery = $this->buildSelectStatement() . ', "" as errors '
+            . 'FROM ' . $this->type . ' '
+            . 'WHERE '
+            . $this->buildPagesClause()
+            . $this->buildTcaWhereClause()
+            . $this->buildUserWhereClause();
+
+        try {
+            if ($connectionPool->getConnectionForTable($this->type)->getParams() === $connectionPool->getConnectionForTable('tx_meilisearch_indexqueue_item')->getParams()) {
+                // If both tables are in the same DB, send only one query to copy all datas from one table to the other
+                $initializationQuery = 'INSERT INTO tx_meilisearch_indexqueue_item (root, item_type, item_uid, indexing_configuration, indexing_priority, changed, errors) ' . $fetchItemsQuery;
+                $logData = ['query' => $initializationQuery];
+                $logData['rows'] = $this->queueItemRepository->initializeByNativeSQLStatement($initializationQuery);
+            } else {
+                // If tables are using distinct connections, start by fetching items matching criteria
+                $logData = ['query' => $fetchItemsQuery];
+                $items = $connectionPool->getConnectionForTable($this->type)->fetchAll($fetchItemsQuery);
+                $logData['rows'] = count($items);
+
+                if (count($items)) {
+                    // Add items to the queue (if any)
+                    $logData['rows'] = $connectionPool
+                        ->getConnectionForTable('tx_meilisearch_indexqueue_item')
+                        ->bulkInsert('tx_meilisearch_indexqueue_item', $items, array_keys($items[0]));
+                }
+            }
+        } catch (DBALException $DBALException) {
+            $logData['error'] = $DBALException->getCode() . ': ' . $DBALException->getMessage();
+        }
+
+        $this->logInitialization($logData);
+        return true;
+    }
+
+    /**
+     * Builds the SELECT part of the Index Queue initialization query.
+     *
+     */
+    protected function buildSelectStatement()
+    {
+        $changedField = $GLOBALS['TCA'][$this->type]['ctrl']['tstamp'];
+        if (!empty($GLOBALS['TCA'][$this->type]['ctrl']['enablecolumns']['starttime'])) {
+            $changedField = 'GREATEST(' . $GLOBALS['TCA'][$this->type]['ctrl']['enablecolumns']['starttime'] . ',' . $GLOBALS['TCA'][$this->type]['ctrl']['tstamp'] . ')';
+        }
+        $select = 'SELECT '
+            . '\'' . $this->site->getRootPageId() . '\' as root, '
+            . '\'' . $this->type . '\' AS item_type, '
+            . 'uid AS item_uid, '
+            . '\'' . $this->indexingConfigurationName . '\' as indexing_configuration, '
+            . $this->getIndexingPriority() . ' AS indexing_priority, '
+            . $changedField . ' AS changed';
+
+        return $select;
+    }
+
+    // initialization query building
+
+    /**
+     * Reads the indexing priority for an indexing configuration.
+     *
+     * @return int Indexing priority
+     */
+    protected function getIndexingPriority()
+    {
+        $priority = 0;
+
+        if (!empty($this->indexingConfiguration['indexingPriority'])) {
+            $priority = (int)$this->indexingConfiguration['indexingPriority'];
+        }
+
+        return $priority;
+    }
+
+    /**
+     * Builds a part of the WHERE clause of the Index Queue initialization
+     * query. This part selects the limits items to be selected from the pages
+     * in a site only, plus additional pages that may have been configured.
+     *
+     */
+    protected function buildPagesClause()
+    {
+        $pages = $this->getPages();
+        $pageIdField = ($this->type === 'pages') ? 'uid' : 'pid';
+
+        return $pageIdField . ' IN(' . implode(',', $pages) . ')';
+    }
+
+    /**
+     * Gets the pages in a site plus additional pages that may have been
+     * configured.
+     *
+     * @return array A (sorted) array of page IDs in a site
+     */
+    protected function getPages()
+    {
+        $pages = $this->site->getPages();
+        $additionalPageIds = [];
+        if (!empty($this->indexingConfiguration['additionalPageIds'])) {
+            $additionalPageIds = GeneralUtility::intExplode(',', $this->indexingConfiguration['additionalPageIds']);
+        }
+
+        $pages = array_merge($pages, $additionalPageIds);
+        sort($pages, SORT_NUMERIC);
+
+        return $pages;
+    }
+
+    /**
+     * Builds the WHERE clauses of the Index Queue initialization query based
+     * on TCA information for the type to be initialized.
+     *
+     * @return string Conditions to only add indexable items to the Index Queue
+     */
+    protected function buildTcaWhereClause()
+    {
+        $tcaWhereClause = '';
+        $conditions = [];
+
+        if (isset($GLOBALS['TCA'][$this->type]['ctrl']['delete'])) {
+            $conditions['delete'] = $GLOBALS['TCA'][$this->type]['ctrl']['delete'] . ' = 0';
+        }
+
+        if (isset($GLOBALS['TCA'][$this->type]['ctrl']['enablecolumns']['disabled'])) {
+            $conditions['disabled'] = $GLOBALS['TCA'][$this->type]['ctrl']['enablecolumns']['disabled'] . ' = 0';
+        }
+
+        if (isset($GLOBALS['TCA'][$this->type]['ctrl']['enablecolumns']['endtime'])) {
+            // only include records with a future endtime or default value (0)
+            $endTimeFieldName = $GLOBALS['TCA'][$this->type]['ctrl']['enablecolumns']['endtime'];
+            $conditions['endtime'] = '(' . $endTimeFieldName . ' > ' . time() . ' OR ' . $endTimeFieldName . ' = 0)';
+        }
+
+        if (BackendUtility::isTableLocalizable($this->type)) {
+            $conditions['languageField'] = [
+                $GLOBALS['TCA'][$this->type]['ctrl']['languageField'] . ' = 0',
+                // default language
+                $GLOBALS['TCA'][$this->type]['ctrl']['languageField'] . ' = -1'
+                // all languages
+            ];
+            if (isset($GLOBALS['TCA'][$this->type]['ctrl']['transOrigPointerField'])) {
+                $conditions['languageField'][] = $GLOBALS['TCA'][$this->type]['ctrl']['transOrigPointerField'] . ' = 0'; // translations without original language source
+            }
+            $conditions['languageField'] = '(' . implode(' OR ',
+                    $conditions['languageField']) . ')';
+        }
+
+        if (!empty($GLOBALS['TCA'][$this->type]['ctrl']['versioningWS'])) {
+            // versioning is enabled for this table: exclude draft workspace records
+            /* @see \TYPO3\CMS\Core\Database\Query\Restriction\WorkspaceRestriction::buildExpression */
+            $conditions['versioningWS'] = 't3ver_wsid = 0';
+        }
+
+        if (count($conditions)) {
+            $tcaWhereClause = ' AND ' . implode(' AND ', $conditions);
+        }
+
+        return $tcaWhereClause;
+    }
+
+    /**
+     * Builds the WHERE clauses of the Index Queue initialization query based
+     * on TypoScript configuration for the type to be initialized.
+     *
+     * @return string Conditions to add items to the Index Queue based on TypoScript configuration
+     */
+    protected function buildUserWhereClause()
+    {
+        $condition = '';
+
+        // FIXME replace this with the mechanism described below
+        if (isset($this->indexingConfiguration['additionalWhereClause'])) {
+            $condition = ' AND ' . $this->indexingConfiguration['additionalWhereClause'];
+        }
+
+        return $condition;
+
+        // TODO add a query builder implementation based on TypoScript configuration
+
+        /* example TypoScript
+
+                @see http://docs.jboss.org/drools/release/5.4.0.Final/drools-expert-docs/html_single/index.html
+                @see The Java Rule Engine API (JSR94)
+
+                tt_news {
+
+                        // RULES cObject provided by EXT:rules, simply evaluates to boolean TRUE or FALSE
+                    conditions = RULES
+                    conditions {
+
+                        and {
+
+                            10 {
+                                field = pid
+                                value = 2,3,5
+                                condition = in / equals / notEquals / greaterThan / lessThan / greaterThanOrEqual / lessThanOrEqual
+                            }
+
+                            20 {
+                                field = ...
+                                value = ...
+                                condition = ...
+
+                                or {
+                                    10 {
+                                        field = ...
+                                        value = ...
+                                        condition =  ...
+                                    }
+
+                                    20 {
+                                        field = ...
+                                        value = ...
+                                        condition = ...
+                                    }
+                                }
+                            }
+
+                        }
+
+                    }
+
+                    fields {
+                        // field mapping
+                    }
+                }
+        */
+    }
+
+    /**
+     * Writes the passed log data to the log.
+     *
+     * @param array $logData
+     */
+    protected function logInitialization(array $logData)
+    {
+        if (!$this->site->getSolrConfiguration()->getLoggingIndexingIndexQueueInitialization()) {
+            return;
+        }
+
+        $logSeverity = isset($logData['error']) ? SolrLogManager::ERROR : SolrLogManager::NOTICE;
+        $logData = array_merge($logData, [
+            'site' => $this->site->getLabel(),
+            'indexing configuration name' => $this->indexingConfigurationName,
+            'type' => $this->type,
+        ]);
+
+        $message = 'Index Queue initialized for indexing configuration ' . $this->indexingConfigurationName;
+        $this->logger->log($logSeverity, $message, $logData);
+    }
+}

Classes/IndexQueue/Initializer/IndexQueueInitializer.php

@@ -0,0 +1,75 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue\Initializer;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2011-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *  A copy is found in the textfile GPL.txt and important notices to the license
+ *  from the author is found in LICENSE.txt distributed with these scripts.
+ *
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+use WapplerSystems\Meilisearch\Domain\Site\Site;
+
+/**
+ * Interface to initialize items in the Index Queue.
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+interface IndexQueueInitializer
+{
+
+    /**
+     * Sets the site for the initializer.
+     *
+     * @param Site $site The site to initialize Index Queue items for.
+     */
+    public function setSite(Site $site);
+
+    /**
+     * Set the type (usually a Db table name) of items to initialize.
+     *
+     * @param string $type Type to initialize.
+     */
+    public function setType($type);
+
+    /**
+     * Sets the name of the indexing configuration to initialize.
+     *
+     * @param string $indexingConfigurationName Indexing configuration name
+     */
+    public function setIndexingConfigurationName($indexingConfigurationName);
+
+    /**
+     * Sets the configuration for how to index a type of items.
+     *
+     * @param array $indexingConfiguration Indexing configuration from TypoScript
+     */
+    public function setIndexingConfiguration(array $indexingConfiguration);
+
+    /**
+     * Initializes Index Queue items for a certain site and indexing
+     * configuration.
+     *
+     * @return bool TRUE if initialization was successful, FALSE on error.
+     */
+    public function initialize();
+}

Classes/IndexQueue/Initializer/Page.php

@@ -0,0 +1,342 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue\Initializer;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2011-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *  A copy is found in the textfile GPL.txt and important notices to the license
+ *  from the author is found in LICENSE.txt distributed with these scripts.
+ *
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+use WapplerSystems\Meilisearch\Domain\Index\Queue\QueueItemRepository;
+use WapplerSystems\Meilisearch\Domain\Site\SiteRepository;
+use WapplerSystems\Meilisearch\IndexQueue\Item;
+use WapplerSystems\Meilisearch\IndexQueue\Queue;
+use WapplerSystems\Meilisearch\System\Logging\SolrLogManager;
+use WapplerSystems\Meilisearch\System\Records\Pages\PagesRepository;
+use Doctrine\DBAL\DBALException;
+use TYPO3\CMS\Backend\Utility\BackendUtility;
+use TYPO3\CMS\Core\Database\Connection;
+use TYPO3\CMS\Core\Database\ConnectionPool;
+use TYPO3\CMS\Core\Messaging\FlashMessage;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+/**
+ * Index Queue initializer for pages which also covers resolution of mount
+ * pages.
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+class Page extends AbstractInitializer
+{
+    /**
+     * The type of items this initializer is handling.
+     * @var string
+     */
+    protected $type = 'pages';
+
+    /**
+     * @var PagesRepository
+     */
+    protected $pagesRepository;
+
+    /**
+     * Constructor, sets type and indexingConfigurationName to "pages".
+     *
+     * @param QueueItemRepository|null $queueItemRepository
+     * @param PagesRepository|null $pagesRepository
+     */
+    public function __construct(QueueItemRepository $queueItemRepository = null, PagesRepository $pagesRepository = null)
+    {
+        parent::__construct($queueItemRepository);
+        $this->pagesRepository = $pagesRepository ?? GeneralUtility::makeInstance(PagesRepository::class);
+    }
+
+    /**
+     * Overrides the general setType() implementation, forcing type to "pages".
+     *
+     * @param string $type Type to initialize (ignored).
+     */
+    public function setType($type)
+    {
+        $this->type = 'pages';
+    }
+
+    /**
+     * Initializes Index Queue page items for a site. Includes regular pages
+     * and mounted pages - no nested mount page structures though.
+     *
+     * @return bool TRUE if initialization was successful, FALSE on error.
+     */
+    public function initialize()
+    {
+        $pagesInitialized = parent::initialize();
+        $mountPagesInitialized = $this->initializeMountPages();
+
+        return ($pagesInitialized && $mountPagesInitialized);
+    }
+
+    /**
+     * Initialize a single page that is part of a mounted tree.
+     *
+     * @param array $mountProperties Array of mount point properties mountPageSource, mountPageDestination, and mountPageOverlayed
+     * @param int $mountPageId The ID of the mounted page
+     */
+    public function initializeMountedPage(array $mountProperties, $mountPageId)
+    {
+        $mountedPages = [$mountPageId];
+
+        $this->addMountedPagesToIndexQueue($mountedPages, $mountProperties);
+        $this->addIndexQueueItemIndexingProperties($mountProperties, $mountedPages);
+    }
+
+    /**
+     * Initializes Mount Pages to be indexed through the Index Queue. The Mount
+     * Pages are searched and their mounted virtual sub-trees are then resolved
+     * and added to the Index Queue as if they were actually present below the
+     * Mount Page.
+     *
+     * @return bool TRUE if initialization of the Mount Pages was successful, FALSE otherwise
+     */
+    protected function initializeMountPages()
+    {
+        $mountPagesInitialized = false;
+        $mountPages = $this->pagesRepository->findAllMountPagesByWhereClause($this->buildPagesClause() . $this->buildTcaWhereClause() . ' AND doktype = 7 AND no_search = 0');
+
+        if (empty($mountPages)) {
+            $mountPagesInitialized = true;
+            return $mountPagesInitialized;
+        }
+
+        $databaseConnection = $this->queueItemRepository->getConnectionForAllInTransactionInvolvedTables(
+            'tx_meilisearch_indexqueue_item',
+            'tx_meilisearch_indexqueue_indexing_property'
+        );
+
+        foreach ($mountPages as $mountPage) {
+            if (!$this->validateMountPage($mountPage)) {
+                continue;
+            }
+
+            $mountedPages = $this->resolveMountPageTree($mountPage);
+
+            // handling mount_pid_ol behavior
+            if ($mountPage['mountPageOverlayed']) {
+                // the page shows the mounted page's content
+                $mountedPages[] = $mountPage['mountPageSource'];
+            } else {
+                // Add page like a regular page, as only the sub tree is
+                // mounted. The page itself has its own content.
+                $indexQueue = GeneralUtility::makeInstance(Queue::class);
+                $indexQueue->updateItem($this->type, $mountPage['uid']);
+            }
+
+            // This can happen when the mount point does not show the content of the
+            // mounted page and the mounted page does not have any subpages.
+            if (empty($mountedPages)) {
+                continue;
+            }
+
+            $databaseConnection->beginTransaction();
+            try {
+                $this->addMountedPagesToIndexQueue($mountedPages, $mountPage);
+                $this->addIndexQueueItemIndexingProperties($mountPage, $mountedPages);
+
+                $databaseConnection->commit();
+                $mountPagesInitialized = true;
+            } catch (\Exception $e) {
+                $databaseConnection->rollBack();
+
+                $this->logger->log(
+                    SolrLogManager::ERROR,
+                    'Index Queue initialization failed for mount pages',
+                    [
+                        $e->__toString()
+                    ]
+                );
+                break;
+            }
+        }
+
+        return $mountPagesInitialized;
+    }
+
+    /**
+     * Checks whether a Mount Page is properly configured.
+     *
+     * @param array $mountPage A mount page
+     * @return bool TRUE if the Mount Page is OK, FALSE otherwise
+     */
+    protected function validateMountPage(array $mountPage)
+    {
+        $isValidMountPage = true;
+
+        if (empty($mountPage['mountPageSource'])) {
+            $isValidMountPage = false;
+
+            $flashMessage = GeneralUtility::makeInstance(
+                FlashMessage::class,
+                'Property "Mounted page" must not be empty. Invalid Mount Page configuration for page ID ' . $mountPage['uid'] . '.',
+                'Failed to initialize Mount Page tree. ',
+                FlashMessage::ERROR
+            );
+            // @extensionScannerIgnoreLine
+            $this->flashMessageQueue->addMessage($flashMessage);
+        }
+
+        if (!$this->mountedPageExists($mountPage['mountPageSource'])) {
+            $isValidMountPage = false;
+
+            $flashMessage = GeneralUtility::makeInstance(
+                FlashMessage::class,
+                'The mounted page must be accessible in the frontend. '
+                . 'Invalid Mount Page configuration for page ID '
+                . $mountPage['uid'] . ', the mounted page with ID '
+                . $mountPage['mountPageSource']
+                . ' is not accessible in the frontend.',
+                'Failed to initialize Mount Page tree. ',
+                FlashMessage::ERROR
+            );
+            // @extensionScannerIgnoreLine
+            $this->flashMessageQueue->addMessage($flashMessage);
+        }
+
+        return $isValidMountPage;
+    }
+
+    /**
+     * Checks whether the mounted page (mount page source) exists. That is,
+     * whether it accessible in the frontend. So the record must exist
+     * (deleted = 0) and must not be hidden (hidden = 0).
+     *
+     * @param int $mountedPageId Mounted page ID
+     * @return bool TRUE if the page is accessible in the frontend, FALSE otherwise.
+     */
+    protected function mountedPageExists($mountedPageId)
+    {
+        $mountedPageExists = false;
+
+        $mountedPage = BackendUtility::getRecord('pages', $mountedPageId, 'uid', ' AND hidden = 0');
+        if (!empty($mountedPage)) {
+            $mountedPageExists = true;
+        }
+
+        return $mountedPageExists;
+    }
+
+    /**
+     * Adds the virtual / mounted pages to the Index Queue as if they would
+     * belong to the same site where they are mounted.
+     *
+     * @param array $mountedPages An array of mounted page IDs
+     * @param array $mountProperties Array with mount point properties (mountPageSource, mountPageDestination, mountPageOverlayed)
+     */
+    protected function addMountedPagesToIndexQueue(array $mountedPages, array $mountProperties)
+    {
+        $mountPointIdentifier = $this->getMountPointIdentifier($mountProperties);
+        $mountPointPageIsWithExistingQueueEntry = $this->queueItemRepository->findPageIdsOfExistingMountPagesByMountIdentifier($mountPointIdentifier);
+        $mountedPagesThatNeedToBeAdded = array_diff($mountedPages, $mountPointPageIsWithExistingQueueEntry);
+
+        if (count($mountedPagesThatNeedToBeAdded) === 0) {
+            //nothing to do
+            return;
+        }
+
+        /* @var Connection $connection */
+        $connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable('tx_meilisearch_indexqueue_item');
+
+        $mountIdentifier = $this->getMountPointIdentifier($mountProperties);
+        $initializationQuery = 'INSERT INTO tx_meilisearch_indexqueue_item (root, item_type, item_uid, indexing_configuration, indexing_priority, changed, has_indexing_properties, pages_mountidentifier, errors) '
+            . $this->buildSelectStatement() . ', 1, ' . $connection->quote($mountIdentifier, \PDO::PARAM_STR) . ',""'
+            . 'FROM pages '
+            . 'WHERE '
+            . 'uid IN(' . implode(',', $mountedPagesThatNeedToBeAdded) . ') '
+            . $this->buildTcaWhereClause()
+            . $this->buildUserWhereClause();
+        $logData = ['query' => $initializationQuery];
+
+        try {
+            $logData['rows'] = $this->queueItemRepository->initializeByNativeSQLStatement($initializationQuery);
+        } catch (DBALException $DBALException) {
+            $logData['error'] = $DBALException->getCode() . ': ' . $DBALException->getMessage();
+        }
+
+        $this->logInitialization($logData);
+    }
+
+    /**
+     * Adds Index Queue item indexing properties for mounted pages. The page
+     * indexer later needs to know that he's dealing with a mounted page, the
+     * indexing properties will let make it possible for the indexer to
+     * distinguish the mounted pages.
+     *
+     * @param array $mountPage An array with information about the root/destination Mount Page
+     * @param array $mountedPages An array of mounted page IDs
+     */
+    protected function addIndexQueueItemIndexingProperties(array $mountPage, array $mountedPages)
+    {
+        $mountIdentifier = $this->getMountPointIdentifier($mountPage);
+        $mountPageItems = $this->queueItemRepository->findAllIndexQueueItemsByRootPidAndMountIdentifierAndMountedPids($this->site->getRootPageId(), $mountIdentifier, $mountedPages);
+
+        foreach ($mountPageItems as $mountPageItemRecord) {
+            /* @var Item $mountPageItem */
+            $mountPageItem = GeneralUtility::makeInstance(Item::class, /** @scrutinizer ignore-type */ $mountPageItemRecord);
+            $mountPageItem->setIndexingProperty('mountPageSource', $mountPage['mountPageSource']);
+            $mountPageItem->setIndexingProperty('mountPageDestination', $mountPage['mountPageDestination']);
+            $mountPageItem->setIndexingProperty('isMountedPage', '1');
+
+            $mountPageItem->storeIndexingProperties();
+        }
+    }
+
+    /**
+     * Builds an identifier of the given mount point properties.
+     *
+     * @param array $mountProperties Array with mount point properties (mountPageSource, mountPageDestination, mountPageOverlayed)
+     * @return string String consisting of mountPageSource-mountPageDestination-mountPageOverlayed
+     */
+    protected function getMountPointIdentifier(array $mountProperties)
+    {
+        return $mountProperties['mountPageSource']
+        . '-' . $mountProperties['mountPageDestination']
+        . '-' . $mountProperties['mountPageOverlayed'];
+    }
+
+    // Mount Page resolution
+    /**
+     * Gets all the pages from a mounted page tree.
+     *
+     * @param array $mountPage
+     * @return array An array of page IDs in the mounted page tree
+     */
+    protected function resolveMountPageTree($mountPage)
+    {
+        $mountPageSourceId = $mountPage['mountPageSource'];
+        $mountPageIdentifier = $this->getMountPointIdentifier($mountPage);
+
+        $siteRepository = GeneralUtility::makeInstance(SiteRepository::class);
+        /* @var $siteRepository SiteRepository */
+        $mountedSite = $siteRepository->getSiteByPageId($mountPageSourceId, $mountPageIdentifier);
+
+        return $mountedSite ? $mountedSite->getPages($mountPageSourceId) : [];
+    }
+}

Classes/IndexQueue/Initializer/Record.php

@@ -0,0 +1,40 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue\Initializer;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2011-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *  A copy is found in the textfile GPL.txt and important notices to the license
+ *  from the author is found in LICENSE.txt distributed with these scripts.
+ *
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+/**
+ * Simple Index Queue initializer for records as found in tables configured
+ * through TCA.
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+class Record extends AbstractInitializer
+{
+
+    // just the default behavior as in the abstract class
+}

Classes/IndexQueue/InvalidFieldNameException.php

@@ -0,0 +1,35 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+/**
+ * Exception that is thrown when trying to add a field to a Solr document using
+ * a reserved name.
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+class InvalidFieldNameException extends \RuntimeException
+{
+}

Classes/IndexQueue/Item.php

@@ -0,0 +1,528 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2009-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+use WapplerSystems\Meilisearch\Domain\Index\Queue\IndexQueueIndexingPropertyRepository;
+use WapplerSystems\Meilisearch\Domain\Index\Queue\QueueItemRepository;
+use WapplerSystems\Meilisearch\Domain\Site\SiteRepository;
+use WapplerSystems\Meilisearch\Domain\Site\Site;
+use TYPO3\CMS\Backend\Utility\BackendUtility;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+/**
+ * Representation of an index queue item, carrying meta data and the record to be
+ * indexed.
+ *
+ * @todo: Loose coupling from Repos
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+class Item
+{
+    const STATE_BLOCKED = -1;
+
+    const STATE_PENDING = 0;
+
+    const STATE_INDEXED = 1;
+
+    /**
+     * The item's uid in the index queue (tx_meilisearch_indexqueue_item.uid)
+     *
+     * @var int
+     */
+    protected $indexQueueUid;
+
+    /**
+     * The root page uid of the tree the item is located in (tx_meilisearch_indexqueue_item.root)
+     *
+     * @var int
+     */
+    protected $rootPageUid;
+
+    /**
+     * The record's type, usually a table name, but could also be a file type (tx_meilisearch_indexqueue_item.item_type)
+     *
+     * @var string
+     */
+    protected $type;
+
+    /**
+     * The name of the indexing configuration that should be used when indexing (tx_meilisearch_indexqueue_item.indexing_configuration)
+     * the item.
+     *
+     * @var string
+     */
+    protected $indexingConfigurationName;
+
+    /**
+     * The unix timestamp when the record was last changed (tx_meilisearch_indexqueue_item.changed)
+     *
+     * @var int
+     */
+    protected $changed;
+
+    /**
+     * The unix timestamp when the record was last indexed (tx_meilisearch_indexqueue_item.indexed)
+     *
+     * @var int
+     */
+    protected $indexed;
+
+    /**
+     * Indexing properties to provide additional information for the item's
+     * indexer / how to index the item.
+     *
+     * @var array
+     */
+    protected $indexingProperties = [];
+
+    /**
+     * Flag for lazy loading indexing properties.
+     *
+     * @var bool
+     */
+    protected $indexingPropertiesLoaded = false;
+
+    /**
+     * Flag, whether indexing properties exits for this item.
+     *
+     * @var bool
+     */
+    protected $hasIndexingProperties = false;
+
+    /**
+     * The record's uid.
+     *
+     * @var int
+     */
+    protected $recordUid = 0;
+
+    /**
+     * The record itself
+     *
+     * @var array
+     */
+    protected $record;
+
+    /**
+     * Moint point identifier.
+     *
+     * @var string
+     */
+    protected $mountPointIdentifier;
+
+    /**
+     * @var string
+     */
+    protected $errors = '';
+
+    /**
+     * @var IndexQueueIndexingPropertyRepository
+     */
+    protected $indexQueueIndexingPropertyRepository;
+
+    /**
+     * @var QueueItemRepository
+     */
+    protected $queueItemRepository;
+
+    /**
+     * Constructor, takes item meta data information and resolves that to the full record.
+     *
+     * @param array $itemMetaData Metadata describing the item to index using the index queue. Is expected to contain a record from table tx_meilisearch_indexqueue_item
+     * @param array $fullRecord Optional full record for the item. If provided, can save some SQL queries.
+     * @param IndexQueueIndexingPropertyRepository|null $indexQueueIndexingPropertyRepository
+     */
+    public function __construct(array $itemMetaData, array $fullRecord = [], IndexQueueIndexingPropertyRepository $indexQueueIndexingPropertyRepository = null, QueueItemRepository $queueItemRepository = null)
+    {
+        $this->indexQueueUid = $itemMetaData['uid'];
+        $this->rootPageUid = $itemMetaData['root'];
+        $this->type = $itemMetaData['item_type'];
+        $this->recordUid = $itemMetaData['item_uid'];
+        $this->mountPointIdentifier = (string) empty($itemMetaData['pages_mountidentifier']) ? '' : $itemMetaData['pages_mountidentifier'];
+        $this->changed = $itemMetaData['changed'];
+        $this->indexed = $itemMetaData['indexed'];
+        $this->errors = (string) empty($itemMetaData['errors']) ? '' : $itemMetaData['errors'];
+
+        $this->indexingConfigurationName = $itemMetaData['indexing_configuration'];
+        $this->hasIndexingProperties = (boolean)$itemMetaData['has_indexing_properties'];
+
+        if (!empty($fullRecord)) {
+            $this->record = $fullRecord;
+        }
+
+        $this->indexQueueIndexingPropertyRepository = $indexQueueIndexingPropertyRepository ?? GeneralUtility::makeInstance(IndexQueueIndexingPropertyRepository::class);
+        $this->queueItemRepository = $queueItemRepository ?? GeneralUtility::makeInstance(QueueItemRepository::class);
+    }
+
+    /**
+     * Getter for Index Queue UID
+     *
+     * @return integer
+     */
+    public function getIndexQueueUid()
+    {
+        return $this->indexQueueUid;
+    }
+
+    /**
+     * Gets the item's root page ID (uid)
+     *
+     * @return int root page ID
+     */
+    public function getRootPageUid()
+    {
+        return $this->rootPageUid;
+    }
+
+    /**
+     * Returns mount point identifier
+     *
+     * @return string
+     */
+    public function getMountPointIdentifier()
+    {
+        return $this->mountPointIdentifier;
+    }
+
+    /**
+     * @param integer $uid
+     */
+    public function setRootPageUid($uid)
+    {
+        $this->rootPageUid = intval($uid);
+    }
+
+    /**
+     * @return string
+     */
+    public function getErrors()
+    {
+        return $this->errors;
+    }
+
+    /**
+     * @return boolean
+     */
+    public function getHasErrors()
+    {
+        return trim($this->errors) !== '';
+    }
+
+    /**
+     * @return int
+     */
+    public function getState()
+    {
+        if ($this->getHasErrors()) {
+            return self::STATE_BLOCKED;
+        }
+
+        if ($this->getIndexed() > $this->getChanged()) {
+            return self::STATE_INDEXED;
+        }
+
+        return self::STATE_PENDING;
+    }
+
+    /**
+     * Gets the site the item belongs to.
+     *
+     * @return Site Site instance the item belongs to.
+     */
+    public function getSite()
+    {
+        $siteRepository = GeneralUtility::makeInstance(SiteRepository::class);
+        return $siteRepository->getSiteByRootPageId($this->rootPageUid);
+    }
+
+    /**
+     * Returns the type/tablename of the queue record.
+     *
+     * @return mixed|string
+     */
+    public function getType()
+    {
+        return $this->type;
+    }
+
+    /**
+     * @param $type
+     */
+    public function setType($type)
+    {
+        $this->type = $type;
+    }
+
+    /**
+     * Returns the name of the index configuration that was used to create this record.
+     *
+     * @return mixed|string
+     */
+    public function getIndexingConfigurationName()
+    {
+        return $this->indexingConfigurationName;
+    }
+
+    /**
+     * @param string $indexingConfigurationName
+     */
+    public function setIndexingConfigurationName($indexingConfigurationName)
+    {
+        $this->indexingConfigurationName = $indexingConfigurationName;
+    }
+
+    /**
+     * Returns the timestamp when this queue item was changed.
+     *
+     * @return int|mixed
+     */
+    public function getChanged()
+    {
+        return $this->changed;
+    }
+
+    /**
+     * Returns the timestamp when this queue item was indexed.
+     *
+     * @return int|mixed
+     */
+    public function getIndexed()
+    {
+        return $this->indexed;
+    }
+
+    /**
+     * Used to set the timestamp when the related item was changed.
+     *
+     * @param int $changed
+     */
+    public function setChanged($changed)
+    {
+        $this->changed = intval($changed);
+    }
+
+    /**
+     * Returns the uid of related record (item_uid).
+     *
+     * @return mixed
+     */
+    public function getRecordUid()
+    {
+        $this->getRecord();
+
+        return $this->record['uid'];
+    }
+
+    /**
+     * Gets the item's full record.
+     *
+     * Uses lazy loading.
+     *
+     * @return array The item's DB record.
+     */
+    public function getRecord()
+    {
+        if (empty($this->record)) {
+            $this->record = (array)BackendUtility::getRecord(
+                $this->type,
+                $this->recordUid,
+                '*',
+                '',
+                false
+            );
+        }
+
+        return $this->record;
+    }
+
+    /**
+     * Can be used to set the related record.
+     *
+     * @param array $record
+     */
+    public function setRecord(array $record)
+    {
+        $this->record = $record;
+    }
+
+    /**
+     * Retrieves the page id where the related record is stored.
+     *
+     * @return int
+     */
+    public function getRecordPageId()
+    {
+        $this->getRecord();
+
+        return $this->record['pid'];
+    }
+
+    /**
+     * Stores the indexing properties.
+     *
+     */
+    public function storeIndexingProperties()
+    {
+        $this->indexQueueIndexingPropertyRepository->removeByRootPidAndIndexQueueUid(intval($this->rootPageUid), intval($this->indexQueueUid));
+
+        if ($this->hasIndexingProperties()) {
+            $this->writeIndexingProperties();
+        }
+
+        $this->queueItemRepository->updateHasIndexingPropertiesFlagByItemUid($this->indexQueueUid, $this->hasIndexingProperties);
+    }
+
+    /**
+     * @return bool
+     */
+    public function hasIndexingProperties()
+    {
+        return $this->hasIndexingProperties;
+    }
+
+    /**
+     * Writes all indexing properties.
+     */
+    protected function writeIndexingProperties()
+    {
+        $properties = [];
+        foreach ($this->indexingProperties as $propertyKey => $propertyValue) {
+            $properties[] = [
+                'root' => $this->rootPageUid,
+                'item_id' => $this->indexQueueUid,
+                'property_key' => $propertyKey,
+                'property_value' => $propertyValue
+            ];
+        }
+        if (empty($properties)) {
+            return;
+        }
+        $this->indexQueueIndexingPropertyRepository->bulkInsert($properties);
+    }
+
+    /**
+     * @param string $key
+     * @return bool
+     */
+    public function hasIndexingProperty($key)
+    {
+        $this->loadIndexingProperties();
+
+        return array_key_exists($key, $this->indexingProperties);
+    }
+
+    /**
+     * Loads the indexing properties for the item - if not already loaded.
+     */
+    public function loadIndexingProperties()
+    {
+        if ($this->indexingPropertiesLoaded) {
+            return;
+        }
+
+        $indexingProperties = $this->indexQueueIndexingPropertyRepository->findAllByIndexQueueUid(intval($this->indexQueueUid));
+        $this->indexingPropertiesLoaded = true;
+        if (empty($indexingProperties)) {
+            return;
+        }
+
+        foreach ($indexingProperties as $indexingProperty) {
+            $this->indexingProperties[$indexingProperty['property_key']] = $indexingProperty['property_value'];
+        }
+    }
+
+    /**
+     * Sets an indexing property for the item.
+     *
+     * @param string $key Indexing property name
+     * @param string|int|float $value Indexing property value
+     * @throws \InvalidArgumentException when $value is not string, integer or float
+     */
+    public function setIndexingProperty($key, $value)
+    {
+        // make sure to not interfere with existing indexing properties
+        $this->loadIndexingProperties();
+
+        $key = (string)$key; // Scalar typehints now!
+
+        if (!is_string($value) && !is_int($value) && !is_float($value)) {
+            throw new \InvalidArgumentException(
+                'Cannot set indexing property "' . $key
+                . '", its value must be string, integer or float, '
+                . 'type given was "' . gettype($value) . '"',
+                1323173209
+            );
+        }
+
+        $this->indexingProperties[$key] = $value;
+        $this->hasIndexingProperties = true;
+    }
+
+    /**
+     * Gets a specific indexing property by its name/key.
+     *
+     * @param string $key Indexing property name/key.
+     * @throws \InvalidArgumentException when the given $key does not exist.
+     * @return string
+     */
+    public function getIndexingProperty($key)
+    {
+        $this->loadIndexingProperties();
+
+        if (!array_key_exists($key, $this->indexingProperties)) {
+            throw new \InvalidArgumentException(
+                'No indexing property "' . $key . '".',
+                1323174143
+            );
+        }
+
+        return $this->indexingProperties[$key];
+    }
+
+    /**
+     * Gets all indexing properties set for this item.
+     *
+     * @return array Array of indexing properties.
+     */
+    public function getIndexingProperties()
+    {
+        $this->loadIndexingProperties();
+
+        return $this->indexingProperties;
+    }
+
+    /**
+     * Gets the names/keys of the item's indexing properties.
+     *
+     * @return array Array of indexing property names/keys
+     */
+    public function getIndexingPropertyKeys()
+    {
+        $this->loadIndexingProperties();
+
+        return array_keys($this->indexingProperties);
+    }
+}

Classes/IndexQueue/NoPidException.php

@@ -0,0 +1,35 @@
+<?php
+
+namespace WapplerSystems\Meilisearch\IndexQueue;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2008-2016 Timo Hund <timo.hund@dkd.de>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+/**
+ * This Exception is thrown when the RecordMonitor handles a record without a valid pid.
+ *
+ * @author Timo Hund <timo.hund@dkd.de>
+ */
+class NoPidException extends \Exception
+{
+}

Classes/IndexQueue/PageIndexer.php

@@ -0,0 +1,401 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2009-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+use WapplerSystems\Meilisearch\Access\Rootline;
+use WapplerSystems\Meilisearch\Access\RootlineElement;
+use WapplerSystems\Meilisearch\Domain\Index\PageIndexer\Helper\UriBuilder\AbstractUriStrategy;
+use WapplerSystems\Meilisearch\Domain\Index\PageIndexer\Helper\UriStrategyFactory;
+use WapplerSystems\Meilisearch\System\Logging\SolrLogManager;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+/**
+ * A special purpose indexer to index pages.
+ *
+ * In the case of pages we can't directly index the page records, we need to
+ * retrieve the content that belongs to a page from tt_content, too. Also
+ * plugins may be included on a page and thus may need to be executed.
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+class PageIndexer extends Indexer
+{
+    /**
+     * Indexes an item from the indexing queue.
+     *
+     * @param Item $item An index queue item
+     * @return bool Whether indexing was successful
+     */
+    public function index(Item $item)
+    {
+        $this->setLogging($item);
+
+        // check whether we should move on at all
+        if (!$this->isPageIndexable($item)) {
+            return false;
+        }
+
+        $solrConnections = $this->getSolrConnectionsByItem($item);
+        foreach ($solrConnections as $systemLanguageUid => $solrConnection) {
+            $contentAccessGroups = $this->getAccessGroupsFromContent($item, $systemLanguageUid);
+
+            if (empty($contentAccessGroups)) {
+                // might be an empty page w/no content elements or some TYPO3 error / bug
+                // FIXME logging needed
+                continue;
+            }
+
+            foreach ($contentAccessGroups as $userGroup) {
+                $this->indexPage($item, $systemLanguageUid, $userGroup);
+            }
+        }
+
+        return true;
+    }
+
+    /**
+     * Checks whether we can index this page.
+     *
+     * @param Item $item The page we want to index encapsulated in an index queue item
+     * @return bool True if we can index this page, FALSE otherwise
+     */
+    protected function isPageIndexable(Item $item)
+    {
+
+        // TODO do we still need this?
+        // shouldn't those be sorted out by the record monitor / garbage collector already?
+
+        $isIndexable = true;
+        $record = $item->getRecord();
+
+        if (isset($GLOBALS['TCA']['pages']['ctrl']['enablecolumns']['disabled'])
+            && $record[$GLOBALS['TCA']['pages']['ctrl']['enablecolumns']['disabled']]
+        ) {
+            $isIndexable = false;
+        }
+
+        return $isIndexable;
+    }
+
+    /**
+     * Gets the Solr connections applicable for a page.
+     *
+     * The connections include the default connection and connections to be used
+     * for translations of a page.
+     *
+     * @param Item $item An index queue item
+     * @return array An array of WapplerSystems\Meilisearch\System\Solr\SolrConnection connections, the array's keys are the sys_language_uid of the language of the connection
+     */
+    protected function getSolrConnectionsByItem(Item $item)
+    {
+        $solrConnections = parent::getSolrConnectionsByItem($item);
+
+        $page = $item->getRecord();
+        // may use \TYPO3\CMS\Core\Utility\GeneralUtility::hideIfDefaultLanguage($page['l18n_cfg']) with TYPO3 4.6
+        if ($page['l18n_cfg'] & 1) {
+            // page is configured to hide the default translation -> remove Solr connection for default language
+            unset($solrConnections[0]);
+        }
+
+        if (GeneralUtility::hideIfNotTranslated($page['l18n_cfg'])) {
+            $accessibleSolrConnections = [];
+            if (isset($solrConnections[0])) {
+                $accessibleSolrConnections[0] = $solrConnections[0];
+            }
+
+            $translationOverlays = $this->pagesRepository->findTranslationOverlaysByPageId((int)$page['uid']);
+
+            foreach ($translationOverlays as $overlay) {
+                $languageId = $overlay['sys_language_uid'];
+                if (array_key_exists($languageId, $solrConnections)) {
+                    $accessibleSolrConnections[$languageId] = $solrConnections[$languageId];
+                }
+            }
+
+            $solrConnections = $accessibleSolrConnections;
+        }
+
+        return $solrConnections;
+    }
+
+    /**
+     * Finds the FE user groups used on a page including all groups of content
+     * elements and groups of records of extensions that have correctly been
+     * pushed through ContentObjectRenderer during rendering.
+     *
+     * @param Item $item Index queue item representing the current page to get the user groups from
+     * @param int $language The sys_language_uid language ID
+     * @return array Array of user group IDs
+     */
+    protected function getAccessGroupsFromContent(Item $item, $language = 0)
+    {
+        static $accessGroupsCache;
+
+        $accessGroupsCacheEntryId = $item->getRecordUid() . '|' . $language;
+        if (!isset($accessGroupsCache[$accessGroupsCacheEntryId])) {
+            $request = $this->buildBasePageIndexerRequest();
+            $request->setIndexQueueItem($item);
+            $request->addAction('findUserGroups');
+
+            $indexRequestUrl = $this->getDataUrl($item, $language);
+            $response = $request->send($indexRequestUrl);
+
+            $groups = $response->getActionResult('findUserGroups');
+            if (is_array($groups)) {
+                $accessGroupsCache[$accessGroupsCacheEntryId] = $groups;
+            }
+
+            if ($this->loggingEnabled) {
+                $this->logger->log(
+                    SolrLogManager::INFO,
+                    'Page Access Groups',
+                    [
+                        'item' => (array)$item,
+                        'language' => $language,
+                        'index request url' => $indexRequestUrl,
+                        'request' => (array)$request,
+                        'response' => (array)$response,
+                        'groups' => $groups
+                    ]
+                );
+            }
+        }
+
+        return $accessGroupsCache[$accessGroupsCacheEntryId];
+    }
+
+    // Utility methods
+
+    /**
+     * Builds a base page indexer request with configured headers and other
+     * parameters.
+     *
+     * @return PageIndexerRequest Base page indexer request
+     */
+    protected function buildBasePageIndexerRequest()
+    {
+        $request = $this->getPageIndexerRequest();
+        $request->setParameter('loggingEnabled', $this->loggingEnabled);
+
+        if (!empty($this->options['authorization.'])) {
+            $request->setAuthorizationCredentials(
+                $this->options['authorization.']['username'],
+                $this->options['authorization.']['password']
+            );
+        }
+
+        if (!empty($this->options['frontendDataHelper.']['headers.'])) {
+            foreach ($this->options['frontendDataHelper.']['headers.'] as $headerValue) {
+                $request->addHeader($headerValue);
+            }
+        }
+
+        if (!empty($this->options['frontendDataHelper.']['requestTimeout'])) {
+            $request->setTimeout((float)$this->options['frontendDataHelper.']['requestTimeout']);
+        }
+
+        return $request;
+    }
+
+    /**
+     * @return PageIndexerRequest
+     */
+    protected function getPageIndexerRequest()
+    {
+        return GeneralUtility::makeInstance(PageIndexerRequest::class);
+    }
+
+    /**
+     * Determines a page ID's URL.
+     *
+     * Tries to find a domain record to use to build an URL for a given page ID
+     * and then actually build and return the page URL.
+     *
+     * @param Item $item Item to index
+     * @param int $language The language id
+     * @return string URL to send the index request to
+     * @throws \RuntimeException
+     */
+    protected function getDataUrl(Item $item, $language = 0)
+    {
+        $pageId = $item->getRecordUid();
+        $strategy = $this->getUriStrategy($pageId);
+        $mountPointParameter = $this->getMountPageDataUrlParameter($item);
+        $dataUrl = $strategy->getPageIndexingUriFromPageItemAndLanguageId($item, $language, $mountPointParameter, $this->options);
+
+        return $dataUrl;
+    }
+
+    /**
+     * @param int $pageId
+     * @return AbstractUriStrategy
+     */
+    protected function getUriStrategy($pageId)
+    {
+        return GeneralUtility::makeInstance(UriStrategyFactory::class)->getForPageId($pageId);
+    }
+
+    /**
+     * Generates the MP URL parameter needed to access mount pages. If the item
+     * is identified as being a mounted page, the &MP parameter is generated.
+     *
+     * @param Item $item Item to get an &MP URL parameter for
+     * @return string &MP URL parameter if $item is a mounted page
+     */
+    protected function getMountPageDataUrlParameter(Item $item)
+    {
+        if (!$item->hasIndexingProperty('isMountedPage')) {
+            return '';
+        }
+
+        return $item->getIndexingProperty('mountPageSource') . '-' . $item->getIndexingProperty('mountPageDestination');
+    }
+
+    #
+    # Frontend User Groups Access
+    #
+
+    /**
+     * Creates a single Solr Document for a page in a specific language and for
+     * a specific frontend user group.
+     *
+     * @param Item $item The index queue item representing the page.
+     * @param int $language The language to use.
+     * @param int $userGroup The frontend user group to use.
+     * @return PageIndexerResponse Page indexer response
+     * @throws \RuntimeException if indexing an item failed
+     */
+    protected function indexPage(Item $item, $language = 0, $userGroup = 0)
+    {
+        $accessRootline = $this->getAccessRootline($item, $language, $userGroup);
+        $request = $this->buildBasePageIndexerRequest();
+        $request->setIndexQueueItem($item);
+        $request->addAction('indexPage');
+        $request->setParameter('accessRootline', (string)$accessRootline);
+
+        $indexRequestUrl = $this->getDataUrl($item, $language);
+        $response = $request->send($indexRequestUrl);
+        $indexActionResult = $response->getActionResult('indexPage');
+
+        if ($this->loggingEnabled) {
+            $logSeverity = SolrLogManager::INFO;
+            $logStatus = 'Info';
+            if ($indexActionResult['pageIndexed']) {
+                $logSeverity = SolrLogManager::NOTICE;
+                $logStatus = 'Success';
+            }
+
+            $this->logger->log(
+                $logSeverity,
+                'Page Indexer: ' . $logStatus,
+                [
+                    'item' => (array)$item,
+                    'language' => $language,
+                    'user group' => $userGroup,
+                    'index request url' => $indexRequestUrl,
+                    'request' => (array)$request,
+                    'request headers' => $request->getHeaders(),
+                    'response' => (array)$response
+                ]
+            );
+        }
+
+        if (!$indexActionResult['pageIndexed']) {
+            $message = 'Failed indexing page Index Queue item: ' . $item->getIndexQueueUid() . ' url: ' . $indexRequestUrl;
+
+            throw new \RuntimeException($message, 1331837081);
+        }
+
+        return $response;
+    }
+
+    /**
+     * Generates a page document's "Access Rootline".
+     *
+     * The Access Rootline collects frontend user group access restrictions set
+     * for pages up in a page's rootline extended to sub-pages.
+     *
+     * The format is like this:
+     * pageId1:group1,group2|groupId2:group3|c:group1,group4,groupN
+     *
+     * The single elements of the access rootline are separated by a pipe
+     * character. All but the last elements represent pages, the last element
+     * defines the access restrictions applied to the page's content elements
+     * and records shown on the page.
+     * Each page element is composed by the page ID of the page setting frontend
+     * user access restrictions, a colon, and a comma separated list of frontend
+     * user group IDs restricting access to the page.
+     * The content access element does not have a page ID, instead it replaces
+     * the ID by a lower case C.
+     *
+     * @param Item $item Index queue item representing the current page
+     * @param int $language The sys_language_uid language ID
+     * @param int $contentAccessGroup The user group to use for the content access rootline element. Optional, will be determined automatically if not set.
+     * @return string An Access Rootline.
+     */
+    protected function getAccessRootline(Item $item, $language = 0, $contentAccessGroup = null)
+    {
+        static $accessRootlineCache;
+
+        $mountPointParameter = $this->getMountPageDataUrlParameter($item);
+
+        $accessRootlineCacheEntryId = $item->getRecordUid() . '|' . $language;
+        if ($mountPointParameter !== '') {
+            $accessRootlineCacheEntryId .= '|' . $mountPointParameter;
+        }
+        if (!is_null($contentAccessGroup)) {
+            $accessRootlineCacheEntryId .= '|' . $contentAccessGroup;
+        }
+
+        if (!isset($accessRootlineCache[$accessRootlineCacheEntryId])) {
+            $accessRootline = $this->getAccessRootlineByPageId($item->getRecordUid(), $mountPointParameter);
+
+            // current page's content access groups
+            $contentAccessGroups = [$contentAccessGroup];
+            if (is_null($contentAccessGroup)) {
+                $contentAccessGroups = $this->getAccessGroupsFromContent($item, $language);
+            }
+            $element = GeneralUtility::makeInstance(RootlineElement::class, /** @scrutinizer ignore-type */ 'c:' . implode(',', $contentAccessGroups));
+            $accessRootline->push($element);
+
+            $accessRootlineCache[$accessRootlineCacheEntryId] = $accessRootline;
+        }
+
+        return $accessRootlineCache[$accessRootlineCacheEntryId];
+    }
+
+    /**
+     * Returns the access rootLine for a certain pageId.
+     *
+     * @param int $pageId
+     * @param string $mountPointparameter
+     * @return Rootline
+     */
+    protected function getAccessRootlineByPageId($pageId, $mountPointParameter)
+    {
+        return Rootline::getAccessRootlineByPageId($pageId, $mountPointParameter);
+    }
+
+}

Classes/IndexQueue/PageIndexerDataUrlModifier.php

@@ -0,0 +1,47 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2010-2011 Markus Goldbach <markus.goldbach@dkd.de>
+ *  (c) 2012-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *  A copy is found in the textfile GPL.txt and important notices to the license
+ *  from the author is found in LICENSE.txt distributed with these scripts.
+ *
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+/**
+ * Allows to modify the data url before call the frontend form the index queue
+ *
+ * @author Markus Goldbach <markus.goldbach@dkd.de>
+ */
+interface PageIndexerDataUrlModifier
+{
+
+    /**
+     * Modifies the given data url
+     *
+     * @param string $pageUrl the current data url.
+     * @param array $urlData An array of url data
+     * @return string the final data url
+     */
+    public function modifyDataUrl($pageUrl, array $urlData);
+}

Classes/IndexQueue/PageIndexerDocumentsModifier.php

@@ -0,0 +1,48 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2010-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *  A copy is found in the textfile GPL.txt and important notices to the license
+ *  from the author is found in LICENSE.txt distributed with these scripts.
+ *
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+/**
+ * IndexQueuePageIndexerDocumentsModifier interface, allows to modify documents
+ * before adding them to the Solr index in the index queue page indexer.
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+interface PageIndexerDocumentsModifier
+{
+
+    /**
+     * Modifies the given documents
+     *
+     * @param Item $item The currently being indexed item.
+     * @param int $language The language uid of the documents
+     * @param array $documents An array of documents to be indexed
+     * @return array An array of modified documents
+     */
+    public function modifyDocuments(Item $item, $language, array $documents);
+}

Classes/IndexQueue/PageIndexerRequest.php

@@ -0,0 +1,449 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2010-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+use WapplerSystems\Meilisearch\System\Configuration\ExtensionConfiguration;
+use WapplerSystems\Meilisearch\System\Logging\SolrLogManager;
+use Psr\Http\Message\ResponseInterface;
+use GuzzleHttp\Exception\ServerException;
+use GuzzleHttp\Exception\ClientException;
+use TYPO3\CMS\Core\Http\RequestFactory;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+/**
+ * Index Queue Page Indexer request with details about which actions to perform.
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+class PageIndexerRequest
+{
+
+    const SOLR_INDEX_HEADER = 'X-Tx-Solr-Iq';
+
+    /**
+     * List of actions to perform during page rendering.
+     *
+     * @var array
+     */
+    protected $actions = [];
+
+    /**
+     * Parameters as sent from the Index Queue page indexer.
+     *
+     * @var array
+     */
+    protected $parameters = [];
+
+    /**
+     * Headers as sent from the Index Queue page indexer.
+     *
+     * @var array
+     */
+    protected $header = [];
+
+    /**
+     * Unique request ID.
+     *
+     * @var string
+     */
+    protected $requestId;
+
+    /**
+     * Username to use for basic auth protected URLs.
+     *
+     * @var string
+     */
+    protected $username = '';
+
+    /**
+     * Password to use for basic auth protected URLs.
+     *
+     * @var string
+     */
+    protected $password = '';
+
+    /**
+     * An Index Queue item related to this request.
+     *
+     * @var Item
+     */
+    protected $indexQueueItem = null;
+
+    /**
+     * Request timeout in seconds
+     *
+     * @var float
+     */
+    protected $timeout;
+
+    /**
+     * @var \WapplerSystems\Meilisearch\System\Logging\SolrLogManager
+     */
+    protected $logger = null;
+
+    /**
+     * @var ExtensionConfiguration
+     */
+    protected $extensionConfiguration;
+
+    /**
+     * @var RequestFactory
+     */
+    protected $requestFactory;
+
+    /**
+     * PageIndexerRequest constructor.
+     *
+     * @param string $jsonEncodedParameters json encoded header
+     * @param SolrLogManager|null $solrLogManager
+     * @param ExtensionConfiguration|null $extensionConfiguration
+     * @param RequestFactory|null $requestFactory
+     */
+    public function __construct($jsonEncodedParameters = null, SolrLogManager $solrLogManager = null, ExtensionConfiguration $extensionConfiguration = null, RequestFactory $requestFactory = null)
+    {
+        $this->requestId = uniqid();
+        $this->timeout = (float)ini_get('default_socket_timeout');
+
+        $this->logger = $solrLogManager ?? GeneralUtility::makeInstance(SolrLogManager::class, /** @scrutinizer ignore-type */ __CLASS__);
+        $this->extensionConfiguration = $extensionConfiguration ?? GeneralUtility::makeInstance(ExtensionConfiguration::class);
+        $this->requestFactory = $requestFactory ?? GeneralUtility::makeInstance(RequestFactory::class);
+
+        if (is_null($jsonEncodedParameters)) {
+            return;
+        }
+
+        $this->parameters = (array)json_decode($jsonEncodedParameters, true);
+        $this->requestId = $this->parameters['requestId'];
+        unset($this->parameters['requestId']);
+
+        $actions = explode(',', $this->parameters['actions']);
+        foreach ($actions as $action) {
+            $this->addAction($action);
+        }
+        unset($this->parameters['actions']);
+    }
+
+    /**
+     * Adds an action to perform during page rendering.
+     *
+     * @param string $action Action name.
+     */
+    public function addAction($action)
+    {
+        $this->actions[] = $action;
+    }
+
+    /**
+     * Executes the request.
+     *
+     * Uses headers to submit additional data and avoiding to have these
+     * arguments integrated into the URL when created by RealURL.
+     *
+     * @param string $url The URL to request.
+     * @return PageIndexerResponse Response
+     */
+    public function send($url)
+    {
+        /** @var $response PageIndexerResponse */
+        $response = GeneralUtility::makeInstance(PageIndexerResponse::class);
+        $decodedResponse = $this->getUrlAndDecodeResponse($url, $response);
+
+        if ($decodedResponse['requestId'] != $this->requestId) {
+            throw new \RuntimeException(
+                'Request ID mismatch. Request ID was ' . $this->requestId . ', received ' . $decodedResponse['requestId'] . '. Are requests cached?',
+                1351260655
+            );
+        }
+
+        $response->setRequestId($decodedResponse['requestId']);
+
+        if (!is_array($decodedResponse['actionResults'])) {
+            // nothing to parse
+            return $response;
+        }
+
+        foreach ($decodedResponse['actionResults'] as $action => $actionResult) {
+            $response->addActionResult($action, $actionResult);
+        }
+
+        return $response;
+    }
+
+    /**
+     * This method is used to retrieve an url from the frontend and decode the response.
+     *
+     * @param string $url
+     * @param PageIndexerResponse $response
+     * @return mixed
+     */
+    protected function getUrlAndDecodeResponse($url, PageIndexerResponse $response)
+    {
+        $headers = $this->getHeaders();
+        $rawResponse = $this->getUrl($url, $headers, $this->timeout);
+        // convert JSON response to response object properties
+        $decodedResponse = $response->getResultsFromJson($rawResponse->getBody()->getContents());
+
+        if ($rawResponse === false || $decodedResponse === false) {
+            $this->logger->log(
+                SolrLogManager::ERROR,
+                'Failed to execute Page Indexer Request. Request ID: ' . $this->requestId,
+                [
+                    'request ID' => $this->requestId,
+                    'request url' => $url,
+                    'request headers' => $headers,
+                    'response headers' => $rawResponse->getHeaders(),
+                    'raw response body' => $rawResponse->getBody()->getContents()
+                ]
+            );
+
+            throw new \RuntimeException('Failed to execute Page Indexer Request. See log for details. Request ID: ' . $this->requestId, 1319116885);
+        }
+        return $decodedResponse;
+    }
+
+    /**
+     * Generates the headers to be send with the request.
+     *
+     * @return string[] Array of HTTP headers.
+     */
+    public function getHeaders()
+    {
+        $headers = $this->header;
+        $headers[] = 'User-Agent: ' . $this->getUserAgent();
+        $itemId = $this->indexQueueItem->getIndexQueueUid();
+        $pageId = $this->indexQueueItem->getRecordUid();
+
+        $indexerRequestData = [
+            'requestId' => $this->requestId,
+            'item' => $itemId,
+            'page' => $pageId,
+            'actions' => implode(',', $this->actions),
+            'hash' => md5(
+                $itemId . '|' .
+                $pageId . '|' .
+                $GLOBALS['TYPO3_CONF_VARS']['SYS']['encryptionKey']
+            )
+        ];
+
+        $indexerRequestData = array_merge($indexerRequestData, $this->parameters);
+        $headers[] = self::SOLR_INDEX_HEADER . ': ' . json_encode($indexerRequestData, JSON_HEX_TAG | JSON_HEX_APOS | JSON_HEX_AMP | JSON_HEX_QUOT | JSON_UNESCAPED_SLASHES);
+
+        return $headers;
+    }
+
+    /**
+     * @return string
+     */
+    protected function getUserAgent()
+    {
+        return $GLOBALS['TYPO3_CONF_VARS']['HTTP']['headers']['User-Agent'] ?? 'TYPO3';
+    }
+
+    /**
+     * Adds an HTTP header to be send with the request.
+     *
+     * @param string $header HTTP header
+     */
+    public function addHeader($header)
+    {
+        $this->header[] = $header;
+    }
+
+    /**
+     * Checks whether this is a legitimate request coming from the Index Queue
+     * page indexer worker task.
+     *
+     * @return bool TRUE if it's a legitimate request, FALSE otherwise.
+     */
+    public function isAuthenticated()
+    {
+        $authenticated = false;
+
+        if (is_null($this->parameters)) {
+            return $authenticated;
+        }
+
+        $calculatedHash = md5(
+            $this->parameters['item'] . '|' .
+            $this->parameters['page'] . '|' .
+            $GLOBALS['TYPO3_CONF_VARS']['SYS']['encryptionKey']
+        );
+
+        if ($this->parameters['hash'] === $calculatedHash) {
+            $authenticated = true;
+        }
+
+        return $authenticated;
+    }
+
+    /**
+     * Gets the list of actions to perform during page rendering.
+     *
+     * @return array List of actions
+     */
+    public function getActions()
+    {
+        return $this->actions;
+    }
+
+    /**
+     * Gets the request's parameters.
+     *
+     * @return array Request parameters.
+     */
+    public function getParameters()
+    {
+        return $this->parameters;
+    }
+
+    /**
+     * Gets the request's unique ID.
+     *
+     * @return string Unique request ID.
+     */
+    public function getRequestId()
+    {
+        return $this->requestId;
+    }
+
+    /**
+     * Gets a specific parameter's value.
+     *
+     * @param string $parameterName The parameter to retrieve.
+     * @return mixed NULL if a parameter was not set or it's value otherwise.
+     */
+    public function getParameter($parameterName)
+    {
+        return isset($this->parameters[$parameterName]) ? $this->parameters[$parameterName] : null;
+    }
+
+    /**
+     * Sets a request's parameter and its value.
+     *
+     * @param string $parameter Parameter name
+     * @param string $value Parameter value.
+     */
+    public function setParameter($parameter, $value)
+    {
+        if (is_bool($value)) {
+            $value = $value ? '1' : '0';
+        }
+
+        $this->parameters[$parameter] = $value;
+    }
+
+    /**
+     * Sets username and password to be used for a basic auth request header.
+     *
+     * @param string $username username.
+     * @param string $password password.
+     */
+    public function setAuthorizationCredentials($username, $password)
+    {
+        $this->username = $username;
+        $this->password = $password;
+    }
+
+    /**
+     * Sets the Index Queue item this request is related to.
+     *
+     * @param Item $item Related Index Queue item.
+     */
+    public function setIndexQueueItem(Item $item)
+    {
+        $this->indexQueueItem = $item;
+    }
+
+    /**
+     * Returns the request timeout in seconds
+     *
+     * @return float
+     */
+    public function getTimeout()
+    {
+        return $this->timeout;
+    }
+
+    /**
+     * Sets the request timeout in seconds
+     *
+     * @param float $timeout Timeout seconds
+     */
+    public function setTimeout($timeout)
+    {
+        $this->timeout = (float)$timeout;
+    }
+
+    /**
+     * Fetches a page by sending the configured headers.
+     *
+     * @param string $url
+     * @param string[] $headers
+     * @param float $timeout
+     * @return ResponseInterface
+     * @throws \Exception
+     */
+    protected function getUrl($url, $headers, $timeout): ResponseInterface
+    {
+        try {
+            $options = $this->buildGuzzleOptions($headers, $timeout);
+            $response = $this->requestFactory->request($url, 'GET', $options);
+        } catch (ClientException $e) {
+            $response = $e->getResponse();
+        } catch (ServerException $e) {
+            $response = $e->getResponse();
+        }
+
+        return $response;
+    }
+
+    /**
+     * Build the options array for the guzzle client.
+     *
+     * @param array $headers
+     * @param float $timeout
+     * @return array
+     */
+    protected function buildGuzzleOptions($headers, $timeout)
+    {
+        $finalHeaders = [];
+
+        foreach ($headers as $header) {
+            list($name, $value) = explode(':', $header, 2);
+            $finalHeaders[$name] = trim($value);
+        }
+
+        $options = ['headers' => $finalHeaders, 'timeout' => $timeout];
+        if (!empty($this->username) && !empty($this->password)) {
+            $options['auth'] = [$this->username, $this->password];
+        }
+
+        if ($this->extensionConfiguration->getIsSelfSignedCertificatesEnabled()) {
+            $options['verify'] = false;
+        }
+
+        return $options;
+    }
+}

Classes/IndexQueue/PageIndexerRequestHandler.php

@@ -0,0 +1,118 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2010-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+use WapplerSystems\Meilisearch\IndexQueue\FrontendHelper\Dispatcher;
+use TYPO3\CMS\Core\SingletonInterface;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+/**
+ * Checks for Index Queue page indexer requests and handles the actions
+ * requested by them.
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+class PageIndexerRequestHandler implements SingletonInterface
+{
+
+    /**
+     * Index Queue page indexer request.
+     *
+     * @var PageIndexerRequest
+     */
+    protected $request;
+
+    /**
+     * Index Queue page indexer response.
+     *
+     * @var PageIndexerResponse
+     */
+    protected $response;
+
+    /**
+     * Index Queue page indexer frontend helper dispatcher.
+     *
+     * @var Dispatcher
+     */
+    protected $dispatcher;
+
+    /**
+     * Constructor.
+     *
+     * Initializes request, response, and dispatcher.
+     * @param string|null $jsonEncodedParameters
+     */
+    public function __construct(string $jsonEncodedParameters = null)
+    {
+        $this->dispatcher = GeneralUtility::makeInstance(Dispatcher::class);
+        $this->request = GeneralUtility::makeInstance(PageIndexerRequest::class, /** @scrutinizer ignore-type */ $jsonEncodedParameters);
+        $this->response = GeneralUtility::makeInstance(PageIndexerResponse::class);
+        $this->response->setRequestId($this->request->getRequestId());
+    }
+
+
+    /**
+     * Authenticates the request, runs the frontend helpers defined by the
+     * request, and registers its own shutdown() method for execution at
+     * hook_eofe in tslib/class.tslib_fe.php.
+     *
+     * @return void
+     */
+    public function run()
+    {
+        $this->dispatcher->dispatch($this->request, $this->response);
+    }
+
+    /**
+     * Completes the Index Queue page indexer request and returns the response
+     * with the collected results.
+     *
+     * @return void
+     */
+    public function shutdown()
+    {
+        $this->dispatcher->shutdown();
+    }
+
+    /**
+     * Gets the Index Queue page indexer request.
+     *
+     * @return PageIndexerRequest
+     */
+    public function getRequest()
+    {
+        return $this->request;
+    }
+
+    /**
+     * Gets the Index Queue page indexer response.
+     *
+     * @return PageIndexerResponse
+     */
+    public function getResponse()
+    {
+        return $this->response;
+    }
+}

Classes/IndexQueue/PageIndexerResponse.php

@@ -0,0 +1,158 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2010-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+/**
+ * Index Queue Page Indexer response to provide data for requested actions.
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+class PageIndexerResponse
+{
+
+    /**
+     * Unique request ID.
+     *
+     * @var string
+     */
+    protected $requestId = null;
+
+    /**
+     * The actions' results as action => result pairs.
+     *
+     * @var array
+     */
+    protected $results = [];
+
+    /**
+     * Turns a JSON encoded result string back into its PHP representation.
+     *
+     * @param string $jsonEncodedResponse JSON encoded result string
+     * @return array|bool An array of action => result pairs or FALSE if the response could not be decoded
+     */
+    public static function getResultsFromJson($jsonEncodedResponse)
+    {
+        $responseData = json_decode($jsonEncodedResponse, true);
+
+        if (is_array($responseData['actionResults'])) {
+            foreach ($responseData['actionResults'] as $action => $serializedActionResult) {
+                $responseData['actionResults'][$action] = unserialize($serializedActionResult);
+            }
+        } elseif (is_null($responseData)) {
+            $responseData = false;
+        }
+
+        return $responseData;
+    }
+
+    /**
+     * Adds an action's result.
+     *
+     * @param string $action The action name.
+     * @param mixed $result The action's result.
+     * @throws \RuntimeException if $action is null
+     */
+    public function addActionResult($action, $result)
+    {
+        if (is_null($action)) {
+            throw new \RuntimeException(
+                'Attempt to provide a result without providing an action',
+                1294080509
+            );
+        }
+
+        $this->results[$action] = $result;
+    }
+
+    /**
+     * Gets the complete set of results or a specific action's results.
+     *
+     * @param string $action Optional action name.
+     * @return array
+     */
+    public function getActionResult($action = null)
+    {
+        $result = $this->results;
+
+        if (!empty($action)) {
+            $result = $this->results[$action];
+        }
+
+        return $result;
+    }
+
+    /**
+     * Compiles the response's content so that it can be sent back to the
+     * Index Queue page indexer.
+     *
+     * @return string The response content
+     */
+    public function getContent()
+    {
+        return $this->toJson();
+    }
+
+    /**
+     * Converts the response's data to JSON.
+     *
+     * @return string JSON representation of the results.
+     */
+    protected function toJson()
+    {
+        $serializedActionResults = [];
+
+        foreach ($this->results as $action => $result) {
+            $serializedActionResults[$action] = serialize($result);
+        }
+
+        $responseData = [
+            'requestId' => $this->requestId,
+            'actionResults' => $serializedActionResults
+        ];
+
+        return json_encode($responseData);
+    }
+
+    /**
+     * Gets the Id of the request this response belongs to.
+     *
+     * @return string Request Id.
+     */
+    public function getRequestId()
+    {
+        return $this->requestId;
+    }
+
+    /**
+     * Sets the Id of the request this response belongs to.
+     *
+     * @param string $requestId Request Id.
+     * @return void
+     */
+    public function setRequestId($requestId)
+    {
+        $this->requestId = $requestId;
+    }
+}

Classes/IndexQueue/Queue.php

@@ -0,0 +1,610 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2009-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+use WapplerSystems\Meilisearch\Domain\Index\Queue\QueueInitializationService;
+use WapplerSystems\Meilisearch\Domain\Index\Queue\QueueItemRepository;
+use WapplerSystems\Meilisearch\Domain\Index\Queue\RecordMonitor\Helper\ConfigurationAwareRecordService;
+use WapplerSystems\Meilisearch\Domain\Index\Queue\RecordMonitor\Helper\RootPageResolver;
+use WapplerSystems\Meilisearch\Domain\Index\Queue\Statistic\QueueStatistic;
+use WapplerSystems\Meilisearch\Domain\Index\Queue\Statistic\QueueStatisticsRepository;
+use WapplerSystems\Meilisearch\Domain\Site\Site;
+use WapplerSystems\Meilisearch\FrontendEnvironment;
+use WapplerSystems\Meilisearch\System\Cache\TwoLevelCache;
+use WapplerSystems\Meilisearch\System\Logging\SolrLogManager;
+use TYPO3\CMS\Backend\Utility\BackendUtility;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+/**
+ * The Indexing Queue. It allows us to decouple from frontend indexing and
+ * reacting to changes faster.
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+class Queue
+{
+    /**
+     * @var RootPageResolver
+     */
+    protected $rootPageResolver;
+
+    /**
+     * @var ConfigurationAwareRecordService
+     */
+    protected $recordService;
+
+    /**
+     * @var \WapplerSystems\Meilisearch\System\Logging\SolrLogManager
+     */
+    protected $logger = null;
+
+    /**
+     * @var QueueItemRepository
+     */
+    protected $queueItemRepository;
+
+    /**
+     * @var QueueStatisticsRepository
+     */
+    protected $queueStatisticsRepository;
+
+    /**
+     * @var QueueInitializationService
+     */
+    protected $queueInitializationService;
+
+    /**
+     * @var FrontendEnvironment
+     */
+    protected $frontendEnvironment = null;
+
+    /**
+     * Queue constructor.
+     * @param RootPageResolver|null $rootPageResolver
+     * @param ConfigurationAwareRecordService|null $recordService
+     * @param QueueItemRepository|null $queueItemRepository
+     * @param QueueStatisticsRepository|null $queueStatisticsRepository
+     * @param QueueInitializationService|null $queueInitializationService
+     */
+    public function __construct(
+        RootPageResolver $rootPageResolver = null,
+        ConfigurationAwareRecordService $recordService = null,
+        QueueItemRepository $queueItemRepository = null,
+        QueueStatisticsRepository $queueStatisticsRepository = null,
+        QueueInitializationService $queueInitializationService = null,
+        FrontendEnvironment $frontendEnvironment = null
+    )
+    {
+        $this->logger = GeneralUtility::makeInstance(SolrLogManager::class, /** @scrutinizer ignore-type */ __CLASS__);
+        $this->rootPageResolver = $rootPageResolver ?? GeneralUtility::makeInstance(RootPageResolver::class);
+        $this->recordService = $recordService ?? GeneralUtility::makeInstance(ConfigurationAwareRecordService::class);
+        $this->queueItemRepository = $queueItemRepository ?? GeneralUtility::makeInstance(QueueItemRepository::class);
+        $this->queueStatisticsRepository = $queueStatisticsRepository ??  GeneralUtility::makeInstance(QueueStatisticsRepository::class);
+        $this->queueInitializationService = $queueInitializationService ?? GeneralUtility::makeInstance(QueueInitializationService::class, /** @scrutinizer ignore-type */ $this);
+        $this->frontendEnvironment = $frontendEnvironment ?? GeneralUtility::makeInstance(FrontendEnvironment::class);
+    }
+
+    // FIXME some of the methods should be renamed to plural forms
+    // FIXME singular form methods should deal with exactly one item only
+
+    /**
+     * Returns the timestamp of the last indexing run.
+     *
+     * @param int $rootPageId The root page uid for which to get
+     *      the last indexed item id
+     * @return int Timestamp of last index run.
+     */
+    public function getLastIndexTime($rootPageId)
+    {
+        $lastIndexTime = 0;
+
+        $lastIndexedRow = $this->queueItemRepository->findLastIndexedRow($rootPageId);
+
+        if ($lastIndexedRow[0]['indexed']) {
+            $lastIndexTime = $lastIndexedRow[0]['indexed'];
+        }
+
+        return $lastIndexTime;
+    }
+
+    /**
+     * Returns the uid of the last indexed item in the queue
+     *
+     * @param int $rootPageId The root page uid for which to get
+     *      the last indexed item id
+     * @return int The last indexed item's ID.
+     */
+    public function getLastIndexedItemId($rootPageId)
+    {
+        $lastIndexedItemId = 0;
+
+        $lastIndexedItemRow = $this->queueItemRepository->findLastIndexedRow($rootPageId);
+        if ($lastIndexedItemRow[0]['uid']) {
+            $lastIndexedItemId = $lastIndexedItemRow[0]['uid'];
+        }
+
+        return $lastIndexedItemId;
+    }
+
+    /**
+     * @return QueueInitializationService
+     */
+    public function getInitializationService()
+    {
+        return $this->queueInitializationService;
+    }
+
+    /**
+     * Marks an item as needing (re)indexing.
+     *
+     * Like with Solr itself, there's no add method, just a simple update method
+     * that handles the adds, too.
+     *
+     * The method creates or updates the index queue items for all related rootPageIds.
+     *
+     * @param string $itemType The item's type, usually a table name.
+     * @param string $itemUid The item's uid, usually an integer uid, could be a different value for non-database-record types.
+     * @param int $forcedChangeTime The change time for the item if set, otherwise value from getItemChangedTime() is used.
+     * @return int Number of updated/created items
+     */
+    public function updateItem($itemType, $itemUid, $forcedChangeTime = 0)
+    {
+        $updateCount = $this->updateOrAddItemForAllRelatedRootPages($itemType, $itemUid, $forcedChangeTime);
+        $updateCount = $this->postProcessIndexQueueUpdateItem($itemType, $itemUid, $updateCount, $forcedChangeTime);
+
+        return $updateCount;
+    }
+
+    /**
+     * Updates or add's the item for all relevant root pages.
+     *
+     * @param string $itemType The item's type, usually a table name.
+     * @param string $itemUid The item's uid, usually an integer uid, could be a different value for non-database-record types.
+     * @param int $forcedChangeTime The change time for the item if set, otherwise value from getItemChangedTime() is used.
+     * @return int
+     */
+    protected function updateOrAddItemForAllRelatedRootPages($itemType, $itemUid, $forcedChangeTime): int
+    {
+        $updateCount = 0;
+        $rootPageIds = $this->rootPageResolver->getResponsibleRootPageIds($itemType, $itemUid);
+        foreach ($rootPageIds as $rootPageId) {
+            $skipInvalidRootPage = $rootPageId === 0;
+            if ($skipInvalidRootPage) {
+                continue;
+            }
+
+            $solrConfiguration = $this->frontendEnvironment->getSolrConfigurationFromPageId($rootPageId);
+            $indexingConfiguration = $this->recordService->getIndexingConfigurationName($itemType, $itemUid, $solrConfiguration);
+            if ($indexingConfiguration === null) {
+                continue;
+            }
+            $itemInQueueForRootPage = $this->containsItemWithRootPageId($itemType, $itemUid, $rootPageId);
+            if ($itemInQueueForRootPage) {
+                // update changed time if that item is in the queue already
+                $changedTime = ($forcedChangeTime > 0) ? $forcedChangeTime : $this->getItemChangedTime($itemType, $itemUid);
+                $updatedRows = $this->queueItemRepository->updateExistingItemByItemTypeAndItemUidAndRootPageId($itemType, $itemUid, $rootPageId, $changedTime, $indexingConfiguration);
+            } else {
+                // add the item since it's not in the queue yet
+                $updatedRows = $this->addNewItem($itemType, $itemUid, $indexingConfiguration, $rootPageId);
+            }
+
+            $updateCount += $updatedRows;
+        }
+
+        return $updateCount;
+    }
+
+    /**
+     * Executes the updateItem post processing hook.
+     *
+     * @param string $itemType
+     * @param int $itemUid
+     * @param int $updateCount
+     * @param int $forcedChangeTime
+     * @return int
+     */
+    protected function postProcessIndexQueueUpdateItem($itemType, $itemUid, $updateCount, $forcedChangeTime = 0)
+    {
+        if (!is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['postProcessIndexQueueUpdateItem'])) {
+            return $updateCount;
+        }
+
+        foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['postProcessIndexQueueUpdateItem'] as $classReference) {
+            $updateHandler = $this->getHookImplementation($classReference);
+            $updateCount = $updateHandler->postProcessIndexQueueUpdateItem($itemType, $itemUid, $updateCount, $forcedChangeTime);
+        }
+
+        return $updateCount;
+    }
+
+    /**
+     * @param string $classReference
+     * @return object
+     */
+    protected function getHookImplementation($classReference)
+    {
+        return GeneralUtility::makeInstance($classReference);
+    }
+
+    /**
+     * Finds indexing errors for the current site
+     *
+     * @param Site $site
+     * @return array Error items for the current site's Index Queue
+     */
+    public function getErrorsBySite(Site $site)
+    {
+        return $this->queueItemRepository->findErrorsBySite($site);
+    }
+
+    /**
+     * Resets all the errors for all index queue items.
+     *
+     * @return mixed
+     */
+    public function resetAllErrors()
+    {
+        return $this->queueItemRepository->flushAllErrors();
+    }
+
+    /**
+     * Resets the errors in the index queue for a specific site
+     *
+     * @param Site $site
+     * @return mixed
+     */
+    public function resetErrorsBySite(Site $site)
+    {
+        return $this->queueItemRepository->flushErrorsBySite($site);
+    }
+
+    /**
+     * Resets the error in the index queue for a specific item
+     *
+     * @param Item $item
+     * @return mixed
+     */
+    public function resetErrorByItem(Item $item)
+    {
+        return $this->queueItemRepository->flushErrorByItem($item);
+    }
+
+    /**
+     * Adds an item to the index queue.
+     *
+     * Not meant for public use.
+     *
+     * @param string $itemType The item's type, usually a table name.
+     * @param string $itemUid The item's uid, usually an integer uid, could be a
+     *      different value for non-database-record types.
+     * @param string $indexingConfiguration The item's indexing configuration to use.
+     *      Optional, overwrites existing / determined configuration.
+     * @param $rootPageId
+     * @return int
+     */
+    private function addNewItem($itemType, $itemUid, $indexingConfiguration, $rootPageId)
+    {
+        $additionalRecordFields = '';
+        if ($itemType === 'pages') {
+            $additionalRecordFields = ', doktype, uid';
+        }
+
+        $record = $this->getRecordCached($itemType, $itemUid, $additionalRecordFields);
+
+        if (empty($record) || ($itemType === 'pages' && !$this->frontendEnvironment->isAllowedPageType($record, $indexingConfiguration))) {
+            return 0;
+        }
+
+        $changedTime = $this->getItemChangedTime($itemType, $itemUid);
+
+        return $this->queueItemRepository->add($itemType, $itemUid, $rootPageId, $changedTime, $indexingConfiguration);
+    }
+
+    /**
+     * Get record to be added in addNewItem
+     *
+     * @param string $itemType The item's type, usually a table name.
+     * @param string $itemUid The item's uid, usually an integer uid, could be a
+     *      different value for non-database-record types.
+     * @param string $additionalRecordFields for sql-query
+     *
+     * @return array|NULL
+     */
+    protected function getRecordCached($itemType, $itemUid, $additionalRecordFields)
+    {
+        $cache = GeneralUtility::makeInstance(TwoLevelCache::class, /** @scrutinizer ignore-type */ 'cache_runtime');
+        $cacheId = md5('Queue' . ':' . 'getRecordCached' . ':' . $itemType . ':' . $itemUid . ':' . 'pid' . $additionalRecordFields);
+
+        $record = $cache->get($cacheId);
+        if (empty($record)) {
+            $record = BackendUtility::getRecord($itemType, $itemUid, 'pid' . $additionalRecordFields);
+            $cache->set($cacheId, $record);
+        }
+
+        return $record;
+    }
+
+    /**
+     * Determines the time for when an item should be indexed. This timestamp
+     * is then stored in the changed column in the Index Queue.
+     *
+     * The changed timestamp usually is now - time(). For records which are set
+     * to published at a later time, this timestamp is the start time. So if a
+     * future start time has been set, that will be used to delay indexing
+     * of an item.
+     *
+     * @param string $itemType The item's table name.
+     * @param string $itemUid The item's uid, usually an integer uid, could be a
+     *      different value for non-database-record types.
+     * @return int Timestamp of the item's changed time or future start time
+     */
+    protected function getItemChangedTime($itemType, $itemUid)
+    {
+        $itemTypeHasStartTimeColumn = false;
+        $changedTimeColumns = $GLOBALS['TCA'][$itemType]['ctrl']['tstamp'];
+        $startTime = 0;
+        $pageChangedTime = 0;
+
+        if (!empty($GLOBALS['TCA'][$itemType]['ctrl']['enablecolumns']['starttime'])) {
+            $itemTypeHasStartTimeColumn = true;
+            $changedTimeColumns .= ', ' . $GLOBALS['TCA'][$itemType]['ctrl']['enablecolumns']['starttime'];
+        }
+        if ($itemType === 'pages') {
+            // does not carry time information directly, but needed to support
+            // canonical pages
+            $changedTimeColumns .= ', content_from_pid';
+        }
+
+        $record = BackendUtility::getRecord($itemType, $itemUid, $changedTimeColumns);
+        $itemChangedTime = $record[$GLOBALS['TCA'][$itemType]['ctrl']['tstamp']];
+
+        if ($itemTypeHasStartTimeColumn) {
+            $startTime = $record[$GLOBALS['TCA'][$itemType]['ctrl']['enablecolumns']['starttime']];
+        }
+
+        if ($itemType === 'pages') {
+            $record['uid'] = $itemUid;
+            // overrule the page's last changed time with the most recent
+            //content element change
+            $pageChangedTime = $this->getPageItemChangedTime($record);
+        }
+
+        $localizationsChangedTime = $this->queueItemRepository->getLocalizableItemChangedTime($itemType, (int)$itemUid);
+
+        // if start time exists and start time is higher than last changed timestamp
+        // then set changed to the future start time to make the item
+        // indexed at a later time
+        $changedTime = max(
+            $itemChangedTime,
+            $pageChangedTime,
+            $localizationsChangedTime,
+            $startTime
+        );
+
+        return $changedTime;
+    }
+
+    /**
+     * Gets the most recent changed time of a page's content elements
+     *
+     * @param array $page Partial page record
+     * @return int Timestamp of the most recent content element change
+     */
+    protected function getPageItemChangedTime(array $page)
+    {
+        if (!empty($page['content_from_pid'])) {
+            // canonical page, get the original page's last changed time
+            return $this->queueItemRepository->getPageItemChangedTimeByPageUid((int)$page['content_from_pid']);
+        }
+        return $this->queueItemRepository->getPageItemChangedTimeByPageUid((int)$page['uid']);
+    }
+
+    /**
+     * Checks whether the Index Queue contains a specific item.
+     *
+     * @param string $itemType The item's type, usually a table name.
+     * @param string $itemUid The item's uid, usually an integer uid, could be a
+     *      different value for non-database-record types.
+     * @return bool TRUE if the item is found in the queue, FALSE otherwise
+     */
+    public function containsItem($itemType, $itemUid)
+    {
+        return $this->queueItemRepository->containsItem($itemType, (int)$itemUid);
+    }
+
+    /**
+     * Checks whether the Index Queue contains a specific item.
+     *
+     * @param string $itemType The item's type, usually a table name.
+     * @param string $itemUid The item's uid, usually an integer uid, could be a
+     *      different value for non-database-record types.
+     * @param integer $rootPageId
+     * @return bool TRUE if the item is found in the queue, FALSE otherwise
+     */
+    public function containsItemWithRootPageId($itemType, $itemUid, $rootPageId)
+    {
+        return $this->queueItemRepository->containsItemWithRootPageId($itemType, (int)$itemUid, (int)$rootPageId);
+    }
+
+    /**
+     * Checks whether the Index Queue contains a specific item that has been
+     * marked as indexed.
+     *
+     * @param string $itemType The item's type, usually a table name.
+     * @param string $itemUid The item's uid, usually an integer uid, could be a
+     *      different value for non-database-record types.
+     * @return bool TRUE if the item is found in the queue and marked as
+     *      indexed, FALSE otherwise
+     */
+    public function containsIndexedItem($itemType, $itemUid)
+    {
+        return $this->queueItemRepository->containsIndexedItem($itemType, (int)$itemUid);
+    }
+
+    /**
+     * Removes an item from the Index Queue.
+     *
+     * @param string $itemType The type of the item to remove, usually a table name.
+     * @param int $itemUid The uid of the item to remove
+     */
+    public function deleteItem($itemType, $itemUid)
+    {
+        $this->queueItemRepository->deleteItem($itemType, (int)$itemUid);
+    }
+
+    /**
+     * Removes all items of a certain type from the Index Queue.
+     *
+     * @param string $itemType The type of items to remove, usually a table name.
+     */
+    public function deleteItemsByType($itemType)
+    {
+        $this->queueItemRepository->deleteItemsByType($itemType);
+    }
+
+    /**
+     * Removes all items of a certain site from the Index Queue. Accepts an
+     * optional parameter to limit the deleted items by indexing configuration.
+     *
+     * @param Site $site The site to remove items for.
+     * @param string $indexingConfigurationName Name of a specific indexing
+     *      configuration
+     */
+    public function deleteItemsBySite(Site $site, $indexingConfigurationName = '')
+    {
+        $this->queueItemRepository->deleteItemsBySite($site, $indexingConfigurationName);
+    }
+
+    /**
+     * Removes all items from the Index Queue.
+     *
+     */
+    public function deleteAllItems()
+    {
+        $this->queueItemRepository->deleteAllItems();
+    }
+
+    /**
+     * Gets a single Index Queue item by its uid.
+     *
+     * @param int $itemId Index Queue item uid
+     * @return Item The request Index Queue item or NULL if no item with $itemId was found
+     */
+    public function getItem($itemId)
+    {
+        return $this->queueItemRepository->findItemByUid($itemId);
+    }
+
+    /**
+     * Gets Index Queue items by type and uid.
+     *
+     * @param string $itemType item type, usually  the table name
+     * @param int $itemUid item uid
+     * @return Item[] An array of items matching $itemType and $itemUid
+     */
+    public function getItems($itemType, $itemUid)
+    {
+        return $this->queueItemRepository->findItemsByItemTypeAndItemUid($itemType, (int)$itemUid);
+    }
+
+    /**
+     * Returns all items in the queue.
+     *
+     * @return Item[] An array of items
+     */
+    public function getAllItems()
+    {
+        return $this->queueItemRepository->findAll();
+    }
+
+    /**
+     * Returns the number of items for all queues.
+     *
+     * @return int
+     */
+    public function getAllItemsCount()
+    {
+        return $this->queueItemRepository->count();
+    }
+
+    /**
+     * Extracts the number of pending, indexed and erroneous items from the
+     * Index Queue.
+     *
+     * @param Site $site
+     * @param string $indexingConfigurationName
+     *
+     * @return QueueStatistic
+     */
+    public function getStatisticsBySite(Site $site, $indexingConfigurationName = '')
+    {
+        return $this->queueStatisticsRepository->findOneByRootPidAndOptionalIndexingConfigurationName($site->getRootPageId(), $indexingConfigurationName);
+    }
+
+    /**
+     * Gets $limit number of items to index for a particular $site.
+     *
+     * @param Site $site TYPO3 site
+     * @param int $limit Number of items to get from the queue
+     * @return Item[] Items to index to the given solr server
+     */
+    public function getItemsToIndex(Site $site, $limit = 50)
+    {
+        return $this->queueItemRepository->findItemsToIndex($site, $limit);
+    }
+
+    /**
+     * Marks an item as failed and causes the indexer to skip the item in the
+     * next run.
+     *
+     * @param int|Item $item Either the item's Index Queue uid or the complete item
+     * @param string $errorMessage Error message
+     */
+    public function markItemAsFailed($item, $errorMessage = '')
+    {
+        $this->queueItemRepository->markItemAsFailed($item, $errorMessage);
+    }
+
+    /**
+     * Sets the timestamp of when an item last has been indexed.
+     *
+     * @param Item $item
+     */
+    public function updateIndexTimeByItem(Item $item)
+    {
+        $this->queueItemRepository->updateIndexTimeByItem($item);
+    }
+
+    /**
+     * Sets the change timestamp of an item.
+     *
+     * @param Item $item
+     * @param int $forcedChangeTime The change time for the item
+     */
+    public function setForcedChangeTimeByItem(Item $item, $forcedChangeTime)
+    {
+        $this->queueItemRepository->updateChangedTimeByItem($item, $forcedChangeTime);
+    }
+}

Classes/IndexQueue/RecordMonitor.php

@@ -0,0 +1,601 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2009-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+use WapplerSystems\Meilisearch\AbstractDataHandlerListener;
+use WapplerSystems\Meilisearch\Domain\Index\Queue\RecordMonitor\Helper\ConfigurationAwareRecordService;
+use WapplerSystems\Meilisearch\Domain\Index\Queue\RecordMonitor\Helper\MountPagesUpdater;
+use WapplerSystems\Meilisearch\Domain\Index\Queue\RecordMonitor\Helper\RootPageResolver;
+use WapplerSystems\Meilisearch\FrontendEnvironment;
+use WapplerSystems\Meilisearch\GarbageCollector;
+use WapplerSystems\Meilisearch\System\Configuration\ExtensionConfiguration;
+use WapplerSystems\Meilisearch\System\Configuration\TypoScriptConfiguration;
+use WapplerSystems\Meilisearch\System\Logging\SolrLogManager;
+use WapplerSystems\Meilisearch\System\Records\Pages\PagesRepository;
+use WapplerSystems\Meilisearch\System\TCA\TCAService;
+use WapplerSystems\Meilisearch\Util;
+use TYPO3\CMS\Backend\Utility\BackendUtility;
+use TYPO3\CMS\Core\DataHandling\DataHandler;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Core\Utility\MathUtility;
+
+/**
+ * A class that monitors changes to records so that the changed record gets
+ * passed to the index queue to update the according index document.
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+class RecordMonitor extends AbstractDataHandlerListener
+{
+    /**
+     * Index Queue
+     *
+     * @var Queue
+     */
+    protected $indexQueue;
+
+    /**
+     * Mount Page Updater
+     *
+     * @var MountPagesUpdater
+     */
+    protected $mountPageUpdater;
+
+    /**
+     * TCA Service
+     *
+     * @var TCAService
+     */
+    protected $tcaService;
+
+    /**
+     * RootPageResolver
+     *
+     * @var RootPageResolver
+     */
+    protected $rootPageResolver;
+
+    /**
+     * @var PagesRepository
+     */
+    protected $pagesRepository;
+
+    /**
+     * @var SolrLogManager
+     */
+    protected $logger = null;
+
+    /**
+     * @var FrontendEnvironment
+     */
+    protected $frontendEnvironment = null;
+
+    /**
+     * RecordMonitor constructor.
+     *
+     * @param Queue|null $indexQueue
+     * @param MountPagesUpdater|null $mountPageUpdater
+     * @param TCAService|null $TCAService
+     * @param RootPageResolver $rootPageResolver
+     * @param PagesRepository|null $pagesRepository
+     * @param SolrLogManager|null $solrLogManager
+     * @param ConfigurationAwareRecordService|null $recordService
+     */
+    public function __construct(
+        Queue $indexQueue = null,
+        MountPagesUpdater $mountPageUpdater = null,
+        TCAService $TCAService = null,
+        RootPageResolver $rootPageResolver = null,
+        PagesRepository $pagesRepository = null,
+        SolrLogManager $solrLogManager = null,
+        ConfigurationAwareRecordService $recordService = null,
+        FrontendEnvironment $frontendEnvironment = null
+    )
+    {
+        parent::__construct($recordService);
+        $this->indexQueue = $indexQueue ?? GeneralUtility::makeInstance(Queue::class);
+        $this->mountPageUpdater = $mountPageUpdater ?? GeneralUtility::makeInstance(MountPagesUpdater::class);
+        $this->tcaService = $TCAService ?? GeneralUtility::makeInstance(TCAService::class);
+        $this->rootPageResolver = $rootPageResolver ?? GeneralUtility::makeInstance(RootPageResolver::class);
+        $this->pagesRepository = $pagesRepository ?? GeneralUtility::makeInstance(PagesRepository::class);
+        $this->logger = $solrLogManager ?? GeneralUtility::makeInstance(SolrLogManager::class, /** @scrutinizer ignore-type */ __CLASS__);
+        $this->frontendEnvironment = $frontendEnvironment ?? GeneralUtility::makeInstance(FrontendEnvironment::class);
+    }
+
+    /**
+     * @param SolrLogManager $logger
+     */
+    public function setLogger(SolrLogManager $logger)
+    {
+        $this->logger = $logger;
+    }
+
+    /**
+     * Holds the configuration when a recursive page queing should be triggered.
+     *
+     * Note: The SQL transaction is already committed, so the current state covers only "non"-changed fields.
+     *
+     * @var array
+     * @return array
+     */
+    protected function getUpdateSubPagesRecursiveTriggerConfiguration()
+    {
+        return [
+            // the current page has the both fields "extendToSubpages" and "hidden" set from 1 to 0 => requeue subpages
+            'HiddenAndExtendToSubpageWereDisabled' => [
+                'changeSet' => [
+                    'hidden' => '0',
+                    'extendToSubpages' => '0'
+                ]
+            ],
+            // the current page has the field "extendToSubpages" enabled and the field "hidden" was set to 0 => requeue subpages
+            'extendToSubpageEnabledAndHiddenFlagWasRemoved' => [
+                'currentState' =>  ['extendToSubpages' => '1'],
+                'changeSet' => ['hidden' => '0']
+            ],
+            // the current page has the field "hidden" enabled and the field "extendToSubpages" was set to 0 => requeue subpages
+            'hiddenIsEnabledAndExtendToSubPagesWasRemoved' => [
+                'currentState' =>  ['hidden' => '1'],
+                'changeSet' => ['extendToSubpages' => '0']
+            ]
+        ];
+    }
+
+    /**
+     * Hooks into TCE main and tracks record deletion commands.
+     *
+     * @param string $command The command.
+     * @param string $table The table the record belongs to
+     * @param int $uid The record's uid
+     * @param string $value
+     * @param DataHandler $tceMain TYPO3 Core Engine parent object
+     */
+    public function processCmdmap_preProcess(
+        $command,
+        $table,
+        $uid,
+        /** @noinspection PhpUnusedParameterInspection */
+        $value,
+        DataHandler $tceMain
+    ) {
+        if ($command === 'delete' && $table === 'tt_content' && $GLOBALS['BE_USER']->workspace == 0) {
+            // skip workspaces: index only LIVE workspace
+            $pid = $this->getValidatedPid($tceMain, $table, $uid);
+            $this->indexQueue->updateItem('pages', $pid, time());
+        }
+    }
+
+    /**
+     * Hooks into TCE main and tracks workspace publish/swap events and
+     * page move commands in LIVE workspace.
+     *
+     * @param string $command The command.
+     * @param string $table The table the record belongs to
+     * @param int $uid The record's uid
+     * @param string $value
+     * @param DataHandler $tceMain TYPO3 Core Engine parent object
+     */
+    public function processCmdmap_postProcess($command, $table, $uid, $value, DataHandler $tceMain)
+    {
+        if ($this->isDraftRecord($table, $uid)) {
+            // skip workspaces: index only LIVE workspace
+            return;
+        }
+
+        // track publish / swap events for records (workspace support)
+        // command "version"
+        if ($command === 'version' && $value['action'] === 'swap') {
+            $this->applyVersionSwap($table, $uid, $tceMain);
+        }
+
+        // moving pages/records in LIVE workspace
+        if ($command === 'move' && $GLOBALS['BE_USER']->workspace == 0) {
+            if ($table === 'pages') {
+                $this->applyPageChangesToQueue($uid);
+            } else {
+                $this->applyRecordChangesToQueue($table, $uid, $value);
+            }
+
+        }
+    }
+
+    /**
+     * Apply's version swap to the IndexQueue.
+     *
+     * @param string $table
+     * @param integer $uid
+     * @param DataHandler $tceMain
+     */
+    protected function applyVersionSwap($table, $uid, DataHandler $tceMain)
+    {
+        $isPageRelatedRecord = $table === 'tt_content' || $table === 'pages';
+        if($isPageRelatedRecord) {
+            $uid = $table === 'tt_content' ? $this->getValidatedPid($tceMain, $table, $uid) : $uid;
+            $this->applyPageChangesToQueue($uid);
+        } else {
+            $recordPageId = $this->getValidatedPid($tceMain, $table, $uid);
+            $this->applyRecordChangesToQueue($table, $uid, $recordPageId);
+        }
+    }
+
+    /**
+     * Add's a page to the queue and updates mounts, when it is enabled, otherwise ensure that the page is removed
+     * from the queue.
+     *
+     * @param integer $uid
+     */
+    protected function applyPageChangesToQueue($uid)
+    {
+        $solrConfiguration = $this->getSolrConfigurationFromPageId($uid);
+        $record = $this->configurationAwareRecordService->getRecord('pages', $uid, $solrConfiguration);
+        if (!empty($record) && $this->tcaService->isEnabledRecord('pages', $record)) {
+            $this->mountPageUpdater->update($uid);
+            $this->indexQueue->updateItem('pages', $uid);
+        } else {
+            // TODO should be moved to garbage collector
+            $this->removeFromIndexAndQueueWhenItemInQueue('pages', $uid);
+        }
+    }
+
+    /**
+     * Add's a record to the queue if it is monitored and enabled, otherwise it removes the record from the queue.
+     *
+     * @param string $table
+     * @param integer $uid
+     * @param integer $pid
+     */
+    protected function applyRecordChangesToQueue($table, $uid, $pid)
+    {
+        $solrConfiguration = $this->getSolrConfigurationFromPageId($pid);
+        $isMonitoredTable = $solrConfiguration->getIndexQueueIsMonitoredTable($table);
+
+        if ($isMonitoredTable) {
+            $record = $this->configurationAwareRecordService->getRecord($table, $uid, $solrConfiguration);
+
+            if (!empty($record) && $this->tcaService->isEnabledRecord($table, $record)) {
+                $uid = $this->tcaService->getTranslationOriginalUidIfTranslated($table, $record, $uid);
+                $this->indexQueue->updateItem($table, $uid);
+            } else {
+                // TODO should be moved to garbage collector
+                $this->removeFromIndexAndQueueWhenItemInQueue($table, $uid);
+            }
+        }
+    }
+
+    /**
+     * Hooks into TCE Main and watches all record creations and updates. If it
+     * detects that the new/updated record belongs to a table configured for
+     * indexing through Solr, we add the record to the index queue.
+     *
+     * @param string $status Status of the current operation, 'new' or 'update'
+     * @param string $table The table the record belongs to
+     * @param mixed $uid The record's uid, [integer] or [string] (like 'NEW...')
+     * @param array $fields The record's data
+     * @param DataHandler $tceMain TYPO3 Core Engine parent object
+     * @return void
+     */
+    public function processDatamap_afterDatabaseOperations($status, $table, $uid, array $fields, DataHandler $tceMain) {
+        $recordTable = $table;
+        $recordUid = $uid;
+
+        if ($this->skipMonitoringOfTable($table)) {
+            return;
+        }
+
+        if ($status === 'new' && !MathUtility::canBeInterpretedAsInteger($recordUid)) {
+            $recordUid = $tceMain->substNEWwithIDs[$recordUid];
+        }
+        if ($this->isDraftRecord($table, $recordUid)) {
+            // skip workspaces: index only LIVE workspace
+            return;
+        }
+
+        try {
+            $recordPageId = $this->getRecordPageId($status, $recordTable, $recordUid, $uid, $fields, $tceMain);
+
+            // when a content element changes we need to updated the page instead
+            if ($recordTable === 'tt_content') {
+                $recordTable = 'pages';
+                $recordUid = $recordPageId;
+            }
+
+            $this->processRecord($recordTable, $recordPageId, $recordUid, $fields);
+        } catch (NoPidException $e) {
+            $message = 'Record without valid pid was processed ' . $table . ':' . $uid;
+            $this->logger->log(SolrLogManager::WARNING, $message);
+        }
+    }
+
+    /**
+     * Check if the provided table is explicitly configured for monitoring
+     *
+     * @param string $table
+     * @return bool
+     */
+    protected function skipMonitoringOfTable($table)
+    {
+        static $configurationMonitorTables;
+
+        if (empty($configurationMonitorTables)) {
+            $configuration = GeneralUtility::makeInstance(ExtensionConfiguration::class);
+            $configurationMonitorTables = $configuration->getIsUseConfigurationMonitorTables();
+        }
+
+        // No explicit configuration => all tables should be monitored
+        if (empty($configurationMonitorTables)) {
+            return false;
+        }
+
+        return !in_array($table, $configurationMonitorTables);
+    }
+
+    /**
+     * Process the record located in processDatamap_afterDatabaseOperations
+     *
+     * @param string $recordTable The table the record belongs to
+     * @param int $recordPageId pageid
+     * @param mixed $recordUid The record's uid, [integer] or [string] (like 'NEW...')
+     * @param array $fields The record's data
+     */
+    protected function processRecord($recordTable, $recordPageId, $recordUid, $fields)
+    {
+        if ($recordTable === 'pages') {
+            $configurationPageId = $this->getConfigurationPageId($recordTable, $recordPageId, $recordUid);
+            if ($configurationPageId === 0) {
+                return;
+            }
+            $rootPageIds = [$configurationPageId];
+        } else {
+            try {
+                $rootPageIds = $this->rootPageResolver->getResponsibleRootPageIds($recordTable, $recordUid);
+                if (empty($rootPageIds)) {
+                    $this->removeFromIndexAndQueueWhenItemInQueue($recordTable, $recordUid);
+                    return;
+                }
+            } catch ( \InvalidArgumentException $e) {
+                $this->removeFromIndexAndQueueWhenItemInQueue($recordTable, $recordUid);
+                return;
+            }
+        }
+        foreach ($rootPageIds as $configurationPageId) {
+            $solrConfiguration = $this->getSolrConfigurationFromPageId($configurationPageId);
+            $isMonitoredRecord = $solrConfiguration->getIndexQueueIsMonitoredTable($recordTable);
+            if (!$isMonitoredRecord) {
+                // when it is a non monitored record, we can skip it.
+                continue;
+            }
+
+            $record = $this->configurationAwareRecordService->getRecord($recordTable, $recordUid, $solrConfiguration);
+            if (empty($record)) {
+                // TODO move this part to the garbage collector
+                // check if the item should be removed from the index because it no longer matches the conditions
+                $this->removeFromIndexAndQueueWhenItemInQueue($recordTable, $recordUid);
+                continue;
+            }
+            // Clear existing index queue items to prevent mount point duplicates.
+            // This needs to be done before the overlay handling, because handling an overlay record should
+            // not trigger a deletion.
+            $isTranslation = !empty($record['sys_language_uid']) && $record['sys_language_uid'] !== 0;
+            if ($recordTable === 'pages' && !$isTranslation) {
+                $this->indexQueue->deleteItem('pages', $recordUid);
+            }
+
+            // only update/insert the item if we actually found a record
+            $isLocalizedRecord = $this->tcaService->isLocalizedRecord($recordTable, $record);
+            $recordUid = $this->tcaService->getTranslationOriginalUidIfTranslated($recordTable, $record, $recordUid);
+
+            if ($isLocalizedRecord && !$this->getIsTranslationParentRecordEnabled($recordTable, $recordUid)) {
+                // we have a localized record without a visible parent record. Nothing to do.
+                continue;
+            }
+
+            if ($this->tcaService->isEnabledRecord($recordTable, $record)) {
+                $this->indexQueue->updateItem($recordTable, $recordUid);
+            }
+
+            if ($recordTable === 'pages') {
+                $this->doPagesPostUpdateOperations($fields, $recordUid);
+            }
+        }
+    }
+
+    /**
+     * This method is used to determine the pageId that should be used to retrieve the index queue configuration.
+     *
+     * @param string $recordTable
+     * @param integer $recordPageId
+     * @param integer $recordUid
+     * @return integer
+     */
+    protected function getConfigurationPageId($recordTable, $recordPageId, $recordUid)
+    {
+        $rootPageId = $this->rootPageResolver->getRootPageId($recordPageId);
+        if ($this->rootPageResolver->getIsRootPageId($rootPageId)) {
+            return $recordPageId;
+        }
+
+        $alternativeSiteRoots = $this->rootPageResolver->getAlternativeSiteRootPagesIds($recordTable, $recordUid, $recordPageId);
+        $lastRootPage = array_pop($alternativeSiteRoots);
+        return empty($lastRootPage) ? 0 : $lastRootPage;
+    }
+
+    /**
+     * Checks if the parent record of the translated record is enabled.
+     *
+     * @param string $recordTable
+     * @param integer $recordUid
+     * @return bool
+     */
+    protected function getIsTranslationParentRecordEnabled($recordTable, $recordUid)
+    {
+        $l10nParentRecord = (array)BackendUtility::getRecord($recordTable, $recordUid, '*', '', false);
+        return $this->tcaService->isEnabledRecord($recordTable, $l10nParentRecord);
+    }
+
+    /**
+     * Applies needed updates when a pages record was processed by the RecordMonitor.
+     *
+     * @param array $fields
+     * @param int $recordUid
+     */
+    protected function doPagesPostUpdateOperations(array $fields, $recordUid)
+    {
+        $this->updateCanonicalPages($recordUid);
+        $this->mountPageUpdater->update($recordUid);
+
+        if ($this->isRecursivePageUpdateRequired($recordUid, $fields)) {
+            $treePageIds = $this->getSubPageIds($recordUid);
+            $this->updatePageIdItems($treePageIds);
+        }
+    }
+
+    /**
+     * Determines the recordPageId (pid) of a record.
+     *
+     * @param string $status
+     * @param string $recordTable
+     * @param int $recordUid
+     * @param int $originalUid
+     * @param array $fields
+     * @param DataHandler $tceMain
+     *
+     * @return int
+     */
+    protected function getRecordPageId($status, $recordTable, $recordUid, $originalUid, array $fields, DataHandler $tceMain)
+    {
+        if ($recordTable === 'pages' && isset($fields['l10n_parent']) && intval($fields['l10n_parent']) > 0) {
+            return $fields['l10n_parent'];
+        }
+
+        if ($status === 'update' && !isset($fields['pid'])) {
+            $recordPageId = $this->getValidatedPid($tceMain, $recordTable, $recordUid);
+            if (($recordTable === 'pages') && ($this->rootPageResolver->getIsRootPageId($recordUid))) {
+                $recordPageId = $originalUid;
+            }
+
+            return $recordPageId;
+        }
+
+        return $fields['pid'];
+    }
+
+    /**
+     * Applies the updateItem instruction on a collection of pageIds.
+     *
+     * @param array $treePageIds
+     */
+    protected function updatePageIdItems(array $treePageIds)
+    {
+        foreach ($treePageIds as $treePageId) {
+            $this->indexQueue->updateItem('pages', $treePageId);
+        }
+    }
+
+    /**
+     * Removes record from the index queue and from the solr index
+     *
+     * @param string $recordTable Name of table where the record lives
+     * @param int $recordUid Id of record
+     */
+    protected function removeFromIndexAndQueue($recordTable, $recordUid)
+    {
+        $garbageCollector = GeneralUtility::makeInstance(GarbageCollector::class);
+        $garbageCollector->collectGarbage($recordTable, $recordUid);
+    }
+
+    /**
+     * Removes record from the index queue and from the solr index when the item is in the queue.
+     *
+     * @param string $recordTable Name of table where the record lives
+     * @param int $recordUid Id of record
+     */
+    protected function removeFromIndexAndQueueWhenItemInQueue($recordTable, $recordUid)
+    {
+        if (!$this->indexQueue->containsItem($recordTable, $recordUid)) {
+            return;
+        }
+
+        $this->removeFromIndexAndQueue($recordTable, $recordUid);
+    }
+
+    // Handle pages showing content from another page
+
+    /**
+     * Triggers Index Queue updates for other pages showing content from the
+     * page currently being updated.
+     *
+     * @param int $pageId UID of the page currently being updated
+     */
+    protected function updateCanonicalPages($pageId)
+    {
+        $canonicalPages = $this->pagesRepository->findPageUidsWithContentsFromPid((int)$pageId);
+        foreach ($canonicalPages as $page) {
+            $this->indexQueue->updateItem('pages', $page['uid']);
+        }
+    }
+
+    /**
+     * Retrieves the pid of a record and throws an exception when getPid returns false.
+     *
+     * @param DataHandler $tceMain
+     * @param string $table
+     * @param integer $uid
+     * @throws NoPidException
+     * @return integer
+     */
+    protected function getValidatedPid(DataHandler $tceMain, $table, $uid)
+    {
+        $pid = $tceMain->getPID($table, $uid);
+        if ($pid === false) {
+            throw new NoPidException('Pid should not be false');
+        }
+
+        $pid = intval($pid);
+        return $pid;
+    }
+
+    /**
+     * Checks if the record is a draft record.
+     *
+     * @param string $table
+     * @param int $uid
+     * @return bool
+     */
+    protected function isDraftRecord($table, $uid)
+    {
+        return Util::isDraftRecord($table, $uid);
+    }
+
+    /**
+     * @param $pageId
+     * @param bool $initializeTsfe
+     * @param int $language
+     * @return TypoScriptConfiguration
+     */
+    protected function getSolrConfigurationFromPageId($pageId)
+    {
+        return $this->frontendEnvironment->getSolrConfigurationFromPageId($pageId);
+    }
+}

Classes/IndexQueue/SerializedValueDetector.php

@@ -0,0 +1,51 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2014-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *  A copy is found in the textfile GPL.txt and important notices to the license
+ *  from the author is found in LICENSE.txt distributed with these scripts.
+ *
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+/**
+ * Serialized value detector interface
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+interface SerializedValueDetector
+{
+
+    /**
+     * Uses a field's configuration to detect whether its value returned by a
+     * content object is expected to be serialized and thus needs to be
+     * unserialized.
+     *
+     * @param array $indexingConfiguration Current item's indexing configuration
+     * @param string $solrFieldName Current field being indexed
+     * @return bool TRUE if the value is expected to be serialized, FALSE otherwise
+     */
+    public function isSerializedValue(
+        array $indexingConfiguration,
+        $solrFieldName
+    );
+}