first commit

Classes/IndexQueue/FrontendHelper/AbstractFrontendHelper.php

@@ -0,0 +1,123 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue\FrontendHelper;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2010-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+use WapplerSystems\Meilisearch\IndexQueue\PageIndexerRequest;
+use WapplerSystems\Meilisearch\IndexQueue\PageIndexerResponse;
+use WapplerSystems\Meilisearch\System\Logging\SolrLogManager;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController;
+
+/**
+ * Index Queue page indexer frontend helper base class implementing common
+ * functionality.
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+abstract class AbstractFrontendHelper implements FrontendHelper
+{
+
+    /**
+     * Index Queue page indexer request.
+     *
+     * @var PageIndexerRequest
+     */
+    protected $request;
+
+    /**
+     * Index Queue page indexer response.
+     *
+     * @var PageIndexerResponse
+     */
+    protected $response;
+
+    /**
+     * The action a frontend helper executes.
+     */
+    protected $action = null;
+
+    /**
+     * @var SolrLogManager
+     */
+    protected $logger = null;
+
+    /**
+     * Disables the frontend output for index queue requests.
+     *
+     * @param array $parameters Parameters from frontend
+     */
+    public function disableFrontendOutput(&$parameters)
+    {
+        $parameters['enableOutput'] = false;
+    }
+
+    /**
+     * Disables caching for page generation to get reliable results.
+     *
+     * @param array $parameters Parameters from frontend
+     * @param TypoScriptFrontendController $parentObject TSFE object
+     */
+    public function disableCaching(
+        /** @noinspection PhpUnusedParameterInspection */
+        &$parameters,
+        $parentObject
+    ) {
+        $parentObject->no_cache = true;
+    }
+
+    /**
+     * Starts the execution of a frontend helper.
+     *
+     * @param PageIndexerRequest $request Page indexer request
+     * @param PageIndexerResponse $response Page indexer response
+     */
+    public function processRequest(
+        PageIndexerRequest $request,
+        PageIndexerResponse $response
+    ) {
+        $this->request = $request;
+        $this->response = $response;
+        $this->logger = GeneralUtility::makeInstance(SolrLogManager::class, /** @scrutinizer ignore-type */ __CLASS__);
+
+        if ($request->getParameter('loggingEnabled')) {
+            $this->logger->log(
+                SolrLogManager::INFO,
+                'Page indexer request received',
+                [
+                    'request' => (array)$request,
+                ]
+            );
+        }
+    }
+
+    /**
+     * Deactivates a frontend helper by unregistering from hooks and releasing
+     * resources.
+     */
+    public function deactivate()
+    {
+        $this->response->addActionResult($this->action, $this->getData());
+    }
+}

Classes/IndexQueue/FrontendHelper/AuthorizationService.php

@@ -0,0 +1,126 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue\FrontendHelper;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2011-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+use WapplerSystems\Meilisearch\Access\Rootline;
+use WapplerSystems\Meilisearch\IndexQueue\PageIndexerRequestHandler;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Core\Authentication\AbstractAuthenticationService;
+
+/**
+ * Authentication service to authorize the Index Queue page indexer to access
+ * protected pages.
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+class AuthorizationService extends AbstractAuthenticationService
+{
+
+    /**
+     * User used when authenticating the page indexer for protected pages,
+     * to allow the indexer to access and protected content. May also allow to
+     * identify requests by the page indexer.
+     *
+     * @var string
+     */
+    const SOLR_INDEXER_USERNAME = '__SolrIndexerUser__';
+
+    /**
+     * Gets a fake frontend user record to allow access to protected pages.
+     *
+     * @return array An array representing a frontend user.
+     */
+    public function getUser()
+    {
+        return [
+            'uid' => 0,
+            'username' => self::SOLR_INDEXER_USERNAME,
+            'authenticated' => true
+        ];
+    }
+
+    /**
+     * Authenticates the page indexer frontend user to grant it access to
+     * protected pages and page content.
+     *
+     * Returns 200 which automatically grants access for the current fake page
+     * indexer user. A status of >= 200 also tells TYPO3 that it doesn't need to
+     * conduct other services that might be registered for "their opinion"
+     * whether a user is authenticated.
+     *
+     * @see \TYPO3\CMS\Core\Authentication\AbstractUserAuthentication::checkAuthentication()
+     * @param array $user Array of user data
+     * @return int Returns 200 to grant access for the page indexer.
+     */
+    public function authUser($user)
+    {
+        // shouldn't happen, but in case we get a regular user we just
+        // pass it on to another (regular) auth service
+        $authenticationLevel = 100;
+
+        if ($user['username'] == self::SOLR_INDEXER_USERNAME) {
+            $authenticationLevel = 200;
+        }
+
+        return $authenticationLevel;
+    }
+
+    /**
+     * Creates user group records so that the page indexer is granted access to
+     * protected pages.
+     *
+     * @param array $user Data of user.
+     * @param array $knownGroups Group data array of already known groups. This is handy if you want select other related groups. Keys in this array are unique IDs of those groups.
+     * @return mixed Groups array, keys = uid which must be unique
+     */
+    public function getGroups(
+        $user,
+        /** @noinspection PhpUnusedParameterInspection */
+        $knownGroups
+    ) {
+        $groupData = [];
+
+            /** @var $requestHandler PageIndexerRequestHandler */
+        $requestHandler = GeneralUtility::makeInstance(PageIndexerRequestHandler::class);
+        $accessRootline = $requestHandler->getRequest()->getParameter('accessRootline');
+
+        if ($user['username'] == self::SOLR_INDEXER_USERNAME && !empty($accessRootline)) {
+            $accessRootline = GeneralUtility::makeInstance(Rootline::class, /** @scrutinizer ignore-type */ $accessRootline);
+            $groups = $accessRootline->getGroups();
+
+            foreach ($groups as $groupId) {
+                // faking a user group record
+                $groupData[] = [
+                    'uid' => $groupId,
+                    'pid' => 0,
+                    'title' => '__SolrIndexerGroup__',
+                    'TSconfig' => ''
+                ];
+            }
+        }
+
+        return $groupData;
+    }
+}

Classes/IndexQueue/FrontendHelper/Dispatcher.php

@@ -0,0 +1,89 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue\FrontendHelper;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2010-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+use WapplerSystems\Meilisearch\IndexQueue\PageIndexerRequest;
+use WapplerSystems\Meilisearch\IndexQueue\PageIndexerResponse;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+/**
+ * Dispatches the actions requested to the matching frontend helpers.
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+class Dispatcher
+{
+
+    /**
+     * Frontend helper manager.
+     *
+     * @var Manager
+     */
+    protected $frontendHelperManager;
+
+    /**
+     * Constructor
+     *
+     */
+    public function __construct()
+    {
+        $this->frontendHelperManager = GeneralUtility::makeInstance(Manager::class);
+    }
+
+    /**
+     * Takes the request's actions and hands them of to the according frontend
+     * helpers.
+     *
+     * @param PageIndexerRequest $request The request to dispatch
+     * @param PageIndexerResponse $response The request's response
+     */
+    public function dispatch(
+        PageIndexerRequest $request,
+        PageIndexerResponse $response
+    ) {
+        $actions = $request->getActions();
+
+        foreach ($actions as $action) {
+            $frontendHelper = $this->frontendHelperManager->resolveAction($action);
+            $frontendHelper->activate();
+            $frontendHelper->processRequest($request, $response);
+        }
+    }
+
+    /**
+     * Sends a shutdown signal to all activated frontend helpers.
+     *
+     * @return void
+     */
+    public function shutdown()
+    {
+        $frontendHelpers = $this->frontendHelperManager->getActivatedFrontendHelpers();
+
+        foreach ($frontendHelpers as $frontendHelper) {
+            /** @var FrontendHelper $frontendHelper */
+            $frontendHelper->deactivate();
+        }
+    }
+}

Classes/IndexQueue/FrontendHelper/FrontendHelper.php

@@ -0,0 +1,70 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue\FrontendHelper;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2010-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *  A copy is found in the textfile GPL.txt and important notices to the license
+ *  from the author is found in LICENSE.txt distributed with these scripts.
+ *
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+use WapplerSystems\Meilisearch\IndexQueue\PageIndexerRequest;
+use WapplerSystems\Meilisearch\IndexQueue\PageIndexerResponse;
+
+/**
+ * Index Queue Frontend Helper interface.
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+interface FrontendHelper
+{
+
+    /**
+     * Activates a frontend helper by registering for hooks and other
+     * resources required by the frontend helper to work.
+     */
+    public function activate();
+
+    /**
+     * Deactivates a frontend helper by unregistering from hooks and releasing
+     * resources.
+     */
+    public function deactivate();
+
+    /**
+     * Starts the execution of a frontend helper.
+     *
+     * @param PageIndexerRequest $request Page indexer request
+     * @param PageIndexerResponse $response Page indexer response
+     */
+    public function processRequest(
+        PageIndexerRequest $request,
+        PageIndexerResponse $response
+    );
+
+    /**
+     * Returns the collected data.
+     *
+     * @return array Collected data.
+     */
+    public function getData();
+}

Classes/IndexQueue/FrontendHelper/Manager.php

@@ -0,0 +1,98 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue\FrontendHelper;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2010-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+use RuntimeException;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+/**
+ * Index Queue Page Indexer frontend helper manager.
+ *
+ * Manages frontend helpers and creates instances.
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+class Manager
+{
+
+    /**
+     * Frontend helper descriptions.
+     *
+     * @var array
+     */
+    protected static $frontendHelperRegistry = [];
+
+    /**
+     * Instances of activated frontend helpers.
+     *
+     * @var array
+     */
+    protected $activatedFrontendHelpers = [];
+
+    /**
+     * Registers a frontend helper class for a certain action.
+     *
+     * @param string $action Action to register.
+     * @param string $class Class to register for an action.
+     */
+    public static function registerFrontendHelper($action, $class)
+    {
+        self::$frontendHelperRegistry[$action] = $class;
+    }
+
+    /**
+     * Tries to find a frontend helper for a given action. If found, creates an
+     * instance of the helper.
+     *
+     * @param string $action The action to get a frontend helper for.
+     * @return FrontendHelper Index Queue page indexer frontend helper
+     * @throws RuntimeException if the class registered for an action is not an implementation of WapplerSystems\Meilisearch\IndexQueue\FrontendHelper\FrontendHelper
+     */
+    public function resolveAction($action)
+    {
+        if (!array_key_exists($action, self::$frontendHelperRegistry)) {
+            return null;
+        }
+
+        $frontendHelper = GeneralUtility::makeInstance(self::$frontendHelperRegistry[$action]);
+        if (!$frontendHelper instanceof FrontendHelper) {
+            $message = self::$frontendHelperRegistry[$action] . ' is not an implementation of WapplerSystems\Meilisearch\IndexQueue\FrontendHelper\FrontendHelper';
+            throw new RuntimeException($message, 1292497896);
+        }
+
+        $this->activatedFrontendHelpers[$action] = $frontendHelper;
+        return $frontendHelper;
+    }
+
+    /**
+     * Gets an array with references to activated frontend helpers.
+     *
+     * @return array Array of references to activated frontend helpers.
+     */
+    public function getActivatedFrontendHelpers()
+    {
+        return $this->activatedFrontendHelpers;
+    }
+}

Classes/IndexQueue/FrontendHelper/PageFieldMappingIndexer.php

@@ -0,0 +1,164 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue\FrontendHelper;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2011-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+// TODO use/extend WapplerSystems\Meilisearch\IndexQueue\AbstractIndexer
+use WapplerSystems\Meilisearch\IndexQueue\AbstractIndexer;
+use WapplerSystems\Meilisearch\IndexQueue\InvalidFieldNameException;
+use WapplerSystems\Meilisearch\SubstitutePageIndexer;
+use WapplerSystems\Meilisearch\System\Configuration\TypoScriptConfiguration;
+use WapplerSystems\Meilisearch\System\Solr\Document\Document;
+use WapplerSystems\Meilisearch\Util;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Frontend\ContentObject\ContentObjectRenderer;
+
+/**
+ * Indexer to add / overwrite page document fields as defined in
+ * plugin.tx_meilisearch.index.queue.pages.fields.
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+class PageFieldMappingIndexer implements SubstitutePageIndexer
+{
+    /**
+     * @var \WapplerSystems\Meilisearch\System\Configuration\TypoScriptConfiguration
+     */
+    protected $configuration;
+
+    /**
+     * @var string
+     */
+    protected $pageIndexingConfigurationName = 'pages';
+
+    /**
+     * @param TypoScriptConfiguration $configuration
+     */
+    public function __construct(TypoScriptConfiguration $configuration = null)
+    {
+        $this->configuration = $configuration == null ? Util::getSolrConfiguration() : $configuration;
+    }
+
+    /**
+     * @param string $pageIndexingConfigurationName
+     */
+    public function setPageIndexingConfigurationName($pageIndexingConfigurationName)
+    {
+        $this->pageIndexingConfigurationName = $pageIndexingConfigurationName;
+    }
+
+    /**
+     * Returns a substitute document for the currently being indexed page.
+     *
+     * Uses the original document and adds fields as defined in
+     * plugin.tx_meilisearch.index.queue.pages.fields.
+     *
+     * @param Document $pageDocument The original page document.
+     * @return Document A Apache Solr Document object that replace the default page document
+     */
+    public function getPageDocument(Document $pageDocument)
+    {
+        $substitutePageDocument = clone $pageDocument;
+
+
+        $mappedFields = $this->getMappedFields($pageDocument);
+        foreach ($mappedFields as $fieldName => $fieldValue) {
+            if (isset($substitutePageDocument->{$fieldName})) {
+                // reset = overwrite, especially important to not make fields
+                // multi valued where they may not accept multiple values
+                unset($substitutePageDocument->{$fieldName});
+            }
+
+            // add new field / overwrite field if it was set before
+            if ($fieldValue !== '' && $fieldValue !== null) {
+                $substitutePageDocument->setField($fieldName, $fieldValue);
+            }
+        }
+
+        return $substitutePageDocument;
+    }
+
+    /**
+     * Gets the mapped fields as an array mapping field names to values.
+     *
+     * @throws InvalidFieldNameException
+     * @param Document $pageDocument The original page document.
+     * @return array An array mapping field names to their values.
+     */
+    protected function getMappedFields(Document $pageDocument)
+    {
+        $fields = [];
+
+        $mappedFieldNames = $this->configuration->getIndexQueueMappedFieldsByConfigurationName($this->pageIndexingConfigurationName);
+
+        foreach ($mappedFieldNames as $mappedFieldName) {
+            if (!AbstractIndexer::isAllowedToOverrideField($mappedFieldName)) {
+                throw new InvalidFieldNameException(
+                    'Must not overwrite field "type".',
+                    1435441863
+                );
+            }
+            $fields[$mappedFieldName] = $this->resolveFieldValue($mappedFieldName, $pageDocument);
+        }
+
+        return $fields;
+    }
+
+    /**
+     * Resolves a field mapping to its value depending on its configuration.
+     *
+     * Allows to put the page record through cObj processing if wanted / needed.
+     * Otherwise the plain page record field value is used.
+     *
+     * @param string $solrFieldName The Solr field name to resolve the value from the item's record
+     * @return string The resolved string value to be indexed
+     */
+    protected function resolveFieldValue($solrFieldName, Document $pageDocument)
+    {
+        $pageRecord = $GLOBALS['TSFE']->page;
+
+        $pageIndexingConfiguration = $this->configuration->getIndexQueueFieldsConfigurationByConfigurationName($this->pageIndexingConfigurationName);
+
+        if (isset($pageIndexingConfiguration[$solrFieldName . '.'])) {
+            $pageRecord = AbstractIndexer::addVirtualContentFieldToRecord($pageDocument, $pageRecord);
+
+            // configuration found => need to resolve a cObj
+            $contentObject = GeneralUtility::makeInstance(ContentObjectRenderer::class);
+            $contentObject->start($pageRecord, 'pages');
+
+            $fieldValue = $contentObject->cObjGetSingle(
+                $pageIndexingConfiguration[$solrFieldName],
+                $pageIndexingConfiguration[$solrFieldName . '.']
+            );
+
+            if (AbstractIndexer::isSerializedValue($pageIndexingConfiguration, $solrFieldName)) {
+                $fieldValue = unserialize($fieldValue);
+            }
+        } else {
+            $fieldValue = $pageRecord[$pageIndexingConfiguration[$solrFieldName]];
+        }
+
+        return $fieldValue;
+    }
+}

Classes/IndexQueue/FrontendHelper/PageIndexer.php

@@ -0,0 +1,364 @@
+<?php
+
+namespace WapplerSystems\Meilisearch\IndexQueue\FrontendHelper;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2011-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+use WapplerSystems\Meilisearch\Access\Rootline;
+use WapplerSystems\Meilisearch\ConnectionManager;
+use WapplerSystems\Meilisearch\IndexQueue\Item;
+use WapplerSystems\Meilisearch\IndexQueue\Queue;
+use WapplerSystems\Meilisearch\NoSolrConnectionFoundException;
+use WapplerSystems\Meilisearch\System\Logging\SolrLogManager;
+use WapplerSystems\Meilisearch\System\Solr\SolrConnection;
+use WapplerSystems\Meilisearch\Typo3PageIndexer;
+use WapplerSystems\Meilisearch\Util;
+use Exception;
+use TYPO3\CMS\Core\Log\Logger;
+use TYPO3\CMS\Core\SingletonInterface;
+use TYPO3\CMS\Core\Utility\ExtensionManagementUtility;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Frontend\ContentObject\ContentObjectRenderer;
+use TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController;
+use UnexpectedValueException;
+
+/**
+ * Index Queue Page Indexer frontend helper to ask the frontend page indexer to
+ * index the page.
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+class PageIndexer extends AbstractFrontendHelper implements SingletonInterface
+{
+
+    /**
+     * This frontend helper's executed action.
+     *
+     * @var string
+     */
+    protected $action = 'indexPage';
+
+    /**
+     * the page currently being indexed.
+     *
+     * @var TypoScriptFrontendController
+     */
+    protected $page;
+
+    /**
+     * Response data
+     *
+     * @var array
+     */
+    protected $responseData = [];
+
+    /**
+     * @var Logger
+     */
+    protected $logger = null;
+
+    /**
+     * Activates a frontend helper by registering for hooks and other
+     * resources required by the frontend helper to work.
+     *
+     * @noinspection PhpUnused
+     */
+    public function activate()
+    {
+        $pageIndexingHookRegistration = PageIndexer::class;
+
+        $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['tslib/class.tslib_fe.php']['pageIndexing'][__CLASS__] = $pageIndexingHookRegistration;
+
+        // indexes fields defined in plugin.tx_meilisearch.index.queue.pages.fields
+        $GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['solr']['Indexer']['indexPageSubstitutePageDocument'][PageFieldMappingIndexer::class] = PageFieldMappingIndexer::class;
+
+        $this->registerAuthorizationService();
+    }
+
+    /**
+     * Returns the status of whether a page was indexed.
+     *
+     * @return array Page indexed status.
+     * @noinspection PhpUnused
+     */
+    public function getData()
+    {
+        return $this->responseData;
+    }
+
+    #
+    # Indexer authorisation for access restricted pages / content
+    #
+
+    /**
+     * Fakes a logged in user to retrieve access restricted content.
+     *
+     * @return void
+     * @noinspection PhpUnused
+     */
+    public function authorizeFrontendUser()
+    {
+        $accessRootline = $this->getAccessRootline();
+        $stringAccessRootline = (string)$accessRootline;
+
+        if (empty($stringAccessRootline)) {
+            return;
+        }
+
+        if (!is_array($GLOBALS['TSFE']->fe_user->user)) {
+            $GLOBALS['TSFE']->fe_user->user = [];
+        }
+
+        $groups = $accessRootline->getGroups();
+        $groupList = implode(',', $groups);
+
+        $GLOBALS['TSFE']->fe_user->user['username'] = AuthorizationService::SOLR_INDEXER_USERNAME;
+        $GLOBALS['TSFE']->fe_user->user['usergroup'] = $groupList;
+
+        $this->responseData['authorization'] = [
+            'username' => $GLOBALS['TSFE']->fe_user->user['username'],
+            'usergroups' => $GLOBALS['TSFE']->fe_user->user['usergroup']
+        ];
+    }
+
+    /**
+     * Gets the access rootline as defined by the request.
+     *
+     * @return Rootline The access rootline to use for indexing.
+     */
+    protected function getAccessRootline()
+    {
+        $stringAccessRootline = '';
+
+        if ($this->request->getParameter('accessRootline')) {
+            $stringAccessRootline = $this->request->getParameter('accessRootline');
+        }
+
+        /** @noinspection PhpIncompatibleReturnTypeInspection */
+        return GeneralUtility::makeInstance(Rootline::class, /** @scrutinizer ignore-type */ $stringAccessRootline);
+    }
+
+    /**
+     * Registers an authentication service to authorize / grant the indexer to
+     * access protected pages.
+     *
+     * @return void
+     */
+    protected function registerAuthorizationService()
+    {
+        $overrulingPriority = $this->getHighestAuthenticationServicePriority() + 1;
+
+        ExtensionManagementUtility::addService(
+            'solr', // extension key
+            'auth', // service type
+            AuthorizationService::class,
+            // service key
+            [// service meta data
+                'title' => 'Solr Indexer Authorization',
+                'description' => 'Authorizes the Solr Index Queue indexer to access protected pages.',
+
+                'subtype' => 'getUserFE,authUserFE,getGroupsFE',
+
+                'available' => true,
+                'priority' => $overrulingPriority,
+                'quality' => 100,
+
+                'os' => '',
+                'exec' => '',
+
+                'classFile' => ExtensionManagementUtility::extPath('solr') . 'Classes/IndexQueue/FrontendHelper/AuthorizationService.php',
+                'className' => AuthorizationService::class,
+            ]
+        );
+    }
+
+    /**
+     * Determines the highest priority of all registered authentication
+     * services.
+     *
+     * @return int Highest priority of all registered authentication service
+     */
+    protected function getHighestAuthenticationServicePriority()
+    {
+        $highestPriority = 0;
+
+        if (is_array($GLOBALS['T3_SERVICES']['auth'])) {
+            foreach ($GLOBALS['T3_SERVICES']['auth'] as $service) {
+                if ($service['priority'] > $highestPriority) {
+                    $highestPriority = $service['priority'];
+                }
+            }
+        }
+
+        return $highestPriority;
+    }
+
+    #
+    # Indexing
+    #
+
+    /**
+     * Generates the current page's URL.
+     *
+     * Uses the provided GET parameters, page id and language id.
+     *
+     * @return string URL of the current page.
+     */
+    protected function generatePageUrl()
+    {
+        if ($this->request->getParameter('overridePageUrl')) {
+            return $this->request->getParameter('overridePageUrl');
+        }
+
+            /** @var $contentObject ContentObjectRenderer */
+        $contentObject = GeneralUtility::makeInstance(ContentObjectRenderer::class);
+
+        $typolinkConfiguration = [
+            'parameter' => intval($this->page->id),
+            'linkAccessRestrictedPages' => '1'
+        ];
+
+        $language = GeneralUtility::_GET('L');
+        if (!empty($language)) {
+            $typolinkConfiguration['additionalParams'] = '&L=' . $language;
+        }
+
+        $url = $contentObject->typoLink_URL($typolinkConfiguration);
+
+        // clean up
+        if ($url == '') {
+            $url = '/';
+        }
+
+        return $url;
+    }
+
+    /**
+     * Handles the indexing of the page content during post processing of a
+     * generated page.
+     *
+     * @param TypoScriptFrontendController $page TypoScript frontend
+     * @noinspection PhpUnused
+     */
+    public function hook_indexContent(TypoScriptFrontendController $page)
+    {
+        $this->logger = GeneralUtility::makeInstance(SolrLogManager::class, /** @scrutinizer ignore-type */ __CLASS__);
+
+        $this->page = $page;
+        $configuration = Util::getSolrConfiguration();
+
+        $logPageIndexed = $configuration->getLoggingIndexingPageIndexed();
+        if (!$this->page->config['config']['index_enable']) {
+            if ($logPageIndexed) {
+                $this->logger->log(
+                    SolrLogManager::ERROR,
+                    'Indexing is disabled. Set config.index_enable = 1 .'
+                );
+            }
+            return;
+        }
+
+        try {
+            $indexQueueItem = $this->getIndexQueueItem();
+            if (is_null($indexQueueItem)) {
+                throw new UnexpectedValueException('Can not get index queue item', 1482162337);
+            }
+
+            $solrConnection = $this->getSolrConnection($indexQueueItem);
+
+            /** @var $indexer Typo3PageIndexer */
+            $indexer = GeneralUtility::makeInstance(Typo3PageIndexer::class, /** @scrutinizer ignore-type */ $page);
+            $indexer->setSolrConnection($solrConnection);
+            $indexer->setPageAccessRootline($this->getAccessRootline());
+            $indexer->setPageUrl($this->generatePageUrl());
+            $indexer->setMountPointParameter($GLOBALS['TSFE']->MP);
+            $indexer->setIndexQueueItem($indexQueueItem);
+
+            $this->responseData['pageIndexed'] = (int)$indexer->indexPage();
+            $this->responseData['originalPageDocument'] = (array)$indexer->getPageSolrDocument();
+            $this->responseData['solrConnection'] = [
+                'rootPage' => $indexQueueItem->getRootPageUid(),
+                'sys_language_uid' => Util::getLanguageUid(),
+                'solr' => (string)$solrConnection->getNode('write')
+            ];
+
+            $documentsSentToSolr = $indexer->getDocumentsSentToSolr();
+            foreach ($documentsSentToSolr as $document) {
+                $this->responseData['documentsSentToSolr'][] = (array)$document;
+            }
+        } catch (Exception $e) {
+            if ($configuration->getLoggingExceptions()) {
+                $this->logger->log(
+                    SolrLogManager::ERROR,
+                    'Exception while trying to index page ' . $page->id,
+                    [
+                        $e->__toString()
+                    ]
+                );
+            }
+        }
+
+        if ($logPageIndexed) {
+            $success = $this->responseData['pageIndexed'] ? 'Success' : 'Failed';
+            $severity = $this->responseData['pageIndexed'] ? SolrLogManager::NOTICE : SolrLogManager::ERROR;
+
+            $this->logger->log(
+                $severity,
+                'Page indexed: ' . $success,
+                $this->responseData
+            );
+        }
+    }
+
+    /**
+     * Gets the solr connection to use for indexing the page based on the
+     * Index Queue item's properties.
+     *
+     * @param Item $indexQueueItem
+     * @return SolrConnection Solr server connection
+     * @throws NoSolrConnectionFoundException
+     */
+    protected function getSolrConnection(Item $indexQueueItem)
+    {
+        /** @var $connectionManager ConnectionManager */
+        $connectionManager = GeneralUtility::makeInstance(ConnectionManager::class);
+
+        return $connectionManager->getConnectionByRootPageId(
+            $indexQueueItem->getRootPageUid(),
+            Util::getLanguageUid()
+        );
+    }
+
+    /**
+     * This method retrieves the item from the index queue, that is indexed in this request.
+     *
+     * @return Item
+     */
+    protected function getIndexQueueItem()
+    {
+        /** @var $indexQueue Queue */
+        $indexQueue = GeneralUtility::makeInstance(Queue::class);
+        return $indexQueue->getItem($this->request->getParameter('item'));
+    }
+}

Classes/IndexQueue/FrontendHelper/UserGroupDetector.php

@@ -0,0 +1,253 @@
+<?php
+namespace WapplerSystems\Meilisearch\IndexQueue\FrontendHelper;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) 2010-2015 Ingo Renner <ingo@typo3.org>
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+use WapplerSystems\Meilisearch\System\Logging\SolrLogManager;
+use TYPO3\CMS\Core\SingletonInterface;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Frontend\ContentObject\ContentObjectPostInitHookInterface;
+use TYPO3\CMS\Frontend\ContentObject\ContentObjectRenderer;
+use TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController;
+use TYPO3\CMS\Frontend\Page\PageRepository;
+use TYPO3\CMS\Frontend\Page\PageRepositoryGetPageHookInterface;
+use TYPO3\CMS\Frontend\Page\PageRepositoryGetPageOverlayHookInterface;
+
+/**
+ * The UserGroupDetector is responsible to identify the fe_group references on records that are visible on the page (not the page itself).
+ *
+ * @author Ingo Renner <ingo@typo3.org>
+ */
+class UserGroupDetector extends AbstractFrontendHelper implements
+    SingletonInterface,
+    ContentObjectPostInitHookInterface,
+    PageRepositoryGetPageHookInterface,
+    PageRepositoryGetPageOverlayHookInterface
+{
+
+    /**
+     * This frontend helper's executed action.
+     */
+    protected $action = 'findUserGroups';
+
+    /**
+     * Holds the original, unmodified TCA during user group detection
+     *
+     * @var array
+     */
+    protected $originalTca = null;
+
+    /**
+     * Collects the usergroups used on a page.
+     *
+     * @var array
+     */
+    protected $frontendGroups = [];
+
+    /**
+     * @var \WapplerSystems\Meilisearch\System\Logging\SolrLogManager
+     */
+    protected $logger = null;
+
+    // activation
+
+    /**
+     * Activates a frontend helper by registering for hooks and other
+     * resources required by the frontend helper to work.
+     */
+    public function activate()
+    {
+        // register hooks
+        $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['tslib/class.tslib_fe.php']['isOutputting'][__CLASS__] = UserGroupDetector::class . '->disableFrontendOutput';
+        // disable TSFE cache for TYPO3 v9
+        $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['tslib/class.tslib_fe.php']['tslib_fe-PostProc'][__CLASS__] = UserGroupDetector::class . '->disableCaching';
+        $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['tslib/class.tslib_fe.php']['configArrayPostProc'][__CLASS__] = UserGroupDetector::class . '->deactivateTcaFrontendGroupEnableFields';
+        $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['tslib/class.tslib_fe.php']['hook_checkEnableFields'][__CLASS__] = UserGroupDetector::class . '->checkEnableFields';
+
+        $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['t3lib/class.t3lib_page.php']['getPage'][__CLASS__] = UserGroupDetector::class;
+        $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['t3lib/class.t3lib_page.php']['getPageOverlay'][__CLASS__] = UserGroupDetector::class;
+
+        $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['tslib/class.tslib_content.php']['postInit'][__CLASS__] = UserGroupDetector::class;
+    }
+
+    /**
+     * Disables the group access check by resetting the fe_group field in the given page table row.
+     * Will be called by the hook in the TypoScriptFrontendController in the checkEnableFields() method.
+     *
+     * @param array $parameters
+     * @param TypoScriptFrontendController $tsfe
+     * @see \TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController::checkEnableFields()
+     */
+    public function checkEnableFields(
+        $parameters,
+        /** @noinspection PhpUnusedParameterInspection */
+        $tsfe
+    ) {
+        $parameters['row']['fe_group'] = '';
+    }
+
+    /**
+     * Deactivates the frontend user group fields in TCA so that no access
+     * restrictions apply during page rendering.
+     *
+     * @param array $parameters Parameters from frontend
+     * @param TypoScriptFrontendController $parentObject TSFE object
+     */
+    public function deactivateTcaFrontendGroupEnableFields(
+        /** @noinspection PhpUnusedParameterInspection */
+        &$parameters,
+        /** @noinspection PhpUnusedParameterInspection */
+        $parentObject
+    ) {
+        $this->originalTca = $GLOBALS['TCA'];
+
+        foreach ($GLOBALS['TCA'] as $tableName => $tableConfiguration) {
+            if (isset($GLOBALS['TCA'][$tableName]['ctrl']['enablecolumns']['fe_group'])) {
+                unset($GLOBALS['TCA'][$tableName]['ctrl']['enablecolumns']['fe_group']);
+            }
+        }
+    }
+
+    // manipulation
+
+    /**
+     * Modifies the database query parameters so that access checks for pages
+     * are not performed any longer.
+     *
+     * @param int $uid The page ID
+     * @param bool $disableGroupAccessCheck If set, the check for group access is disabled. VERY rarely used
+     * @param PageRepository $parentObject parent \TYPO3\CMS\Frontend\Page\PageRepository object
+     */
+    public function getPage_preProcess(
+        &$uid,
+        &$disableGroupAccessCheck,
+        PageRepository $parentObject
+    ) {
+        $disableGroupAccessCheck = true;
+        $parentObject->where_groupAccess = ''; // just to be on the safe side
+    }
+
+    /**
+     * Modifies page records so that when checking for access through fe groups
+     * no groups or extendToSubpages flag is found and thus access is granted.
+     *
+     * @param array $pageRecord Page record
+     * @param int $languageUid Overlay language ID
+     * @param PageRepository $parentObject Parent \TYPO3\CMS\Frontend\Page\PageRepository object
+     */
+    public function getPageOverlay_preProcess(
+        &$pageRecord,
+        &$languageUid,
+        PageRepository $parentObject
+    ) {
+        if (is_array($pageRecord)) {
+            $pageRecord['fe_group'] = '';
+            $pageRecord['extendToSubpages'] = '0';
+        }
+    }
+
+    // execution
+
+    /**
+     * Hook for post processing the initialization of ContentObjectRenderer
+     *
+     * @param ContentObjectRenderer $parentObject parent content object
+     */
+    public function postProcessContentObjectInitialization(ContentObjectRenderer &$parentObject)
+    {
+        if (!empty($parentObject->currentRecord)) {
+            list($table) = explode(':', $parentObject->currentRecord);
+
+            if (!empty($table) && $table != 'pages') {
+                $this->findFrontendGroups($parentObject->data, $table);
+            }
+        }
+    }
+
+    /**
+     * Tracks user groups access restriction applied to records.
+     *
+     * @param array $record A record as an array of fieldname => fieldvalue mappings
+     * @param string $table Table name the record belongs to
+     */
+    protected function findFrontendGroups($record, $table)
+    {
+        if ($this->originalTca[$table]['ctrl']['enablecolumns']['fe_group']) {
+            $frontendGroups = $record[$this->originalTca[$table]['ctrl']['enablecolumns']['fe_group']];
+
+            if (empty($frontendGroups)) {
+                // default = public access
+                $frontendGroups = 0;
+            } else {
+                if ($this->request->getParameter('loggingEnabled')) {
+                    $this->logger->log(
+                        SolrLogManager::INFO,
+                        'Access restriction found',
+                        [
+                            'groups' => $frontendGroups,
+                            'record' => $record,
+                            'record type' => $table,
+                        ]
+                    );
+                }
+            }
+
+            $this->frontendGroups[] = $frontendGroups;
+        }
+    }
+
+    /**
+     * Returns an array of user groups that have been tracked during page
+     * rendering.
+     *
+     * @return array Array of user group IDs
+     */
+    protected function getFrontendGroups()
+    {
+        $frontendGroupsList = implode(',', $this->frontendGroups);
+        $frontendGroups = GeneralUtility::trimExplode(',', $frontendGroupsList,
+            true);
+
+        // clean up: filter double groups
+        $frontendGroups = array_unique($frontendGroups);
+        $frontendGroups = array_values($frontendGroups);
+
+        if (empty($frontendGroups)) {
+            // most likely an empty page with no content elements => public
+            $frontendGroups[] = '0';
+        }
+
+        return $frontendGroups;
+    }
+
+    /**
+     * Returns the user groups found.
+     *
+     * @return array Array of user groups.
+     */
+    public function getData()
+    {
+        return $this->getFrontendGroups();
+    }
+}