TYPO3 v15 dev-main snapshot ()

Classes/TypoScript/AST/AbstractAstBuilder.php

@@ -0,0 +1,276 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\AST;
+
+use Psr\EventDispatcher\EventDispatcherInterface;
+use TYPO3\CMS\Core\TypoScript\AST\CurrentObjectPath\CurrentObjectPath;
+use TYPO3\CMS\Core\TypoScript\AST\Event\EvaluateModifierFunctionEvent;
+use TYPO3\CMS\Core\TypoScript\AST\Node\ChildNode;
+use TYPO3\CMS\Core\TypoScript\AST\Node\ChildNodeInterface;
+use TYPO3\CMS\Core\TypoScript\AST\Node\NodeInterface;
+use TYPO3\CMS\Core\TypoScript\AST\Node\ReferenceChildNode;
+use TYPO3\CMS\Core\TypoScript\AST\Node\RootNode;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierCopyLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierReferenceLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierUnsetLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\ConstantAwareTokenStream;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\IdentifierTokenStream;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\Token;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenStreamInterface;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+/**
+ * Common methods of both AST builders.
+ *
+ * @internal: Internal AST structure.
+ */
+abstract class AbstractAstBuilder
+{
+    /**
+     * @var array<string, string>
+     */
+    protected array $flatConstants = [];
+    protected EventDispatcherInterface $eventDispatcher;
+
+    protected function handleIdentifierUnsetLine(IdentifierUnsetLine $line, CurrentObjectPath $currentObjectPath): void
+    {
+        $node = $currentObjectPath->getFirst();
+        $identifierStream = $line->getIdentifierTokenStream()->reset();
+        while ($identifierToken = $identifierStream->getNext()) {
+            if (!$foundNode = $node->getChildByName($identifierToken->getValue())) {
+                break;
+            }
+            $nextIdentifierToken = $identifierStream->peekNext();
+            if ($nextIdentifierToken) {
+                $node = $foundNode;
+                continue;
+            }
+            $node->removeChildByName($identifierToken->getValue());
+            break;
+        }
+    }
+
+    protected function handleIdentifierCopyLine(IdentifierCopyLine $line, RootNode $rootNode, CurrentObjectPath $currentObjectPath): ?NodeInterface
+    {
+        $sourceIdentifierStream = $line->getValueTokenStream()->reset();
+        $sourceNode = $rootNode;
+        if ($sourceIdentifierStream->isRelative()) {
+            // Entry node is current node from current object path if relative, otherwise RootNode.
+            $sourceNode = $currentObjectPath->getLast();
+        }
+        while ($identifierToken = $sourceIdentifierStream->getNext()) {
+            // Go through source token stream and locate the sourceNode to copy from.
+            if (!$sourceNode = $sourceNode->getChildByName($identifierToken->getValue())) {
+                // Source node not found - nothing to do for this line
+                return null;
+            }
+        }
+        $isSourceNodeValueNull = true;
+        if ($sourceNode->getValue() !== null) {
+            // When the source node value is not null, it will override the target node value if that exists.
+            $isSourceNodeValueNull = false;
+        }
+
+        // Locate/create the targets parent node the copied source should be added as child to,
+        // and get the name of the node we're dealing with.
+        $targetIdentifierTokenStream = $line->getIdentifierTokenStream()->reset();
+        $targetParentNode = $currentObjectPath->getFirst();
+        $targetTokenName = null;
+        while ($targetToken = $targetIdentifierTokenStream->getNext()) {
+            $targetTokenName = $targetToken->getValue();
+            if (!($targetIdentifierTokenStream->peekNext() ?? false)) {
+                break;
+            }
+            if (!$foundNode = $targetParentNode->getChildByName($targetTokenName)) {
+                // Add new node as new child of current last element in path
+                $foundNode = new ChildNode($targetTokenName);
+                $targetParentNode->addChild($foundNode);
+            }
+            $targetParentNode = $foundNode;
+        }
+
+        $existingTarget = null;
+        if ($isSourceNodeValueNull) {
+            // When the node to copy has no value, but the existing target has,
+            // the value from the existing target is kept. Also, if the existing
+            // node is a ReferenceChildNode and the source does not override this,
+            // source children are added to the existing reference instead of
+            // dropping the existing target.
+            $existingTarget = $targetParentNode->getChildByName($targetTokenName);
+            $existingTargetNodeValue = $existingTarget?->getValue();
+        } else {
+            // Blindly remove existing target node if exists and the value is not overridden by source.
+            $targetParentNode->removeChildByName($targetTokenName);
+        }
+        if ($existingTarget instanceof ReferenceChildNode) {
+            // When existing target is a ReferenceChildNode, keep it and
+            // copy children from source into existing target.
+            $targetNode = $existingTarget;
+            foreach ($sourceNode->getNextChild() as $sourceChild) {
+                $targetNode->addChild(clone $sourceChild);
+            }
+        } else {
+            // Clone full source node tree, update name and add as child to parent node.
+            /** @var ChildNodeInterface $targetNode */
+            $targetNode = clone $sourceNode;
+            $targetNode->updateName($targetTokenName);
+            $targetParentNode->addChild($targetNode);
+        }
+        if ($isSourceNodeValueNull && $existingTargetNodeValue) {
+            // If value of old existing target should be kept, set in now.
+            $targetNode->setValue($existingTargetNodeValue);
+        }
+
+        return $targetNode;
+    }
+
+    /**
+     * "foo =< bar": Prepare a reference resolving.
+     * Note this does *not* resolve "=<" itself at this point since this operator can only be
+     * evaluated after the full AST has been established. Also, having a full AST-traverser run
+     * that does this is *very* expensive and "=<" is only done for "tt_content.myElement" and
+     * "lib.parseFunc" anyways. As such, "=<" is NOT a language construct itself and the AST-parser
+     * only marks nodes that use it by using the special node "ObjectReference".
+     * Resolving then happens "lazy" and "on demand" in ContentObjectRenderer cObjGetSingle()
+     * and mergeTSRef() for frontend "setup" TypoScript.
+     */
+    protected function handleIdentifierReferenceLine(IdentifierReferenceLine $line, CurrentObjectPath $currentObjectPath): NodeInterface
+    {
+        $tokenStream = $line->getIdentifierTokenStream();
+        $node = $currentObjectPath->getFirst();
+        $identifierStream = $tokenStream->reset();
+        while ($identifierToken = $identifierStream->getNext()) {
+            $nextIdentifier = $identifierStream->peekNext();
+            $identifierTokenValue = $identifierToken->getValue();
+            if (!($node->getChildByName($identifierTokenValue)) && $nextIdentifier) {
+                // Add new node as new child of current last element in path
+                $foundNode = new ChildNode($identifierTokenValue);
+                $node->addChild($foundNode);
+            } elseif (!$node->getChildByName($identifierTokenValue) && $nextIdentifier === null) {
+                // Parent of target node exists, but target node does not. Add new reference child.
+                $foundNode = new ReferenceChildNode($identifierTokenValue);
+                $foundNode->setReferenceSourceStream($line->getValueTokenStream());
+                $node->addChild($foundNode);
+            } elseif (($foundNode = $node->getChildByName($identifierTokenValue)) && $nextIdentifier === null) {
+                // Target node exists already. We create a new one, remove old, but transfer existing children from old to new.
+                $newNode = new ReferenceChildNode($identifierTokenValue);
+                $newNode->setReferenceSourceStream($line->getValueTokenStream());
+                foreach ($foundNode->getNextChild() as $existingNodeChild) {
+                    $newNode->addChild($existingNodeChild);
+                }
+                $node->removeChildByName($identifierTokenValue);
+                $node->addChild($newNode);
+                $foundNode = $newNode;
+            }
+            $node = $foundNode;
+        }
+        return $node;
+    }
+
+    protected function getOrAddNodeFromIdentifierStream(CurrentObjectPath $currentObjectPath, IdentifierTokenStream $tokenStream): NodeInterface
+    {
+        $node = $currentObjectPath->getFirst();
+        $identifierStream = $tokenStream->reset();
+        while ($identifierToken = $identifierStream->getNext()) {
+            $identifierTokenValue = $identifierToken->getValue();
+            if (!$foundNode = $node->getChildByName($identifierTokenValue)) {
+                // Add new node as new child of current last element in path
+                $foundNode = new ChildNode($identifierTokenValue);
+                $node->addChild($foundNode);
+            }
+            $node = $foundNode;
+        }
+        return $node;
+    }
+
+    /**
+     * Evaluate operator functions, example TypoScript:
+     * "page.10.value := appendString(foo)"
+     */
+    protected function evaluateValueModifier(Token $functionNameToken, TokenStreamInterface $functionArgumentTokenStream, ?string $originalValue): ?string
+    {
+        $functionName = $functionNameToken->getValue();
+        // Constants are evaluated via __toString() of ConstantAwareTokenStream and thus need current constants.
+        // This implements constants in function arguments: "foo := addToList({$my.constant})"
+        if ($functionArgumentTokenStream instanceof ConstantAwareTokenStream) {
+            $functionArgumentTokenStream->setFlatConstants($this->flatConstants);
+        }
+        $functionArgument = (string)$functionArgumentTokenStream;
+        switch ($functionName) {
+            case 'prependString':
+                return $functionArgument . $originalValue;
+            case 'appendString':
+                return $originalValue . $functionArgument;
+            case 'removeString':
+                return str_replace($functionArgument, '', $originalValue);
+            case 'replaceString':
+                $functionValueArray = explode('|', $functionArgument, 2);
+                $fromStr = $functionValueArray[0];
+                $toStr = $functionValueArray[1] ?? '';
+                return str_replace($fromStr, $toStr, $originalValue);
+            case 'addToList':
+                return ($originalValue !== null ? $originalValue . ',' : '') . $functionArgument;
+            case 'removeFromList':
+                $existingElements = GeneralUtility::trimExplode(',', $originalValue ?? '');
+                $removeElements = GeneralUtility::trimExplode(',', $functionArgument);
+                if (!empty($removeElements)) {
+                    return implode(',', array_diff($existingElements, $removeElements));
+                }
+                return $originalValue;
+            case 'uniqueList':
+                $elements = GeneralUtility::trimExplode(',', $originalValue ?? '');
+                return implode(',', array_unique($elements));
+            case 'reverseList':
+                $elements = GeneralUtility::trimExplode(',', $originalValue ?? '');
+                return implode(',', array_reverse($elements));
+            case 'sortList':
+                $elements = GeneralUtility::trimExplode(',', $originalValue ?? '');
+                $arguments = GeneralUtility::trimExplode(',', $functionArgument);
+                $arguments = array_map('strtolower', $arguments);
+                $sortFlags = SORT_REGULAR;
+                if (in_array('numeric', $arguments)) {
+                    $sortFlags = SORT_NUMERIC;
+                    // If the sorting modifier "numeric" is given, all values
+                    // are checked and an exception is thrown if a non-numeric value is given
+                    // otherwise there is a different behaviour between PHP 7 and PHP 5.x
+                    // See also the warning on http://us.php.net/manual/en/function.sort.php
+                    foreach ($elements as $element) {
+                        if (!is_numeric($element)) {
+                            throw new \InvalidArgumentException(
+                                'The list "' . $originalValue . '" should be sorted numerically but contains a non-numeric value',
+                                1650893781
+                            );
+                        }
+                    }
+                }
+                sort($elements, $sortFlags);
+                if (in_array('descending', $arguments)) {
+                    $elements = array_reverse($elements);
+                }
+                return implode(',', $elements);
+            case 'getEnv':
+                $environmentValue = getenv(trim($functionArgument));
+                if ($environmentValue !== false) {
+                    return $environmentValue;
+                }
+                return $originalValue;
+            default:
+                return $this->eventDispatcher->dispatch(new EvaluateModifierFunctionEvent($functionName, $functionArgument, $originalValue))->getValue() ?? $originalValue;
+        }
+    }
+}

Classes/TypoScript/AST/AstBuilder.php

@@ -0,0 +1,109 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\AST;
+
+use Psr\EventDispatcher\EventDispatcherInterface;
+use Symfony\Component\DependencyInjection\Attribute\AsAlias;
+use Symfony\Component\DependencyInjection\Attribute\Autoconfigure;
+use TYPO3\CMS\Core\TypoScript\AST\CurrentObjectPath\CurrentObjectPath;
+use TYPO3\CMS\Core\TypoScript\AST\CurrentObjectPath\CurrentObjectPathStack;
+use TYPO3\CMS\Core\TypoScript\AST\Node\RootNode;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\BlockCloseLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierAssignmentLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierBlockOpenLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierCopyLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierFunctionLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierReferenceLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierUnsetLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\LineStream;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\ConstantAwareTokenStream;
+
+/**
+ * The main TypoScript AST builder.
+ *
+ * This creates a tree of Nodes, starting with the root node. Each node can have
+ * children. The implementation basically iterates a LineStream created by the
+ * tokenizers, and creates AST depending on the line type. It handles all the
+ * different operator lines like "=", "<" and so on.
+ *
+ * @internal: Internal AST structure.
+ */
+#[Autoconfigure(public: true), AsAlias(AstBuilderInterface::class)]
+final class AstBuilder extends AbstractAstBuilder implements AstBuilderInterface
+{
+    public function __construct(
+        EventDispatcherInterface $eventDispatcher,
+    ) {
+        $this->eventDispatcher = $eventDispatcher;
+    }
+
+    /**
+     * @param array<string, string> $flatConstants
+     */
+    public function build(LineStream $lineStream, RootNode $ast, array $flatConstants = []): RootNode
+    {
+        $this->flatConstants = $flatConstants;
+
+        $currentObjectPath = new CurrentObjectPath($ast);
+        $currentObjectPathStack = new CurrentObjectPathStack();
+        $currentObjectPathStack->push($currentObjectPath);
+
+        foreach ($lineStream->getNextLine() as $line) {
+            if ($line instanceof IdentifierAssignmentLine) {
+                // "foo = bar" and "foo ( bar )": Single and multi line assignments
+                $this->handleIdentifierAssignmentLine($line, $currentObjectPath);
+            } elseif ($line instanceof IdentifierBlockOpenLine) {
+                // "foo {": Opening a block - push to object path stack
+                $node = $this->getOrAddNodeFromIdentifierStream($currentObjectPath, $line->getIdentifierTokenStream());
+                $currentObjectPath = (new CurrentObjectPath($node));
+                $currentObjectPathStack->push($currentObjectPath);
+            } elseif ($line instanceof BlockCloseLine) {
+                // "}": Closing a block - pop from object path stack
+                $currentObjectPath = $currentObjectPathStack->pop();
+            } elseif ($line instanceof IdentifierUnsetLine) {
+                // "foo >": Remove a path
+                $this->handleIdentifierUnsetLine($line, $currentObjectPath);
+            } elseif ($line instanceof IdentifierCopyLine) {
+                // "foo < bar": Copy a node source path to a target path
+                $this->handleIdentifierCopyLine($line, $ast, $currentObjectPath);
+            } elseif ($line instanceof IdentifierFunctionLine) {
+                // "foo := addToList(42)": Evaluate functions
+                $node = $this->getOrAddNodeFromIdentifierStream($currentObjectPath, $line->getIdentifierTokenStream());
+                $node->setValue($this->evaluateValueModifier($line->getFunctionNameToken(), $line->getFunctionValueTokenStream(), $node->getValue()));
+            } elseif ($line instanceof IdentifierReferenceLine) {
+                // "foo =< bar": Prepare a reference resolving
+                $this->handleIdentifierReferenceLine($line, $currentObjectPath);
+            }
+        }
+
+        return $ast;
+    }
+
+    private function handleIdentifierAssignmentLine(IdentifierAssignmentLine $line, CurrentObjectPath $currentObjectPath): void
+    {
+        $node = $this->getOrAddNodeFromIdentifierStream($currentObjectPath, $line->getIdentifierTokenStream());
+        $valueTokenStream = $line->getValueTokenStream();
+        if ($valueTokenStream instanceof ConstantAwareTokenStream) {
+            $valueTokenStream = clone $valueTokenStream;
+            $valueTokenStream->setFlatConstants($this->flatConstants);
+            $node->setValue((string)$valueTokenStream);
+            return;
+        }
+        $node->setValue((string)$valueTokenStream);
+    }
+}

Classes/TypoScript/AST/AstBuilderInterface.php

@@ -0,0 +1,39 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\AST;
+
+use TYPO3\CMS\Core\TypoScript\AST\Node\RootNode;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\LineStream;
+
+/**
+ * The main TypoScript AST builder.
+ *
+ * This creates a tree of Nodes, starting with the root node. Each node can have
+ * children. The implementation basically iterates a LineStream created by the
+ * tokenizers, and creates AST depending on the line type. It handles all the
+ * different operator lines like "=", "<" and so on.
+ *
+ * @internal: Internal AST structure.
+ */
+interface AstBuilderInterface
+{
+    /**
+     * @param array<string, string> $flatConstants
+     */
+    public function build(LineStream $lineStream, RootNode $ast, array $flatConstants = []): RootNode;
+}

Classes/TypoScript/AST/CommentAwareAstBuilder.php

@@ -0,0 +1,187 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\AST;
+
+use Psr\EventDispatcher\EventDispatcherInterface;
+use TYPO3\CMS\Core\TypoScript\AST\CurrentObjectPath\CurrentObjectPath;
+use TYPO3\CMS\Core\TypoScript\AST\CurrentObjectPath\CurrentObjectPathStack;
+use TYPO3\CMS\Core\TypoScript\AST\Node\NodeInterface;
+use TYPO3\CMS\Core\TypoScript\AST\Node\RootNode;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\BlockCloseLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\CommentLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\EmptyLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierAssignmentLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierBlockOpenLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierCopyLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierFunctionLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierReferenceLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierUnsetLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\LineStream;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\ConstantAwareTokenStream;
+
+/**
+ * Secondary TypoScript AST builder.
+ *
+ * This creates a tree of Nodes, starting with the root node. Each node can have
+ * children. The implementation basically iterates a LineStream created by the
+ * tokenizers, and creates AST depending on the line type. It handles all the
+ * different operator lines like "=", "<" and so on.
+ *
+ * This AST builder is comment aware: Comments are assigned to nodes. This is used
+ * in ext:tstemplate and page TSconfig backend modules to add the comment related
+ * TypoScript functionality.
+ *
+ * This AST builder variant adds runtime overhead and is slower than the main
+ * AstBuilder class.
+ *
+ * @internal: Internal AST structure.
+ */
+final class CommentAwareAstBuilder extends AbstractAstBuilder implements AstBuilderInterface
+{
+    public function __construct(
+        EventDispatcherInterface $eventDispatcher,
+    ) {
+        $this->eventDispatcher = $eventDispatcher;
+    }
+
+    /**
+     * @param array<string, string> $flatConstants
+     */
+    public function build(LineStream $lineStream, RootNode $ast, array $flatConstants = []): RootNode
+    {
+        $this->flatConstants = $flatConstants;
+
+        $currentObjectPath = new CurrentObjectPath($ast);
+        $currentObjectPathStack = new CurrentObjectPathStack();
+        $currentObjectPathStack->push($currentObjectPath);
+
+        $previousLineComments = [];
+        while ($line = $lineStream->getNext()) {
+            $node = null;
+            if ($line instanceof IdentifierAssignmentLine) {
+                // "foo = bar" and "foo ( bar )": Single and multi line assignments
+                $node = $this->handleIdentifierAssignmentLine($line, $currentObjectPath);
+                if ($previousLineComments) {
+                    foreach ($previousLineComments as $previousLineComment) {
+                        $node->addComment($previousLineComment);
+                    }
+                    $previousLineComments = [];
+                }
+            } elseif ($line instanceof IdentifierBlockOpenLine) {
+                // "foo {": Opening a block - push to object path stack
+                $node = $this->getOrAddNodeFromIdentifierStream($currentObjectPath, $line->getIdentifierTokenStream());
+                if ($previousLineComments) {
+                    foreach ($previousLineComments as $previousLineComment) {
+                        $node->addComment($previousLineComment);
+                    }
+                    $previousLineComments = [];
+                }
+                $currentObjectPath = (new CurrentObjectPath($node));
+                $currentObjectPathStack->push($currentObjectPath);
+            } elseif ($line instanceof BlockCloseLine) {
+                // "}": Closing a block - pop from object path stack
+                $currentObjectPath = $currentObjectPathStack->pop();
+            } elseif ($line instanceof IdentifierUnsetLine) {
+                // "foo >": Remove a path
+                $this->handleIdentifierUnsetLine($line, $currentObjectPath);
+            } elseif ($line instanceof IdentifierCopyLine) {
+                // "foo < bar": Copy a node source path to a target path
+                $node = $this->handleIdentifierCopyLine($line, $ast, $currentObjectPath);
+                if ($node && $previousLineComments) {
+                    foreach ($previousLineComments as $previousLineComment) {
+                        $node->addComment($previousLineComment);
+                    }
+                    $previousLineComments = [];
+                }
+            } elseif ($line instanceof IdentifierFunctionLine) {
+                // "foo := addToList(42)": Evaluate functions
+                $node = $this->getOrAddNodeFromIdentifierStream($currentObjectPath, $line->getIdentifierTokenStream());
+                $functionValueTokenStream = $line->getFunctionValueTokenStream();
+                $node->setValue($this->evaluateValueModifier($line->getFunctionNameToken(), clone $functionValueTokenStream, $node->getValue()));
+                if ($functionValueTokenStream instanceof ConstantAwareTokenStream) {
+                    // @todo: This is a bit unfortunate. When multiple functions manipulate a value after each other,
+                    //        only the stream of the last one is preserved, previous ones are lost. This way, the BE modules
+                    //        can not reflect when previous functions used constants. One idea to solve this is to turn existing
+                    //        nodes into special "function" nodes that park single operations, which are then executed lazy when
+                    //        a node value is string'ified. The BE modules could then render full lists of value manipulations and
+                    //        show how values evolve. Another idea is to change setOriginalValueTokenStream() to gather multiple
+                    //        streams - probably together with the current value at this point in time, which would also allow
+                    //        rendering how a value evolves over time as well.
+                    $node->setOriginalValueTokenStream($functionValueTokenStream);
+                }
+                if ($previousLineComments) {
+                    foreach ($previousLineComments as $previousLineComment) {
+                        $node->addComment($previousLineComment);
+                    }
+                    $previousLineComments = [];
+                }
+            } elseif ($line instanceof IdentifierReferenceLine) {
+                // "foo =< bar": Prepare a reference resolving
+                $node = $this->handleIdentifierReferenceLine($line, $currentObjectPath);
+                if ($previousLineComments) {
+                    foreach ($previousLineComments as $previousLineComment) {
+                        $node->addComment($previousLineComment);
+                    }
+                    $previousLineComments = [];
+                }
+            } elseif ($line instanceof CommentLine) {
+                $nextLine = $lineStream->peekNext();
+                if ($currentObjectPath->getLast() instanceof RootNode && ($nextLine === null || $nextLine instanceof EmptyLine)) {
+                    $previousLineComments[] = $line->getTokenStream();
+                    foreach ($previousLineComments as $commentLineTokenStream) {
+                        $ast->addComment($commentLineTokenStream);
+                    }
+                    $previousLineComments = [];
+                }
+                if ($nextLine instanceof CommentLine) {
+                    $previousLineComments[] = $line->getTokenStream();
+                }
+                if ($nextLine instanceof IdentifierAssignmentLine
+                    || $nextLine instanceof IdentifierBlockOpenLine
+                    || $nextLine instanceof IdentifierCopyLine
+                    || $nextLine instanceof IdentifierFunctionLine
+                    || $nextLine instanceof IdentifierReferenceLine
+                ) {
+                    $previousLineComments[] = $line->getTokenStream();
+                }
+            }
+        }
+
+        return $ast;
+    }
+
+    /**
+     * Slightly different from AstBuilder since it sets 'previousValue'
+     */
+    private function handleIdentifierAssignmentLine(IdentifierAssignmentLine $line, CurrentObjectPath $currentObjectPath): NodeInterface
+    {
+        $node = $this->getOrAddNodeFromIdentifierStream($currentObjectPath, $line->getIdentifierTokenStream());
+        $valueTokenStream = $line->getValueTokenStream();
+        if ($valueTokenStream instanceof ConstantAwareTokenStream) {
+            $node->setOriginalValueTokenStream($valueTokenStream);
+            $valueTokenStream = clone $valueTokenStream;
+            $valueTokenStream->setFlatConstants($this->flatConstants);
+            $node->setPreviousValue($node->getValue());
+            $node->setValue((string)$valueTokenStream);
+            return $node;
+        }
+        $node->setPreviousValue($node->getValue());
+        $node->setValue((string)$valueTokenStream);
+        return $node;
+    }
+}

Classes/TypoScript/AST/CurrentObjectPath/CurrentObjectPath.php

@@ -0,0 +1,99 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\AST\CurrentObjectPath;
+
+use TYPO3\CMS\Core\TypoScript\AST\Node\NodeInterface;
+use TYPO3\CMS\Core\TypoScript\AST\Node\RootNode;
+
+/**
+ * Internal state class to track the current hierarchy in tree.
+ * This is important in combination with block open "{" and block
+ * close "}" brackets.
+ * Also used in BE Template Object Browser tree rendering.
+ *
+ * @internal: Internal AST structure.
+ */
+final class CurrentObjectPath
+{
+    /**
+     * @var NodeInterface[]
+     */
+    private array $path;
+
+    public function __construct(NodeInterface ...$path)
+    {
+        $this->path = $path;
+    }
+
+    public function append(NodeInterface $node): void
+    {
+        $this->path[] = $node;
+    }
+
+    /**
+     * @return NodeInterface[]
+     */
+    public function getAll(): array
+    {
+        return $this->path;
+    }
+
+    /**
+     * Turn current object path into a string. Quote dots in keys.
+     * Used in BE Template Object Browser tree, expand and search handling.
+     * Not implementing __toString() here since Fluid can't call this.
+     *
+     * Example:
+     * page.10.foo\.bar.baz
+     */
+    public function getPathAsString(): string
+    {
+        $flatArray = [];
+        foreach ($this->getAll() as $pathNode) {
+            if ($pathNode instanceof RootNode) {
+                continue;
+            }
+            $name = $pathNode->getName();
+            if ($name === '') {
+                throw new \RuntimeException('Node names must not be empty string', 1658578645);
+            }
+            $flatArray[] = addcslashes($name, '.');
+        }
+        return implode('.', $flatArray);
+    }
+
+    public function getFirst(): NodeInterface
+    {
+        return reset($this->path);
+    }
+
+    public function getLast(): NodeInterface
+    {
+        return array_last($this->path);
+    }
+
+    public function getSecondLast(): NodeInterface
+    {
+        return array_slice($this->path, -2, 1)[0];
+    }
+
+    public function removeLast(): void
+    {
+        array_pop($this->path);
+    }
+}

Classes/TypoScript/AST/CurrentObjectPath/CurrentObjectPathStack.php

@@ -0,0 +1,63 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\AST\CurrentObjectPath;
+
+/**
+ * A stack for CurrentObjectPath: When opening a block "{",
+ * CurrentObjectPath is pushed, when closing a block "}", it
+ * is popped from this stack.
+ *
+ * @internal: Internal AST structure.
+ */
+final class CurrentObjectPathStack
+{
+    /**
+     * @var CurrentObjectPath[]
+     */
+    private array $stack = [];
+    private int $stackSize = 0;
+
+    public function push(CurrentObjectPath $path): void
+    {
+        $this->stack[] = $path;
+        $this->stackSize++;
+    }
+
+    public function pop(): CurrentObjectPath
+    {
+        if ($this->stackSize === 1) {
+            // Never pop the very last element off from the stack. This is the
+            // RootNode. This prevents errors when TypoScript has a closing
+            // curly bracket '}' too much.
+            return $this->getCurrent();
+        }
+        array_pop($this->stack);
+        $this->stackSize--;
+        return $this->getCurrent();
+    }
+
+    public function getCurrent(): CurrentObjectPath
+    {
+        return array_last($this->stack);
+    }
+
+    public function getFirst(): CurrentObjectPath
+    {
+        return reset($this->stack);
+    }
+}

Classes/TypoScript/AST/Event/EvaluateModifierFunctionEvent.php

@@ -0,0 +1,85 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\AST\Event;
+
+/**
+ * Listeners to this event are able to implement own ":=" TypoScript modifier functions, example:
+ *
+ * foo = myOriginalValue
+ * foo := myNewFunction(myFunctionArgument)
+ *
+ * Listeners should take care function names can not overlap with function names
+ * from other extensions and should thus namespace, example naming: "extNewsSortFunction()"
+ */
+final class EvaluateModifierFunctionEvent
+{
+    private ?string $value = null;
+
+    public function __construct(
+        private readonly string $functionName,
+        private readonly string $functionArgument,
+        private readonly ?string $originalValue,
+    ) {}
+
+    /**
+     * The function name, for example "extNewsSortFunction" when using "foo := extNewsSortFunction()"
+     */
+    public function getFunctionName(): string
+    {
+        return $this->functionName;
+    }
+
+    /**
+     * Optional function argument, for example "myArgument" when using "foo := extNewsSortFunction(myArgument)"
+     * If the argument contained constants, those have been resolved at this point.
+     */
+    public function getFunctionArgument(): string
+    {
+        return $this->functionArgument;
+    }
+
+    /**
+     * Original / current value, for example "fooValue" when using:
+     * foo = fooValue
+     * foo := extNewsSortFunction(myArgument)
+     */
+    public function getOriginalValue(): ?string
+    {
+        return $this->originalValue;
+    }
+
+    /**
+     * Set the updated value calculated by a listener.
+     * Note you can not set to null to "unset", since getValue() falls back to
+     * originalValue in this case. Set to empty string instead for this edge case.
+     */
+    public function setValue(string $value): void
+    {
+        $this->value = $value;
+    }
+
+    /**
+     * Used by AstBuilder to fetch the updated value, falls back to given original value.
+     * Can be used by Listeners to see if a previous listener changed the value already
+     * by comparing with getOriginalValue().
+     */
+    public function getValue(): ?string
+    {
+        return $this->value;
+    }
+}

Classes/TypoScript/AST/Merger/SetupConfigMerger.php

@@ -0,0 +1,66 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\AST\Merger;
+
+use TYPO3\CMS\Core\TypoScript\AST\Node\ChildNodeInterface;
+use TYPO3\CMS\Core\TypoScript\AST\Node\NodeInterface;
+use TYPO3\CMS\Core\TypoScript\AST\Node\RootNode;
+
+/**
+ * Frontend TypoScript 'setup' has the main 'config' section for global config,
+ * plus a per type / typeNum specific PAGE 'config' (often page.config) that can
+ * override global 'config' per type / typeNum.
+ *
+ * This class merges both into the final 'config', later available in Request
+ * attribute 'frontend.typoscript' getConfigTree() and getConfigArray().
+ *
+ * @internal: Internal AST structure.
+ */
+final readonly class SetupConfigMerger
+{
+    public function merge(?ChildNodeInterface $config, ?ChildNodeInterface $pageConfig): RootNode
+    {
+        $configResult = new RootNode();
+        if ($config) {
+            foreach ($config->getNextChild() as $child) {
+                $configResult->addChild($child);
+            }
+        }
+        if (!$pageConfig) {
+            return $configResult;
+        }
+        $this->mergeRecursive($pageConfig, $configResult);
+        return $configResult;
+    }
+
+    private function mergeRecursive(ChildNodeInterface $mergeFrom, NodeInterface $mergeTo): void
+    {
+        foreach ($mergeFrom->getNextChild() as $mergeFromChild) {
+            $mergeToChild = $mergeTo->getChildByName($mergeFromChild->getName());
+            if (!$mergeToChild) {
+                $mergeTo->addChild($mergeFromChild);
+                continue;
+            }
+            $mergeFromChildValue = $mergeFromChild->getValue();
+            if ($mergeFromChildValue !== null && $mergeFromChildValue !== $mergeToChild->getValue()) {
+                $mergeToChild->setValue($mergeFromChildValue);
+            }
+            $this->mergeRecursive($mergeFromChild, $mergeToChild);
+        }
+    }
+}

Classes/TypoScript/AST/Node/AbstractChildNode.php

@@ -0,0 +1,86 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\AST\Node;
+
+/**
+ * Generic child node. Implements common methods of NodeInterface used
+ * in all Node classes.
+ *
+ * @internal: Internal AST structure.
+ */
+abstract class AbstractChildNode extends AbstractNode implements ChildNodeInterface
+{
+    public function __construct(protected string $name) {}
+
+    /**
+     * Dereference children on clone().
+     * Used with '<' operator to create a deep-copy of the tree to copy.
+     */
+    public function __clone(): void
+    {
+        foreach ($this->children as $childName => $child) {
+            $this->children[$childName] = clone $child;
+        }
+    }
+
+    public function getName(): string
+    {
+        return $this->name;
+    }
+
+    public function toArray(): ?array
+    {
+        if (!$this->hasChildren()) {
+            return null;
+        }
+        $result = [];
+        foreach ($this->getNextChild() as $child) {
+            $childName = $child->getName();
+            $childValue = $child->getValue();
+            if ($child instanceof ReferenceChildNode) {
+                // Hack for b/w compat parsing of `=<` operator. See ContentObjectRenderer cObjGetSingle() and mergeTSRef()
+                // @todo: adding the whitespace after '<' is another bit of a hack here ... maybe solve in tokenizer?
+                //        compare this for what happens when doing 'foo = bar' in old parser: Is the whitespace kept for
+                //        value to not trigger the ref lookup to often if doing 'foo = <div...' ?
+                // @todo: same situation in RootNode!
+                $childValue = '< ' . $child->getReferenceSourceStream();
+            }
+            if ($childValue !== null) {
+                $result[$child->getName()] = $childValue;
+            }
+            $grandChildren = $child->toArray();
+            if ($grandChildren !== null) {
+                $result[$childName . '.'] = $grandChildren;
+            }
+        }
+        return $result;
+    }
+
+    public function flatten(string $prefix = ''): array
+    {
+        $flatArray = [];
+        $prefixedQuotedNodeName = $prefix . addcslashes($this->getName(), '.');
+        if (!$this->isValueNull()) {
+            $flatArray[$prefixedQuotedNodeName] = $this->getValue();
+        }
+        foreach ($this->getNextChild() as $child) {
+            $flatArray = array_merge($flatArray, $child->flatten($prefixedQuotedNodeName . '.'));
+        }
+        return $flatArray;
+    }
+}

Classes/TypoScript/AST/Node/AbstractNode.php

@@ -0,0 +1,185 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\AST\Node;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenStreamInterface;
+
+/**
+ * Generic node. Implements common methods of NodeInterface used
+ * in all Node classes.
+ *
+ * @internal: Internal AST structure.
+ */
+abstract class AbstractNode implements NodeInterface
+{
+    private ?string $identifier = null;
+    protected string $name;
+    private ?string $value = null;
+    private ?string $previousValue = null;
+
+    /**
+     * @var array<string, ChildNodeInterface>
+     */
+    protected array $children = [];
+    private ?TokenStreamInterface $originalValueTokenStream = null;
+    private array $comments = [];
+
+    /**
+     * When storing to cache, we only store FE relevant properties and skip
+     * various BE related properties which then default to class defaults when
+     * unserialized. This is done to create smaller php cache files.
+     */
+    final public function __serialize(): array
+    {
+        return $this->serialize();
+    }
+
+    protected function serialize(): array
+    {
+        $result = [
+            'name' => $this->name,
+            'children' => $this->children,
+        ];
+        if ($this->value !== null) {
+            $result['value'] = $this->value;
+        }
+        return $result;
+    }
+
+    public function setIdentifier(string $identifier): void
+    {
+        $this->identifier = hash('xxh3', $identifier);
+        $childCounter = 0;
+        foreach ($this->getNextChild() as $child) {
+            $child->setIdentifier($this->identifier . $childCounter);
+            $childCounter++;
+        }
+    }
+
+    /**
+     * This forces $this->name NOT to be readonly.
+     * Used with '<' operator on tree root to copy:
+     *      foo = value
+     *      bar < foo
+     * The 'foo' object node is copied, but added to AST as name 'bar'
+     */
+    public function updateName(string $name): void
+    {
+        $this->name = $name;
+    }
+
+    public function getIdentifier(): string
+    {
+        if ($this->identifier === null) {
+            throw new \RuntimeException(
+                'Identifier has not been initialized. This happens when getIdentifier() is called on'
+                . ' trees retrieved from cache. The identifier is not supposed to be used in this context.',
+                1674620169
+            );
+        }
+        return $this->identifier;
+    }
+
+    public function addChild(ChildNodeInterface $node): void
+    {
+        $this->children[$node->getName()] = $node;
+    }
+
+    public function getChildByName(string $name): ?ChildNodeInterface
+    {
+        return $this->children[$name] ?? null;
+    }
+
+    /**
+     * Note this does *not* choke if that child does not exist, so we can "blindly" remove without error.
+     */
+    public function removeChildByName(string $name): void
+    {
+        unset($this->children[$name]);
+    }
+
+    public function hasChildren(): bool
+    {
+        return !empty($this->children);
+    }
+
+    public function getNextChild(): iterable
+    {
+        foreach ($this->children as $child) {
+            yield $child;
+        }
+    }
+
+    public function sortChildren(): void
+    {
+        ksort($this->children, SORT_FLAG_CASE | SORT_STRING);
+    }
+
+    public function setValue(?string $value): void
+    {
+        $this->value = $value;
+    }
+
+    public function appendValue(string $value): void
+    {
+        $this->value .= $value;
+    }
+
+    public function getValue(): ?string
+    {
+        return $this->value;
+    }
+
+    public function isValueNull(): bool
+    {
+        return $this->value === null;
+    }
+
+    public function setPreviousValue(?string $value): void
+    {
+        $this->previousValue = $value;
+    }
+
+    public function getPreviousValue(): ?string
+    {
+        return $this->previousValue;
+    }
+
+    public function setOriginalValueTokenStream(?TokenStreamInterface $tokenStream): void
+    {
+        $this->originalValueTokenStream = $tokenStream;
+    }
+
+    public function getOriginalValueTokenStream(): ?TokenStreamInterface
+    {
+        return $this->originalValueTokenStream;
+    }
+
+    public function addComment(TokenStreamInterface $tokenStream): void
+    {
+        $this->comments[] = $tokenStream;
+    }
+
+    /**
+     * @return TokenStreamInterface[]
+     */
+    public function getComments(): array
+    {
+        return $this->comments;
+    }
+}

Classes/TypoScript/AST/Node/ChildNode.php

@@ -0,0 +1,23 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\AST\Node;
+
+/**
+ * A generic child node that is not the node root.
+ */
+final class ChildNode extends AbstractChildNode {}

Classes/TypoScript/AST/Node/ChildNodeInterface.php

@@ -0,0 +1,26 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\AST\Node;
+
+/**
+ * The created AST consists of a NodeRoot object with nested NodeObject children.
+ * This is the interface implemented by all children nodes (not RootNode).
+ *
+ * @internal: Internal AST structure.
+ */
+interface ChildNodeInterface extends NodeInterface {}

Classes/TypoScript/AST/Node/NodeInterface.php

@@ -0,0 +1,125 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\AST\Node;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenStreamInterface;
+
+/**
+ * The created AST consists of a NodeRoot object with nested NodeObject children.
+ * This is the main interface to any node type.
+ *
+ * Example TypoScript:
+ * "foo = fooValue"
+ * "foo.bar = barValue"
+ * This creates a RootNode with one ChildNode name "foo" and value "fooValue",
+ * that has a child ChildNode name "bar" and value "barValue".
+ *
+ * @internal: Internal AST structure.
+ */
+interface NodeInterface
+{
+    /**
+     * An identifier for this node. Typically, a hash of some kind. This identifier
+     * is unique within the tree, by being created from the parent identifier plus
+     * the name. This identifier is used in the backend, when referencing single nodes.
+     * Calculating identifiers is initiated by calling setIdentifier() on RootNode, which
+     * will recurse the tree. Call this on the final tree, after AST calculation finished,
+     * so AST building itself does not need to fiddle with identifier updates when for
+     * instance tree parts are cloned using '<' operator.
+     * Note this value is skipped when persisting to caches since it's a Backend related
+     * thing that does not use cached context: When retrieving nodes from cache (e.g. in Frontend),
+     * the identifier is null and calling the getter will throw an exception.
+     */
+    public function setIdentifier(string $identifier): void;
+    public function getIdentifier(): string;
+
+    /**
+     * Helper methods for node name.
+     */
+    public function getName(): ?string;
+    public function updateName(string $name): void;
+
+    /**
+     * Helper methods to operate on children.
+     */
+    public function addChild(ChildNodeInterface $node): void;
+    public function getChildByName(string $name): ?ChildNodeInterface;
+    public function removeChildByName(string $name): void;
+    public function hasChildren(): bool;
+    /**
+     * @return iterable<ChildNodeInterface>
+     */
+    public function getNextChild(): iterable;
+    public function sortChildren(): void;
+
+    /**
+     * Helper methods for value.
+     */
+    public function setValue(?string $value): void;
+    public function appendValue(string $value): void;
+    public function getValue(): ?string;
+    public function isValueNull(): bool;
+
+    /**
+     * Previous value is only set by comment aware ast builder. It is used in
+     * constant editor to see if a value has been changed.
+     */
+    public function setPreviousValue(?string $value): void;
+    public function getPreviousValue(): ?string;
+
+    /**
+     * Helper method for backend object browser to retrieve the original
+     * stream when a constant substitution happened, only set by CommentAwareAstBuilder.
+     */
+    public function setOriginalValueTokenStream(?TokenStreamInterface $tokenStream): void;
+    public function getOriginalValueTokenStream(): ?TokenStreamInterface;
+
+    /**
+     * Helper methods to attach TypoScript tokens to a node.
+     * This is used in ext:tstemplate "Constant Editor" and "Object Browser" and handled
+     * by CommentAwareAstBuilder.
+     */
+    public function addComment(TokenStreamInterface $tokenStream): void;
+    /**
+     * @return TokenStreamInterface[]
+     */
+    public function getComments(): array;
+
+    /**
+     * b/w compat method to turn AST into an array.
+     * Note we're NOT using magic __toArray() here to avoid calling array-cast of AST by
+     * accident: toArray() should be called explicitly if needed, which makes it much easier
+     * to drop this b/w compat method when we later want to drop that layer.
+     *
+     * Note RootNode *always* returns an array, while ObjectNode's may return null.
+     */
+    public function toArray(): ?array;
+
+    /**
+     * Flatten the tree. A RootNode with a ChildNode "foo" and value "fooValue", with this
+     * ChildNode again having a ChildNode "bar" and value "barValue" becomes:
+     * [
+     *      'foo' => 'fooValue',
+     *      'foo.bar'  => 'barValue',
+     * ]
+     *
+     * Flattening a TypoScript tree is especially used for constants to quickly look
+     * up constants when parsing setup node value streams that use T_CONSTANT tokens.
+     */
+    public function flatten(string $prefix = ''): array;
+}

Classes/TypoScript/AST/Node/ReferenceChildNode.php

@@ -0,0 +1,60 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\AST\Node;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\IdentifierTokenStream;
+
+/**
+ * A node object created for LineIdentifierReference lines which use the T_OPERATOR_REFERENCE
+ * operator and have a TokenStreamIdentifier stream for "the right side" of the expression.
+ *
+ * The reference operator is nasty, since it's no "true" reference / pointer:
+ * foo.bar = barValue1
+ * baz =< foo
+ * baz.bar = barValue2
+ * This ends up with "barValue1" for "foo.bar", and "barValue2" for "baz.bar". "barValue1"
+ * for "foo.bar" is kept!
+ *
+ * Note the reference operator *only* works for TS "setup" code, not for "constants", and it
+ * is only resolved in these cases. See ContentObjectRenderer->cObjGetSingle() for details.
+ *
+ * @internal: Internal AST structure.
+ */
+final class ReferenceChildNode extends AbstractChildNode
+{
+    private ?IdentifierTokenStream $referenceSourceStream;
+
+    protected function serialize(): array
+    {
+        $result = parent::serialize();
+        if ($this->referenceSourceStream !== null) {
+            $result['referenceSourceStream'] = $this->referenceSourceStream;
+        }
+        return $result;
+    }
+
+    public function setReferenceSourceStream(?IdentifierTokenStream $referenceSourceStream): void
+    {
+        $this->referenceSourceStream = $referenceSourceStream;
+    }
+
+    public function getReferenceSourceStream(): IdentifierTokenStream
+    {
+        return $this->referenceSourceStream;
+    }
+}

Classes/TypoScript/AST/Node/RootNode.php

@@ -0,0 +1,109 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\AST\Node;
+
+/**
+ * AST entry node.
+ *
+ * @internal: Internal AST structure.
+ */
+final class RootNode extends AbstractNode
+{
+    /**
+     * Attempting to clone the RootNode indicates a bug in AstBuilder.
+     * It should never happen.
+     */
+    public function __clone(): void
+    {
+        throw new \LogicException('Can not clone RootNode', 1655988945);
+    }
+
+    /**
+     * RootNode has no properties to cache, just children.
+     */
+    protected function serialize(): array
+    {
+        return [
+            'children' => $this->children,
+        ];
+    }
+
+    public function getName(): ?string
+    {
+        return null;
+    }
+
+    public function updateName(string $name): void
+    {
+        throw new \RuntimeException('RootNode has no name. Don\'t call updateName().', 1653743453);
+    }
+
+    public function setValue(?string $value): void
+    {
+        throw new \RuntimeException('RootNode has no value. Don\'t call setValue().', 1653743454);
+    }
+
+    public function appendValue(string $value): void
+    {
+        throw new \RuntimeException('RootNode has no value. Don\'t call appendValue().', 1653743455);
+    }
+
+    public function getValue(): ?string
+    {
+        return null;
+    }
+
+    public function isValueNull(): bool
+    {
+        return true;
+    }
+
+    /**
+     * @return array<string, mixed>
+     */
+    public function toArray(): array
+    {
+        $result = [];
+        foreach ($this->getNextChild() as $child) {
+            $childName = $child->getName();
+            if ($child instanceof ReferenceChildNode) {
+                // Hack for b/w compat parsing of `=<` operator. See ContentObjectRenderer cObjGetSingle() and mergeTSRef()
+                $childValue = '< ' . $child->getReferenceSourceStream();
+            } else {
+                $childValue = $child->getValue();
+            }
+            if ($childValue !== null) {
+                $result[$childName] = $childValue;
+            }
+            $grandChildren = $child->toArray();
+            if ($grandChildren !== null) {
+                $result[$childName . '.'] = $grandChildren;
+            }
+        }
+        return $result;
+    }
+
+    public function flatten(string $prefix = ''): array
+    {
+        $flatArray = [];
+        foreach ($this->getNextChild() as $child) {
+            $flatArray = array_merge($flatArray, $child->flatten(''));
+        }
+        return $flatArray;
+    }
+}

Classes/TypoScript/AST/Traverser/AstTraverser.php

@@ -0,0 +1,69 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\AST\Traverser;
+
+use TYPO3\CMS\Core\TypoScript\AST\CurrentObjectPath\CurrentObjectPath;
+use TYPO3\CMS\Core\TypoScript\AST\Node\NodeInterface;
+use TYPO3\CMS\Core\TypoScript\AST\Node\RootNode;
+use TYPO3\CMS\Core\TypoScript\AST\Visitor\AstVisitorInterface;
+
+/**
+ * Traverse the entire AST.
+ *
+ * @internal: Internal AST structure.
+ */
+final class AstTraverser
+{
+    /**
+     * @param AstVisitorInterface[] $visitors
+     */
+    public function traverse(RootNode $rootNode, array $visitors): void
+    {
+        foreach ($visitors as $visitor) {
+            if (!$visitor instanceof AstVisitorInterface) {
+                throw new \RuntimeException(
+                    'Visitors must implement AstTreeVisitorInterface',
+                    1689244842
+                );
+            }
+        }
+        $currentObjectPath = new CurrentObjectPath();
+        $this->traverseRecursive($visitors, $rootNode, $rootNode, $currentObjectPath, 0);
+    }
+
+    /**
+     * @param AstVisitorInterface[] $visitors
+     */
+    private function traverseRecursive(array $visitors, RootNode $nodeRoot, NodeInterface $node, CurrentObjectPath $currentObjectPath, int $currentDepth): void
+    {
+        $currentObjectPath->append($node);
+        foreach ($visitors as $visitor) {
+            $visitor->visitBeforeChildren($nodeRoot, $node, $currentObjectPath, $currentDepth);
+        }
+        foreach ($node->getNextChild() as $child) {
+            $this->traverseRecursive($visitors, $nodeRoot, $child, $currentObjectPath, $currentDepth + 1);
+            foreach ($visitors as $visitor) {
+                $visitor->visit($nodeRoot, $child, $currentObjectPath, $currentDepth);
+            }
+        }
+        foreach ($visitors as $visitor) {
+            $visitor->visitAfterChildren($nodeRoot, $node, $currentObjectPath, $currentDepth);
+        }
+        $currentObjectPath->removeLast();
+    }
+}

Classes/TypoScript/AST/Visitor/AstConstantCommentVisitor.php

@@ -0,0 +1,506 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\AST\Visitor;
+
+use TYPO3\CMS\Core\Localization\LanguageService;
+use TYPO3\CMS\Core\TypoScript\AST\CurrentObjectPath\CurrentObjectPath;
+use TYPO3\CMS\Core\TypoScript\AST\Node\NodeInterface;
+use TYPO3\CMS\Core\TypoScript\AST\Node\RootNode;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenStream;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenStreamInterface;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenType;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+/**
+ * Gather comments suitable for constant editor.
+ *
+ * @internal This is a specific Backend implementation and is not considered part of the Public TYPO3 API.
+ */
+final class AstConstantCommentVisitor implements AstVisitorInterface
+{
+    private array $categories = [
+        'basic' => [
+            'label' => 'Basic',
+            'usageCount' => 0,
+        ],
+        'menu' => [
+            'label' => 'Menu',
+            'usageCount' => 0,
+        ],
+        'content' => [
+            'label' => 'Content',
+            'usageCount' => 0,
+        ],
+        'page' => [
+            'label' => 'Page',
+            'usageCount' => 0,
+        ],
+        'advanced' => [
+            'label' => 'Advanced',
+            'usageCount' => 0,
+        ],
+        'all' => [
+            'label' => 'All',
+            'usageCount' => 0,
+        ],
+    ];
+
+    private array $subCategories = [
+        'enable' => [
+            'label' => 'Enable features',
+            'sorting' => 'a',
+        ],
+        'dims' => [
+            'label' => 'Dimensions, widths, heights, pixels',
+            'sorting' => 'b',
+        ],
+        'file' => [
+            'label' => 'Files',
+            'sorting' => 'c',
+        ],
+        'typo' => [
+            'label' => 'Typography',
+            'sorting' => 'd',
+        ],
+        'color' => [
+            'label' => 'Colors',
+            'sorting' => 'e',
+        ],
+        'links' => [
+            'label' => 'Links and targets',
+            'sorting' => 'f',
+        ],
+        'language' => [
+            'label' => 'Language specific constants',
+            'sorting' => 'g',
+        ],
+        'cheader' => [
+            'label' => 'Content: \'Header\'',
+            'sorting' => 'ma',
+        ],
+        'cheader_g' => [
+            'label' => 'Content: \'Header\', Graphical',
+            'sorting' => 'ma',
+        ],
+        'ctext' => [
+            'label' => 'Content: \'Text\'',
+            'sorting' => 'mb',
+        ],
+        'cimage' => [
+            'label' => 'Content: \'Image\'',
+            'sorting' => 'md',
+        ],
+        'ctextmedia' => [
+            'label' => 'Content: \'Textmedia\'',
+            'sorting' => 'ml',
+        ],
+        'cbullets' => [
+            'label' => 'Content: \'Bullet list\'',
+            'sorting' => 'me',
+        ],
+        'ctable' => [
+            'label' => 'Content: \'Table\'',
+            'sorting' => 'mf',
+        ],
+        'cuploads' => [
+            'label' => 'Content: \'Filelinks\'',
+            'sorting' => 'mg',
+        ],
+        'cmultimedia' => [
+            'label' => 'Content: \'Multimedia\'',
+            'sorting' => 'mh',
+        ],
+        'cmedia' => [
+            'label' => 'Content: \'Media\'',
+            'sorting' => 'mr',
+        ],
+        'cmailform' => [
+            'label' => 'Content: \'Form\'',
+            'sorting' => 'mi',
+        ],
+        'csearch' => [
+            'label' => 'Content: \'Search\'',
+            'sorting' => 'mj',
+        ],
+        'clogin' => [
+            'label' => 'Content: \'Login\'',
+            'sorting' => 'mk',
+        ],
+        'cmenu' => [
+            'label' => 'Content: \'Menu/Sitemap\'',
+            'sorting' => 'mm',
+        ],
+        'cshortcut' => [
+            'label' => 'Content: \'Insert records\'',
+            'sorting' => 'mn',
+        ],
+        'clist' => [
+            'label' => 'Content: \'List of records\'',
+            'sorting' => 'mo',
+        ],
+        'chtml' => [
+            'label' => 'Content: \'HTML\'',
+            'sorting' => 'mq',
+        ],
+    ];
+
+    /**
+     * Helper hack variable to have a unique sub category order if no sub category is given.
+     */
+    private int $subCategoryCounter = 0;
+
+    private array $currentTemplateFlatConstants = [];
+
+    private array $constants = [];
+
+    public function visitBeforeChildren(RootNode $rootNode, NodeInterface $node, CurrentObjectPath $currentObjectPath, int $currentDepth): void
+    {
+        if ($node instanceof RootNode) {
+            $rootNodeComments = $rootNode->getComments();
+            foreach ($rootNodeComments as $comment) {
+                $this->subCategoryCounter++;
+                // Additional custom categories are attached as comments to root node
+                $this->parseCustomCategoryAndSubCategories($comment);
+            }
+        } else {
+            $nodeComments = $node->getComments();
+            foreach ($nodeComments as $comment) {
+                $this->subCategoryCounter++;
+                $parsedCommentArray = $this->parseNodeComment($comment, $node->getName(), $node->getValue());
+                if (empty($parsedCommentArray)) {
+                    continue;
+                }
+                $currentDottedPath = $currentObjectPath->getPathAsString();
+                if (array_key_exists($currentDottedPath, $this->constants)) {
+                    // A constant definition can be defined only once. Stop when trying to override.
+                    continue;
+                }
+                $parsedCommentArray['name'] = $currentDottedPath;
+                $parsedCommentArray['idName'] = str_replace('.', '-', $currentDottedPath);
+                $parsedCommentArray['value'] = $node->getValue();
+                $parsedCommentArray['default_value'] = $node->getPreviousValue() ?? $node->getValue() ?? '[Empty]';
+                $parsedCommentArray['isInCurrentTemplate'] = false;
+                if (array_key_exists($currentDottedPath, $this->currentTemplateFlatConstants)) {
+                    $parsedCommentArray['isInCurrentTemplate'] = true;
+                }
+                $this->constants[$currentDottedPath] = $parsedCommentArray;
+            }
+        }
+    }
+
+    public function setCurrentTemplateFlatConstants(array $currentTemplateFlatConstants)
+    {
+        $this->currentTemplateFlatConstants = $currentTemplateFlatConstants;
+    }
+
+    public function getConstants(): array
+    {
+        return $this->constants;
+    }
+
+    public function getCategories(): array
+    {
+        return $this->categories;
+    }
+
+    private function parseNodeComment(TokenStreamInterface $commentTokenStream, string $nodeName, ?string $currentValue = null): array
+    {
+        $languageService = $this->getLanguageService();
+        $parsedCommentArray = [];
+        $commentTokenStream->reset();
+        $trimmedTokenStream = new TokenStream();
+        while ($token = $commentTokenStream->getNext()) {
+            if ($token->getType() !== TokenType::T_BLANK) {
+                $trimmedTokenStream->append($token);
+            }
+        }
+        $firstTokenType = $trimmedTokenStream->peekNext()->getType();
+        if ($firstTokenType !== TokenType::T_COMMENT_ONELINE_HASH && $firstTokenType !== TokenType::T_COMMENT_ONELINE_DOUBLESLASH) {
+            // Ignore multiline comments, only '#' and '//' allowed here
+            return $parsedCommentArray;
+        }
+        $commentString = trim((string)$trimmedTokenStream);
+        // Get rid of '#' and '//'
+        $commentString = trim(preg_replace('/^[#\\/]*/', '', $commentString));
+        if (empty($commentString)) {
+            return $parsedCommentArray;
+        }
+        // "# cat=my custom: custom1/customsub1; type=string; label=custom1 customsub1 test1"
+        $commentParts = explode(';', $commentString);
+        foreach ($commentParts as $commentPart) {
+            if (!str_contains($commentPart, '=')) {
+                // Whatever it is, we ignore it.
+                continue;
+            }
+            $partArray = explode('=', $commentPart, 2);
+            $partKey = strtolower(trim($partArray[0]));
+            $partValue = trim($partArray[1] ?? '');
+            if (empty($partKey) || empty($partValue)) {
+                continue;
+            }
+            if ($partKey === 'type') {
+                if (str_starts_with($partValue, 'int+')) {
+                    $parsedCommentArray['type'] = 'int+';
+                    $parsedCommentArray['typeIntPlusMin'] = 0;
+                    preg_match('/int\+\[(.*)\]/is', $partValue, $typeMatches);
+                    if (!empty($typeMatches[1]) && str_contains($typeMatches[1], '-')) {
+                        $intPlusExplodedRange = GeneralUtility::intExplode('-', $typeMatches[1]);
+                        $parsedCommentArray['typeIntPlusMin'] = $intPlusExplodedRange[0];
+                        $parsedCommentArray['typeHint'] = 'Greater than ' . $intPlusExplodedRange[0];
+                        if ($intPlusExplodedRange[1] > 0) {
+                            $parsedCommentArray['typeIntPlusMax'] = $intPlusExplodedRange[1];
+                            $parsedCommentArray['typeHint'] = 'Range ' . $intPlusExplodedRange[0] . ' - ' . $intPlusExplodedRange[1];
+                        }
+                    }
+                } elseif (str_starts_with($partValue, 'int')) {
+                    preg_match('/int\[(.*)\]/is', $partValue, $typeMatches);
+                    $parsedCommentArray['type'] = 'int';
+                    if (!empty($typeMatches[1]) && str_contains($typeMatches[1], '-')) {
+                        $rangeArray = mb_str_split($typeMatches[1]);
+                        $negativeStart = false;
+                        $negativeStop = false;
+                        $gotSeparatorDash = false;
+                        $start = null;
+                        $stop = null;
+                        foreach ($rangeArray as $index => $char) {
+                            if ($index === 0 && $char === '-') {
+                                $negativeStart = true;
+                            } elseif ($char === '-' && !$gotSeparatorDash) {
+                                $gotSeparatorDash = true;
+                            } elseif (!$gotSeparatorDash) {
+                                $start .= $char;
+                            } elseif ($stop === null && $char === '-') {
+                                $negativeStop = true;
+                            } else {
+                                $stop .= $char;
+                            }
+                        }
+                        if ($start !== null) {
+                            if ($negativeStart) {
+                                $start = (int)$start * -1;
+                            }
+                            $parsedCommentArray['typeIntMin'] = (string)$start;
+                            $parsedCommentArray['typeHint'] = 'Greater than ' . $start;
+                        }
+                        if ($stop !== null) {
+                            if ($negativeStop) {
+                                $stop = (int)$stop * -1;
+                            }
+                            $parsedCommentArray['typeIntMax'] = (string)$stop;
+                            $parsedCommentArray['typeHint'] = 'Range ' . $start . ' - ' . $stop;
+                        }
+                    }
+                } elseif ($partValue === 'wrap') {
+                    $parsedCommentArray['type'] = 'wrap';
+                    $splitValue = explode('|', $currentValue ?? '');
+                    $parsedCommentArray['wrapStart'] = $splitValue[0];
+                    $parsedCommentArray['wrapEnd'] = $splitValue[1] ?? '';
+                } elseif (str_starts_with($partValue, 'offset')) {
+                    $parsedCommentArray['type'] = 'offset';
+                    preg_match('/offset\[(.*)\]/is', $partValue, $typeMatches);
+                    $labelArray = explode(',', $typeMatches[1] ?? '');
+                    $valueArray = explode(',', $currentValue ?? '');
+                    $parsedCommentArray['labelValueArray'] = [
+                        [
+                            'label' => (!empty($labelArray[0])) ? $labelArray[0] : 'x',
+                            'value' => (!empty($valueArray[0])) ? $valueArray[0] : '',
+                        ],
+                        [
+                            'label' => $labelArray[1] ?? 'y',
+                            'value' => $valueArray[1] ?? '',
+                        ],
+                    ];
+                    for ($i = 2; $i <= 5; $i++) {
+                        if (!($labelArray[$i] ?? false)) {
+                            break;
+                        }
+                        $parsedCommentArray['labelValueArray'][] = [
+                            'label' => $labelArray[$i],
+                            'value' => $valueArray[$i] ?? '',
+                        ];
+                    }
+                } elseif (str_starts_with($partValue, 'options')) {
+                    preg_match('/options\\s*\[(.*)\]/is', $partValue, $typeMatches);
+                    if (!empty($typeMatches[1] ?? '')) {
+                        $parsedCommentArray['type'] = 'options';
+                        $labelValueStringArray = GeneralUtility::trimExplode(',', $typeMatches[1], true);
+                        foreach ($labelValueStringArray as $labelValueString) {
+                            $labelValueArray = explode('=', $labelValueString, 2);
+                            $label = $labelValueArray[0];
+                            $value = $labelValueArray[1] ?? $labelValueArray[0];
+                            $selected = false;
+                            if ($value === $currentValue) {
+                                $selected = true;
+                            }
+                            $parsedCommentArray['labelValueArray'][] = [
+                                'label' => $languageService->sL($label),
+                                'value' => $value,
+                                'selected' => $selected,
+                            ];
+                        }
+                    }
+                } elseif (str_starts_with($partValue, 'boolean')) {
+                    $parsedCommentArray['type'] = 'boolean';
+                    preg_match('/boolean\\s*\[(.*)\]/is', $partValue, $typeMatches);
+                    $parsedCommentArray['trueValue'] = '1';
+                    if (!empty($typeMatches[1] ?? '')) {
+                        $parsedCommentArray['trueValue'] = $typeMatches[1];
+                    }
+                } elseif (str_starts_with($partValue, 'user')) {
+                    preg_match('/user\\s*\[(.*)\]/is', $partValue, $typeMatches);
+                    if (!empty($typeMatches[1] ?? '')) {
+                        $parsedCommentArray['type'] = 'user';
+                        $userFunction = $typeMatches[1];
+                        $userFunctionParams = [
+                            'fieldName' => $nodeName,
+                            'fieldValue' => $currentValue,
+                        ];
+                        $parsedCommentArray['html'] = (string)GeneralUtility::callUserFunction(
+                            $userFunction,
+                            $userFunctionParams
+                        );
+                    }
+                } elseif ($partValue === 'comment') {
+                    $parsedCommentArray['type'] = 'comment';
+                } elseif ($partValue === 'color') {
+                    $parsedCommentArray['type'] = 'color';
+                } else {
+                    $parsedCommentArray['type'] = 'string';
+                }
+            } elseif ($partKey === 'cat') {
+                $categorySplitArray = explode('/', strtolower($partValue));
+                $mainCategory = strtolower(trim($categorySplitArray[0]));
+                if (empty($mainCategory)) {
+                    return [];
+                }
+                if (isset($this->categories[$mainCategory])) {
+                    $this->categories[$mainCategory]['usageCount']++;
+                } else {
+                    $this->categories[$mainCategory] = [
+                        'usageCount' => 1,
+                        'label' => $mainCategory,
+                    ];
+                }
+                $parsedCommentArray['cat'] = $mainCategory;
+                $subCategory = trim($categorySplitArray[1] ?? '');
+                $subCategoryOrder = trim($categorySplitArray[2] ?? '');
+                if ($subCategory && array_key_exists($subCategory, $this->subCategories)) {
+                    $parsedCommentArray['subcat_name'] = $subCategory;
+                    $parsedCommentArray['subcat_label'] = $languageService->sL($this->subCategories[$subCategory]['label']);
+                    $sortIdentifier = empty($subCategoryOrder) ? $this->subCategoryCounter : $subCategoryOrder;
+                    $parsedCommentArray['subcat_sorting_first'] = $this->subCategories[$subCategory]['sorting'];
+                    $parsedCommentArray['subcat_sorting_second'] = $sortIdentifier . 'z';
+                } elseif ($subCategoryOrder !== '') {
+                    // "0" is a valid key for an assignment like "# cat=foo//0; type=boolean; label=some config"
+                    $parsedCommentArray['subcat_name'] = 'other';
+                    $parsedCommentArray['subcat_label'] = 'Other';
+                    $parsedCommentArray['subcat_sorting_first'] = 'o';
+                    $parsedCommentArray['subcat_sorting_second'] = $subCategoryOrder . 'z';
+                } else {
+                    $parsedCommentArray['subcat_name'] = 'other';
+                    $parsedCommentArray['subcat_label'] = 'Other';
+                    $parsedCommentArray['subcat_sorting_first'] = 'o';
+                    $parsedCommentArray['subcat_sorting_second'] = $this->subCategoryCounter . 'z';
+                }
+            } elseif ($partKey === 'label') {
+                $fullLabel = $languageService->sL($partValue);
+                $splitLabelArray = explode(':', $fullLabel, 2);
+                $parsedCommentArray['label'] = $splitLabelArray[0];
+                $parsedCommentArray['description'] = $splitLabelArray[1] ?? '';
+            }
+        }
+        if (!array_key_exists('cat', $parsedCommentArray)) {
+            // At least 'category' must be there, everything else is optional.
+            return [];
+        }
+        $parsedCommentArray['type'] ??= 'string';
+        return $parsedCommentArray;
+    }
+
+    /**
+     * Parse RootNode comments for additional custom categories and sub categories
+     * and register them in $this properties.
+     */
+    private function parseCustomCategoryAndSubCategories(TokenStreamInterface $commentTokenStream): void
+    {
+        $languageService = $this->getLanguageService();
+        $firstTokenType = $commentTokenStream->peekNext()->getType();
+        if ($firstTokenType !== TokenType::T_COMMENT_ONELINE_HASH && $firstTokenType !== TokenType::T_COMMENT_ONELINE_DOUBLESLASH) {
+            // Ignore multiline comments, only '#' and '//' allowed here
+            return;
+        }
+        $commentString = trim((string)$commentTokenStream);
+        // Get rid of '#' and '//'
+        $commentString = trim(preg_replace('/^[#\\/]*/', '', $commentString));
+        if (empty($commentString)) {
+            return;
+        }
+        // "# customcategory=myCustomCategoryKey=My custom category label"
+        if (str_contains($commentString, '=') && str_starts_with(strtolower($commentString), 'customcategory')) {
+            $customCategoryArray = explode('=', $commentString, 3);
+            if (strtolower(trim($customCategoryArray[0])) !== 'customcategory'
+                || empty(trim($customCategoryArray[1]))
+                || empty(trim($customCategoryArray[2]))
+            ) {
+                return;
+            }
+            $categoryKey = strtolower($customCategoryArray[1]);
+            $categoryLabel = $customCategoryArray[2];
+            if (!isset($this->categories[$categoryKey])) {
+                $this->categories[$categoryKey] = [
+                    'usageCount' => 0,
+                    'label' => $languageService->sL($categoryLabel),
+                ];
+            }
+            return;
+        }
+        // "customsubcategory=120=My custom sub category label"
+        if (str_contains($commentString, '=') && str_starts_with(strtolower($commentString), 'customsubcategory')) {
+            $customSubCategoryArray = explode('=', $commentString, 3);
+            if (strtolower(trim($customSubCategoryArray[0])) !== 'customsubcategory'
+                || empty(trim($customSubCategoryArray[1]))
+                || empty(trim($customSubCategoryArray[2]))
+            ) {
+                return;
+            }
+            $subCategoryKey = strtolower($customSubCategoryArray[1]);
+            $subCategoryLabel = $customSubCategoryArray[2];
+            if (!isset($this->subCategories[$subCategoryKey])) {
+                $this->subCategories[$subCategoryKey] = [
+                    'label' => $languageService->sL($subCategoryLabel),
+                    'sorting' => $this->subCategoryCounter,
+                ];
+            }
+        }
+    }
+
+    public function visit(RootNode $rootNode, NodeInterface $node, CurrentObjectPath $currentObjectPath, int $currentDepth): void
+    {
+        // Implement interface
+    }
+
+    public function visitAfterChildren(RootNode $rootNode, NodeInterface $node, CurrentObjectPath $currentObjectPath, int $currentDepth): void
+    {
+        // Implement interface
+    }
+
+    private function getLanguageService(): LanguageService
+    {
+        return $GLOBALS['LANG'];
+    }
+}

Classes/TypoScript/AST/Visitor/AstNodeFinderVisitor.php

@@ -0,0 +1,67 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\AST\Visitor;
+
+use TYPO3\CMS\Core\TypoScript\AST\CurrentObjectPath\CurrentObjectPath;
+use TYPO3\CMS\Core\TypoScript\AST\Node\NodeInterface;
+use TYPO3\CMS\Core\TypoScript\AST\Node\RootNode;
+
+/**
+ * Find a single node in tree identified by node identifier.
+ *
+ * @internal This is a specific Backend implementation and is not considered part of the Public TYPO3 API.
+ */
+final class AstNodeFinderVisitor implements AstVisitorInterface
+{
+    private string $nodeIdentifier;
+    private ?NodeInterface $foundNode = null;
+    private ?CurrentObjectPath $foundNodeCurrentObjectPath = null;
+
+    public function setNodeIdentifier(string $nodeIdentifier)
+    {
+        $this->nodeIdentifier = $nodeIdentifier;
+    }
+
+    public function getFoundNode(): ?NodeInterface
+    {
+        return $this->foundNode;
+    }
+
+    public function getFoundNodeCurrentObjectPath(): ?CurrentObjectPath
+    {
+        return $this->foundNodeCurrentObjectPath;
+    }
+
+    public function visitBeforeChildren(RootNode $rootNode, NodeInterface $node, CurrentObjectPath $currentObjectPath, int $currentDepth): void
+    {
+        if ($node->getIdentifier() === $this->nodeIdentifier) {
+            $this->foundNode = $node;
+            $this->foundNodeCurrentObjectPath = clone $currentObjectPath;
+        }
+    }
+
+    public function visit(RootNode $rootNode, NodeInterface $node, CurrentObjectPath $currentObjectPath, int $currentDepth): void
+    {
+        // Implement interface
+    }
+
+    public function visitAfterChildren(RootNode $rootNode, NodeInterface $node, CurrentObjectPath $currentObjectPath, int $currentDepth): void
+    {
+        // Implement interface
+    }
+}

Classes/TypoScript/AST/Visitor/AstSortChildrenVisitor.php

@@ -0,0 +1,45 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\AST\Visitor;
+
+use TYPO3\CMS\Core\TypoScript\AST\CurrentObjectPath\CurrentObjectPath;
+use TYPO3\CMS\Core\TypoScript\AST\Node\NodeInterface;
+use TYPO3\CMS\Core\TypoScript\AST\Node\RootNode;
+
+/**
+ * Sort all children alphabetically. Used in backend Object Browser.
+ *
+ * @internal: Internal AST structure.
+ */
+final class AstSortChildrenVisitor implements AstVisitorInterface
+{
+    public function visitBeforeChildren(RootNode $rootNode, NodeInterface $node, CurrentObjectPath $currentObjectPath, int $currentDepth): void
+    {
+        $node->sortChildren();
+    }
+
+    public function visit(RootNode $rootNode, NodeInterface $node, CurrentObjectPath $currentObjectPath, int $currentDepth): void
+    {
+        // Implement interface
+    }
+
+    public function visitAfterChildren(RootNode $rootNode, NodeInterface $node, CurrentObjectPath $currentObjectPath, int $currentDepth): void
+    {
+        // Implement interface
+    }
+}

Classes/TypoScript/AST/Visitor/AstVisitorInterface.php

@@ -0,0 +1,36 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\AST\Visitor;
+
+use TYPO3\CMS\Core\TypoScript\AST\CurrentObjectPath\CurrentObjectPath;
+use TYPO3\CMS\Core\TypoScript\AST\Node\NodeInterface;
+use TYPO3\CMS\Core\TypoScript\AST\Node\RootNode;
+
+/**
+ * An interface implemented by all visitors of AstTraverser.
+ *
+ * @internal: Internal AST structure.
+ */
+interface AstVisitorInterface
+{
+    public function visitBeforeChildren(RootNode $rootNode, NodeInterface $node, CurrentObjectPath $currentObjectPath, int $currentDepth): void;
+
+    public function visit(RootNode $rootNode, NodeInterface $node, CurrentObjectPath $currentObjectPath, int $currentDepth): void;
+
+    public function visitAfterChildren(RootNode $rootNode, NodeInterface $node, CurrentObjectPath $currentObjectPath, int $currentDepth): void;
+}

Classes/TypoScript/FrontendTypoScript.php

@@ -0,0 +1,308 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript;
+
+use TYPO3\CMS\Core\TypoScript\AST\Node\RootNode;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\RootInclude;
+
+/**
+ * This class contains the TypoScript set up by the PrepareTypoScriptFrontendRendering
+ * Frontend middleware. It can be accessed in content objects:
+ *
+ * $frontendTypoScript = $request->getAttribute('frontend.typoscript');
+ */
+final class FrontendTypoScript
+{
+    private ?RootInclude $setupIncludeTree = null;
+    private ?RootNode $setupTree = null;
+    private ?array $setupArray = null;
+    private ?RootNode $configTree = null;
+    private ?array $configArray = null;
+    private ?RootNode $pageTree = null;
+    private ?array $pageArray = null;
+
+    public function __construct(
+        private readonly RootNode $settingsTree,
+        private readonly array $settingsConditionList,
+        private readonly array $flatSettings,
+        private readonly array $setupConditionList,
+    ) {}
+
+    /**
+     * The settings ("constants") AST.
+     *
+     * @internal Internal for now until the AST API stabilized.
+     */
+    public function getSettingsTree(): RootNode
+    {
+        return $this->settingsTree;
+    }
+
+    /**
+     * List of settings conditions with verdicts. Used internally for
+     * page cache identifier calculation.
+     *
+     * @internal
+     */
+    public function getSettingsConditionList(): array
+    {
+        return $this->settingsConditionList;
+    }
+
+    /**
+     * This is *always* set up by the middleware / factory: Current settings ("constants")
+     * are needed for page cache identifier calculation.
+     * This is a "flattened" array of all settings, as example, consider these settings TypoScript:
+     *
+     * ```
+     * mySettings {
+     *     foo = fooValue
+     *     bar = barValue
+     * }
+     * ```
+     *
+     * This will result in this array:
+     *
+     * ```
+     * $flatSettings = [
+     *     'mySettings.foo' => 'fooValue',
+     *     'mySettings.bar' => 'barValue',
+     * ];
+     * ```
+     */
+    public function getFlatSettings(): array
+    {
+        return $this->flatSettings;
+    }
+
+    /**
+     * List of setup conditions with verdicts. Used internally for
+     * page cache identifier calculation.
+     *
+     * @internal
+     */
+    public function getSetupConditionList(): array
+    {
+        return $this->setupConditionList;
+    }
+
+    /**
+     * @internal
+     */
+    public function setSetupIncludeTree(RootInclude $setupIncludeTree): void
+    {
+        $this->setupIncludeTree = $setupIncludeTree;
+    }
+
+    /**
+     * A tree of all TypoScript setup includes. Used internally within
+     * FrontendTypoScriptFactory to suppress calculating the include tree
+     * twice.
+     *
+     * @internal
+     */
+    public function getSetupIncludeTree(): ?RootInclude
+    {
+        return $this->setupIncludeTree;
+    }
+
+    /**
+     * @internal
+     */
+    public function setSetupTree(RootNode $setupTree): void
+    {
+        $this->setupTree = $setupTree;
+    }
+
+    /**
+     * When a page is retrieved from cache and does not contain COA_INT or USER_INT objects,
+     * Frontend TypoScript setup is not calculated, AST and the array representation aren't set.
+     * Calling getSetupTree() or getSetupArray() will then throw an exception.
+     *
+     * To avoid the exception, consumers can call hasSetup() beforehand.
+     *
+     * Note casual content objects do not need to do this, since setup TypoScript is always
+     * set up when content objects need to be calculated.
+     *
+     * @internal
+     */
+    public function hasSetup(): bool
+    {
+        return $this->setupTree !== null;
+    }
+
+    /**
+     * @internal Internal for now until the AST API stabilized.
+     */
+    public function getSetupTree(): RootNode
+    {
+        if ($this->setupTree === null) {
+            throw new \RuntimeException(
+                'Setup tree has not been initialized. This happens in cached Frontend scope where full TypoScript'
+                . ' is not needed by the system.',
+                1666513644
+            );
+        }
+        return $this->setupTree;
+    }
+
+    /**
+     * @internal
+     */
+    public function setSetupArray(array $setupArray): void
+    {
+        $this->setupArray = $setupArray;
+    }
+
+    /**
+     * The full Frontend TypoScript array.
+     *
+     * This is always set up as soon as the Frontend rendering needs to actually render something and
+     * can not get the *full* content from page cache. This is the case when a page cache entry does
+     * not exist, or when the page contains COA_INT or USER_INT objects.
+     */
+    public function getSetupArray(): array
+    {
+        if ($this->setupArray === null) {
+            throw new \RuntimeException(
+                'Setup array has not been initialized. This happens in cached Frontend scope where full TypoScript'
+                . ' is not needed by the system.',
+                1666513645
+            );
+        }
+        return $this->setupArray;
+    }
+
+    /**
+     * @internal
+     */
+    public function setConfigTree(RootNode $setupConfig): void
+    {
+        $this->configTree = $setupConfig;
+    }
+
+    /**
+     * The merged TypoScript 'config.'.
+     *
+     * This is the result of the "global" TypoScript 'config' section, merged with
+     * the 'config' section of the determined PAGE object which can override
+     * "global" 'config' per type / typeNum.
+     *
+     * This is *always* needed within casual Frontend rendering by FrontendTypoScriptFactory and
+     * has a dedicated cache layer to be quick to retrieve. It is needed even in fully cached pages
+     * context to for instance know if debug headers should be added ("config.debug=1") to a response.
+     *
+     * @internal Internal for now until the AST API stabilized.
+     */
+    public function getConfigTree(): RootNode
+    {
+        if ($this->configTree === null) {
+            throw new \RuntimeException(
+                'Setup "config." not initialized. FrontendTypoScriptFactory->createSetupConfigOrFullSetup() not called?',
+                1710666154
+            );
+        }
+        return $this->configTree;
+    }
+
+    /**
+     * @internal
+     */
+    public function setConfigArray(array $configArray): void
+    {
+        $this->configArray = $configArray;
+    }
+
+    /**
+     * Array representation of getConfigTree().
+     */
+    public function getConfigArray(): array
+    {
+        if ($this->configArray === null) {
+            throw new \RuntimeException(
+                'Setup "config." not initialized. FrontendTypoScriptFactory->createSetupConfigOrFullSetup() not called?',
+                1710666123
+            );
+        }
+        return $this->configArray;
+    }
+
+    /**
+     * @internal
+     */
+    public function setPageTree(RootNode $pageTree): void
+    {
+        $this->pageTree = $pageTree;
+    }
+
+    /**
+     * The determined PAGE object from main TypoScript 'setup' that depends
+     * on type / typeNum.
+     *
+     * This is used internally by RequestHandler for page generation.
+     * It is *not* set in full cached page scenarios without _INT object.
+     *
+     * @internal
+     */
+    public function getPageTree(): RootNode
+    {
+        if ($this->pageTree === null) {
+            throw new \RuntimeException(
+                'PAGE node has not been initialized. This happens in cached Frontend scope where full TypoScript'
+                . ' is not needed by the system, and if a PAGE object for given type could not be determined.'
+                . ' Test with hasPage().',
+                1710399966
+            );
+        }
+        return $this->pageTree;
+    }
+
+    /**
+     * @internal
+     */
+    public function hasPage(): bool
+    {
+        return $this->pageTree !== null;
+    }
+
+    /**
+     * @internal
+     */
+    public function setPageArray(array $pageArray): void
+    {
+        $this->pageArray = $pageArray;
+    }
+
+    /**
+     * Array representation of getPageTree().
+     *
+     * @internal
+     */
+    public function getPageArray(): array
+    {
+        if ($this->pageArray === null) {
+            throw new \RuntimeException(
+                'PAGE array has not been initialized. This happens in cached Frontend scope where full TypoScript'
+                . ' is not needed by the system, and if a PAGE object for given type could not be determined.'
+                . ' Test with hasPage().',
+                1710399967
+            );
+        }
+        return $this->pageArray;
+    }
+}

Classes/TypoScript/FrontendTypoScriptFactory.php

@@ -0,0 +1,484 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript;
+
+use Psr\Container\ContainerInterface;
+use Psr\EventDispatcher\EventDispatcherInterface;
+use Psr\Http\Message\ServerRequestInterface;
+use TYPO3\CMS\Core\Cache\Frontend\PhpFrontend;
+use TYPO3\CMS\Core\Site\Entity\Site;
+use TYPO3\CMS\Core\Site\Entity\SiteInterface;
+use TYPO3\CMS\Core\TypoScript\AST\Merger\SetupConfigMerger;
+use TYPO3\CMS\Core\TypoScript\AST\Node\ChildNode;
+use TYPO3\CMS\Core\TypoScript\AST\Node\RootNode;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\RootInclude;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\SysTemplateTreeBuilder;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\Traverser\ConditionVerdictAwareIncludeTreeTraverser;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\Traverser\IncludeTreeTraverser;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\Visitor\IncludeTreeAstBuilderVisitor;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\Visitor\IncludeTreeConditionIncludeListAccumulatorVisitor;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\Visitor\IncludeTreeConditionMatcherVisitor;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\Visitor\IncludeTreeSetupConditionConstantSubstitutionVisitor;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\LossyTokenizer;
+use TYPO3\CMS\Frontend\Event\ModifyTypoScriptConfigEvent;
+use TYPO3\CMS\Frontend\Event\ModifyTypoScriptConstantsEvent;
+
+/**
+ * Create FrontendTypoScript with its details. This is typically used by a Frontend middleware
+ * to calculate the TypoScript needed to satisfy rendering details of the specific Request.
+ *
+ * @internal Methods signatures and detail implementations are still subject to change.
+ */
+final readonly class FrontendTypoScriptFactory
+{
+    public function __construct(
+        private ContainerInterface $container,
+        private EventDispatcherInterface $eventDispatcher,
+        private SysTemplateTreeBuilder $treeBuilder,
+        private LossyTokenizer $tokenizer,
+        private IncludeTreeTraverser $includeTreeTraverser,
+        private ConditionVerdictAwareIncludeTreeTraverser $includeTreeTraverserConditionVerdictAware,
+    ) {}
+
+    /**
+     * First step of TypoScript calculations.
+     * This is *always* called, even in FE fully cached pages context since the page
+     * cache entry depends on setup condition verdicts, which depends on settings.
+     *
+     * Returns the FrontendTypoScript object with these parameters set:
+     * * settingsTree: The full settings ("constants") AST
+     * * flatSettings: Flattened list of settings, derived from settings tree
+     * * settingsConditionList: Settings conditions with verdicts of this Request
+     * * setupConditionList: Setup conditions with verdicts of this Request
+     * * (sometimes) setupIncludeTree: The setup include tree *if* it had to be calculated
+     */
+    public function createSettingsAndSetupConditions(
+        SiteInterface $site,
+        array $sysTemplateRows,
+        array $expressionMatcherVariables,
+        ?PhpFrontend $typoScriptCache,
+    ): FrontendTypoScript {
+        $settingsDetails = $this->createSettings(
+            $site,
+            $sysTemplateRows,
+            $expressionMatcherVariables,
+            $typoScriptCache
+        );
+        $setupDetails = $this->createSetupConditionList(
+            $site,
+            $sysTemplateRows,
+            $expressionMatcherVariables,
+            $typoScriptCache,
+            $settingsDetails['flatSettings'],
+            $settingsDetails['settingsConditionList'],
+        );
+        $frontendTypoScript = new FrontendTypoScript(
+            $settingsDetails['settingsTree'],
+            $settingsDetails['settingsConditionList'],
+            $settingsDetails['flatSettings'],
+            $setupDetails['setupConditionList'],
+        );
+        if ($setupDetails['setupIncludeTree']) {
+            $frontendTypoScript->setSetupIncludeTree($setupDetails['setupIncludeTree']);
+        }
+        return $frontendTypoScript;
+    }
+
+    /**
+     * Calculate settings (formerly "constants").
+     *
+     * The page cache entry identifier depends on setup TypoScript: A single page with two different
+     * setup TypoScript AST will probably render different results, thus two page-cache entries.
+     * Setup TypoScript can be different when setup conditions match differently.
+     * Setup conditions can use settings "[{$foo} = 42]".
+     *
+     * All FE requests thus need the current list of settings, and settings can have conditions, too.
+     * We thus *always* need the current list of settings, even in fully cached pages context.
+     *
+     * The method calculates settings and uses caches as much as possible:
+     * * settingsTree: The full settings AST
+     * * flatSettings: Flattened list of settings, derived from settings AST
+     * * settingsConditionList: Settings conditions with verdicts of this Request
+     *
+     * @return array{settingsTree: RootNode, flatSettings: array, settingsConditionList: array}
+     */
+    private function createSettings(
+        SiteInterface $site,
+        array $sysTemplateRows,
+        array $expressionMatcherVariables,
+        ?PhpFrontend $typoScriptCache,
+    ): array {
+        $cacheCriteria = [
+            'sysTemplateRows' => $sysTemplateRows,
+        ];
+        if ($site instanceof Site && $site->isTypoScriptRoot()) {
+            $cacheCriteria['siteIdentifier'] = $site->getIdentifier();
+        }
+        $conditionTreeCacheIdentifier = 'settings-condition-tree-' . hash('xxh3', json_encode($cacheCriteria, JSON_THROW_ON_ERROR));
+
+        if ($conditionTree = $typoScriptCache?->require($conditionTreeCacheIdentifier)) {
+            // Got the (flat) include tree of all settings conditions for this TypoScript combination from cache.
+            // Good. Traverse this list to calculate "current" condition verdicts. Hash this list together with a
+            // hash of the TypoScript sys_templates, and try to retrieve the full settings TypoScript AST from cache.
+            // Note: Working with the derived condition tree that *only* contains conditions, but not the full
+            // include tree is a trick: We only need the condition verdicts to know the AST cache identifier,
+            // and traversing the flat condition tree is quicker than traversing the entire settings include tree,
+            // since it only scales with the number of settings conditions and not with the full amount of TypoScript
+            // settings. The same trick is used for the setup AST cache later.
+            $conditionMatcherVisitor = $this->container->get(IncludeTreeConditionMatcherVisitor::class);
+            $conditionMatcherVisitor->initializeExpressionMatcherWithVariables($expressionMatcherVariables);
+            // It does not matter if we use IncludeTreeTraverser or ConditionVerdictAwareIncludeTreeTraverser here:
+            // Conditions list is flat, not nested. IncludeTreeTraverser has an if() less, so we use that one.
+            $this->includeTreeTraverser->traverse($conditionTree, [$conditionMatcherVisitor]);
+            $conditionList = $conditionMatcherVisitor->getConditionListWithVerdicts();
+            $settings = $typoScriptCache->require(
+                'settings-' . hash('xxh3', $conditionTreeCacheIdentifier . json_encode($conditionList, JSON_THROW_ON_ERROR))
+            );
+            if (is_array($settings)) {
+                return [
+                    'settingsTree' => $settings['ast'],
+                    'flatSettings' => $settings['flatSettings'],
+                    'settingsConditionList' => $conditionList,
+                ];
+            }
+        }
+
+        // We did not get settings from cache, or are not allowed to use cache. Build settings from scratch.
+        // We fetch the full settings include tree (from cache if possible), register the condition
+        // matcher and register the AST builder and traverse include tree to retrieve settings AST and derive
+        // 'flat settings' from it. Both are cached if allowed afterward for the above 'if' to kick in next time.
+        $includeTree = $this->treeBuilder->getTreeBySysTemplateRowsAndSite('constants', $sysTemplateRows, $this->tokenizer, $site, $typoScriptCache);
+        $conditionMatcherVisitor = $this->container->get(IncludeTreeConditionMatcherVisitor::class);
+        $conditionMatcherVisitor->initializeExpressionMatcherWithVariables($expressionMatcherVariables);
+        $visitors = [];
+        $visitors[] = $conditionMatcherVisitor;
+        $astBuilderVisitor = $this->container->get(IncludeTreeAstBuilderVisitor::class);
+        $visitors[] = $astBuilderVisitor;
+        // We must use ConditionVerdictAwareIncludeTreeTraverser here: This one does not walk into
+        // children for not matching conditions, which is important to create the correct AST.
+        $this->includeTreeTraverserConditionVerdictAware->traverse($includeTree, $visitors);
+        $tree = $astBuilderVisitor->getAst();
+        // @internal Dispatch an experimental event allowing listeners to still change the settings AST,
+        //           to for instance implement nested constants if really needed. Note this event may change
+        //           or vanish later without further notice.
+        $tree = $this->eventDispatcher->dispatch(new ModifyTypoScriptConstantsEvent($tree))->getConstantsAst();
+        $flatSettings = $tree->flatten();
+
+        // Prepare the full list of settings conditions in order to cache this list, avoiding the
+        // settings AST building next time. We need all conditions of the entire include tree, but the
+        // above ConditionVerdictAwareIncludeTreeTraverser did not find nested conditions if an upper
+        // condition did not match. We thus have to traverse include tree a second time with the
+        // IncludeTreeTraverser. This one does traverse into not matching conditions.
+        $visitors = [];
+        $conditionMatcherVisitor = $this->container->get(IncludeTreeConditionMatcherVisitor::class);
+        $conditionMatcherVisitor->initializeExpressionMatcherWithVariables($expressionMatcherVariables);
+        $visitors[] = $conditionMatcherVisitor;
+        $conditionTreeAccumulatorVisitor = null;
+        if (!$conditionTree && $typoScriptCache) {
+            // If the settingsConditionTree did not come from cache above and if we are allowed to cache,
+            // register the visitor that creates the settings condition include tree, to cache it.
+            $conditionTreeAccumulatorVisitor = $this->container->get(IncludeTreeConditionIncludeListAccumulatorVisitor::class);
+            $visitors[] = $conditionTreeAccumulatorVisitor;
+        }
+        $this->includeTreeTraverser->traverse($includeTree, $visitors);
+        $conditionList = $conditionMatcherVisitor->getConditionListWithVerdicts();
+
+        if ($conditionTreeAccumulatorVisitor) {
+            // Cache the flat condition include tree for next run.
+            $conditionTree = $conditionTreeAccumulatorVisitor->getConditionIncludes();
+            $typoScriptCache?->set(
+                $conditionTreeCacheIdentifier,
+                'return unserialize(\'' . addcslashes(serialize($conditionTree), '\'\\') . '\');'
+            );
+        }
+        $typoScriptCache?->set(
+            // Cache full AST and the derived 'flattened' variant for next run, which will kick in if
+            // the sys_templates and condition verdicts are identical with another Request.
+            'settings-' . hash('xxh3', $conditionTreeCacheIdentifier . json_encode($conditionList, JSON_THROW_ON_ERROR)),
+            'return unserialize(\'' . addcslashes(serialize(['ast' => $tree, 'flatSettings' => $flatSettings]), '\'\\') . '\');'
+        );
+
+        return [
+            'settingsTree' => $tree,
+            'flatSettings' => $flatSettings,
+            'settingsConditionList' => $conditionList,
+        ];
+    }
+
+    /**
+     * Calculate setup condition verdicts.
+     *
+     * With settings being done, the list of matching setup condition verdicts is calculated,
+     * which depend on settings. Setup conditions with their verdicts are part of the page
+     * cache identifier, they are *always* needed in the FE rendering chain.
+     *
+     * The cached variant uses a similar trick as with the settings calculation above: We
+     * calculate a flat tree of all conditions and cache this, so the traverser only needs
+     * to iterate the conditions to calculate their verdicts, but not the entire include
+     * tree next time.
+     *
+     * The method returns:
+     * * 'setupConditionList': Setup conditions with verdicts of this Request
+     * * (sometimes) setupIncludeTree: The setup include tree *if* it had to be calculated. Used internally
+     *                                 to suppress a second calculation in createSetupConfigOrFullSetup().
+     *
+     * @return array{setupConditionList: array, setupIncludeTree: RootInclude|null}
+     */
+    private function createSetupConditionList(
+        SiteInterface $site,
+        array $sysTemplateRows,
+        array $expressionMatcherVariables,
+        ?PhpFrontend $typoScriptCache,
+        array $flatSettings,
+        array $settingsConditionList,
+    ): array {
+        $conditionTreeCacheIdentifier = 'setup-condition-tree-' . hash(
+            'xxh3',
+            json_encode($sysTemplateRows, JSON_THROW_ON_ERROR)
+            . json_encode($site instanceof Site && $site->isTypoScriptRoot() ? $site->getSets() : '', JSON_THROW_ON_ERROR)
+            . json_encode($settingsConditionList, JSON_THROW_ON_ERROR)
+        );
+
+        if ($conditionTree = $typoScriptCache?->require($conditionTreeCacheIdentifier)) {
+            // We got the flat list of all setup conditions for this TypoScript combination from cache. Good. We traverse
+            // this list to calculate "current" condition verdicts, which we need as hash to be part of page cache identifier.
+            // We're done and return. Note 'setupIncludeTree' is *not* returned in this case since it is not needed and
+            // may or may not be needed later, depending on if we can get a page cache entry later and if it has _INT objects.
+            $visitors = [];
+            $conditionConstantSubstitutionVisitor = $this->container->get(IncludeTreeSetupConditionConstantSubstitutionVisitor::class);
+            $conditionConstantSubstitutionVisitor->setFlattenedConstants($flatSettings);
+            $visitors[] = $conditionConstantSubstitutionVisitor;
+            $conditionMatcherVisitor = $this->container->get(IncludeTreeConditionMatcherVisitor::class);
+            $conditionMatcherVisitor->initializeExpressionMatcherWithVariables($expressionMatcherVariables);
+            $visitors[] = $conditionMatcherVisitor;
+            // It does not matter if we use IncludeTreeTraverser or ConditionVerdictAwareIncludeTreeTraverser here:
+            // Condition list is flat, not nested. IncludeTreeTraverser has an if() less, so we use that one.
+            $this->includeTreeTraverser->traverse($conditionTree, $visitors);
+            return [
+                'setupConditionList' => $conditionMatcherVisitor->getConditionListWithVerdicts(),
+                'setupIncludeTree' => null,
+            ];
+        }
+
+        // We did not get setup condition list from cache, or are not allowed to use cache. We have to build setup
+        // condition list from scratch. This means we'll fetch the full setup include tree (from cache if possible),
+        // register the constant substitution visitor, the condition matcher and the condition accumulator visitor.
+        $includeTree = $this->treeBuilder->getTreeBySysTemplateRowsAndSite('setup', $sysTemplateRows, $this->tokenizer, $site, $typoScriptCache);
+        $visitors = [];
+        $conditionConstantSubstitutionVisitor = $this->container->get(IncludeTreeSetupConditionConstantSubstitutionVisitor::class);
+        $conditionConstantSubstitutionVisitor->setFlattenedConstants($flatSettings);
+        $visitors[] = $conditionConstantSubstitutionVisitor;
+        $conditionMatcherVisitor = $this->container->get(IncludeTreeConditionMatcherVisitor::class);
+        $conditionMatcherVisitor->initializeExpressionMatcherWithVariables($expressionMatcherVariables);
+        $visitors[] = $conditionMatcherVisitor;
+        $conditionTreeAccumulatorVisitor = $this->container->get(IncludeTreeConditionIncludeListAccumulatorVisitor::class);
+        $visitors[] = $conditionTreeAccumulatorVisitor;
+        // It is important to use IncludeTreeTraverser here: We need the condition verdicts of *all* conditions, and
+        // we want to accumulate all of them. The ConditionVerdictAwareIncludeTreeTraverser wouldn't walk into nested
+        // conditions if an upper one does not match, which defeats cache identifier calculations.
+        $this->includeTreeTraverser->traverse($includeTree, $visitors);
+
+        $typoScriptCache?->set(
+            $conditionTreeCacheIdentifier,
+            'return unserialize(\'' . addcslashes(serialize($conditionTreeAccumulatorVisitor->getConditionIncludes()), '\'\\') . '\');'
+        );
+
+        return [
+            'setupConditionList' => $conditionMatcherVisitor->getConditionListWithVerdicts(),
+            'setupIncludeTree' => $includeTree,
+        ];
+    }
+
+    /**
+     * Enrich the given FrontendTypoScript object with TypoScript 'setup' relevant data.
+     *
+     * The method is called in FE after an attempt to retrieve page content from cache has
+     * been done. There are three possible outcomes:
+     * * The page has been retrieved from cache and the content *does not* contain uncached "_INT" objects
+     * * The page has been retrieved from cache and the content *does* contain uncached "_INT" objects
+     * * The page could not be retrieved from cache
+     *
+     * If the page could not be retrieved from cache, or if the cached page content contains "_INT" objects,
+     * flag $needsFullSetup is given true, and the full TypoScript is calculated since at least parts of
+     * the page content has to be rendered, which then needs full TypoScript.
+     * If the page could be retrieved from cache, and contains no "_INT" objects, $needsFullSetup in false, the
+     * rendering chain only needs the "config." part of TypoScript to satisfy the remaining middlewares.
+     *
+     * The method implements these variants and tries to add as little overhead as possible.
+     *
+     * Returns the FrontendTypoScript object:
+     * * configTree: Always set. Global TypoScript 'config.' merged with overrides from given type/typeNum "page.config.".
+     * * configArray: Always set. Array representation of configTree.
+     * * setupTree: Not set if $needsFullSetup=false and configTree could be retrieved from cache. Full TypoScript setup.
+     * * setupArray: Not set if $needsFullSetup=false and configTree could be retrieved from cache.
+     *               Array representation of setupTree.
+     * * pageTree: Not set if $needsFullSetup=false and configTree could be retrieved from cache, or if no PAGE object
+     *             could be determined. The 'PAGE' object tree for given type/typeNum.
+     * * pageArray: Not set if $needsFullSetup=false and configTree could be retrieved from cache, or if no PAGE object
+     *              could be determined. Array representation of PageTree.
+     */
+    public function createSetupConfigOrFullSetup(
+        bool $needsFullSetup,
+        FrontendTypoScript $frontendTypoScript,
+        SiteInterface $site,
+        array $sysTemplateRows,
+        array $expressionMatcherVariables,
+        string $type,
+        ?PhpFrontend $typoScriptCache,
+        ?ServerRequestInterface $request,
+    ): FrontendTypoScript {
+        $setupTypoScriptCacheIdentifier = 'setup-' . hash(
+            'xxh3',
+            json_encode($sysTemplateRows, JSON_THROW_ON_ERROR)
+            . ($site instanceof Site && $site->isTypoScriptRoot() ? $site->getIdentifier() : '')
+            . json_encode($frontendTypoScript->getSettingsConditionList(), JSON_THROW_ON_ERROR)
+            . json_encode($frontendTypoScript->getSetupConditionList(), JSON_THROW_ON_ERROR)
+        );
+        $setupConfigTypoScriptCacheIdentifier = 'setup-config-' . hash('xxh3', $setupTypoScriptCacheIdentifier . $type);
+
+        $gotSetupConfigFromCache = false;
+        if ($setupConfigTypoScriptCache = $typoScriptCache?->require($setupConfigTypoScriptCacheIdentifier)) {
+            $frontendTypoScript->setConfigTree($setupConfigTypoScriptCache['ast']);
+            $frontendTypoScript->setConfigArray($setupConfigTypoScriptCache['array']);
+            if (!$needsFullSetup) {
+                // Fully cached page context without _INT - only 'config' is needed. Return early.
+                return $frontendTypoScript;
+            }
+            $gotSetupConfigFromCache = true;
+        }
+
+        $setupRawConfigAst = null;
+        if (!$typoScriptCache || $needsFullSetup || !$gotSetupConfigFromCache) {
+            // If caching is not allowed, if no page cache entry could be loaded or if the page cache entry has _INT
+            // object, we need the full setup AST. Try to use a cache entry for setup AST, which especially up _INT
+            // parsing. In unavailable, calculate full setup AST and cache it if allowed.
+            $gotSetupFromCache = false;
+            if ($setupTypoScriptCache = $typoScriptCache?->require($setupTypoScriptCacheIdentifier)) {
+                // We need AST, and we got it from cache.
+                $frontendTypoScript->setSetupTree($setupTypoScriptCache['ast']);
+                $frontendTypoScript->setSetupArray($setupTypoScriptCache['array']);
+                $setupRawConfigAst = $setupTypoScriptCache['ast']->getChildByName('config');
+                $gotSetupFromCache = true;
+            }
+            if (!$typoScriptCache || !$gotSetupFromCache) {
+                // We need AST and couldn't get it from cache or are now allowed to. We thus need the full setup
+                // IncludeTree, which we can get from cache again if allowed, or is calculated a-new if not.
+                $setupIncludeTree = $frontendTypoScript->getSetupIncludeTree();
+                if (!$typoScriptCache || $setupIncludeTree === null) {
+                    // A previous method *may* have calculated setup include tree already. Calculate now if not.
+                    $setupIncludeTree = $this->treeBuilder->getTreeBySysTemplateRowsAndSite('setup', $sysTemplateRows, $this->tokenizer, $site, $typoScriptCache);
+                }
+                $visitors = [];
+                $conditionConstantSubstitutionVisitor = $this->container->get(IncludeTreeSetupConditionConstantSubstitutionVisitor::class);
+                $conditionConstantSubstitutionVisitor->setFlattenedConstants($frontendTypoScript->getFlatSettings());
+                $visitors[] = $conditionConstantSubstitutionVisitor;
+                $conditionMatcherVisitor = $this->container->get(IncludeTreeConditionMatcherVisitor::class);
+                $conditionMatcherVisitor->initializeExpressionMatcherWithVariables($expressionMatcherVariables);
+                $visitors[] = $conditionMatcherVisitor;
+                $astBuilderVisitor = $this->container->get(IncludeTreeAstBuilderVisitor::class);
+                $astBuilderVisitor->setFlatConstants($frontendTypoScript->getFlatSettings());
+                $visitors[] = $astBuilderVisitor;
+                $this->includeTreeTraverserConditionVerdictAware->traverse($setupIncludeTree, $visitors);
+                $setupAst = $astBuilderVisitor->getAst();
+                // @todo: It would be good to actively remove 'config' from AST and array here
+                //        to prevent people from using the unmerged variant. The same
+                //        is already done for the determined PAGE 'config' below. This works, but
+                //        is currently blocked by functional tests that assert details?
+                //        Also, we need to still cache with full 'config' to handle multiple types.
+                $setupRawConfigAst = $setupAst->getChildByName('config');
+                $frontendTypoScript->setSetupTree($setupAst);
+                $frontendTypoScript->setSetupArray($setupAst->toArray());
+
+                // Write cache entry for AST and its array representation.
+                $typoScriptCache?->set(
+                    $setupTypoScriptCacheIdentifier,
+                    'return unserialize(\'' . addcslashes(serialize(['ast' => $setupAst, 'array' => $setupAst->toArray()]), '\'\\') . '\');'
+                );
+            }
+
+            $setupAst = $frontendTypoScript->getSetupTree();
+            $rawSetupPageNodeFromType = null;
+            $pageNodeFoundByType = false;
+            foreach ($setupAst->getNextChild() as $potentialPageNode) {
+                // Find the PAGE object that matches given type/typeNum
+                if ($potentialPageNode->getValue() === 'PAGE') {
+                    // @todo: We could potentially remove *all* PAGE objects from setup here. This prevents people
+                    //        from accessing other ones than the determined one in $frontendTypoScript->getSetupArray().
+                    $typeNumChild = $potentialPageNode->getChildByName('typeNum');
+                    if ($typeNumChild && $type === $typeNumChild->getValue()) {
+                        $rawSetupPageNodeFromType = $potentialPageNode;
+                        $pageNodeFoundByType = true;
+                        break;
+                    }
+                    if (!$typeNumChild && $type === '0') {
+                        // The first PAGE node that has no typeNum is considered '0' automatically.
+                        $rawSetupPageNodeFromType = $potentialPageNode;
+                        $pageNodeFoundByType = true;
+                        break;
+                    }
+                }
+            }
+            if (!$pageNodeFoundByType) {
+                $rawSetupPageNodeFromType = new RootNode();
+            }
+            $setupPageAst = new RootNode();
+            foreach ($rawSetupPageNodeFromType->getNextChild() as $child) {
+                $setupPageAst->addChild($child);
+            }
+
+            if (!$gotSetupConfigFromCache) {
+                // If we did not get merged 'config.' from cache above, create it now and cache it.
+                $mergedSetupConfigAst = (new SetupConfigMerger())->merge($setupRawConfigAst, $setupPageAst->getChildByName('config'));
+                if ($mergedSetupConfigAst->getChildByName('absRefPrefix') === null) {
+                    // Make sure config.absRefPrefix is set, fallback to 'auto'.
+                    $absRefPrefixNode = new ChildNode('absRefPrefix');
+                    $absRefPrefixNode->setValue('auto');
+                    $mergedSetupConfigAst->addChild($absRefPrefixNode);
+                }
+                if ($mergedSetupConfigAst->getChildByName('doctype') === null) {
+                    // Make sure config.doctype is set, fallback to 'html5'.
+                    $doctypeNode = new ChildNode('doctype');
+                    $doctypeNode->setValue('html5');
+                    $mergedSetupConfigAst->addChild($doctypeNode);
+                }
+                if ($request) {
+                    // Dispatch ModifyTypoScriptConfigEvent before config is cached and if Request is given.
+                    $mergedSetupConfigAst = $this->eventDispatcher
+                        ->dispatch(new ModifyTypoScriptConfigEvent($request, $setupAst, $mergedSetupConfigAst))->getConfigTree();
+                }
+                $frontendTypoScript->setConfigTree($mergedSetupConfigAst);
+                $setupConfigArray = $mergedSetupConfigAst->toArray();
+                $frontendTypoScript->setConfigArray($setupConfigArray);
+                $typoScriptCache?->set(
+                    $setupConfigTypoScriptCacheIdentifier,
+                    'return unserialize(\'' . addcslashes(serialize(['ast' => $mergedSetupConfigAst, 'array' => $setupConfigArray]), '\'\\') . '\');'
+                );
+            }
+
+            if ($pageNodeFoundByType) {
+                // Remove "page.config" to prevent people from working with the not merged variant.
+                // We do *not* set page if it could not be determined (important for hasPage() later
+                // to return an early "no PAGE for type found" Response.
+                $setupPageAst->removeChildByName('config');
+                $frontendTypoScript->setPageTree($setupPageAst);
+                $frontendTypoScript->setPageArray($setupPageAst->toArray());
+            }
+        }
+        return $frontendTypoScript;
+    }
+}

Classes/TypoScript/IncludeTree/Event/AfterTemplatesHaveBeenDeterminedEvent.php

@@ -0,0 +1,63 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\Event;
+
+use Psr\Http\Message\ServerRequestInterface;
+use TYPO3\CMS\Core\Site\Entity\SiteInterface;
+
+/**
+ * A PSR-14 event fired when sys_template rows have been fetched.
+ *
+ * This event is intended to add own rows based on given rows or site resolution.
+ */
+final class AfterTemplatesHaveBeenDeterminedEvent
+{
+    public function __construct(
+        private readonly array $rootline,
+        private readonly ?ServerRequestInterface $request,
+        private array $templateRows,
+    ) {}
+
+    public function getRootline(): array
+    {
+        return $this->rootline;
+    }
+
+    public function getRequest(): ?ServerRequestInterface
+    {
+        return $this->request;
+    }
+
+    /**
+     * Convenience method to directly retrieve the Site. May be null though!
+     */
+    public function getSite(): ?SiteInterface
+    {
+        return $this->request?->getAttribute('site');
+    }
+
+    public function getTemplateRows(): array
+    {
+        return $this->templateRows;
+    }
+
+    public function setTemplateRows(array $templateRows): void
+    {
+        $this->templateRows = $templateRows;
+    }
+}

Classes/TypoScript/IncludeTree/Event/BeforeLoadedPageTsConfigEvent.php

@@ -0,0 +1,45 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\Event;
+
+/**
+ * Extensions can add global page TSconfig right before they are loaded from other sources
+ * like the global page.tsconfig file.
+ *
+ * Note: The added config should not depend on runtime / request. This is considered static
+ *       config and thus should be identical on every request.
+ */
+final class BeforeLoadedPageTsConfigEvent
+{
+    public function __construct(private array $tsConfig = []) {}
+
+    public function getTsConfig(): array
+    {
+        return $this->tsConfig;
+    }
+
+    public function addTsConfig(string $tsConfig): void
+    {
+        $this->tsConfig[] = $tsConfig;
+    }
+
+    public function setTsConfig(array $tsConfig): void
+    {
+        $this->tsConfig = $tsConfig;
+    }
+}

Classes/TypoScript/IncludeTree/Event/BeforeLoadedUserTsConfigEvent.php

@@ -0,0 +1,45 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\Event;
+
+/**
+ * Extensions can add global user TSconfig right before they are loaded from other sources
+ * like the global user.tsconfig file.
+ *
+ * Note: The added config should not depend on runtime / request. This is considered static
+ *       config and thus should be identical on every request.
+ */
+final class BeforeLoadedUserTsConfigEvent
+{
+    public function __construct(private array $tsConfig = []) {}
+
+    public function getTsConfig(): array
+    {
+        return $this->tsConfig;
+    }
+
+    public function addTsConfig(string $tsConfig): void
+    {
+        $this->tsConfig[] = $tsConfig;
+    }
+
+    public function setTsConfig(array $tsConfig): void
+    {
+        $this->tsConfig = $tsConfig;
+    }
+}

Classes/TypoScript/IncludeTree/Event/ModifyLoadedPageTsConfigEvent.php

@@ -0,0 +1,46 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\Event;
+
+/**
+ * Extensions can modify page TSconfig entries that can be overridden or added, based on the root line
+ */
+final class ModifyLoadedPageTsConfigEvent
+{
+    public function __construct(private array $tsConfig, private readonly array $rootLine) {}
+
+    public function getTsConfig(): array
+    {
+        return $this->tsConfig;
+    }
+
+    public function addTsConfig(string $tsConfig): void
+    {
+        $this->tsConfig[] = $tsConfig;
+    }
+
+    public function setTsConfig(array $tsConfig): void
+    {
+        $this->tsConfig = $tsConfig;
+    }
+
+    public function getRootLine(): array
+    {
+        return $this->rootLine;
+    }
+}

Classes/TypoScript/IncludeTree/IncludeNode/AbstractConditionInclude.php

@@ -0,0 +1,84 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\Token;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenType;
+
+/**
+ * Base implementation of condition nodes.
+ *
+ * @internal: Internal tree structure.
+ */
+abstract class AbstractConditionInclude extends AbstractInclude implements IncludeConditionInterface
+{
+    protected Token $conditionValueToken;
+    protected ?Token $originalConditionValueToken = null;
+    protected bool $verdict;
+
+    /**
+     * Add the condition token to cache when serialized. See __serialize() of AbstractInclude.
+     */
+    protected function serialize(): array
+    {
+        $result = parent::serialize();
+        $result['conditionValueToken'] = $this->conditionValueToken;
+        return $result;
+    }
+
+    public function setConditionToken(Token $token): void
+    {
+        if ($token->getType() !== TokenType::T_VALUE) {
+            throw new \LogicException('Token must be of type T_VALUE', 1655977210);
+        }
+        $this->conditionValueToken = $token;
+    }
+
+    public function getConditionToken(): Token
+    {
+        return $this->conditionValueToken;
+    }
+
+    public function setOriginalConditionToken(Token $token): void
+    {
+        if ($token->getType() !== TokenType::T_VALUE) {
+            throw new \LogicException('Token must be of type T_VALUE', 1655977211);
+        }
+        $this->originalConditionValueToken = $token;
+    }
+
+    public function getOriginalConditionToken(): ?Token
+    {
+        return $this->originalConditionValueToken;
+    }
+
+    public function isConditionNegated(): bool
+    {
+        return false;
+    }
+
+    public function setConditionVerdict(bool $verdict): void
+    {
+        $this->verdict = $verdict;
+    }
+
+    public function getConditionVerdict(): bool
+    {
+        return $this->verdict;
+    }
+}

Classes/TypoScript/IncludeTree/IncludeNode/AbstractInclude.php

@@ -0,0 +1,203 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\LineInterface;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\LineStream;
+
+/**
+ * Base implementation of IncludeInterface.
+ *
+ * @internal: Internal tree structure.
+ */
+abstract class AbstractInclude implements IncludeInterface
+{
+    private ?string $identifier = null;
+    protected string $name = '';
+    protected string $path = '';
+
+    /**
+     * @var array<int, IncludeInterface>
+     */
+    protected array $children = [];
+    protected ?LineStream $lineStream = null;
+    protected ?LineInterface $originalTokenLine = null;
+    protected bool $isSplit = false;
+    protected bool $root = false;
+    protected bool $clear = false;
+    protected ?int $pid = null;
+
+    /**
+     * When storing to cache, we only store FE relevant properties and skip
+     * things like "name", "identifier" and friends. We also don't need the
+     * LineStream when a node is split.
+     */
+    final public function __serialize(): array
+    {
+        return $this->serialize();
+    }
+
+    protected function serialize(): array
+    {
+        $result['children'] = $this->children;
+        if ($this->isSplit()) {
+            $result['isSplit'] = true;
+        }
+        if (!$this->isSplit()) {
+            $result['lineStream'] = $this->lineStream;
+        }
+        if ($this->isRoot()) {
+            $result['root'] = true;
+        }
+        if ($this->isClear()) {
+            $result['clear'] = true;
+        }
+        return $result;
+    }
+
+    public function getType(): string
+    {
+        $classWithNamespace = static::class;
+        $lastBackslash = strrpos($classWithNamespace, '\\');
+        return substr($classWithNamespace, $lastBackslash + 1, -7);
+    }
+
+    public function setIdentifier(string $identifier): void
+    {
+        $this->identifier = hash('xxh3', $identifier);
+        $childCounter = 0;
+        foreach ($this->getNextChild() as $child) {
+            $child->setIdentifier($this->identifier . $childCounter);
+            $childCounter++;
+        }
+    }
+
+    public function getIdentifier(): string
+    {
+        if ($this->identifier === null) {
+            throw new \RuntimeException(
+                'Identifier has not been initialized. This happens when getIdentifier() is called on'
+                . ' trees retrieved from cache. The identifier is not supposed to be used in this context.',
+                1673634853
+            );
+        }
+        return $this->identifier;
+    }
+
+    public function setName(string $name): void
+    {
+        $this->name = $name;
+    }
+
+    public function getName(): string
+    {
+        return $this->name;
+    }
+
+    public function setPath(string $path): void
+    {
+        $this->path = $path;
+    }
+
+    public function getPath(): string
+    {
+        return $this->path;
+    }
+
+    public function addChild(IncludeInterface $node): void
+    {
+        $this->children[] = $node;
+    }
+
+    public function hasChildren(): bool
+    {
+        return !empty($this->children);
+    }
+
+    public function getNextChild(): iterable
+    {
+        foreach ($this->children as $child) {
+            yield $child;
+        }
+    }
+
+    public function isSysTemplateRecord(): bool
+    {
+        return false;
+    }
+
+    public function setLineStream(?LineStream $lineStream): void
+    {
+        $this->lineStream = $lineStream;
+    }
+
+    public function getLineStream(): ?LineStream
+    {
+        return $this->lineStream;
+    }
+
+    public function setOriginalLine(LineInterface $line): void
+    {
+        $this->originalTokenLine = $line;
+    }
+
+    public function getOriginalLine(): ?LineInterface
+    {
+        return $this->originalTokenLine;
+    }
+
+    public function setSplit(): void
+    {
+        $this->isSplit = true;
+    }
+
+    public function isSplit(): bool
+    {
+        return $this->isSplit;
+    }
+
+    public function setRoot(bool $root): void
+    {
+        $this->root = $root;
+    }
+
+    public function isRoot(): bool
+    {
+        return $this->root;
+    }
+
+    public function setClear(bool $clear): void
+    {
+        $this->clear = $clear;
+    }
+
+    public function isClear(): bool
+    {
+        return $this->clear;
+    }
+
+    public function setPid(int $pid): void
+    {
+        $this->pid = $pid;
+    }
+
+    public function getPid(): ?int
+    {
+        return $this->pid;
+    }
+}

Classes/TypoScript/IncludeTree/IncludeNode/AtImportInclude.php

@@ -0,0 +1,27 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode;
+
+/**
+ * A node representing an "@import" include. The LineStream is set
+ * to the content of the included source, which can be split again
+ * if that source contains further conditions or includes.
+ *
+ * @internal: Internal tree structure.
+ */
+final class AtImportInclude extends AbstractInclude {}

Classes/TypoScript/IncludeTree/IncludeNode/ConditionElseInclude.php

@@ -0,0 +1,39 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode;
+
+/**
+ * A node representing the [ELSE] body of a condition:
+ *
+ * [foo = bar]
+ *     ...
+ * [ELSE]
+ *     baz = bazValue
+ *
+ * The LineStream is the body of the else block, the condition token
+ * is set to the token of the condition "[foo = bar]".
+ *
+ * @internal: Internal tree structure.
+ */
+final class ConditionElseInclude extends AbstractConditionInclude
+{
+    public function isConditionNegated(): bool
+    {
+        return true;
+    }
+}

Classes/TypoScript/IncludeTree/IncludeNode/ConditionInclude.php

@@ -0,0 +1,29 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode;
+
+/**
+ * A node representing a condition and its body.
+ *
+ * [foo = bar]
+ *     baz = bazValue
+ * [END]
+ *
+ * @internal: Internal tree structure.
+ */
+final class ConditionInclude extends AbstractConditionInclude {}

Classes/TypoScript/IncludeTree/IncludeNode/ConditionStopInclude.php

@@ -0,0 +1,36 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode;
+
+/**
+ * A simple include representing [END] and [GLOBAL] lines.
+ *
+ * @internal: Internal tree structure.
+ */
+final class ConditionStopInclude extends AbstractInclude
+{
+    public function addChild(IncludeInterface $node): void
+    {
+        throw new \LogicException('ConditionStopInclude can not have children', 1717691734);
+    }
+
+    public function hasChildren(): bool
+    {
+        return false;
+    }
+}

Classes/TypoScript/IncludeTree/IncludeNode/DefaultTypoScriptInclude.php

@@ -0,0 +1,26 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode;
+
+/**
+ * A node created for "default TypoScript" from globals, content from:
+ * $GLOBALS['TYPO3_CONF_VARS']['FE']['defaultTypoScript_[constants|setup]'].
+ *
+ * @internal: Internal tree structure.
+ */
+final class DefaultTypoScriptInclude extends AbstractInclude {}

Classes/TypoScript/IncludeTree/IncludeNode/DefaultTypoScriptMagicKeyInclude.php

@@ -0,0 +1,26 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode;
+
+/**
+ * A node created for "magic" include from globals, when processing
+ * $GLOBALS['TYPO3_CONF_VARS ']['FE']['defaultTypoScript_[constants|setup]']
+ *
+ * @internal: Internal tree structure.
+ */
+final class DefaultTypoScriptMagicKeyInclude extends AbstractInclude {}

Classes/TypoScript/IncludeTree/IncludeNode/ExtensionStaticInclude.php

@@ -0,0 +1,26 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode;
+
+/**
+ * A node created for "extension static" TypoScript auto-include files:
+ * EXT:my_extension/ext_typoscript_[constants|setup].typoscript
+ *
+ * @internal: Internal tree structure.
+ */
+final class ExtensionStaticInclude extends AbstractInclude {}

Classes/TypoScript/IncludeTree/IncludeNode/FileInclude.php

@@ -0,0 +1,28 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode;
+
+/**
+ * A classic include from sys_template "include_static_file":
+ * EXT:/My/Path/[constants|setup].[typoscript|ts|txt]
+ *
+ * This is always a child of an IncludeStaticFileDatabaseInclude.
+ *
+ * @internal: Internal tree structure.
+ */
+final class FileInclude extends AbstractInclude {}

Classes/TypoScript/IncludeTree/IncludeNode/IncludeConditionInterface.php

@@ -0,0 +1,61 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\Token;
+
+/**
+ * Source streams that contain conditions are split smaller parts
+ * and each condition creates a Condition node.
+ *
+ * This interface is implemented by all conditions nodes. It allows
+ * "parking" the main condition token to be evaluated during AST building.
+ *
+ * @internal: Internal tree structure.
+ */
+interface IncludeConditionInterface
+{
+    /**
+     * Set and get the condition token: "[foo = bar]"
+     */
+    public function setConditionToken(Token $token): void;
+    public function getConditionToken(): Token;
+
+    /**
+     * Conditions may use constants: "[foo = {$bar}]". This getter/setter
+     * allows storing the original condition token string.
+     * This is set in backend only in case a constant substitution has taken
+     * place. Otherwise, the "vanilla" condition token is identical,
+     * getOriginalConditionToken() returns null and the condition token should
+     * be fetched from getConditionToken().
+     */
+    public function setOriginalConditionToken(Token $token): void;
+    public function getOriginalConditionToken(): ?Token;
+
+    /**
+     * True for ConditionElseInclude: The [ELSE] node of a condition.
+     */
+    public function isConditionNegated(): bool;
+
+    /**
+     * When a condition is evaluated, this is set to true of false
+     * depending on the condition result.
+     */
+    public function setConditionVerdict(bool $verdict): void;
+    public function getConditionVerdict(): bool;
+}

Classes/TypoScript/IncludeTree/IncludeNode/IncludeInterface.php

@@ -0,0 +1,138 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\LineInterface;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\LineStream;
+
+/**
+ * General interface of IncludeTree tree nodes.
+ *
+ * The TreeBuilder classes return a tree of these nodes, with the root node being
+ * a RootInclude. Each "include type" is represented by an own class: There
+ * is for instance "SysTemplateInclude" for a node that represents a sys_template
+ * row, and DefaultTypoScriptInclude for the default TypoScript string included from
+ * TYPO3_CONF_VARS.
+ *
+ * Nodes may have children, and a single stream of lines from the tokenizer
+ * may be split into multiple children: Each @import creates an own child node,
+ * and conditions trigger splitting as well.
+ *
+ * @internal: Internal tree structure.
+ */
+interface IncludeInterface
+{
+    /**
+     * A human-readable string derived from class name - Used in BE template analyzer
+     */
+    public function getType(): string;
+
+    /**
+     * An identifier for this include. Typically, a hash of some kind. This identifier
+     * is unique within the tree, by being created from the parent identifier plus
+     * something unique for this level like a counter. This identifier is used in the backend,
+     * when referencing single includes to be rendered.
+     * Calculating identifiers is initiated by calling setIdentifier() on RootNode, which
+     * will recurse the tree. Call this on the final tree, after include calculation finished,
+     * so include building itself does not need to fiddle with identifier updates.
+     * Note this value is skipped when persisting to caches since it's a Backend related
+     * thing that does not use cached context: When retrieving includes from cache
+     * (e.g. in Frontend), the identifier is null and calling the getter will throw an exception.
+     */
+    public function setIdentifier(string $identifier): void;
+    public function getIdentifier(): string;
+
+    /**
+     * A human-readable version of the identifier: Used in backend tree rendering.
+     */
+    public function setName(string $name): void;
+    public function getName(): string;
+
+    /**
+     * This is set to a non-empty string for includes that represent files. The file location
+     * is stored here, typically something like "EXT:my_extension/path/to/foo.typoscript".
+     * This is used when resolving file includes relative to a parent include, so a
+     * potential child node knows where to look relative to its parent path.
+     * Note this value is skipped when persisting to caches: The parent path
+     * information is no longer needed when a tree is fetched from cache since
+     * all children were attached already and don't need to be recalculated
+     * depending on their parent path value.
+     */
+    public function setPath(string $path): void;
+    public function getPath(): string;
+
+    /**
+     * Child maintenance methods.
+     */
+    public function addChild(IncludeInterface $node): void;
+    public function hasChildren(): bool;
+
+    /**
+     * @return iterable<IncludeInterface>
+     */
+    public function getNextChild(): iterable;
+
+    /**
+     * True for IncludeTypoScriptInclude - this node represents a sys_template record.
+     * When true, methods like isRoot() and isClear() are relevant.
+     */
+    public function isSysTemplateRecord(): bool;
+
+    /**
+     * The source split into single lines by a tokenizer.
+     */
+    public function setLineStream(?LineStream $lineStream): void;
+    public function getLineStream(): ?LineStream;
+
+    /**
+     * When an imports are handled, such a line is substituted by the included
+     * content. To be able to still output the original line, it is parked here.
+     * Relevant in backend tree and source display only.
+     */
+    public function setOriginalLine(LineInterface $line): void;
+    public function getOriginalLine(): ?LineInterface;
+
+    /**
+     * When included line streams contain conditions or imports, the node is split into
+     * children that contain single segments of the source. The node itself is then just
+     * a container and the LineStream attached is irrelevant for further processing.
+     * This flag is set when a line stream is split and the children fully represent the source.
+     */
+    public function setSplit(): void;
+    public function isSplit(): bool;
+
+    /**
+     * Set to true for IncludeTypoScriptInclude's (sys_template records) when "root" flag is set.
+     */
+    public function setRoot(bool $root): void;
+    public function isRoot(): bool;
+
+    /**
+     * Set to true for IncludeTypoScriptInclude's (sys_template records) when "clear constants"
+     * or "clear setup" is set. Depends on context if currently constants or setup are parsed.
+     */
+    public function setClear(bool $clear): void;
+    public function isClear(): bool;
+
+    /**
+     * Set to the pid of IncludeTypoScriptInclude's (sys_template records). Relevant in backend
+     * tree rendering only.
+     */
+    public function setPid(int $pid): void;
+    public function getPid(): ?int;
+}

Classes/TypoScript/IncludeTree/IncludeNode/IncludeStaticFileDatabaseInclude.php

@@ -0,0 +1,27 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode;
+
+/**
+ * Main node created for sys_template "include_static_file":
+ * This has FileInclude or IncludeStaticFileFileInclude children,
+ * depending on specific string.
+ *
+ * @internal: Internal tree structure.
+ */
+final class IncludeStaticFileDatabaseInclude extends AbstractInclude {}

Classes/TypoScript/IncludeTree/IncludeNode/IncludeStaticFileFileInclude.php

@@ -0,0 +1,26 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode;
+
+/**
+ * Created when a sys_template "static_file_include" includes "include_static_file.txt" files:
+ * EXT:/My/Path/include_static_file.txt
+ *
+ * @internal: Internal tree structure.
+ */
+final class IncludeStaticFileFileInclude extends AbstractInclude {}

Classes/TypoScript/IncludeTree/IncludeNode/RootInclude.php

@@ -0,0 +1,34 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode;
+
+/**
+ * Root of the IncludeTree. Does not contain LineStreams itself,
+ * only children do.
+ *
+ * @internal: Internal tree structure.
+ */
+final class RootInclude extends AbstractInclude
+{
+    protected string $name = 'ROOT';
+
+    public function setName(string $name): void
+    {
+        throw new \LogicException('Can not set name on RootNode', 1656668001);
+    }
+}

Classes/TypoScript/IncludeTree/IncludeNode/SegmentInclude.php

@@ -0,0 +1,32 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode;
+
+/**
+ * When a source stream is split into children because the LineStream contains
+ * conditions or imports, this node represents TypoScript that is not within
+ * condition or import context, the "baz = bazValue" part in the example below:
+ *
+ * [foo=bar]
+ *     ...
+ * [END]
+ * baz = bazValue
+ *
+ * @internal: Internal tree structure.
+ */
+final class SegmentInclude extends AbstractInclude {}

Classes/TypoScript/IncludeTree/IncludeNode/SiteInclude.php

@@ -0,0 +1,25 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode;
+
+/**
+ * Include node created for includes from Site objects. Only relevant for constants.
+ *
+ * @internal: Internal tree structure.
+ */
+final class SiteInclude extends AbstractInclude {}

Classes/TypoScript/IncludeTree/IncludeNode/SiteTemplateInclude.php

@@ -0,0 +1,40 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode;
+
+/**
+ * The main node created for TypoScript from site sets
+ * and %configPath/sites/{constants,setup}.typoscript.
+ *
+ * @internal: Internal tree structure.
+ */
+final class SiteTemplateInclude extends AbstractInclude
+{
+    protected bool $root = true;
+    protected bool $clear = true;
+
+    public function isRoot(): bool
+    {
+        return true;
+    }
+
+    public function isClear(): bool
+    {
+        return true;
+    }
+}

Classes/TypoScript/IncludeTree/IncludeNode/StringInclude.php

@@ -0,0 +1,26 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode;
+
+/**
+ * A simple include type used by StringTreeBuilder when a single entry
+ * TypoScript snipped is parsed.
+ *
+ * @internal: Internal tree structure.
+ */
+final class StringInclude extends AbstractInclude {}

Classes/TypoScript/IncludeTree/IncludeNode/SysTemplateInclude.php

@@ -0,0 +1,31 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode;
+
+/**
+ * The main node created for sys_template rows.
+ *
+ * @internal: Internal tree structure.
+ */
+final class SysTemplateInclude extends AbstractInclude
+{
+    public function isSysTemplateRecord(): bool
+    {
+        return true;
+    }
+}

Classes/TypoScript/IncludeTree/IncludeNode/TsConfigInclude.php

@@ -0,0 +1,25 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode;
+
+/**
+ * An include type used by user and pages TsConfig for single TsConfig snippets.
+ *
+ * @internal: Internal tree structure.
+ */
+final class TsConfigInclude extends AbstractInclude {}

Classes/TypoScript/IncludeTree/StringTreeBuilder.php

@@ -0,0 +1,76 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree;
+
+use Symfony\Component\DependencyInjection\Attribute\Autoconfigure;
+use TYPO3\CMS\Core\Cache\Frontend\PhpFrontend;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\RootInclude;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\StringInclude;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\TokenizerInterface;
+
+/**
+ * Parse a single TypoScript string, supporting imports and conditions.
+ *
+ * This is a relatively simple "tree" builder: It gets a single TypoScript string
+ * snippet, tokenizes it and creates a RootInclude "tree". The string is scanned
+ * for imports and conditions: Those create sub includes, just like the other
+ * TreeBuilder classes do.
+ *
+ * @internal
+ */
+#[Autoconfigure(public: true)]
+final readonly class StringTreeBuilder
+{
+    public function __construct(
+        private TreeFromLineStreamBuilder $treeFromTokenStreamBuilder,
+    ) {}
+
+    /**
+     * Create tree, ready to be traversed. Will cache if $cache is not null.
+     *
+     * @param non-empty-string $name A name used as cache identifier, [a-z,A-Z,-] only
+     */
+    public function getTreeFromString(
+        string $name,
+        string $typoScriptString,
+        TokenizerInterface $tokenizer,
+        ?PhpFrontend $cache = null,
+    ): RootInclude {
+        $lowerCaseName = mb_strtolower($name);
+        $identifier = 'string-' . $lowerCaseName . '-' . hash('xxh3', $typoScriptString);
+        if ($cache) {
+            $includeTree = $cache->require($identifier);
+            if ($includeTree instanceof RootInclude) {
+                return $includeTree;
+            }
+        }
+        $includeTree = new RootInclude();
+        $includeNode = new StringInclude();
+        $includeNode->setName('[string] ' . $name);
+        $includeNode->setLineStream($tokenizer->tokenize($typoScriptString));
+        $this->treeFromTokenStreamBuilder->buildTree($includeNode, 'other', $tokenizer);
+        $includeTree->addChild($includeNode);
+        $cache?->set($identifier, $this->prepareTreeForCache($includeTree));
+        return $includeTree;
+    }
+
+    private function prepareTreeForCache(RootInclude $node): string
+    {
+        return 'return unserialize(\'' . addcslashes(serialize($node), '\'\\') . '\');';
+    }
+}

Classes/TypoScript/IncludeTree/SysTemplateRepository.php

@@ -0,0 +1,220 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree;
+
+use Psr\EventDispatcher\EventDispatcherInterface;
+use Psr\Http\Message\ServerRequestInterface;
+use Symfony\Component\DependencyInjection\Attribute\Autoconfigure;
+use TYPO3\CMS\Core\Context\Context;
+use TYPO3\CMS\Core\Context\VisibilityAspect;
+use TYPO3\CMS\Core\Database\Connection;
+use TYPO3\CMS\Core\Database\ConnectionPool;
+use TYPO3\CMS\Core\Database\Query\Restriction\DefaultRestrictionContainer;
+use TYPO3\CMS\Core\Database\Query\Restriction\EndTimeRestriction;
+use TYPO3\CMS\Core\Database\Query\Restriction\HiddenRestriction;
+use TYPO3\CMS\Core\Database\Query\Restriction\StartTimeRestriction;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\Event\AfterTemplatesHaveBeenDeterminedEvent;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+/**
+ * Fetch relevant sys_template records from database by given page rootline.
+ *
+ * The result sys_template rows are fed to the SysTemplateTreeBuilder for processing.
+ *
+ * @internal: Internal structure. There is optimization potential and especially getSysTemplateRowsByRootline() will probably vanish later.
+ */
+#[Autoconfigure(public: true)]
+final readonly class SysTemplateRepository
+{
+    public function __construct(
+        private EventDispatcherInterface $eventDispatcher,
+        private ConnectionPool $connectionPool,
+        private Context $context,
+    ) {}
+
+    /**
+     * To calculate the TS include tree, we have to find sys_template rows attached to all rootline pages.
+     * When there are multiple active sys_template rows on a page, we pick the one with the lower sorting
+     * value.
+     *
+     * The query implementation below does that with *one* query for all rootline pages at once, not
+     * one query per page. To handle the capabilities mentioned above, the query is a bit nifty, but
+     * the implementation should scale nearly O(1) instead of O(n) with the rootline depth.
+     *
+     * @param ServerRequestInterface|null $request Nullable since Request is not a hard dependency ond just convenient for the Event
+     *
+     * @todo: It's potentially possible to get rid of this method in the frontend by joining sys_template
+     *        into the Page rootline resolving as soon as it uses a CTE: This would save one query in *all* FE
+     *        requests, even for fully-cached page requests.
+     */
+    public function getSysTemplateRowsByRootline(array $rootline, ?ServerRequestInterface $request = null, ?VisibilityAspect $visibility = null): array
+    {
+        if ($rootline === []) {
+            return [];
+        }
+
+        // Site-root node first!
+        $rootLinePageIds = array_reverse(array_column($rootline, 'uid'));
+        $sysTemplateRows = [];
+        $queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_template');
+        $queryBuilder->setRestrictions($this->getSysTemplateQueryRestrictionContainer($visibility));
+        $queryBuilder->select('sys_template.*')->from('sys_template');
+        // Build a value list as joined table to have sorting based on list sorting
+        $valueList = [];
+        foreach ($rootLinePageIds as $sorting => $rootLinePageId) {
+            $valueList[] = sprintf(
+                '%s, %s',
+                $queryBuilder->expr()->castInt(
+                    $queryBuilder->createNamedParameter($rootLinePageId, Connection::PARAM_INT),
+                    'uid',
+                ),
+                $queryBuilder->expr()->castInt(
+                    $queryBuilder->createNamedParameter($sorting, Connection::PARAM_INT),
+                    'sorting',
+                )
+            );
+        }
+        $valueList = 'SELECT ' . implode(' UNION ALL SELECT ', $valueList);
+        $queryBuilder->getConcreteQueryBuilder()->innerJoin(
+            $queryBuilder->quoteIdentifier('sys_template'),
+            sprintf('(%s)', $valueList),
+            $queryBuilder->quoteIdentifier('pidlist'),
+            '(' . $queryBuilder->expr()->eq(
+                'sys_template.pid',
+                $queryBuilder->quoteIdentifier('pidlist.uid')
+            ) . ')'
+        );
+        // Sort by rootline determined depth as sort criteria
+        $queryBuilder->orderBy('pidlist.sorting', 'ASC')
+            ->addOrderBy('sys_template.root', 'DESC')
+            ->addOrderBy('sys_template.sorting', 'ASC');
+        $lastPid = null;
+        $queryResult = $queryBuilder->executeQuery();
+        while ($sysTemplateRow = $queryResult->fetchAssociative()) {
+            // We're retrieving *all* templates per pid, but need the first one only. The
+            // order restriction above at least takes care they're after-each-other per pid.
+            if ($lastPid === (int)$sysTemplateRow['pid']) {
+                continue;
+            }
+            $lastPid = (int)$sysTemplateRow['pid'];
+            $sysTemplateRows[] = $sysTemplateRow;
+        }
+        $event = new AfterTemplatesHaveBeenDeterminedEvent($rootline, $request, $sysTemplateRows);
+        $this->eventDispatcher->dispatch($event);
+        return $event->getTemplateRows();
+    }
+
+    /**
+     * To calculate the TS include tree, we have to find sys_template rows attached to all rootline pages.
+     * When there are multiple active sys_template rows on a page, we pick the one with the lower sorting
+     * value.
+     *
+     * This variant is tailored for ext:tstemplate use. It allows "overriding" the sys_template uid of
+     * the deepest page, which is used when multiple sys_template records on one page are managed in the Backend.
+     *
+     * The query implementation below does that with *one* query for all rootline pages at once, not
+     * one query per page. To handle the capabilities mentioned above, the query is a bit nifty, but
+     * the implementation should scale nearly O(1) instead of O(n) with the rootline depth.
+     */
+    public function getSysTemplateRowsByRootlineWithUidOverride(array $rootline, ?ServerRequestInterface $request, int $templateUidOnDeepestRootline, ?VisibilityAspect $visibility = null): array
+    {
+        // Site-root node first!
+        $rootLinePageIds = array_reverse(array_column($rootline, 'uid'));
+        $templatePidOnDeepestRootline = array_first($rootline)['uid'];
+        $sysTemplateRows = [];
+        $queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_template');
+        $queryBuilder->setRestrictions($this->getSysTemplateQueryRestrictionContainer($visibility));
+        $queryBuilder->select('sys_template.*')->from('sys_template');
+        if ($templateUidOnDeepestRootline && $templatePidOnDeepestRootline) {
+            $queryBuilder->andWhere(
+                $queryBuilder->expr()->or(
+                    $queryBuilder->expr()->neq('sys_template.pid', $queryBuilder->createNamedParameter($templatePidOnDeepestRootline, Connection::PARAM_INT)),
+                    $queryBuilder->expr()->and(
+                        $queryBuilder->expr()->eq('sys_template.pid', $queryBuilder->createNamedParameter($templatePidOnDeepestRootline, Connection::PARAM_INT)),
+                        $queryBuilder->expr()->eq('sys_template.uid', $queryBuilder->createNamedParameter($templateUidOnDeepestRootline, Connection::PARAM_INT)),
+                    ),
+                ),
+            );
+        }
+        // Build a value list as joined table to have sorting based on list sorting
+        $valueList = [];
+        foreach ($rootLinePageIds as $sorting => $rootLinePageId) {
+            $valueList[] = sprintf(
+                '%s, %s',
+                $queryBuilder->expr()->castInt(
+                    $queryBuilder->createNamedParameter($rootLinePageId, Connection::PARAM_INT),
+                    'uid',
+                ),
+                $queryBuilder->expr()->castInt(
+                    $queryBuilder->createNamedParameter($sorting, Connection::PARAM_INT),
+                    'sorting',
+                ),
+            );
+        }
+        $valueList = 'SELECT ' . implode(' UNION ALL SELECT ', $valueList);
+        $queryBuilder->getConcreteQueryBuilder()->innerJoin(
+            $queryBuilder->quoteIdentifier('sys_template'),
+            sprintf('(%s)', $valueList),
+            $queryBuilder->quoteIdentifier('pidlist'),
+            '(' . $queryBuilder->expr()->eq(
+                'sys_template.pid',
+                $queryBuilder->quoteIdentifier('pidlist.uid')
+            ) . ')'
+        );
+        // Sort by rootline determined depth as sort criteria
+        $queryBuilder->orderBy('pidlist.sorting', 'ASC')
+            ->addOrderBy('sys_template.root', 'DESC')
+            ->addOrderBy('sys_template.sorting', 'ASC');
+        $lastPid = null;
+        $queryResult = $queryBuilder->executeQuery();
+        while ($sysTemplateRow = $queryResult->fetchAssociative()) {
+            // We're retrieving *all* templates per pid, but need the first one only. The
+            // order restriction above at least takes care they're after-each-other per pid.
+            if ($lastPid === (int)$sysTemplateRow['pid']) {
+                continue;
+            }
+            $lastPid = (int)$sysTemplateRow['pid'];
+            $sysTemplateRows[] = $sysTemplateRow;
+        }
+        // @todo: This event should be able to be fired even if the sys_template resolving is
+        //        merged into an early middleware like "SiteResolver" which could join / sub-select
+        //        pages together with sys_template directly, which would be possible if we manage
+        //        to switch away from RootlineUtility usage in SiteResolver by using a CTE instead.
+        $event = new AfterTemplatesHaveBeenDeterminedEvent($rootline, $request, $sysTemplateRows);
+        $this->eventDispatcher->dispatch($event);
+        return $event->getTemplateRows();
+    }
+
+    /**
+     * Get sys_template record query builder restrictions.
+     * Allows hidden records if enabled in context.
+     */
+    private function getSysTemplateQueryRestrictionContainer(?VisibilityAspect $visibility = null): DefaultRestrictionContainer
+    {
+        $restrictionContainer = GeneralUtility::makeInstance(DefaultRestrictionContainer::class);
+        $visibility ??= $this->context->getAspect('visibility');
+        if ($visibility->includeHiddenContent()) {
+            $restrictionContainer->removeByType(HiddenRestriction::class);
+        }
+        if ($visibility->includeScheduledRecords()) {
+            $restrictionContainer->removeByType(StartTimeRestriction::class);
+            $restrictionContainer->removeByType(EndTimeRestriction::class);
+        }
+        return $restrictionContainer;
+    }
+}

Classes/TypoScript/IncludeTree/SysTemplateTreeBuilder.php

@@ -0,0 +1,659 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree;
+
+use Symfony\Component\DependencyInjection\Attribute\Autoconfigure;
+use TYPO3\CMS\Core\Cache\Frontend\PhpFrontend;
+use TYPO3\CMS\Core\Context\Context;
+use TYPO3\CMS\Core\Database\Connection;
+use TYPO3\CMS\Core\Database\ConnectionPool;
+use TYPO3\CMS\Core\Database\Query\Restriction\DefaultRestrictionContainer;
+use TYPO3\CMS\Core\Database\Query\Restriction\HiddenRestriction;
+use TYPO3\CMS\Core\Package\PackageManager;
+use TYPO3\CMS\Core\Site\Entity\Site;
+use TYPO3\CMS\Core\Site\Entity\SiteInterface;
+use TYPO3\CMS\Core\Site\Set\SetRegistry;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\DefaultTypoScriptInclude;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\DefaultTypoScriptMagicKeyInclude;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\ExtensionStaticInclude;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\FileInclude;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\IncludeInterface;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\IncludeStaticFileDatabaseInclude;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\IncludeStaticFileFileInclude;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\RootInclude;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\SiteInclude;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\SiteTemplateInclude;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\SysTemplateInclude;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\TokenizerInterface;
+use TYPO3\CMS\Core\Utility\ExtensionManagementUtility;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Core\Utility\PathUtility;
+
+/**
+ * Create a tree representing all TypoScript includes.
+ *
+ * This is the 'middle' part of the TypoScript parsing process: The tokenizers as "lowest"
+ * structure create line streams from TypoScript, the AST builder as "highest" structure create
+ * the TypoScript object tree.
+ *
+ * This structure gathers all TypoScript snippets that have to be tokenized, and creates a
+ * tree with include nodes and sub include nodes.
+ *
+ * It is called in frontend (and backend "Template" module) with the page rootline, gets all
+ * attached sys_template records, gets their content and various sub includes and takes care
+ * of correct include order.
+ *
+ * This class together with TreeFromLineStreamBuilder also takes care of conditions and
+ * imports ("@import"): Those create child nodes in the tree. To evaluate conditions, the
+ * tree is later traversed, condition verdicts (true / false) are determined, to see if
+ * condition's child nodes should be considered in AST.
+ *
+ * The IncludeTree is "runtime stateless": Constants values and conditions are *not* evaluated
+ * here, so the tree is always the same for a given rootline. This makes this structure cache-able:
+ * In frontend, the tree (or sub parts of it) is cached and fetched from cache for next
+ * call. This means the entire tree-building and tokenizing is suppressed. After that runtime
+ * information is added: Conditions are evaluated, and the AST is built from given IncludeTree.
+ *
+ * @internal: Internal tree structure.
+ */
+#[Autoconfigure(public: true)]
+final class SysTemplateTreeBuilder
+{
+    /**
+     * Used in 'basedOn' includes to prevent endless loop: Each sys_template row can
+     * be included only once in 'basedOn'.
+     *
+     * @var array<int, int>
+     */
+    private array $includedSysTemplateUids = [];
+
+    /** @var 'constants'|'setup' */
+    private string $type;
+
+    private TokenizerInterface $tokenizer;
+    private ?PhpFrontend $cache = null;
+
+    private bool $enableStaticMagicIncludes = false;
+
+    public function __construct(
+        private readonly ConnectionPool $connectionPool,
+        private readonly PackageManager $packageManager,
+        private readonly Context $context,
+        private readonly TreeFromLineStreamBuilder $treeFromTokenStreamBuilder,
+        private readonly SetRegistry $setRegistry,
+    ) {}
+
+    /**
+     * @param 'constants'|'setup' $type
+     */
+    public function getTreeBySysTemplateRowsAndSite(
+        string $type,
+        array $sysTemplateRows,
+        TokenizerInterface $tokenizer,
+        ?SiteInterface $site = null,
+        ?PhpFrontend $cache = null
+    ): RootInclude {
+        if (!in_array($type, ['constants', 'setup'], true)) {
+            throw new \RuntimeException('type must be either constants or setup', 1653737656);
+        }
+        $this->tokenizer = $tokenizer;
+        $this->cache = $cache;
+        $this->type = $type;
+        $this->includedSysTemplateUids = [];
+
+        $rootNode = new RootInclude();
+
+        $siteIsTypoScriptRoot = $site instanceof Site ? $site->isTypoScriptRoot() : false;
+        if ($siteIsTypoScriptRoot) {
+            $this->enableStaticMagicIncludes = false;
+            $cacheIdentifier = 'site-template-' . $this->type . '-' . $site->getIdentifier();
+            $includeNode = $this->cache?->require($cacheIdentifier) ?: null;
+            $includeNode ??= $this->createSiteTemplateInclude($site, $cacheIdentifier);
+            $rootNode->addChild($includeNode);
+        }
+
+        if (empty($sysTemplateRows)) {
+            return $rootNode;
+        }
+
+        $this->enableStaticMagicIncludes = true;
+        // Convenience code: Usually, at least one sys_template records needs to have 'clear' set. This resets
+        // the AST and triggers inclusion of "globals" TypoScript. When integrators missed to set the clear flags,
+        // important globals TypoScript is not loaded, leading to pretty hard to find issues in Frontend
+        // rendering. Since the details of the 'clear' flags are rather complex anyway, this code scans the given
+        // sys_template records if the flag is set somewhere and if not, actively sets it dynamically for the
+        // first templates. As a result, integrators do not need to think about the 'clear' flags at all for
+        // simple instances, it 'just works'.
+        $atLeastOneSysTemplateRowHasClearFlag = $siteIsTypoScriptRoot;
+        if (!$atLeastOneSysTemplateRowHasClearFlag) {
+            foreach ($sysTemplateRows as $sysTemplateRow) {
+                if (($this->type === 'constants' && $sysTemplateRow['clear'] & 1) || ($this->type === 'setup' && $sysTemplateRow['clear'] & 2)) {
+                    $atLeastOneSysTemplateRowHasClearFlag = true;
+                    break;
+                }
+            }
+            $firstRow = reset($sysTemplateRows);
+            $firstRow['clear'] = $this->type === 'constants' ? 1 : 2;
+            $sysTemplateRows[array_key_first($sysTemplateRows)] = $firstRow;
+        }
+
+        foreach ($sysTemplateRows as $sysTemplateRow) {
+            $cacheIdentifier = 'sys-template-' . $this->type . '-' . $this->getSysTemplateRowIdentifier($sysTemplateRow, $site);
+            if ($this->cache) {
+                // Get from cache if possible
+                $includeNode = $this->cache->require($cacheIdentifier);
+                if ($includeNode) {
+                    $rootNode->addChild($includeNode);
+                    continue;
+                }
+            }
+            $includeNode = new SysTemplateInclude();
+            $name = '[sys_template:' . $sysTemplateRow['uid'] . '] ' . $sysTemplateRow['title'];
+            $includeNode->setName($name);
+            $includeNode->setPid((int)$sysTemplateRow['pid']);
+            if ($this->type === 'constants') {
+                $includeNode->setLineStream($this->tokenizer->tokenize($sysTemplateRow['constants'] ?? ''));
+            } else {
+                $includeNode->setLineStream($this->tokenizer->tokenize($sysTemplateRow['config'] ?? ''));
+            }
+            if ($sysTemplateRow['root']) {
+                $includeNode->setRoot(true);
+            }
+            $clear = $sysTemplateRow['clear'];
+            if (($this->type === 'constants' && $clear & 1) || ($this->type === 'setup' && $clear & 2)) {
+                $includeNode->setClear(true);
+            }
+            $this->handleSysTemplateRecordInclude($includeNode, $sysTemplateRow, $site);
+            $this->treeFromTokenStreamBuilder->buildTree($includeNode, $this->type, $this->tokenizer);
+            $this->cache?->set($cacheIdentifier, $this->prepareNodeForCache($includeNode));
+            $rootNode->addChild($includeNode);
+        }
+
+        return $rootNode;
+    }
+
+    private function createSiteTemplateInclude(
+        Site $site,
+        string $cacheIdentifier
+    ): SiteTemplateInclude {
+        $includeNode = new SiteTemplateInclude();
+        $includeNode->setRoot(true);
+        $includeNode->setClear(true);
+
+        $this->addScopedStaticsFromGlobals($includeNode, 'siteSets');
+        $this->addContentRenderingFromGlobals($includeNode, 'TYPO3_CONF_VARS defaultContentRendering');
+
+        $sets = $this->setRegistry->getSets(...$site->getSets());
+        if (count($sets) > 0) {
+            $includeSetInclude = new IncludeStaticFileFileInclude();
+            $includeSetInclude->setName('site:' . $site->getIdentifier() . ':sets');
+            $includeSetInclude->setPath('site:' . $site->getIdentifier() . '/');
+            foreach ($sets as $set) {
+                if ($set->typoscript === null) {
+                    continue;
+                }
+                $this->handleSetInclude($includeSetInclude, rtrim($set->typoscript, '/') . '/', 'set:' . $set->name);
+            }
+            $includeNode->addChild($includeSetInclude);
+        }
+
+        if ($this->type === 'constants') {
+            $this->addDefaultTypoScriptConstantsFromSite($includeNode, $site);
+        }
+
+        $siteTypoScript = $site->getTypoScript();
+        $content = $this->type === 'constants' ? $siteTypoScript?->constants : $siteTypoScript?->setup;
+        if ($content !== null) {
+            $includeNode->setLineStream($this->tokenizer->tokenize($content));
+            $this->treeFromTokenStreamBuilder->buildTree($includeNode, $this->type, $this->tokenizer, false);
+        }
+
+        $includeNode->setName(sprintf(
+            '[site:%s%s] %s',
+            $site->getIdentifier(),
+            $content === null ? '' : '/' . $this->type . '.typoscript',
+            $site->getConfiguration()['websiteTitle'] ?? ''
+        ));
+
+        $this->cache?->set($cacheIdentifier, $this->prepareNodeForCache($includeNode));
+
+        return $includeNode;
+    }
+
+    private function handleSetInclude(IncludeInterface $parentNode, string $path, string $label): void
+    {
+        $path = GeneralUtility::getFileAbsFileName($path);
+
+        // '/.../my_extension/Configuration/TypoScript/MyStaticInclude/include_static_file.txt'
+        $includeStaticFileFileIncludePath = $path . 'include_static_file.txt';
+        if (file_exists($path . 'include_static_file.txt')) {
+            $includeStaticFileFileInclude = new IncludeStaticFileFileInclude();
+            $includeStaticFileFileInclude->setName($label . ':include_static_file.txt');
+            $includeStaticFileFileInclude->setPath($path . 'include_static_file.txt');
+            $parentNode->addChild($includeStaticFileFileInclude);
+            $includeStaticFileFileIncludeContent = (string)file_get_contents($includeStaticFileFileIncludePath);
+            // @todo: There is no array_unique() for DB based include_static_file content?!
+            $includeStaticFileFileIncludeArray = array_unique(GeneralUtility::trimExplode(',', $includeStaticFileFileIncludeContent, true));
+            foreach ($includeStaticFileFileIncludeArray as $includeStaticFileFileIncludeString) {
+                $this->handleSingleIncludeStaticFile($includeStaticFileFileInclude, $includeStaticFileFileIncludeString);
+            }
+        }
+
+        $fileName = $path . $this->type . '.typoscript';
+        if (file_exists($fileName)) {
+            $fileContent = file_get_contents($fileName);
+            $fileNode = new FileInclude();
+            $fileNode->setName($label . ':' . $this->type . '.typoscript');
+            $fileNode->setPath($fileName);
+            $fileNode->setLineStream($this->tokenizer->tokenize($fileContent));
+            $this->treeFromTokenStreamBuilder->buildTree($fileNode, $this->type, $this->tokenizer, false);
+            $parentNode->addChild($fileNode);
+        }
+    }
+
+    /**
+     * Add includes defined in a sys_template record.
+     */
+    private function handleSysTemplateRecordInclude(IncludeInterface $parentNode, array $row, ?SiteInterface $site): void
+    {
+        $this->includedSysTemplateUids[] = (int)$row['uid'];
+
+        $isRoot = (bool)$row['root'];
+        $clearConstants = (int)$row['clear'] & 1;
+        $clearSetup = (int)$row['clear'] & 2;
+        $staticFileMode = (int)($row['static_file_mode']);
+        $includeStaticAfterBasedOn = (bool)$row['includeStaticAfterBasedOn'];
+
+        if ($this->type === 'constants' && $clearConstants) {
+            $this->addDefaultTypoScriptFromGlobals($parentNode);
+            $this->addDefaultTypoScriptConstantsFromSite($parentNode, $site);
+        }
+        if ($this->type === 'setup' && $clearSetup) {
+            $this->addDefaultTypoScriptFromGlobals($parentNode);
+        }
+        if ($staticFileMode === 3 && $isRoot) {
+            $this->addExtensionStatics($parentNode);
+        }
+        if (!$includeStaticAfterBasedOn) {
+            $this->handleIncludeStaticFileArray($parentNode, (string)$row['include_static_file']);
+        }
+        if (!empty($row['basedOn'])) {
+            $this->handleIncludeBasedOnTemplates($parentNode, (string)$row['basedOn'], $site);
+        }
+        if ($includeStaticAfterBasedOn) {
+            $this->handleIncludeStaticFileArray($parentNode, (string)$row['include_static_file']);
+        }
+        if ($staticFileMode === 1 || ($staticFileMode === 0 && $isRoot)) {
+            $this->addExtensionStatics($parentNode);
+        }
+    }
+
+    /**
+     * Handle includes defined in a sys_template['include_static_file'] row. Extracted as
+     * methods since it depends on 'includeStaticAfterBasedOn' field if this is included
+     * *before* or *after* other 'basedOn' includes.
+     *
+     * The cache implemented here *does not* take the *content* of files into account.
+     * This means changing a file *does not* automatically void the cache since that would
+     * lead to lots of file_exists() and file_get_contents() calls in production.
+     * Instances in development context should thus set the typoscript-cache to NullFrontend.
+     * Note this cache-usage is the main-cache that kicks in whenever different sys_template
+     * records include the same file. For instance, when multiple sites include ext:seo XmlSitemap,
+     * the cache implementation here takes care the ext:seo subtree is calculated only once.
+     */
+    private function handleIncludeStaticFileArray(IncludeInterface $parentNode, string $includeStaticFileString): void
+    {
+        $includeStaticFileIncludeArray = GeneralUtility::trimExplode(',', $includeStaticFileString, true);
+        foreach ($includeStaticFileIncludeArray as $includeStaticFile) {
+            $cacheIdentifier = preg_replace('/[^[:alnum:]]/u', '-', mb_strtolower($includeStaticFile)) . '-' . $this->type;
+            if ($this->cache) {
+                $node = $this->cache->require($cacheIdentifier);
+                if ($node) {
+                    $parentNode->addChild($node);
+                    continue;
+                }
+            }
+            $node = new IncludeStaticFileDatabaseInclude();
+            $node->setName($includeStaticFile);
+            $this->handleSingleIncludeStaticFile($node, $includeStaticFile);
+            $this->cache?->set($cacheIdentifier, $this->prepareNodeForCache($node));
+            $parentNode->addChild($node);
+        }
+    }
+
+    /**
+     * Handle includes defined in a sys_template['basedOn'] row.
+     * Warning: Calls handleSysTemplateRecordInclude() recursive when another basedOn templates
+     *          record includes things again!
+     */
+    private function handleIncludeBasedOnTemplates(IncludeInterface $parentNode, string $basedOnList, ?SiteInterface $site): void
+    {
+        $basedOnTemplateUids = GeneralUtility::intExplode(',', $basedOnList, true);
+        // Filter uids that have been handled already.
+        $basedOnTemplateUids = array_diff($basedOnTemplateUids, $this->includedSysTemplateUids);
+        if (empty($basedOnTemplateUids)) {
+            return;
+        }
+
+        $basedOnTemplateRows = $this->getBasedOnSysTemplateRowsFromDatabase($basedOnTemplateUids);
+
+        foreach ($basedOnTemplateUids as $basedOnTemplateUid) {
+            if (is_array($basedOnTemplateRows[$basedOnTemplateUid] ?? false)) {
+                $sysTemplateRow = $basedOnTemplateRows[$basedOnTemplateUid];
+                $this->includedSysTemplateUids[] = (int)$sysTemplateRow['uid'];
+                $includeNode = new SysTemplateInclude();
+                $name = '[sys_template:' . $sysTemplateRow['uid'] . '] ' . $sysTemplateRow['title'];
+                $includeNode->setName($name);
+                $includeNode->setPid((int)$sysTemplateRow['pid']);
+                if ($this->type === 'constants') {
+                    $includeNode->setLineStream($this->tokenizer->tokenize($sysTemplateRow['constants'] ?? ''));
+                } else {
+                    $includeNode->setLineStream($this->tokenizer->tokenize($sysTemplateRow['config'] ?? ''));
+                }
+                $this->treeFromTokenStreamBuilder->buildTree($includeNode, $this->type, $this->tokenizer);
+                if ($sysTemplateRow['root']) {
+                    $includeNode->setRoot(true);
+                }
+                $clear = $sysTemplateRow['clear'];
+                if (($this->type === 'constants' && $clear & 1)
+                    || ($this->type === 'setup' && $clear & 2)
+                ) {
+                    $includeNode->setClear(true);
+                }
+                $parentNode->addChild($includeNode);
+                $this->handleSysTemplateRecordInclude($includeNode, $sysTemplateRow, $site);
+            }
+        }
+    }
+
+    /**
+     * Handle a single sys_template ['include_static_file'] include.
+     * Looks up file "EXT:/My/Path/include_static_file.txt' in an extension and includes this.
+     * Also loads "EXT:/My/Path/[constants|setup].[typoscript|ts|txt].
+     * Warning: Recursive since an include_static_file.txt file can include other extension's include_static_file.txt again.
+     * This method has no cache-layer usage on its own: handleSingleIncludeStaticFile() which calls this
+     * method is the cache layer here.
+     */
+    private function handleSingleIncludeStaticFile(IncludeInterface $parentNode, $includeStaticFileString): void
+    {
+        if (!PathUtility::isExtensionPath($includeStaticFileString)) {
+            // Must start with 'EXT:'
+            throw new \RuntimeException(
+                'Single include_static_file does not start with "EXT:": ' . $includeStaticFileString,
+                1651137904
+            );
+        }
+
+        // Cut off 'EXT:'
+        $includeStaticFileWithoutExt = substr($includeStaticFileString, 4);
+        $includeStaticFileExtKeyAndPath = GeneralUtility::trimExplode('/', $includeStaticFileWithoutExt, true, 2);
+        if (empty($includeStaticFileExtKeyAndPath[0]) || empty($includeStaticFileExtKeyAndPath[1])) {
+            throw new \RuntimeException(
+                'Syntax of static includes is "EXT:extension_key/Path". Usually enforced as such by ExtensionManagementUtility::addStaticFile',
+                1651138603
+            );
+        }
+        $extensionKey = $includeStaticFileExtKeyAndPath[0];
+        if (!ExtensionManagementUtility::isLoaded($extensionKey)) {
+            return;
+        }
+        // example: '/.../my_extension/Configuration/TypoScript/MyStaticInclude/'
+        $pathSegmentWithAppendedSlash = rtrim($includeStaticFileExtKeyAndPath[1]) . '/';
+        $path = ExtensionManagementUtility::extPath($extensionKey, $pathSegmentWithAppendedSlash);
+
+        // '/.../my_extension/Configuration/TypoScript/MyStaticInclude/include_static_file.txt'
+        $includeStaticFileFileIncludePath = $path . 'include_static_file.txt';
+        if (file_exists($path . 'include_static_file.txt')) {
+            $includeStaticFileFileInclude = new IncludeStaticFileFileInclude();
+            $name = 'EXT:' . $extensionKey . '/' . $pathSegmentWithAppendedSlash . 'include_static_file.txt';
+            $includeStaticFileFileInclude->setName($name);
+            $includeStaticFileFileInclude->setPath($includeStaticFileString);
+            $parentNode->addChild($includeStaticFileFileInclude);
+            $includeStaticFileFileIncludeContent = (string)file_get_contents($includeStaticFileFileIncludePath);
+            // @todo: There is no array_unique() for DB based include_static_file content?!
+            $includeStaticFileFileIncludeArray = array_unique(GeneralUtility::trimExplode(',', $includeStaticFileFileIncludeContent, true));
+            foreach ($includeStaticFileFileIncludeArray as $includeStaticFileFileIncludeString) {
+                $this->handleSingleIncludeStaticFile($includeStaticFileFileInclude, $includeStaticFileFileIncludeString);
+            }
+        }
+
+        $extensions = ['.typoscript', '.ts', '.txt'];
+        foreach ($extensions as $extension) {
+            // '/.../my_extension/Configuration/TypoScript/MyStaticInclude/[constants|setup]' plus one of the allowed extensions like '.typoscript'
+            $fileName = $path . $this->type . $extension;
+            if (file_exists($fileName)) {
+                $fileContent = file_get_contents($fileName);
+                $fileNode = new FileInclude();
+                $name = 'EXT:' . $extensionKey . '/' . $pathSegmentWithAppendedSlash . $this->type . $extension;
+                $fileNode->setName($name);
+                $fileNode->setPath($name);
+                $fileNode->setLineStream($this->tokenizer->tokenize($fileContent));
+                $this->treeFromTokenStreamBuilder->buildTree($fileNode, $this->type, $this->tokenizer);
+                $parentNode->addChild($fileNode);
+            }
+        }
+
+        if ($this->enableStaticMagicIncludes) {
+            $extensionKeyWithoutUnderscores = str_replace('_', '', $extensionKey);
+            $this->addStaticMagicFromGlobals($parentNode, $extensionKeyWithoutUnderscores . '/' . $pathSegmentWithAppendedSlash);
+        }
+    }
+
+    /**
+     * Load 'EXT:my_extension/ext_typoscript_[constants|setup].typoscript'
+     * of *all* loaded extensions if they exist.
+     */
+    private function addExtensionStatics(IncludeInterface $parentNode): void
+    {
+        foreach ($this->packageManager->getActivePackages() as $package) {
+            $extensionKey = $package->getPackageKey();
+            $extensionKeyWithoutUnderscores = str_replace('_', '', $extensionKey);
+            $file = $package->getPackagePath() . 'ext_typoscript_' . $this->type . '.typoscript';
+            if (file_exists($file)) {
+                $identifier = preg_replace('/[^[:alnum:]]/u', '-', 'ext-' . $extensionKey . '-ext-typoscript-' . $this->type . '-typoscript');
+                if ($this->cache) {
+                    $node = $this->cache->require($identifier);
+                    if ($node) {
+                        $parentNode->addChild($node);
+                        continue;
+                    }
+                }
+                $fileContent = file_get_contents($file);
+                $this->addStaticMagicFromGlobals($parentNode, $extensionKeyWithoutUnderscores);
+                $node = new ExtensionStaticInclude();
+                $node->setName('EXT:' . $extensionKey . '/ext_typoscript_' . $this->type . '.typoscript');
+                $node->setPath('EXT:' . $extensionKey . '/ext_typoscript_' . $this->type . '.typoscript');
+                $node->setLineStream($this->tokenizer->tokenize($fileContent));
+                $this->treeFromTokenStreamBuilder->buildTree($node, $this->type, $this->tokenizer);
+                $this->cache?->set($identifier, $this->prepareNodeForCache($node));
+                $parentNode->addChild($node);
+            }
+        }
+    }
+
+    /**
+     * Load default constants TS from $GLOBALS['TYPO3_CONF_VARS']['FE']['defaultTypoScript_[constants|setup]']
+     * whenever 'root=1' is set for a sys_template.
+     */
+    private function addDefaultTypoScriptFromGlobals(IncludeInterface $parentConstantNode): void
+    {
+        $defaultTypoScriptConstants = $GLOBALS['TYPO3_CONF_VARS']['FE']['defaultTypoScript_' . $this->type] ?? '';
+        if (!empty($defaultTypoScriptConstants)) {
+            $cacheIdentifier = 'globals-defaulttyposcript-' . $this->type . '-' . hash('xxh3', $defaultTypoScriptConstants);
+            if ($this->cache) {
+                $node = $this->cache->require($cacheIdentifier);
+                if ($node) {
+                    $parentConstantNode->addChild($node);
+                    return;
+                }
+            }
+            $node = new DefaultTypoScriptInclude();
+            $node->setName('TYPO3_CONF_VARS[\'FE\'][\'defaultTypoScript_' . $this->type . '\']');
+            $node->setLineStream($this->tokenizer->tokenize($defaultTypoScriptConstants));
+            $this->treeFromTokenStreamBuilder->buildTree($node, $this->type, $this->tokenizer);
+            $this->cache?->set($cacheIdentifier, $this->prepareNodeForCache($node));
+            $parentConstantNode->addChild($node);
+        }
+    }
+
+    /**
+     * Load default TS constants from site configuration if that page has a site in rootline.
+     */
+    private function addDefaultTypoScriptConstantsFromSite(IncludeInterface $parentConstantNode, ?SiteInterface $site): void
+    {
+        if (!$site instanceof Site) {
+            return;
+        }
+        $siteConstants = '';
+        $siteSettings = $site->getSettings();
+        if ($siteSettings->isEmpty()) {
+            return;
+        }
+        $cacheIdentifier = 'site-constants-' . hash('xxh3', json_encode($siteSettings, JSON_THROW_ON_ERROR));
+        if ($this->cache) {
+            $node = $this->cache->require($cacheIdentifier);
+            if ($node) {
+                $parentConstantNode->addChild($node);
+                return;
+            }
+        }
+        $siteSettings = $siteSettings->getAllFlat();
+        foreach ($siteSettings as $nodeIdentifier => $value) {
+            $siteConstants .= $nodeIdentifier . ' = ' . $value . LF;
+        }
+        $node = new SiteInclude();
+        $node->setName('Site constants settings of site "' . $site->getIdentifier() . '"');
+        $node->setLineStream($this->tokenizer->tokenize($siteConstants));
+        $this->cache?->set($cacheIdentifier, $this->prepareNodeForCache($node));
+        $parentConstantNode->addChild($node);
+    }
+
+    private function addScopedStaticsFromGlobals(IncludeInterface $parentNode, string $identifier): void
+    {
+        // defaultTypoScript_constants.' or defaultTypoScript_setup.'
+        $source = $GLOBALS['TYPO3_CONF_VARS']['FE']['defaultTypoScript_' . $this->type . '.'][$identifier] ?? null;
+        if (!empty($source)) {
+            $node = new DefaultTypoScriptMagicKeyInclude();
+            $node->setName('TYPO3_CONF_VARS globals_defaultTypoScript_' . $this->type . '.' . $identifier);
+            $node->setLineStream($this->tokenizer->tokenize($source));
+            $this->treeFromTokenStreamBuilder->buildTree($node, $this->type, $this->tokenizer);
+            $parentNode->addChild($node);
+        }
+    }
+
+    private function addContentRenderingFromGlobals(IncludeInterface $parentNode, string $name): void
+    {
+        $source = $GLOBALS['TYPO3_CONF_VARS']['FE']['defaultTypoScript_' . $this->type . '.']['defaultContentRendering'] ?? null;
+        if (!empty($source)) {
+            $node = new DefaultTypoScriptMagicKeyInclude();
+            $node->setName($name);
+            $node->setLineStream($this->tokenizer->tokenize($source));
+            $this->treeFromTokenStreamBuilder->buildTree($node, $this->type, $this->tokenizer);
+            $parentNode->addChild($node);
+        }
+    }
+
+    /**
+     * A rather weird lookup in $GLOBALS['TYPO3_CONF_VARS']['FE'] for magic includes.
+     * See ExtensionManagementUtility::addTypoScript() for more details on this.
+     */
+    private function addStaticMagicFromGlobals(IncludeInterface $parentNode, string $identifier): void
+    {
+        $this->addScopedStaticsFromGlobals($parentNode, $identifier);
+        // If this is a template of type "default content rendering", see if other extensions have added their TypoScript that should be included.
+        if (in_array($identifier, $GLOBALS['TYPO3_CONF_VARS']['FE']['contentRenderingTemplates'], true)) {
+            $this->addContentRenderingFromGlobals($parentNode, 'TYPO3_CONF_VARS defaultContentRendering ' . $this->type . ' for ' . $identifier);
+        }
+    }
+
+    /**
+     * Get 'basedOn' sys_template sub-rows of sys_templates that use this.
+     * Note the 'IN()' query implementation below delivers rows in *any* order. To preserve
+     * basedOn list order, we re-index result rows by uid and then iterate on the original
+     * order of $basedOnTemplateUids in handleIncludeBasedOnTemplates().
+     */
+    private function getBasedOnSysTemplateRowsFromDatabase(array $basedOnTemplateUids): array
+    {
+        $queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_template');
+        $queryBuilder->setRestrictions($this->getSysTemplateQueryRestrictionContainer());
+        $basedOnTemplateRows = $queryBuilder
+            ->select('*')
+            ->from('sys_template')
+            ->where(
+                $queryBuilder->expr()->in(
+                    'uid',
+                    $queryBuilder->createNamedParameter($basedOnTemplateUids, Connection::PARAM_INT_ARRAY)
+                )
+            )
+            ->executeQuery()
+            ->fetchAllAssociative();
+        return array_combine(array_column($basedOnTemplateRows, 'uid'), $basedOnTemplateRows);
+    }
+
+    /**
+     * Calculate a cache identifier for a sys_template row.
+     * This is a bit nifty: There are instances in the wild that add the same TypoScript
+     * sys_template over and over again in a page tree to for instance toggle a single value.
+     * Those content-identical template rows create only one cache entry: We create a hash
+     * from the relevant row fields like 'constants' and 'config', but we do NOT include
+     * the sys_template row 'uid' and 'pid'. So different sys_template rows with the same content
+     * lead to the same identifier, and we cache that just once.
+     *
+     * One additional dependency influences the identifier as well: If the 'clear constants'
+     * flag is set, this row will later trigger loading of constants from given site settings.
+     * When two "first" template rows have the exact same field content in different sites, the
+     * site identifier needs to be added to the hash to still create two different cache entries.
+     */
+    private function getSysTemplateRowIdentifier(array $sysTemplateRow, ?SiteInterface $site): string
+    {
+        $siteIdentifier = 'dummy';
+        if ($this->type === 'constants' && ((int)$sysTemplateRow['clear'] & 1) && $site !== null) {
+            $siteIdentifier = $site->getIdentifier();
+        }
+        $cacheRelevantSysTemplateRowValues = [
+            'root' => (int)$sysTemplateRow['root'],
+            'clear' => (int)$sysTemplateRow['clear'],
+            'include_static_file' => (string)$sysTemplateRow['include_static_file'],
+            'constants' => (string)$sysTemplateRow['constants'],
+            'config' => (string)$sysTemplateRow['config'],
+            'basedOn' => (string)$sysTemplateRow['basedOn'],
+            'includeStaticAfterBasedOn' => (int)$sysTemplateRow['includeStaticAfterBasedOn'],
+            'static_file_mode' => (int)$sysTemplateRow['static_file_mode'],
+            'siteIdentifier' => $siteIdentifier,
+        ];
+        return hash('xxh3', json_encode($cacheRelevantSysTemplateRowValues, JSON_THROW_ON_ERROR));
+    }
+
+    private function prepareNodeForCache(IncludeInterface $node): string
+    {
+        return 'return unserialize(\'' . addcslashes(serialize($node), '\'\\') . '\');';
+    }
+
+    /**
+     * Get sys_template record query builder restrictions.
+     * Allows hidden records if enabled in context.
+     */
+    private function getSysTemplateQueryRestrictionContainer(): DefaultRestrictionContainer
+    {
+        $restrictionContainer = GeneralUtility::makeInstance(DefaultRestrictionContainer::class);
+        if ($this->context->getPropertyFromAspect('visibility', 'includeHiddenContent', false)) {
+            $restrictionContainer->removeByType(HiddenRestriction::class);
+        }
+        return $restrictionContainer;
+    }
+}

Classes/TypoScript/IncludeTree/Traverser/ConditionVerdictAwareIncludeTreeTraverser.php

@@ -0,0 +1,68 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\Traverser;
+
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\IncludeConditionInterface;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\IncludeInterface;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\RootInclude;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\Visitor\IncludeTreeVisitorInterface;
+
+/**
+ * An optimized traverser that does not traverse children when a node is
+ * a condition node that evaluate false.
+ *
+ * This is pretty clever: When adding the ConditionMatcherVisitor as first visitor, it
+ * sets the condition verdict of a ConditionInterface node in visitBeforeChildren().
+ * Adding the AstBuilderVisitor as second visitor, the AstBuilderVisitor will not be
+ * called for ConditionInterface children that did not evaluate to true.
+ * This way, we can both evaluate conditions and build the AST in only one traversing round.
+ *
+ * @internal: Internal tree structure.
+ */
+final class ConditionVerdictAwareIncludeTreeTraverser implements IncludeTreeTraverserInterface
+{
+    public function traverse(RootInclude $rootInclude, array $visitors): void
+    {
+        foreach ($visitors as $visitor) {
+            if (!$visitor instanceof IncludeTreeVisitorInterface) {
+                throw new \RuntimeException(
+                    'Visitors must implement IncludeTreeVisitorInterface',
+                    1689244840
+                );
+            }
+        }
+        $this->traverseRecursive($rootInclude, $visitors, 0);
+    }
+
+    private function traverseRecursive(IncludeInterface $include, array $visitors, int $currentDepth): void
+    {
+        foreach ($visitors as $visitor) {
+            $visitor->visitBeforeChildren($include, $currentDepth);
+        }
+        if ($include instanceof IncludeConditionInterface && !$include->getConditionVerdict()) {
+            // Don't traverse children if condition did not match.
+            return;
+        }
+        foreach ($include->getNextChild() as $child) {
+            $this->traverseRecursive($child, $visitors, $currentDepth + 1);
+            foreach ($visitors as $visitor) {
+                $visitor->visit($child, $currentDepth);
+            }
+        }
+    }
+}

Classes/TypoScript/IncludeTree/Traverser/IncludeTreeTraverser.php

@@ -0,0 +1,56 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\Traverser;
+
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\IncludeInterface;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\RootInclude;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\Visitor\IncludeTreeVisitorInterface;
+
+/**
+ * Traverse all nodes of a RootInclude. Used mostly in backend "Template" module.
+ *
+ * @internal: Internal tree structure.
+ */
+final class IncludeTreeTraverser implements IncludeTreeTraverserInterface
+{
+    public function traverse(RootInclude $rootInclude, array $visitors): void
+    {
+        foreach ($visitors as $visitor) {
+            if (!$visitor instanceof IncludeTreeVisitorInterface) {
+                throw new \RuntimeException(
+                    'Visitors must implement IncludeTreeVisitorInterface',
+                    1689244841
+                );
+            }
+        }
+        $this->traverseRecursive($rootInclude, $visitors, 0);
+    }
+
+    private function traverseRecursive(IncludeInterface $include, array $visitors, int $currentDepth): void
+    {
+        foreach ($visitors as $visitor) {
+            $visitor->visitBeforeChildren($include, $currentDepth);
+        }
+        foreach ($include->getNextChild() as $child) {
+            $this->traverseRecursive($child, $visitors, $currentDepth + 1);
+            foreach ($visitors as $visitor) {
+                $visitor->visit($child, $currentDepth);
+            }
+        }
+    }
+}

Classes/TypoScript/IncludeTree/Traverser/IncludeTreeTraverserInterface.php

@@ -0,0 +1,36 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\Traverser;
+
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\RootInclude;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\Visitor\IncludeTreeVisitorInterface;
+
+/**
+ * Interface implemented by include tree traversers.
+ *
+ * Visitors can be attached and are called for each traversed node.
+ *
+ * @internal: Internal tree structure.
+ */
+interface IncludeTreeTraverserInterface
+{
+    /**
+     * @param IncludeTreeVisitorInterface[] $visitors
+     */
+    public function traverse(RootInclude $rootInclude, array $visitors): void;
+}

Classes/TypoScript/IncludeTree/TreeFromLineStreamBuilder.php

@@ -0,0 +1,421 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree;
+
+use TYPO3\CMS\Core\Resource\Security\FileNameValidator;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\AtImportInclude;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\ConditionElseInclude;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\ConditionInclude;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\ConditionStopInclude;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\DefaultTypoScriptMagicKeyInclude;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\IncludeInterface;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\SegmentInclude;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ConditionElseLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ConditionLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ConditionStopLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ImportLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\LineInterface;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\LineStream;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\Token;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\TokenizerInterface;
+use TYPO3\CMS\Core\Utility\ExtensionManagementUtility;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+/**
+ * Helper class of TreeBuilder classes: This class gets a node with a LineStream - a node
+ * created from a sys_template 'constants' or 'setup' field, or created from a
+ * file import or a string. It then looks for conditions and imports in the attached LineStream
+ * and splits the node into child nodes if needed.
+ *
+ * So while SysTemplateTreeBuilder is all about creating includes from sys_template records
+ * in correct order, this class takes care of conditions and @import within single
+ * source streams.
+ *
+ * This class has no cache-implementation itself: The higher level class caches
+ * include trees of token streams.
+ *
+ * @internal: Internal tree structure.
+ */
+final class TreeFromLineStreamBuilder
+{
+    /** @var 'constants'|'setup'|'other' */
+    private string $type;
+    private TokenizerInterface $tokenizer;
+    private bool $enableMagicIncludes = false;
+
+    /**
+     * Using "@import" with wildcards, the file ending depends on the given type:
+     * With Frontend TypoScript, .typoscript is allowed, with TsConfig, .tsconfig
+     * and .typoscript is allowed. This property maps types to their file suffixes.
+     *
+     * @var array<string, array<int, string>>
+     */
+    private array $atImportTypeToSuffixMap = [
+        'constants' => ['typoscript'],
+        'setup' => ['typoscript'],
+        'other' => ['typoscript'],
+        'tsconfig' => ['typoscript', 'tsconfig'],
+    ];
+
+    public function __construct(
+        private readonly FileNameValidator $fileNameValidator,
+    ) {}
+
+    public function buildTree(IncludeInterface $node, string $type, TokenizerInterface $tokenizer, bool $enableMagicIncludes = true): void
+    {
+        if (!in_array($type, ['constants', 'setup', 'tsconfig', 'other'], true)) {
+            // Type "constants" and "setup" trigger the weird addStaticMagicFromGlobals() resolving, while "other" ignores it.
+            throw new \RuntimeException('type must be either "constants", "setup", "tsconfig" or "other"', 1652741356);
+        }
+        $this->type = $type;
+        $this->tokenizer = $tokenizer;
+        $this->enableMagicIncludes = $enableMagicIncludes;
+        $this->buildTreeInternal($node);
+    }
+
+    /**
+     * This method is a bit tricky and not too easy to follow: It loops over
+     * a given source stream of lines exactly once, but creates a two-level
+     * include node structure from it:
+     *
+     * For instance, when a condition is encountered, it creates a node for the
+     * condition, and the "body" lines of the condition are child nodes of the
+     * condition node. The $previousNode <-> $node juggling handles this: When
+     * the condition body ends (new condition, or [end] or similar), the
+     * next include needs to be attached to the former parent node again.
+     *
+     * Essentially, a single source stream is split into multiple child nodes
+     * when there are conditions or imports. A node that is "split" into
+     * child nodes gets the "split" toggle set, indicating that the entire
+     * source stream is represented by its child nodes.
+     *
+     * A condition body may have more than one child: When there are multiple
+     * file includes, each one creates an own node, which may have children
+     * again. This also means the method is called recursive, since the source
+     * stream of an included file may need to be split into segments again, so
+     * it calls this method again with itself as entry node.
+     */
+    private function buildTreeInternal(IncludeInterface $node): void
+    {
+        $parentNode = $node;
+        $givenTokenLineStream = $node->getLineStream();
+        $lineStream = new LineStream();
+        $childNode = new SegmentInclude();
+        $childNode->setName($node->getName());
+        $childNode->setPath($node->getPath());
+
+        foreach ($givenTokenLineStream->getNextLine() as $line) {
+            if ($line instanceof ConditionLine && $node instanceof ConditionInclude) {
+                // Finish current condition when this line is another condition
+                $node->setSplit();
+                if (!$lineStream->isEmpty()) {
+                    $childNode->setLineStream($lineStream);
+                    $node->addChild($childNode);
+                    $lineStream = new LineStream();
+                }
+                $node = $parentNode;
+            }
+
+            if ($line instanceof ConditionLine) {
+                // A new condition not yet in condition context
+                $node->setSplit();
+                $conditionValueToken = $line->getTokenValue();
+                if (!$lineStream->isEmpty()) {
+                    $childNode->setLineStream($lineStream);
+                    $node->addChild($childNode);
+                    $lineStream = new LineStream();
+                }
+                $childNode = new ConditionInclude();
+                $childNode->setSplit();
+                $childNode->setName($node->getName());
+                $childNode->setPath($node->getPath());
+                $childNode->setConditionToken($conditionValueToken);
+                $lineStream->append($line);
+                $childNode->setLineStream($lineStream);
+                $node->addChild($childNode);
+                $parentNode = $node;
+                $node = $childNode;
+                $childNode = new SegmentInclude();
+                $childNode->setName($node->getName());
+                $childNode->setPath($node->getPath());
+                $lineStream = new LineStream();
+                continue;
+            }
+
+            if (($node instanceof ConditionInclude || $node instanceof ConditionElseInclude)
+                && $line instanceof ConditionStopLine
+            ) {
+                // Finish condition segment due to [end] or [global] line
+                $node->setSplit();
+                $childNode->setLineStream($lineStream);
+                $node->addChild($childNode);
+                $node = $parentNode;
+                $childNode = new ConditionStopInclude();
+                $childNode->setName($node->getName());
+                $childNode->setLineStream((new LineStream())->append($line));
+                $node->addChild($childNode);
+                $childNode = new SegmentInclude();
+                $childNode->setName($node->getName());
+                $childNode->setPath($node->getPath());
+                $lineStream = new LineStream();
+                continue;
+            }
+
+            if ($line instanceof ConditionStopLine) {
+                // [end] or [global] not within open condition context. Fishy. Still finish current
+                // segment, mark node split, add new ConditionStopInclude(), open a new segment.
+                $node->setSplit();
+                if (!$lineStream->isEmpty()) {
+                    $childNode->setLineStream($lineStream);
+                    $node->addChild($childNode);
+                }
+                $childNode = new ConditionStopInclude();
+                $childNode->setName($node->getName());
+                $childNode->setLineStream((new LineStream())->append($line));
+                $node->addChild($childNode);
+                $childNode = new SegmentInclude();
+                $childNode->setName($node->getName());
+                $childNode->setPath($node->getPath());
+                $lineStream = new LineStream();
+                continue;
+            }
+
+            if ($node instanceof ConditionInclude && $line instanceof ConditionElseLine) {
+                // Active condition into [else] condition
+                $node->setSplit();
+                if (!$lineStream->isEmpty()) {
+                    $childNode->setLineStream($lineStream);
+                    $node->addChild($childNode);
+                }
+                $conditionToken = $node->getConditionToken();
+                $node = $parentNode;
+                $childNode = new ConditionElseInclude();
+                $childNode->setSplit();
+                $childNode->setName($node->getName());
+                $childNode->setPath($node->getPath());
+                $childNode->setConditionToken($conditionToken);
+                $lineStream = new LineStream();
+                $lineStream->append($line);
+                $childNode->setLineStream($lineStream);
+                $node->addChild($childNode);
+                $parentNode = $node;
+                $node = $childNode;
+                $childNode = new SegmentInclude();
+                $childNode->setName($node->getName());
+                $childNode->setPath($node->getPath());
+                $lineStream = new LineStream();
+                continue;
+            }
+
+            if ($line instanceof ImportLine) {
+                $node->setSplit();
+                $atImportValueToken = $line->getValueToken();
+                if (!$lineStream->isEmpty()) {
+                    $childNode->setLineStream($lineStream);
+                    $node->addChild($childNode);
+                    $lineStream = new LineStream();
+                }
+                $childNode = new SegmentInclude();
+                $childNode->setName($node->getName());
+                $childNode->setPath($node->getPath());
+                $allowedSuffixes = $this->atImportTypeToSuffixMap[$this->type];
+                foreach ($allowedSuffixes as $allowedSuffix) {
+                    $this->processAtImport($allowedSuffix, $node, $atImportValueToken, $line);
+                }
+                continue;
+            }
+
+            $lineStream->append($line);
+        }
+
+        if ($node->isSplit() && !$lineStream->isEmpty()) {
+            $childNode->setLineStream($lineStream);
+            $node->addChild($childNode);
+        }
+    }
+
+    /**
+     * Process a single '@import'. May add multiple children when '*' wildcards are involved.
+     * Warning: Calls buildTree() recursive for each included file.
+     * Warning: Calls itself recursive for 'relative' lookups.
+     */
+    private function processAtImport(string $fileSuffix, IncludeInterface $node, Token $atImportValueToken, LineInterface $atImportLine, bool $tryRelative = false): void
+    {
+        $atImportValue = $atImportValueToken->getValue();
+        $atImportName = $atImportValue;
+        if ($tryRelative) {
+            if (empty($node->getPath())) {
+                return;
+            }
+            $parentPath = rtrim(dirname($node->getPath()), '/') . '/';
+            $atImportValue = ltrim($atImportValue, './');
+            $atImportName = preg_replace('#([:/])[^:/]+$#', '$1', $node->getName()) . $atImportValue;
+            $atImportValue = $parentPath . $atImportValue;
+        }
+        $absoluteFileName = rtrim(GeneralUtility::getFileAbsFileName($atImportValue), '/');
+        if ($absoluteFileName === '') {
+            return;
+        }
+        if (str_ends_with($absoluteFileName, '.' . $fileSuffix) && is_file($absoluteFileName)) {
+            // Simple file with allowed file suffix
+            if ($this->fileNameValidator->isValid($absoluteFileName)) {
+                $this->addSingleAtImportFile($node, $absoluteFileName, $atImportValue, $atImportName, $atImportLine);
+                $this->addStaticMagicFromGlobals($node, $atImportValue);
+            }
+        } elseif (is_dir($absoluteFileName)) {
+            // Directories with and without ending /
+            $filesAndDirs = scandir($absoluteFileName);
+            foreach ($filesAndDirs as $potentialInclude) {
+                if (!str_ends_with($potentialInclude, '.' . $fileSuffix)
+                    || is_dir($absoluteFileName . '/' . $potentialInclude)
+                    || !$this->fileNameValidator->isValid($absoluteFileName . '/' . $potentialInclude)
+                ) {
+                    continue;
+                }
+                $singleAbsoluteFileName = $absoluteFileName . '/' . $potentialInclude;
+                $identifier = rtrim($atImportValue, '/') . '/' . $potentialInclude;
+                $this->addSingleAtImportFile($node, $singleAbsoluteFileName, $identifier, $identifier, $atImportLine);
+                $this->addStaticMagicFromGlobals($node, $identifier);
+            }
+        } elseif (is_file($absoluteFileName . '.' . $fileSuffix)) {
+            // File without .typoscript / .tsconfig suffix, but exists when suffix is added
+            if ($this->fileNameValidator->isValid($absoluteFileName . '.' . $fileSuffix)) {
+                $singleAbsoluteFileName = $absoluteFileName . '.' . $fileSuffix;
+                $identifier = $atImportValue . '.' . $fileSuffix;
+                $this->addSingleAtImportFile($node, $singleAbsoluteFileName, $identifier, $identifier, $atImportLine);
+                $this->addStaticMagicFromGlobals($node, $identifier);
+            }
+        } elseif (str_contains($absoluteFileName, '*')) {
+            // Something with *
+            $directory = rtrim(dirname($absoluteFileName) . '/');
+            $directoryExists = is_dir($directory);
+            if (!$directoryExists && str_starts_with($atImportValue, './') && !$tryRelative) {
+                // See if we can import some relative wildcard like "./Setup/*" or "./Setup/*.typoscript"
+                $this->processAtImport($fileSuffix, $node, $atImportValueToken, $atImportLine, true);
+                return;
+            }
+            if (!$directoryExists) {
+                // Absolute directory. There is nothing to import if the directory does not exist.
+                return;
+            }
+            $filePattern = basename($absoluteFileName);
+            if (!str_contains($filePattern, '*')) {
+                // The * wildcard must occur in the filename, wildcards in directories are not handled.
+                return;
+            }
+            if (mb_substr_count($filePattern, '*') > 1) {
+                // Only one wildcard character is allowed, foo*.bar*.typoscript is considered an invalid pattern.
+                return;
+            }
+            // Normalize right side, making sure it always ends with $fileSuffix ".typoscript" / ".tsconfig"
+            if (str_ends_with($filePattern, $fileSuffix)) {
+                $filePattern = mb_substr($filePattern, 0, -1 * strlen($fileSuffix));
+                $filePattern = rtrim($filePattern, '.');
+            }
+            $filePattern = $filePattern . '.' . $fileSuffix;
+            $wildcardPosition = mb_strpos($filePattern, '*');
+            $leftPrefix = mb_substr($filePattern, 0, $wildcardPosition);
+            $rightPrefix = mb_substr($filePattern, $wildcardPosition + 1);
+            $filesAndDirs = scandir($directory);
+            foreach ($filesAndDirs as $potentialInclude) {
+                if ($potentialInclude === '.'
+                    || $potentialInclude === '..'
+                    || !str_starts_with($potentialInclude, $leftPrefix)
+                    || !str_ends_with($potentialInclude, $rightPrefix)
+                    || is_dir($directory . $potentialInclude)
+                    || !$this->fileNameValidator->isValid($directory . $potentialInclude)
+                ) {
+                    continue;
+                }
+                $singleAbsoluteFileName = $directory . $potentialInclude;
+                $identifier = rtrim(dirname($atImportValue), '/') . '/' . $potentialInclude;
+                $this->addSingleAtImportFile($node, $singleAbsoluteFileName, $identifier, $identifier, $atImportLine);
+                $this->addStaticMagicFromGlobals($node, $identifier);
+            }
+        } elseif (!$tryRelative) {
+            // See if we can import relative "./foo.typoscript" or "foo.typoscript"
+            $this->processAtImport($fileSuffix, $node, $atImportValueToken, $atImportLine, true);
+        }
+    }
+
+    /**
+     * Get content of a single @import file and add to current node as child.
+     *
+     * Warning: Recursively calls buildTree() to process includes of included content.
+     */
+    private function addSingleAtImportFile(
+        IncludeInterface $parentNode,
+        string $absoluteFileName,
+        string $path,
+        string $name,
+        LineInterface $atImportLine
+    ): void {
+        $content = file_get_contents($absoluteFileName);
+        $newNode = new AtImportInclude();
+        $newNode->setName($name);
+        $newNode->setPath($path);
+        $newNode->setLineStream($this->tokenizer->tokenize($content));
+        $newNode->setOriginalLine($atImportLine);
+        $this->buildTreeInternal($newNode);
+        $parentNode->addChild($newNode);
+    }
+
+    /**
+     * A rather weird lookup in $GLOBALS['TYPO3_CONF_VARS']['FE'] for magic includes.
+     * See ExtensionManagementUtility::addTypoScript() for more details on this.
+     * Warning: Yes, this is recursive again.
+     */
+    private function addStaticMagicFromGlobals(IncludeInterface $parentNode, string $path): void
+    {
+        if (!in_array($this->type, ['constants', 'setup'], true) || !str_starts_with($path, 'EXT:')) {
+            // This magic method is relevant for Frontend TypoScript only, indicated by
+            // $this->type being either "constants" or "setup".
+            return;
+        }
+        $includeStaticFileWithoutExt = substr($path, 4);
+        $includeStaticFileExtKeyAndPath = GeneralUtility::trimExplode('/', $includeStaticFileWithoutExt, true, 2);
+        $extensionKey = $includeStaticFileExtKeyAndPath[0];
+        $extensionKeyWithoutUnderscores = str_replace('_', '', $extensionKey);
+        if (!$extensionKeyWithoutUnderscores || !ExtensionManagementUtility::isLoaded($extensionKey)) {
+            return;
+        }
+        // example: 'Configuration/TypoScript/MyStaticInclude/'
+        $pathSegmentWithAppendedSlash = rtrim(dirname($includeStaticFileExtKeyAndPath[1])) . '/';
+        $file = basename($path);
+        $type = GeneralUtility::trimExplode('.', $file, false, 2)[0] ?? '';
+        if ($type !== $this->type) {
+            return;
+        }
+        $globalsLookup = $extensionKeyWithoutUnderscores . '/' . $pathSegmentWithAppendedSlash;
+
+        if (!$this->enableMagicIncludes) {
+            return;
+        }
+        // If this is a template of type "default content rendering", see if other extensions have added their TypoScript that should be included.
+        if (in_array($globalsLookup, $GLOBALS['TYPO3_CONF_VARS']['FE']['contentRenderingTemplates'], true)) {
+            $source = $GLOBALS['TYPO3_CONF_VARS']['FE']['defaultTypoScript_' . $type . '.']['defaultContentRendering'] ?? null;
+            if (!empty($source)) {
+                $node = new DefaultTypoScriptMagicKeyInclude();
+                $node->setName('TYPO3_CONF_VARS defaultContentRendering for ' . $path);
+                $node->setLineStream($this->tokenizer->tokenize($source));
+                $this->buildTreeInternal($node);
+                $parentNode->addChild($node);
+            }
+        }
+    }
+}

Classes/TypoScript/IncludeTree/TsConfigTreeBuilder.php

@@ -0,0 +1,342 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree;
+
+use Symfony\Component\DependencyInjection\Attribute\Autoconfigure;
+use TYPO3\CMS\Core\Authentication\BackendUserAuthentication;
+use TYPO3\CMS\Core\Cache\Frontend\PhpFrontend;
+use TYPO3\CMS\Core\Core\Environment;
+use TYPO3\CMS\Core\EventDispatcher\EventDispatcher;
+use TYPO3\CMS\Core\Exception\SiteNotFoundException;
+use TYPO3\CMS\Core\Package\Cache\PackageDependentCacheIdentifier;
+use TYPO3\CMS\Core\Package\PackageManager;
+use TYPO3\CMS\Core\Site\Set\SetRegistry;
+use TYPO3\CMS\Core\Site\SiteFinder;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\Event\BeforeLoadedPageTsConfigEvent;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\Event\BeforeLoadedUserTsConfigEvent;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\Event\ModifyLoadedPageTsConfigEvent;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\RootInclude;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\TsConfigInclude;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\TokenizerInterface;
+use TYPO3\CMS\Core\Utility\ExtensionManagementUtility;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Core\Utility\PathUtility;
+
+/**
+ * Build include tree for user TSconfig and page TSconfig. This is typically used only by
+ * UserTsConfigFactory and PageTsConfigFactory.
+ *
+ * @internal
+ */
+#[Autoconfigure(public: true)]
+final readonly class TsConfigTreeBuilder
+{
+    public function __construct(
+        private TreeFromLineStreamBuilder $treeFromTokenStreamBuilder,
+        private PackageManager $packageManager,
+        private EventDispatcher $eventDispatcher,
+        private SiteFinder $siteFinder,
+        private SetRegistry $setRegistry,
+    ) {}
+
+    public function getUserTsConfigTree(
+        BackendUserAuthentication $backendUser,
+        TokenizerInterface $tokenizer,
+        ?PhpFrontend $cache = null
+    ): RootInclude {
+        $includeTree = new RootInclude();
+
+        $collectedUserTsConfigArray = [];
+        $gotPackagesUserTsConfigFromCache = false;
+        $cacheIdentifier = (new PackageDependentCacheIdentifier($this->packageManager))
+            ->withPrefix('usertsconfig-packages-strings')
+            ->toString();
+        if ($cache) {
+            $collectedUserTsConfigArrayFromCache = $cache->require($cacheIdentifier);
+            if ($collectedUserTsConfigArrayFromCache) {
+                $gotPackagesUserTsConfigFromCache = true;
+                $collectedUserTsConfigArray = $collectedUserTsConfigArrayFromCache;
+            }
+        }
+        if (!$gotPackagesUserTsConfigFromCache) {
+            $event = $this->eventDispatcher->dispatch(new BeforeLoadedUserTsConfigEvent());
+            $collectedUserTsConfigArray = $event->getTsConfig();
+            foreach ($this->packageManager->getActivePackages() as $package) {
+                $packagePath = $package->getPackagePath();
+                $tsConfigFile = null;
+                if (file_exists($packagePath . 'Configuration/user.tsconfig')) {
+                    $tsConfigFile = $packagePath . 'Configuration/user.tsconfig';
+                } elseif (file_exists($packagePath . 'Configuration/User.tsconfig')) {
+                    $tsConfigFile = $packagePath . 'Configuration/User.tsconfig';
+                }
+                if ($tsConfigFile) {
+                    $typoScriptString = @file_get_contents($tsConfigFile);
+                    if (!empty($typoScriptString)) {
+                        $collectedUserTsConfigArray['userTsConfig-package-' . $package->getPackageKey()] = $typoScriptString;
+                    }
+                }
+            }
+            $cache?->set($cacheIdentifier, 'return unserialize(\'' . addcslashes(serialize($collectedUserTsConfigArray), '\'\\') . '\');');
+        }
+        foreach ($collectedUserTsConfigArray as $key => $typoScriptString) {
+            $includeTree->addChild($this->getTreeFromString((string)$key, $typoScriptString, $tokenizer, $cache));
+        }
+
+        foreach ($backendUser->userGroupsUID as $groupId) {
+            // Loop through all groups and add their 'TSconfig' fields
+            if (!empty($backendUser->userGroups[$groupId]['TSconfig'] ?? '')) {
+                $includeTree->addChild($this->getTreeFromString('userTsConfig-group-' . $groupId, $backendUser->userGroups[$groupId]['TSconfig'], $tokenizer, $cache));
+            }
+            if (trim($backendUser->userGroups[$groupId]['tsconfig_includes'] ?? '')) {
+                $includeTsConfigFileList = GeneralUtility::trimExplode(',', $backendUser->userGroups[$groupId]['tsconfig_includes'], true);
+                foreach ($includeTsConfigFileList as $key => $includeTsConfigFile) {
+                    $content = $this->getContentOfTsconfigFile($includeTsConfigFile);
+                    if (!empty($content)) {
+                        $includeTree->addChild($this->getTreeFromString('userTsConfig-include-group' . $key, $content, $tokenizer, $cache));
+                    }
+                }
+            }
+        }
+        if (!empty($backendUser->user['TSconfig'] ?? '')) {
+            $includeTree->addChild($this->getTreeFromString('userTsConfig-user', $backendUser->user['TSconfig'], $tokenizer, $cache));
+        }
+        if (trim($backendUser->user['tsconfig_includes'] ?? '')) {
+            $includeTsConfigFileList = GeneralUtility::trimExplode(',', $backendUser->user['tsconfig_includes'], true);
+            foreach ($includeTsConfigFileList as $key => $includeTsConfigFile) {
+                $content = $this->getContentOfTsconfigFile($includeTsConfigFile);
+                if (!empty($content)) {
+                    $includeTree->addChild($this->getTreeFromString('userTsConfig-include-user' . $key, $content, $tokenizer, $cache));
+                }
+            }
+        }
+
+        return $includeTree;
+    }
+
+    public function getPagesTsConfigTree(
+        array $rootLine,
+        TokenizerInterface $tokenizer,
+        ?PhpFrontend $cache = null
+    ): RootInclude {
+        $collectedPagesTsConfigArray = [];
+
+        $collectedPagesTsConfigArray += $this->getPackagePageTsConfigTree($cache);
+
+        // HEADS up: rootLine may be modified by getSitePagesTsConfigTree
+        $collectedPagesTsConfigArray += $this->getSitePageTsConfigTree($rootLine, $cache);
+
+        $collectedPagesTsConfigArray += $this->getRootlinePageTsConfigTree($rootLine, $cache);
+
+        $event = $this->eventDispatcher->dispatch(new ModifyLoadedPageTsConfigEvent(
+            array_map(static fn(array $descriptor): string => $descriptor['content'], $collectedPagesTsConfigArray),
+            $rootLine
+        ));
+        $collectedPagesTsConfigContentArray = $event->getTsConfig();
+        foreach ($collectedPagesTsConfigContentArray as $key => $content) {
+            $collectedPagesTsConfigArray[$key]['content'] = $content;
+        }
+
+        $includeTree = new RootInclude();
+        foreach ($collectedPagesTsConfigArray as $key => $descriptor) {
+            $typoScriptString = $descriptor['content'];
+            $filename = $descriptor['filename'] ?? null;
+            $includeTree->addChild($this->getTreeFromString((string)$key, $typoScriptString, $tokenizer, $cache, $filename));
+        }
+        return $includeTree;
+    }
+
+    private function getPackagePageTsConfigTree(
+        ?PhpFrontend $cache = null
+    ): array {
+        $collectedPagesTsConfigArray = [];
+        $gotPackagesPagesTsConfigFromCache = false;
+        $cacheIdentifier = (new PackageDependentCacheIdentifier($this->packageManager))
+            ->withPrefix('pagestsconfig-packages-strings')
+            ->toString();
+        if ($cache) {
+            $collectedPagesTsConfigArrayFromCache = $cache->require($cacheIdentifier);
+            if ($collectedPagesTsConfigArrayFromCache) {
+                $gotPackagesPagesTsConfigFromCache = true;
+                $collectedPagesTsConfigArray = $collectedPagesTsConfigArrayFromCache;
+            }
+        }
+        if (!$gotPackagesPagesTsConfigFromCache) {
+            $event = $this->eventDispatcher->dispatch(new BeforeLoadedPageTsConfigEvent());
+            $collectedPagesTsConfigArray = array_map(static fn(string $config): array => ['content' => $config, 'filename' => null], $event->getTsConfig());
+            foreach ($this->packageManager->getActivePackages() as $package) {
+                $packagePath = $package->getPackagePath();
+                $tsConfigFile = null;
+                if (file_exists($packagePath . 'Configuration/page.tsconfig')) {
+                    $tsConfigFile = $packagePath . 'Configuration/page.tsconfig';
+                } elseif (file_exists($packagePath . 'Configuration/Page.tsconfig')) {
+                    $tsConfigFile = $packagePath . 'Configuration/Page.tsconfig';
+                }
+                if ($tsConfigFile) {
+                    $typoScriptString = @file_get_contents($tsConfigFile);
+                    if (!empty($typoScriptString)) {
+                        $collectedPagesTsConfigArray['pagesTsConfig-package-' . $package->getPackageKey()] = [
+                            'filename' => $tsConfigFile,
+                            'content' => $typoScriptString,
+                        ];
+                    }
+                }
+            }
+            $cache?->set($cacheIdentifier, 'return unserialize(\'' . addcslashes(serialize($collectedPagesTsConfigArray), '\'\\') . '\');');
+        }
+        return $collectedPagesTsConfigArray;
+    }
+
+    private function getSitePageTsConfigTree(
+        array &$rootLine,
+        ?PhpFrontend $cache = null
+    ): array {
+        $reverseRootLine = array_reverse($rootLine);
+        $rootlineUntilSite = [];
+        $rootSite = null;
+        foreach ($reverseRootLine as $rootLineEntry) {
+            array_unshift($rootlineUntilSite, $rootLineEntry);
+            $uid = (int)($rootLineEntry['uid'] ?? 0);
+            if ($uid === 0) {
+                continue;
+            }
+            try {
+                $site = $this->siteFinder->getSiteByRootPageId($uid);
+            } catch (SiteNotFoundException) {
+                continue;
+            }
+            if ($site->isTypoScriptRoot()) {
+                $rootSite = $site;
+                $rootLine = $rootlineUntilSite;
+                break;
+            }
+        }
+
+        if ($rootSite === null) {
+            return [];
+        }
+
+        $cacheIdentifier = (new PackageDependentCacheIdentifier($this->packageManager))
+            ->withPrefix('pagestsconfig-site')
+            ->withAdditionalHashedIdentifier($rootSite->getIdentifier())
+            ->toString();
+        $pageTsConfig = $cache?->require($cacheIdentifier) ?: null;
+
+        if ($pageTsConfig === null) {
+            $pageTsConfig = [];
+            $sets = $this->setRegistry->getSets(...$rootSite->getSets());
+            foreach ($sets as $set) {
+                if ($set->pagets === null) {
+                    continue;
+                }
+                $filename = GeneralUtility::getFileAbsFileName($set->pagets);
+                if (!file_exists($filename)) {
+                    continue;
+                }
+                $content = @file_get_contents($filename);
+                if (!empty($content)) {
+                    $pageTsConfig['pageTsConfig-set-' . str_replace('/', '-', $set->name)] = [
+                        'filename' => $filename,
+                        'content' => $content,
+                    ];
+                }
+            }
+
+            $pageTsConfig['pageTsConfig-site-' . $rootSite->getIdentifier()] = [
+                'filename' => GeneralUtility::getFileAbsFileName(Environment::getConfigPath() . '/sites/' . $rootSite->getIdentifier() . '/page.tsconfig'),
+                'content' => $rootSite->getTSconfig()->pageTSconfig ?? '',
+            ];
+            $cache?->set($cacheIdentifier, 'return ' . var_export($pageTsConfig, true) . ';');
+        }
+        return $pageTsConfig;
+    }
+
+    private function getRootlinePageTsConfigTree(
+        array $rootLine,
+        ?PhpFrontend $cache = null
+    ): array {
+        $collectedPagesTsConfigArray = [];
+        foreach ($rootLine as $page) {
+            if (empty($page['uid'])) {
+                // Page 0 can happen when the rootline is given from BE context. It has not TSconfig. Skip this.
+                continue;
+            }
+            if (trim($page['tsconfig_includes'] ?? '')) {
+                $includeTsConfigFileList = GeneralUtility::trimExplode(',', $page['tsconfig_includes'], true);
+                foreach ($includeTsConfigFileList as $key => $includeTsConfigFile) {
+                    $content = $this->getContentOfTsconfigFile($includeTsConfigFile);
+                    if (!empty($content)) {
+                        $collectedPagesTsConfigArray['pagesTsConfig-page-' . $page['uid'] . '-includes-' . $key] = [
+                            'content' => $content,
+                        ];
+                    }
+                }
+            }
+            if (!empty($page['TSconfig'])) {
+                $collectedPagesTsConfigArray['pagesTsConfig-page-' . $page['uid'] . '-tsConfig'] = ['content' => $page['TSconfig']];
+            }
+        }
+        return $collectedPagesTsConfigArray;
+    }
+
+    private function getContentOfTsconfigFile(string $path): string
+    {
+        if (PathUtility::isExtensionPath($path)) {
+            [$includeTsConfigFileExtensionKey, $includeTsConfigFilename] = explode('/', substr($path, 4), 2);
+            if ($includeTsConfigFilename !== ''
+                && $includeTsConfigFileExtensionKey !== ''
+                && ExtensionManagementUtility::isLoaded($includeTsConfigFileExtensionKey)
+            ) {
+                $extensionPath = ExtensionManagementUtility::extPath($includeTsConfigFileExtensionKey);
+                $includeTsConfigFileAndPath = PathUtility::getCanonicalPath($extensionPath . $includeTsConfigFilename);
+                if (str_starts_with($includeTsConfigFileAndPath, $extensionPath) && file_exists($includeTsConfigFileAndPath)) {
+                    return (string)file_get_contents($includeTsConfigFileAndPath);
+                }
+            }
+        }
+        return '';
+    }
+
+    private function getTreeFromString(
+        string $name,
+        string $typoScriptString,
+        TokenizerInterface $tokenizer,
+        ?PhpFrontend $cache = null,
+        ?string $filename = null,
+    ): TsConfigInclude {
+        $lowercaseName = mb_strtolower($name);
+        $identifier = (new PackageDependentCacheIdentifier($this->packageManager))
+            ->withPrefix($lowercaseName)
+            ->withAdditionalHashedIdentifier($typoScriptString)
+            ->toString();
+        if ($cache) {
+            $includeNode = $cache->require($identifier);
+            if ($includeNode instanceof TsConfigInclude) {
+                return $includeNode;
+            }
+        }
+        $includeNode = new TsConfigInclude();
+        $includeNode->setName($name);
+        if ($filename !== null) {
+            $includeNode->setPath($filename);
+        }
+        $includeNode->setLineStream($tokenizer->tokenize($typoScriptString));
+        $this->treeFromTokenStreamBuilder->buildTree($includeNode, 'tsconfig', $tokenizer);
+        $cache?->set($identifier, 'return unserialize(\'' . addcslashes(serialize($includeNode), '\'\\') . '\');');
+        return $includeNode;
+    }
+}

Classes/TypoScript/IncludeTree/Visitor/IncludeTreeAstBuilderVisitor.php

@@ -0,0 +1,96 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\Visitor;
+
+use Symfony\Component\DependencyInjection\Attribute\Autoconfigure;
+use TYPO3\CMS\Core\TypoScript\AST\AstBuilderInterface;
+use TYPO3\CMS\Core\TypoScript\AST\Node\RootNode;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\IncludeInterface;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\SysTemplateInclude;
+
+/**
+ * Main visitor that creates the TypoScript AST: When adding this visitor
+ * and traversing the IncludeTree, the final AST can be fetched using getAst().
+ *
+ * This visitor is usually only used together with ConditionVerdictAwareIncludeTreeTraverser,
+ * and the IncludeTreeConditionMatcherVisitor is added *before* this visitor to determine
+ * condition verdicts, so AST is only extended for conditions with "true" verdict.
+ *
+ * When parsing "setup", "flattened" constants should be assigned to this visitor, so
+ * the AstBuilder can resolve constants.
+ *
+ * @internal: Internal tree structure.
+ */
+
+// Ast builder visitor creates state and should not be re-used
+#[Autoconfigure(public: true, shared: false)]
+final class IncludeTreeAstBuilderVisitor implements IncludeTreeVisitorInterface
+{
+    private RootNode $ast;
+
+    /**
+     * @var array<string, string>
+     */
+    private array $flatConstants = [];
+
+    public function __construct(private readonly AstBuilderInterface $astBuilder)
+    {
+        $this->ast = new RootNode();
+    }
+
+    /**
+     * When 'setup' is parsed, setting resolved flat constants here will make
+     * the AST builder substitute these constants.
+     *
+     * @param array<string, string> $flatConstants
+     */
+    public function setFlatConstants(array $flatConstants): void
+    {
+        $this->flatConstants = $flatConstants;
+    }
+
+    public function getAst(): RootNode
+    {
+        return $this->ast;
+    }
+
+    /**
+     * Reset AST if "clear" flag is set. That's a sys_template record specific thing
+     * to restart with a new RootNode and drop any AST calculated already.
+     */
+    public function visitBeforeChildren(IncludeInterface $include, int $currentDepth): void
+    {
+        if ($include instanceof SysTemplateInclude && $include->isClear()) {
+            // Reset any given AST if this sys_template row has clear flag (constants or setup clear) set.
+            $this->ast = new RootNode();
+        }
+    }
+
+    /**
+     * Extend current AST with given LineStream of include node.
+     */
+    public function visit(IncludeInterface $include, int $currentDepth): void
+    {
+        $lineStream = $include->getLineStream();
+        if ($lineStream && !$include->isSplit()) {
+            // A "split" include means that the entire TypoScript is split into child includes. The
+            // TokenStream of the split include itself must not be parsed, so it's excluded here.
+            $this->ast = $this->astBuilder->build($lineStream, $this->ast, $this->flatConstants);
+        }
+    }
+}

Classes/TypoScript/IncludeTree/Visitor/IncludeTreeCommentAwareAstBuilderVisitor.php

@@ -0,0 +1,89 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\Visitor;
+
+use Symfony\Component\DependencyInjection\Attribute\Autoconfigure;
+use TYPO3\CMS\Core\TypoScript\AST\CommentAwareAstBuilder;
+use TYPO3\CMS\Core\TypoScript\AST\Node\RootNode;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\IncludeInterface;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\SysTemplateInclude;
+
+/**
+ * Secondary visitor that creates the TypoScript AST: When adding this visitor
+ * and traversing the IncludeTree, the final AST can be fetched using getAst().
+ * This is an "extended" version of IncludeTreeAstBuilderVisitor that uses
+ * the CommentAwareAstBuilder instead of the AstBuilder to build the AST: This special
+ * AST builder is comment aware and adds TypoScript comments to nodes.
+ *
+ * This visitor is used in ext:tstemplate TypoScript modules and ext:backend page TSconfig
+ * to allow implementation of the "comment" related functionality.
+ *
+ * When parsing "setup", "flattened" constants should be assigned to this visitor, so
+ * the AstBuilder can resolve constants.
+ *
+ * @internal: Internal tree structure.
+ */
+
+// This Ast builder visitor creates state and should not be re-used
+#[Autoconfigure(public: true, shared: false)]
+final class IncludeTreeCommentAwareAstBuilderVisitor implements IncludeTreeVisitorInterface
+{
+    private RootNode $ast;
+
+    /**
+     * @var array<string, string>
+     */
+    private array $flatConstants = [];
+
+    public function __construct(private readonly CommentAwareAstBuilder $astBuilder)
+    {
+        $this->ast = new RootNode();
+    }
+
+    /**
+     * When 'setup' is parsed, setting resolved flat constants here will make
+     * the AST builder substitute these constants.
+     *
+     * @param array<string, string> $flatConstants
+     */
+    public function setFlatConstants(array $flatConstants): void
+    {
+        $this->flatConstants = $flatConstants;
+    }
+
+    public function getAst(): RootNode
+    {
+        return $this->ast;
+    }
+
+    public function visitBeforeChildren(IncludeInterface $include, int $currentDepth): void
+    {
+        if ($include instanceof SysTemplateInclude && $include->isClear()) {
+            // Reset any given AST if this sys_template row has clear flag (constants or setup clear) set.
+            $this->ast = new RootNode();
+        }
+    }
+
+    public function visit(IncludeInterface $include, int $currentDepth): void
+    {
+        $tokenStream = $include->getLineStream();
+        if ($tokenStream && !$include->isSplit()) {
+            $this->ast = $this->astBuilder->build($tokenStream, $this->ast, $this->flatConstants);
+        }
+    }
+}

Classes/TypoScript/IncludeTree/Visitor/IncludeTreeConditionAggregatorVisitor.php

@@ -0,0 +1,68 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\Visitor;
+
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\IncludeConditionInterface;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\IncludeInterface;
+
+/**
+ * Gather conditions in an IncludeTree.
+ *
+ * This visitor is used in ext:tstemplate TypoScript modules and ext:backend page TSconfig
+ * backend modules to find available conditions and make them toggleable.
+ *
+ * @internal This is a specific Backend implementation and is not considered part of the Public TYPO3 API.
+ */
+final class IncludeTreeConditionAggregatorVisitor implements IncludeTreeVisitorInterface
+{
+    /**
+     * @var array<int, array<string, string>>
+     */
+    private array $conditions = [];
+
+    /**
+     * Get accumulated conditions gathered by visit().
+     */
+    public function getConditions(): array
+    {
+        return $this->conditions;
+    }
+
+    public function visitBeforeChildren(IncludeInterface $include, int $currentDepth): void
+    {
+        // No-op. Magic happens in visit()
+    }
+
+    /**
+     * If the given include is an IncludeConditionInterface, grab it's original (unchanged by constants)
+     * condition token.
+     */
+    public function visit(IncludeInterface $include, int $currentDepth): void
+    {
+        if (!$include instanceof IncludeConditionInterface) {
+            return;
+        }
+        $condition = $include->getConditionToken()->getValue();
+        if (!in_array($condition, array_column($this->conditions, 'value'))) {
+            $this->conditions[] = [
+                'value' => $condition,
+                'originalValue' => $include->getOriginalConditionToken()?->getValue(),
+            ];
+        }
+    }
+}

Classes/TypoScript/IncludeTree/Visitor/IncludeTreeConditionEnforcerVisitor.php

@@ -0,0 +1,59 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\Visitor;
+
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\IncludeConditionInterface;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\IncludeInterface;
+
+/**
+ * Force condition verdicts.
+ *
+ * This visitor is used in ext:tstemplate TypoScript modules and ext:backend page TSconfig
+ * backend modules to toggle on/off selected conditions.
+ *
+ * @internal This is a specific Backend implementation and is not considered part of the Public TYPO3 API.
+ */
+final class IncludeTreeConditionEnforcerVisitor implements IncludeTreeVisitorInterface
+{
+    /**
+     * @var array<int, string>
+     */
+    private array $enabledConditions;
+
+    public function setEnabledConditions(array $enabledConditions): void
+    {
+        $this->enabledConditions = $enabledConditions;
+    }
+
+    public function visitBeforeChildren(IncludeInterface $include, int $currentDepth): void
+    {
+        if (!$include instanceof IncludeConditionInterface) {
+            return;
+        }
+        $conditionValue = $include->getConditionToken()->getValue();
+        if (in_array($conditionValue, $this->enabledConditions) && !$include->isConditionNegated()
+            || !in_array($conditionValue, $this->enabledConditions) && $include->isConditionNegated()
+        ) {
+            $include->setConditionVerdict(true);
+        } else {
+            $include->setConditionVerdict(false);
+        }
+    }
+
+    public function visit(IncludeInterface $include, int $currentDepth): void {}
+}

Classes/TypoScript/IncludeTree/Visitor/IncludeTreeConditionIncludeListAccumulatorVisitor.php

@@ -0,0 +1,64 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\Visitor;
+
+use Symfony\Component\DependencyInjection\Attribute\Autoconfigure;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\IncludeConditionInterface;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\IncludeInterface;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\RootInclude;
+
+/**
+ * This is used in FE to "gather" condition nodes as a flat tree (root + condition nodes).
+ * The FE uses this optimized tree to quickly determine condition verdicts without loading
+ * the full tree.
+ *
+ * @internal: Internal tree structure.
+ */
+
+// This visitor creates state and should not be re-used
+#[Autoconfigure(public: true, shared: false)]
+final class IncludeTreeConditionIncludeListAccumulatorVisitor implements IncludeTreeVisitorInterface
+{
+    private RootInclude $rootInclude;
+
+    public function __construct()
+    {
+        $this->rootInclude = new RootInclude();
+    }
+
+    public function getConditionIncludes(): RootInclude
+    {
+        return $this->rootInclude;
+    }
+
+    public function visitBeforeChildren(IncludeInterface $include, int $currentDepth): void
+    {
+        if (!$include instanceof IncludeConditionInterface) {
+            return;
+        }
+        /** @var IncludeConditionInterface&IncludeInterface $newConditionInclude */
+        $newConditionInclude = (new ($include::class));
+        $newConditionInclude->setConditionToken($include->getConditionToken());
+        $this->rootInclude->addChild($newConditionInclude);
+    }
+
+    public function visit(IncludeInterface $include, int $currentDepth): void
+    {
+        // Noop, just implement interface.
+    }
+}

Classes/TypoScript/IncludeTree/Visitor/IncludeTreeConditionMatcherVisitor.php

@@ -0,0 +1,200 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\Visitor;
+
+use Psr\Log\LoggerInterface;
+use Symfony\Component\DependencyInjection\Attribute\Autoconfigure;
+use Symfony\Component\ExpressionLanguage\SyntaxError;
+use TYPO3\CMS\Backend\Utility\BackendUtility;
+use TYPO3\CMS\Core\Context\Context;
+use TYPO3\CMS\Core\Context\UserAspect;
+use TYPO3\CMS\Core\Context\WorkspaceAspect;
+use TYPO3\CMS\Core\ExpressionLanguage\RequestWrapper;
+use TYPO3\CMS\Core\ExpressionLanguage\Resolver;
+use TYPO3\CMS\Core\Page\PageLayoutResolver;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\IncludeConditionInterface;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\IncludeInterface;
+
+/**
+ * A visitor that looks at IncludeConditionInterface nodes and
+ * evaluates their conditions.
+ *
+ * Condition matching is done in visitBeforeChildren() to be used in combination with
+ * ConditionVerdictAwareIncludeTreeTraverser, so children are only traversed for
+ * conditions that evaluated true.
+ *
+ * @internal: Internal tree structure.
+ */
+
+// This visitor creates state and should not be re-used
+#[Autoconfigure(public: true, shared: false)]
+final class IncludeTreeConditionMatcherVisitor implements IncludeTreeVisitorInterface
+{
+    private Resolver $resolver;
+    private array $conditionList = [];
+
+    public function __construct(
+        private readonly Context $context,
+        private readonly PageLayoutResolver $pageLayoutResolver,
+        private readonly LoggerInterface $logger,
+    ) {}
+
+    /**
+     * Prepare the core expression language Resolver class - our API to symfony
+     * expression language - for typoscript context usage.
+     *
+     * The method gets a series of variables hand over coming from caller scope
+     * like rootline, page array and eventually a request object. These vars are
+     * munged around a bit and enriched with a series of semi-static state variables:
+     * Things that can be injected like derived from context, for example
+     * frontend / backend user, workspace and similar.
+     * This ensures all typoscript 'conditions' receive similar structured data.
+     */
+    public function initializeExpressionMatcherWithVariables(array $variables): void
+    {
+        $context = $this->context;
+        $enrichedVariables = [
+            'context' => $context,
+        ];
+        // Variables derived directly from context are set if context provides according aspects.
+        $frontendUserAspect = $this->context->getAspect('frontend.user');
+        if ($frontendUserAspect instanceof UserAspect) {
+            $frontend = new \stdClass();
+            $frontend->user = new \stdClass();
+            $frontend->user->isLoggedIn = $frontendUserAspect->get('isLoggedIn');
+            $frontend->user->userId = $frontendUserAspect->get('id');
+            $frontend->user->userGroupList = implode(',', $frontendUserAspect->get('groupIds'));
+            $frontend->user->userGroupIds = $frontendUserAspect->get('groupIds');
+            $enrichedVariables['frontend'] = $frontend;
+        }
+        $backendUserAspect = $this->context->getAspect('backend.user');
+        if ($backendUserAspect instanceof UserAspect) {
+            $backend = new \stdClass();
+            $backend->user = new \stdClass();
+            $backend->user->isAdmin = $backendUserAspect->get('isAdmin');
+            $backend->user->isLoggedIn = $backendUserAspect->get('isLoggedIn');
+            $backend->user->userId = $backendUserAspect->get('id');
+            $backend->user->userGroupList = implode(',', $backendUserAspect->get('groupIds'));
+            $backend->user->userGroupIds = $backendUserAspect->get('groupIds');
+            $enrichedVariables['backend'] = $backend;
+        }
+        $workspaceAspect = $this->context->getAspect('workspace');
+        if ($workspaceAspect instanceof WorkspaceAspect) {
+            $workspace = new \stdClass();
+            $workspace->workspaceId = $workspaceAspect->get('id');
+            $workspace->isLive = $workspaceAspect->get('isLive');
+            $workspace->isOffline = $workspaceAspect->get('isOffline');
+            $enrichedVariables['workspace'] = $workspace;
+        }
+
+        $pageId = $variables['pageId'] ?? 0;
+
+        // If rootLine is given, create an object that contains some prepared values.
+        $fullRootLine = $variables['fullRootLine'] ?? null;
+        if ($fullRootLine === null && $pageId > 0) {
+            $fullRootLine = BackendUtility::BEgetRootLine($pageId, '', true);
+            ksort($fullRootLine);
+        }
+        // 'tree' is always exposed to the expression language, even when no rootline could be
+        // determined (e.g. DataHandler CLI operations on orphaned records with a pid pointing to
+        // a non-existing page). Conditions like '[123 in tree.rootLineIds]' must then evaluate
+        // to false instead of raising a SyntaxError for an unknown 'tree' variable.
+        $localRootLine = $variables['localRootLine'] ?? $fullRootLine ?? [];
+        $tree = new \stdClass();
+        $tree->level = count($localRootLine) - 1;
+        $tree->rootLine = $localRootLine;
+        $tree->fullRootLine = $fullRootLine ?? [];
+        $tree->rootLineIds = array_column($localRootLine, 'uid');
+        $tree->rootLineParentIds = array_slice(array_column($localRootLine, 'pid'), 1);
+        $tree->pagelayout = null;
+        if ($localRootLine !== []) {
+            // We're feeding the "full" RootLine here, not the "local" one that stops at sys_template record having 'root' set.
+            // This is to be in-line with backend here: A 'backend_layout_next_level' on a page above sys_template 'root' page should
+            // still be considered. Normally, $fullRootLine is "deepest page first, then up". This is needed for getLayoutForPage() to find
+            // the 'nearest' parent. However, here it is always passed sorted, so it is a top-down rootLine. Hence, this needs to be once
+            // again reversed at this point.
+            $bottomUpFullRootLine = array_reverse($fullRootLine);
+            $tree->pagelayout = $this->pageLayoutResolver->getLayoutIdentifierForPage($variables['page'], $bottomUpFullRootLine);
+        }
+        $enrichedVariables['tree'] = $tree;
+
+        // If a request is given, make sure it is an instance of RequestWrapper,
+        // if not, create an instance from ServerRequestInterface and set it.
+        if (isset($variables['request']) && !($variables['request'] instanceof RequestWrapper)) {
+            $variables['request'] = new RequestWrapper($variables['request']);
+        } elseif (!isset($variables['request'])) {
+            $variables['request'] = new RequestWrapper(null);
+        }
+
+        // We do not expose pageId, rootLine and fullRootLine to conditions directly.
+        unset($variables['pageId'], $variables['localRootLine'], $variables['fullRootLine']);
+
+        $enrichedVariables = array_replace($enrichedVariables, $variables);
+
+        $this->resolver = new Resolver('typoscript', $enrichedVariables);
+    }
+
+    /**
+     * A list of all handled conditions with their verdicts.
+     * This is used in FE since condition verdicts influence page caches.
+     */
+    public function getConditionListWithVerdicts(): array
+    {
+        return $this->conditionList;
+    }
+
+    /**
+     * Let symfony expression language handle the expression, gather expressions
+     * that have been handled since they influence page caching, negate expression
+     * verdicts if they're a [else] expression.
+     */
+    public function visitBeforeChildren(IncludeInterface $include, int $currentDepth): void
+    {
+        if (!$include instanceof IncludeConditionInterface) {
+            return;
+        }
+        $conditionExpression = $include->getConditionToken()->getValue();
+        try {
+            $verdict = (bool)$this->resolver->evaluate($conditionExpression);
+        } catch (SyntaxError $e) {
+            $this->logger->error('TypoScript condition [{expression}] could not be parsed: {error}', [
+                'expression' => $conditionExpression,
+                'error' => $e->getMessage(),
+                'exception' => $e,
+            ]);
+            $verdict = false;
+        } catch (\RuntimeException $e) {
+            throw new \RuntimeException(
+                sprintf('TypoScript condition [%s] could not be evaluated: %s', $conditionExpression, $e->getMessage()),
+                1731486757,
+                $e
+            );
+        }
+        if ($include->isConditionNegated()) {
+            // Honor ConditionElseInclude "[ELSE]" which negates the verdict of the main condition.
+            $verdict = !$verdict;
+        }
+        $this->conditionList[$conditionExpression] = $verdict;
+        $include->setConditionVerdict($verdict);
+    }
+
+    public function visit(IncludeInterface $include, int $currentDepth): void
+    {
+        // Noop, just implement interface
+    }
+}

Classes/TypoScript/IncludeTree/Visitor/IncludeTreeNodeFinderVisitor.php

@@ -0,0 +1,56 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\Visitor;
+
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\IncludeInterface;
+
+/**
+ * Find a single node in tree identified by node identifier.
+ *
+ * This visitor is used in ext:tstemplate TypoScript modules and ext:backend page TSconfig
+ * backend modules to find single nodes, for instance when their source should be rendered.
+ *
+ * @internal This is a specific Backend implementation and is not considered part of the Public TYPO3 API.
+ */
+final class IncludeTreeNodeFinderVisitor implements IncludeTreeVisitorInterface
+{
+    private ?IncludeInterface $foundNode = null;
+    private string $nodeIdentifier;
+
+    public function setNodeIdentifier(string $nodeIdentifier)
+    {
+        $this->nodeIdentifier = $nodeIdentifier;
+    }
+
+    public function getFoundNode(): ?IncludeInterface
+    {
+        return $this->foundNode;
+    }
+
+    public function visitBeforeChildren(IncludeInterface $include, int $currentDepth): void
+    {
+        if ($include->getIdentifier() === $this->nodeIdentifier) {
+            $this->foundNode = $include;
+        }
+    }
+
+    public function visit(IncludeInterface $include, int $currentDepth): void
+    {
+        // Implement interface
+    }
+}

Classes/TypoScript/IncludeTree/Visitor/IncludeTreeSetupConditionConstantSubstitutionVisitor.php

@@ -0,0 +1,95 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\Visitor;
+
+use Symfony\Component\DependencyInjection\Attribute\Autoconfigure;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\IncludeConditionInterface;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\IncludeInterface;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\Token;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenType;
+
+/**
+ * Handle constants within (TS setup) conditions:
+ * When a conditional include is like this: '["{$foo.bar}" == "4711"]', this visitor looks
+ * up 'foo.bar in given (flattened) constants and substitutes it with the constant value.
+ * The 'include' object then contains the substituted condition token for 'getConditionToken()',
+ * while the original token without the substitution is parked in 'getOriginalConditionToken()'.
+ * The latter is done to have the original token available in the backend to show, it is irrelevant in frontend.
+ *
+ * @internal: Internal tree structure.
+ */
+
+// This visitor creates state and should not be re-used
+#[Autoconfigure(public: true, shared: false)]
+final class IncludeTreeSetupConditionConstantSubstitutionVisitor implements IncludeTreeVisitorInterface
+{
+    /**
+     * @var array<string, string>
+     */
+    private array $flattenedConstants;
+
+    /**
+     * Must be set when adding this visitor, to an empty array at least.
+     * Will fatal otherwise, and that's fine, since if not setting this,
+     * this visitor is useless and shouldn't be added at all.
+     *
+     * @param array<string, string> $flattenedConstants
+     */
+    public function setFlattenedConstants(array $flattenedConstants): void
+    {
+        $this->flattenedConstants = $flattenedConstants;
+    }
+
+    /**
+     * Do the magic, see tests for details.
+     * Implementation within 'visitBeforeChildren()' since this allows running *both* this
+     * visitor first, and then IncludeTreeConditionMatcherVisitor directly afterward in the same
+     * traverser cycle!
+     */
+    public function visitBeforeChildren(IncludeInterface $include, int $currentDepth): void
+    {
+        if (!$include instanceof IncludeConditionInterface) {
+            return;
+        }
+        $conditionToken = $include->getConditionToken();
+        $conditionValue = $conditionToken->getValue();
+        $flattenedConstants = $this->flattenedConstants;
+        $hadSubstitution = false;
+        $newConditionValue = preg_replace_callback(
+            '/{\$(.[^}]*)}/',
+            static function ($match) use ($flattenedConstants, &$hadSubstitution) {
+                // Replace {$someConstant} if found, else leave unchanged
+                if (array_key_exists($match[1], $flattenedConstants)) {
+                    $hadSubstitution = true;
+                    return $flattenedConstants[$match[1]];
+                }
+                return $match[0];
+            },
+            $conditionValue
+        );
+        if ($hadSubstitution) {
+            $include->setOriginalConditionToken($conditionToken);
+            $include->setConditionToken(new Token(TokenType::T_VALUE, $newConditionValue, $conditionToken->getLine(), $conditionToken->getColumn()));
+        }
+    }
+
+    public function visit(IncludeInterface $include, int $currentDepth): void
+    {
+        // Noop, just implement interface
+    }
+}

Classes/TypoScript/IncludeTree/Visitor/IncludeTreeSourceAggregatorVisitor.php

@@ -0,0 +1,101 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\Visitor;
+
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\AtImportInclude;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\ConditionElseInclude;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\ConditionInclude;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\IncludeInterface;
+
+/**
+ * Create a TypoScript source back from an IncludeTree. Inline source from
+ * "@import" and friends.
+ *
+ * This visitor is used in ext:tstemplate TypoScript modules and ext:backend page TSconfig
+ * backend modules to show code of single includes with their resolved imports.
+ *
+ * @internal This is a specific Backend implementation and is not considered part of the Public TYPO3 API.
+ */
+final class IncludeTreeSourceAggregatorVisitor implements IncludeTreeVisitorInterface
+{
+    /**
+     * The accumulated source.
+     */
+    private string $source = '';
+
+    /**
+     * Restrict source rendering to specific includes. Used in BE template analyzer
+     * to output source of a single include and its sub includes. Since a single include
+     * could be included multiple times, we track if source for it has been build to
+     * suppress outputting it multiple times.
+     */
+    private string $startNodeIdentifier = '';
+    private bool $startNodeHandled = false;
+    private int $startNodeDepth = 0;
+    private bool $isWithinStartNode = false;
+
+    public function setStartNodeIdentifier(string $startNodeIdentifier)
+    {
+        $this->startNodeIdentifier = $startNodeIdentifier;
+    }
+
+    public function getSource(): string
+    {
+        return $this->source;
+    }
+
+    public function visitBeforeChildren(IncludeInterface $include, int $currentDepth): void
+    {
+        if ($this->startNodeHandled && $currentDepth <= $this->startNodeDepth) {
+            $this->isWithinStartNode = false;
+        }
+        if ($this->startNodeIdentifier === $include->getIdentifier() && !$this->startNodeHandled) {
+            $this->startNodeDepth = $currentDepth;
+            $this->isWithinStartNode = true;
+            $this->startNodeHandled = true;
+        }
+        if (empty($this->startNodeIdentifier) || $this->isWithinStartNode) {
+            $lineStream = $include->getLineStream();
+            if ($lineStream !== null
+                && !$lineStream->isEmpty()
+                && ($include instanceof ConditionInclude || $include instanceof ConditionElseInclude)
+            ) {
+                $this->source .= "\n#\n# Condition from '" . $include->getName() . '\' Line ' . $include->getConditionToken()->getLine() . "\n#\n";
+                $this->source .= $lineStream;
+            }
+            if ($include instanceof AtImportInclude) {
+                $this->source .= "\n#\n# Include from definition '" . trim((string)($include->getOriginalLine()->getTokenStream())) . "'\n#\n";
+            }
+        }
+    }
+
+    public function visit(IncludeInterface $include, int $currentDepth): void
+    {
+        if (empty($this->startNodeIdentifier) || $this->isWithinStartNode) {
+            $lineStream = $include->getLineStream();
+            if ($lineStream === null
+                || $lineStream->isEmpty()
+                || ($include->isSplit())
+            ) {
+                return;
+            }
+            $this->source .= "\n#\n# Content from '" . $include->getName() . "'\n#\n";
+            $this->source .= $lineStream;
+        }
+    }
+}

Classes/TypoScript/IncludeTree/Visitor/IncludeTreeSyntaxScannerVisitor.php

@@ -0,0 +1,178 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\Visitor;
+
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\AtImportInclude;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\ConditionElseInclude;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\ConditionInclude;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\IncludeInterface;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\BlockCloseLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierBlockOpenLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ImportLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\InvalidLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\LineInterface;
+
+/**
+ * This implements a simple TypoScript syntax scanner. It is used in page TSconfig
+ * and TypoScript "include" submodules to find and show broken syntax.
+ *
+ * @internal This is a specific Backend implementation and is not considered part of the Public TYPO3 API.
+ */
+final class IncludeTreeSyntaxScannerVisitor implements IncludeTreeVisitorInterface
+{
+    /**
+     * @var list<array{type: string, include: IncludeInterface, line: LineInterface, lineNumber: int}>
+     */
+    private array $errors = [];
+
+    /**
+     * @return list<array{type: string, include: IncludeInterface, line: LineInterface, lineNumber: int}>
+     */
+    public function getErrors(): array
+    {
+        return $this->errors;
+    }
+
+    public function visitBeforeChildren(IncludeInterface $include, int $currentDepth): void {}
+
+    public function visit(IncludeInterface $include, int $currentDepth): void
+    {
+        $this->brokenLinesAndBraces($include);
+        $this->emptyImports($include);
+
+        // Add the line number of the first token of the line object to the error array.
+        // Not strictly needed, but more convenient in Fluid template to render.
+        foreach ($this->errors as &$error) {
+            /** @var LineInterface $line */
+            $line = $error['line'];
+            $error['lineNumber'] = $line->getTokenStream()->reset()->peekNext()->getLine();
+        }
+
+        // Sort array by line number to list them top->bottom in view.
+        usort($this->errors, fn($a, $b) => $a['lineNumber'] <=> $b['lineNumber']);
+    }
+
+    /**
+     * Scan for invalid lines ("foo.bar <" is invalid since there must be something after "<"),
+     * and scan for "too many" and "not enough" "}" braces.
+     */
+    private function brokenLinesAndBraces(IncludeInterface $include): void
+    {
+        if ($include->isSplit()) {
+            // If this node is split, don't check for syntax errors, this is
+            // done for child nodes.
+            return;
+        }
+        $lineStream = $include->getLineStream();
+        if (!$lineStream) {
+            return;
+        }
+        $braceCount = 0;
+        $lastLine = null;
+        foreach ($lineStream->getNextLine() as $line) {
+            $lastLine = $line;
+            if ($line instanceof InvalidLine) {
+                $this->errors[] = [
+                    'type' => 'line.invalid',
+                    'include' => $include,
+                    'line' => $line,
+                ];
+            }
+            if ($line instanceof IdentifierBlockOpenLine) {
+                $braceCount++;
+            }
+            if ($line instanceof BlockCloseLine) {
+                $braceCount--;
+                if ($braceCount < 0) {
+                    $braceCount = 0;
+                    $this->errors[] = [
+                        'type' => 'brace.excess',
+                        'include' => $include,
+                        'line' => $line,
+                    ];
+                }
+            }
+        }
+        if ($braceCount !== 0) {
+            $this->errors[] = [
+                'type' => 'brace.missing',
+                'include' => $include,
+                'line' => $lastLine,
+            ];
+        }
+    }
+
+    /**
+     * Look for @import that don't find to-include file(s).
+     *
+     * @todo: This code is far more complex than it could be. See #102102 and #102103 for
+     *        changes we should apply to the include tree structure to simplify this.
+     */
+    private function emptyImports(IncludeInterface $include): void
+    {
+        if (!$include->isSplit()) {
+            // Nodes containing @import are always split
+            return;
+        }
+        $lineStream = $include->getLineStream();
+        if (!$lineStream) {
+            // A node that is split should never have an empty line stream,
+            // this may be obsolete, but does not hurt much.
+            return;
+        }
+        // Find @import lines in this include, index by
+        // combination of line number and column position.
+        $allImportLines = [];
+        foreach ($lineStream->getNextLine() as $line) {
+            if ($line instanceof ImportLine) {
+                $valueToken = $line->getValueToken();
+                $allImportLines[$valueToken->getLine() . '-' . $valueToken->getColumn()] = $line;
+            }
+        }
+        // Now iterate children to exclude valid allImportLines, those that included something.
+        foreach ($include->getNextChild() as $child) {
+            if ($child instanceof AtImportInclude) {
+                /** @var ImportLine $originalLine */
+                $originalLine = $child->getOriginalLine();
+                $valueToken = $originalLine->getValueToken();
+                unset($allImportLines[$valueToken->getLine() . '-' . $valueToken->getColumn()]);
+            }
+            // Condition includes don't have the "body" lines itself (or a "body" sub node). This may change,
+            // but until then we'll have to scan the parent node and loop condition includes here to find out
+            // which of them resolved to child nodes.
+            if ($child instanceof ConditionInclude || $child instanceof ConditionElseInclude) {
+                foreach ($child->getNextChild() as $conditionChild) {
+                    if ($conditionChild instanceof AtImportInclude) {
+                        /** @var ImportLine $originalLine */
+                        $originalLine = $conditionChild->getOriginalLine();
+                        $valueToken = $originalLine->getValueToken();
+                        unset($allImportLines[$valueToken->getLine() . '-' . $valueToken->getColumn()]);
+                    }
+                }
+            }
+        }
+        // Everything left are invalid includes
+        foreach ($allImportLines as $importLine) {
+            $this->errors[] = [
+                'type' => 'import.empty',
+                'include' => $include,
+                'line' => $importLine,
+            ];
+        }
+    }
+}

Classes/TypoScript/IncludeTree/Visitor/IncludeTreeVisitorInterface.php

@@ -0,0 +1,40 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\IncludeTree\Visitor;
+
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\IncludeInterface;
+
+/**
+ * A visitor that can be attached to IncludeTreeTraverser's.
+ *
+ * @internal: Internal tree structure.
+ */
+interface IncludeTreeVisitorInterface
+{
+    /**
+     * Gets called by the traversers *before* children are traversed. Useful for
+     * instance for the IncludeTreeConditionMatcherVisitor to evaluate a condition
+     * verdict *before* children are traversed (or not).
+     */
+    public function visitBeforeChildren(IncludeInterface $include, int $currentDepth): void;
+
+    /**
+     * Main visit method called for each node.
+     */
+    public function visit(IncludeInterface $include, int $currentDepth): void;
+}

Classes/TypoScript/PageTsConfig.php

@@ -0,0 +1,52 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript;
+
+use TYPO3\CMS\Core\TypoScript\AST\Node\RootNode;
+
+/**
+ * A data object that carries the final page TSconfig. This is created by PageTsConfigFactory.
+ *
+ * @internal Internal for now until API stabilized. Use BackendUtility::getPagesTSconfig().
+ */
+final readonly class PageTsConfig
+{
+    private array $pageTsConfigArray;
+
+    public function __construct(
+        private RootNode $pageTsConfigTree,
+        private array $conditionListWithVerdicts,
+    ) {
+        $this->pageTsConfigArray = $pageTsConfigTree->toArray();
+    }
+
+    public function getPageTsConfigTree(): RootNode
+    {
+        return $this->pageTsConfigTree;
+    }
+
+    public function getPageTsConfigArray(): array
+    {
+        return $this->pageTsConfigArray;
+    }
+
+    public function getConditionListWithVerdicts(): array
+    {
+        return $this->conditionListWithVerdicts;
+    }
+}

Classes/TypoScript/PageTsConfigFactory.php

@@ -0,0 +1,144 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript;
+
+use Psr\Container\ContainerInterface;
+use Symfony\Component\DependencyInjection\Attribute\Autoconfigure;
+use Symfony\Component\DependencyInjection\Attribute\Autowire;
+use TYPO3\CMS\Backend\Utility\BackendUtility;
+use TYPO3\CMS\Core\Cache\Frontend\PhpFrontend;
+use TYPO3\CMS\Core\Site\Entity\Site;
+use TYPO3\CMS\Core\Site\Entity\SiteInterface;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\RootInclude;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\SiteInclude;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\IncludeNode\TsConfigInclude;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\Traverser\ConditionVerdictAwareIncludeTreeTraverser;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\TsConfigTreeBuilder;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\Visitor\IncludeTreeAstBuilderVisitor;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\Visitor\IncludeTreeConditionMatcherVisitor;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\Visitor\IncludeTreeSetupConditionConstantSubstitutionVisitor;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\TokenizerInterface;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+/**
+ * Calculate page TSconfig. This does the heavy lifting additionally supported by
+ * TsConfigTreeBuilder: Load basic page TSconfig tree, overload with user TSconfig, parse
+ * site settings ("constants"), then build the page TSconfig AST and return page TSconfig DTO.
+ *
+ * @internal Internal for now until API stabilized. Use BackendUtility::getPagesTSconfig().
+ */
+#[Autoconfigure(public: true)]
+final readonly class PageTsConfigFactory
+{
+    public function __construct(
+        private ContainerInterface $container,
+        private TokenizerInterface $tokenizer,
+        private TsConfigTreeBuilder $tsConfigTreeBuilder,
+        #[Autowire(service: 'cache.typoscript')]
+        private PhpFrontend $cache,
+    ) {}
+
+    public function create(
+        array $fullRootLine,
+        SiteInterface $site,
+        ?UserTsConfig $userTsConfig = null
+    ): PageTsConfig {
+        $pagesTsConfigTree = $this->tsConfigTreeBuilder->getPagesTsConfigTree($fullRootLine, $this->tokenizer, $this->cache);
+
+        // Overloading with user TSconfig if hand over
+        if ($userTsConfig !== null) {
+            $userTsConfigAst = $userTsConfig->getUserTsConfigTree();
+            $userTsConfigPageOverrides = '';
+            // @todo: This is ugly and expensive. There should be a better way to do this. Similar in BE page TSconfig controllers.
+            $userTsConfigFlat = $userTsConfigAst->flatten();
+            foreach ($userTsConfigFlat as $userTsConfigIdentifier => $userTsConfigValue) {
+                if (str_starts_with($userTsConfigIdentifier, 'page.')) {
+                    $userTsConfigPageOverrides .= substr($userTsConfigIdentifier, 5) . ' = ' . $userTsConfigValue . chr(10);
+                }
+            }
+            if (!empty($userTsConfigPageOverrides)) {
+                $includeNode = new TsConfigInclude();
+                $includeNode->setName('pageTsConfig-overrides-by-userTsConfig');
+                $includeNode->setLineStream($this->tokenizer->tokenize($userTsConfigPageOverrides));
+                $pagesTsConfigTree->addChild($includeNode);
+            }
+        }
+
+        // Prepare site constants to be substituted
+        $includeTreeTraverserConditionVerdictAware = new ConditionVerdictAwareIncludeTreeTraverser();
+        $siteSettingsFlat = [];
+        if ($site instanceof Site) {
+            $siteSettings = $site->getSettings();
+            if (!$siteSettings->isEmpty()) {
+                $siteSettingsCacheIdentifier = 'site-settings-flat-' . hash('xxh3', json_encode($siteSettings, JSON_THROW_ON_ERROR));
+                $siteSettingsCacheArray = $this->cache->require($siteSettingsCacheIdentifier);
+                if (isset($siteSettingsCacheArray['flatConstants'])) {
+                    $siteSettingsFlat = $siteSettingsCacheArray['flatConstants'];
+                } else {
+                    $siteConstants = '';
+                    $siteSettings = $siteSettings->getAllFlat();
+                    foreach ($siteSettings as $nodeIdentifier => $value) {
+                        $siteConstants .= $nodeIdentifier . ' = ' . $value . LF;
+                    }
+                    $siteSettingsNode = new SiteInclude();
+                    $siteSettingsNode->setName('Site constants settings of site "' . $site->getIdentifier() . '"');
+                    $siteSettingsNode->setLineStream($this->tokenizer->tokenize($siteConstants));
+                    $siteSettingsTreeRoot = new RootInclude();
+                    $siteSettingsTreeRoot->addChild($siteSettingsNode);
+                    $astBuilderVisitor = $this->container->get(IncludeTreeAstBuilderVisitor::class);
+                    $includeTreeTraverserConditionVerdictAware->traverse($siteSettingsTreeRoot, [$astBuilderVisitor]);
+                    $siteSettingsFlat = $astBuilderVisitor->getAst()->flatten();
+                    $this->cache->set($siteSettingsCacheIdentifier, 'return unserialize(\'' . addcslashes(serialize(['flatConstants' => $siteSettingsFlat]), '\'\\') . '\');');
+                }
+            }
+        }
+
+        // Create AST with constants from site and conditions
+        $includeTreeTraverserConditionVerdictAwareVisitors = [];
+        if (!empty($siteSettingsFlat)) {
+            $setupConditionConstantSubstitutionVisitor = new IncludeTreeSetupConditionConstantSubstitutionVisitor();
+            $setupConditionConstantSubstitutionVisitor->setFlattenedConstants($siteSettingsFlat);
+            $includeTreeTraverserConditionVerdictAwareVisitors[] = $setupConditionConstantSubstitutionVisitor;
+        }
+        $lastPageFullRecord = [];
+        $pageId = 0;
+        if (!empty($fullRootLine)) {
+            $lastPage = array_last($fullRootLine);
+            $pageId = $lastPage['uid'];
+            $lastPageFullRecord = BackendUtility::getRecord('pages', $pageId) ?: [];
+        }
+        $conditionMatcherVariables = [
+            'fullRootLine' => $fullRootLine,
+            'site' => $site,
+            // @todo We're using the full page row here to provide all necessary fields (e.g. "backend_layout"),
+            //       which are currently not included in the rows, RootlineUtility provides by default. We might
+            //       want to switch to array_last($fullRootLine) as soon as it contains all fields.
+            'page' => $lastPageFullRecord,
+            'pageId' => $pageId,
+        ];
+        $conditionMatcherVisitor = GeneralUtility::makeInstance(IncludeTreeConditionMatcherVisitor::class);
+        $conditionMatcherVisitor->initializeExpressionMatcherWithVariables($conditionMatcherVariables);
+        $includeTreeTraverserConditionVerdictAwareVisitors[] = $conditionMatcherVisitor;
+        $astBuilderVisitor = $this->container->get(IncludeTreeAstBuilderVisitor::class);
+        $astBuilderVisitor->setFlatConstants($siteSettingsFlat);
+        $includeTreeTraverserConditionVerdictAwareVisitors[] = $astBuilderVisitor;
+        $includeTreeTraverserConditionVerdictAware->traverse($pagesTsConfigTree, $includeTreeTraverserConditionVerdictAwareVisitors);
+
+        return new PageTsConfig($astBuilderVisitor->getAst(), $conditionMatcherVisitor->getConditionListWithVerdicts());
+    }
+}

Classes/TypoScript/Tokenizer/Line/AbstractLine.php

@@ -0,0 +1,41 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenStreamInterface;
+
+/**
+ * Implement main LineInterface methods.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+abstract class AbstractLine implements LineInterface
+{
+    protected TokenStreamInterface $tokenStream;
+
+    public function setTokenStream(TokenStreamInterface $tokenStream): static
+    {
+        $this->tokenStream = $tokenStream;
+        return $this;
+    }
+
+    public function getTokenStream(): TokenStreamInterface
+    {
+        return $this->tokenStream;
+    }
+}

Classes/TypoScript/Tokenizer/Line/BlockCloseLine.php

@@ -0,0 +1,25 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+/**
+ * A block close line, essentially "}".
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class BlockCloseLine extends AbstractLine {}

Classes/TypoScript/Tokenizer/Line/CommentLine.php

@@ -0,0 +1,29 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+/**
+ * A commented TypoScript line: Lines that start with "#", "//" and multiline comments "/* ... *\/"
+ *
+ * Note multiline comments often represent multiple source lines: An opening "/*" as
+ * first source line, then the comment body with one or more source lines, then finally
+ * the closing "*\/". These still create only one "CommentLine".
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class CommentLine extends AbstractLine {}

Classes/TypoScript/Tokenizer/Line/ConditionElseLine.php

@@ -0,0 +1,25 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+/**
+ * "[ELSE]" / "[else]": An else block after a starting ConditionLine.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class ConditionElseLine extends AbstractLine {}

Classes/TypoScript/Tokenizer/Line/ConditionLine.php

@@ -0,0 +1,45 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\Token;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenType;
+
+/**
+ * A condition line: "[foo == bar]".
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class ConditionLine extends AbstractLine
+{
+    private Token $valueToken;
+
+    public function setValueToken(Token $token): static
+    {
+        if ($token->getType() !== TokenType::T_VALUE) {
+            throw new \LogicException('Token must be of type T_VALUE', 1655823705);
+        }
+        $this->valueToken = $token;
+        return $this;
+    }
+
+    public function getTokenValue(): Token
+    {
+        return $this->valueToken;
+    }
+}

Classes/TypoScript/Tokenizer/Line/ConditionStopLine.php

@@ -0,0 +1,26 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+/**
+ * A line stopping current ConditionLine context:
+ * "[END]" / "[end]" / "[GLOBAL]" / "[global]".
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class ConditionStopLine extends AbstractLine {}

Classes/TypoScript/Tokenizer/Line/EmptyLine.php

@@ -0,0 +1,32 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+/**
+ * A completely empty line, or a line consisting of tabs or whitespaces only.
+ *
+ * This is not created when the TypoScript source line is within multiline "("
+ * assignments and multiline "/*" comments: The T_BLANK and T_NEWLINE tokens
+ * are part of the value steram in these contexts.
+ *
+ * Note the LossyTokenizers does not create these and just skips them since
+ * they have no semantic meaning for the resulting TypoScript tree.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class EmptyLine extends AbstractLine {}

Classes/TypoScript/Tokenizer/Line/IdentifierAssignmentLine.php

@@ -0,0 +1,68 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\IdentifierTokenStream;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenStreamInterface;
+
+/**
+ * Simple "=" assignments and multiline "(" assignments: "foo.bar = barValue".
+ *
+ * Each line has two additional token streams: $identifierTokenStream for the
+ * left side ("foo" and "bar" tokens) and $valueTokenStream for the right side
+ * ("barValue" token). Right side is often a single token only, but can be many
+ * tokens when constants and multiline assignments are involved.
+ *
+ * Neither the left, nor the right side streams can be empty: Even with "foo.bar ="
+ * a T_VALUE token with empty value is created for the right side.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class IdentifierAssignmentLine extends AbstractLine
+{
+    private IdentifierTokenStream $identifierTokenStream;
+    private TokenStreamInterface $valueTokenStream;
+
+    public function setIdentifierTokenStream(IdentifierTokenStream $tokenStream): static
+    {
+        if ($tokenStream->isEmpty()) {
+            throw new \LogicException('Identifier token stream must not be empty', 1655824257);
+        }
+        $this->identifierTokenStream = $tokenStream;
+        return $this;
+    }
+
+    public function getIdentifierTokenStream(): IdentifierTokenStream
+    {
+        return $this->identifierTokenStream;
+    }
+
+    public function setValueTokenStream(TokenStreamInterface $tokenStream): static
+    {
+        if ($tokenStream->isEmpty()) {
+            throw new \LogicException('Value token stream must not be empty', 1655824258);
+        }
+        $this->valueTokenStream = $tokenStream;
+        return $this;
+    }
+
+    public function getValueTokenStream(): TokenStreamInterface
+    {
+        return $this->valueTokenStream;
+    }
+}

Classes/TypoScript/Tokenizer/Line/IdentifierBlockOpenLine.php

@@ -0,0 +1,47 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\IdentifierTokenStream;
+
+/**
+ * A block open line: "foo.bar {".
+ *
+ * $identifierTokenStream is a stream of tokens on the left side, "foo"
+ * and "bar" token in the example above. That stream must not be empty.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class IdentifierBlockOpenLine extends AbstractLine
+{
+    private IdentifierTokenStream $identifierTokenStream;
+
+    public function setIdentifierTokenStream(IdentifierTokenStream $tokenStream): static
+    {
+        if ($tokenStream->isEmpty()) {
+            throw new \LogicException('Identifier token stream must not be empty', 1655824621);
+        }
+        $this->identifierTokenStream = $tokenStream;
+        return $this;
+    }
+
+    public function getIdentifierTokenStream(): IdentifierTokenStream
+    {
+        return $this->identifierTokenStream;
+    }
+}

Classes/TypoScript/Tokenizer/Line/IdentifierCopyLine.php

@@ -0,0 +1,68 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\IdentifierTokenStream;
+
+/**
+ * A line using the copy operator: "foo.bar < lib.myLib".
+ *
+ * Contains a stream of tokens for the left side ("foo" and "bar" tokens) and
+ * a stream of tokens for the right side ("lib" and "myLib"). None of these
+ * token streams can be empty, it's an InvalidLine otherwise.
+ *
+ * Note the right side TokenStreamIdentifier can be relative: "foo.bar < .baz".
+ * Flag $relative in TokenStreamIdentifier represents this start dot on the right side.
+ *
+ * None of the two streams can be empty.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class IdentifierCopyLine extends AbstractLine
+{
+    private IdentifierTokenStream $identifierTokenStream;
+    private IdentifierTokenStream $valueTokenStream;
+
+    public function setIdentifierTokenStream(IdentifierTokenStream $tokenStream): static
+    {
+        if ($tokenStream->isEmpty()) {
+            throw new \LogicException('Identifier token stream must not be empty', 1655824946);
+        }
+        $this->identifierTokenStream = $tokenStream;
+        return $this;
+    }
+
+    public function getIdentifierTokenStream(): IdentifierTokenStream
+    {
+        return $this->identifierTokenStream;
+    }
+
+    public function setValueTokenStream(IdentifierTokenStream $tokenStream): static
+    {
+        if ($tokenStream->isEmpty()) {
+            throw new \LogicException('Value token stream must not be empty', 1655824947);
+        }
+        $this->valueTokenStream = $tokenStream;
+        return $this;
+    }
+
+    public function getValueTokenStream(): IdentifierTokenStream
+    {
+        return $this->valueTokenStream;
+    }
+}

Classes/TypoScript/Tokenizer/Line/IdentifierFunctionLine.php

@@ -0,0 +1,87 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\IdentifierTokenStream;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\Token;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenStreamInterface;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenType;
+
+/**
+ * A line with a function assignment using the ":=" operator: "foo.bar := addToList(42)".
+ *
+ * Contains $identifierTokenStream for the left side ("foo" and "bar" token), a single
+ * token for the function name ("addToList"), and an optional token for the value ("42").
+ * Note the value token is optional since there are functions without values (eg. "uniqueList()").
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class IdentifierFunctionLine extends AbstractLine
+{
+    private ?IdentifierTokenStream $identifierTokenStream = null;
+    private ?Token $functionNameToken = null;
+    private ?TokenStreamInterface $functionValueTokenStream = null;
+
+    public function setIdentifierTokenStream(IdentifierTokenStream $tokenStream): IdentifierFunctionLine
+    {
+        if ($tokenStream->isEmpty()) {
+            throw new \LogicException('Identifier token stream must not be empty', 1655825120);
+        }
+        $this->identifierTokenStream = $tokenStream;
+        return $this;
+    }
+
+    public function getIdentifierTokenStream(): IdentifierTokenStream
+    {
+        if ($this->identifierTokenStream === null) {
+            throw new \RuntimeException('Identifier token stream has not been set', 1717495444);
+        }
+        return $this->identifierTokenStream;
+    }
+
+    public function setFunctionNameToken(Token $token): IdentifierFunctionLine
+    {
+        if ($token->getType() !== TokenType::T_FUNCTION_NAME) {
+            throw new \LogicException('Function name token must be of type T_FUNCTION_NAME', 1655825121);
+        }
+        $this->functionNameToken = $token;
+        return $this;
+    }
+
+    public function getFunctionNameToken(): Token
+    {
+        if ($this->functionNameToken === null) {
+            throw new \RuntimeException('Function name token has not been set', 1717495576);
+        }
+        return $this->functionNameToken;
+    }
+
+    public function setFunctionValueTokenStream(TokenStreamInterface $tokenStream): IdentifierFunctionLine
+    {
+        $this->functionValueTokenStream = $tokenStream;
+        return $this;
+    }
+
+    public function getFunctionValueTokenStream(): TokenStreamInterface
+    {
+        if ($this->functionValueTokenStream === null) {
+            throw new \RuntimeException('Function value token stream has not been set', 1717495996);
+        }
+        return $this->functionValueTokenStream;
+    }
+}

Classes/TypoScript/Tokenizer/Line/IdentifierReferenceLine.php

@@ -0,0 +1,66 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\IdentifierTokenStream;
+
+/**
+ * A line using the reference ("=<") operator: "foo.bar =< lib.myLib".
+ *
+ * Contains two non-empty token streams: One for the left side ("foo" and "bar" tokens),
+ * and one for the right side ("lib" and "myLib"). Both streams must not be empty.
+ *
+ * Note the AstBuilder does not directly resolve "=<" operators. This is
+ * not a language construct itself and is only resolved in some special cases
+ * in frontend. See ContentObjectRenderer->cObjGetSingle() for more details.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class IdentifierReferenceLine extends AbstractLine
+{
+    private IdentifierTokenStream $identifierTokenStream;
+    private IdentifierTokenStream $valueTokenStream;
+
+    public function setIdentifierTokenStream(IdentifierTokenStream $tokenStream): static
+    {
+        if ($tokenStream->isEmpty()) {
+            throw new \LogicException('Identifier token stream must not be empty', 1655825891);
+        }
+        $this->identifierTokenStream = $tokenStream;
+        return $this;
+    }
+
+    public function getIdentifierTokenStream(): IdentifierTokenStream
+    {
+        return $this->identifierTokenStream;
+    }
+
+    public function setValueTokenStream(IdentifierTokenStream $tokenStream): static
+    {
+        if ($tokenStream->isEmpty()) {
+            throw new \LogicException('Value token stream must not be empty', 1655825892);
+        }
+        $this->valueTokenStream = $tokenStream;
+        return $this;
+    }
+
+    public function getValueTokenStream(): IdentifierTokenStream
+    {
+        return $this->valueTokenStream;
+    }
+}

Classes/TypoScript/Tokenizer/Line/IdentifierUnsetLine.php

@@ -0,0 +1,47 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\IdentifierTokenStream;
+
+/**
+ * A line using the unset (">") operator: "foo.bar >".
+ *
+ * Has $identifierTokenStream for the stream of tokens on the left
+ * side ("foo" and "bar" tokens).
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class IdentifierUnsetLine extends AbstractLine
+{
+    private IdentifierTokenStream $identifierTokenStream;
+
+    public function setIdentifierTokenStream(IdentifierTokenStream $tokenStream): static
+    {
+        if ($tokenStream->isEmpty()) {
+            throw new \LogicException('Identifier token stream must not be empty', 1655826025);
+        }
+        $this->identifierTokenStream = $tokenStream;
+        return $this;
+    }
+
+    public function getIdentifierTokenStream(): IdentifierTokenStream
+    {
+        return $this->identifierTokenStream;
+    }
+}

Classes/TypoScript/Tokenizer/Line/ImportLine.php

@@ -0,0 +1,49 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\Token;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenType;
+
+/**
+ * A line using the "@import" keyword: "@import 'EXT:my_extension/Configuration/TypoScript/randomfile.typoscript'"
+ *
+ * Contains the $valueToken ("EXT:my_extension/Configuration/TypoScript/randomfile.typoscript"), without the
+ * surrounding tick (') or doubletick ("). The value itself is not parsed further at this point, this
+ * is done by the IncludeTree classes.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class ImportLine extends AbstractLine
+{
+    private Token $valueToken;
+
+    public function setValueToken(Token $token): static
+    {
+        if ($token->getType() !== TokenType::T_VALUE) {
+            throw new \LogicException('Value token must be of type T_VALUE', 1655826193);
+        }
+        $this->valueToken = $token;
+        return $this;
+    }
+
+    public function getValueToken(): Token
+    {
+        return $this->valueToken;
+    }
+}

Classes/TypoScript/Tokenizer/Line/InvalidLine.php

@@ -0,0 +1,33 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+/**
+ * A line that is syntactically invalid.
+ *
+ * This is created by LosslessTokenizer whenever a line does not make sense.
+ * Examples:
+ * "foo.bar" - no operator
+ * "foo.bar <" - right side empty
+ * "@import ''" - no import value
+ *
+ * Note only LosslessTokenizer creates these lines, LossyTokenizer just skips them.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class InvalidLine extends AbstractLine {}

Classes/TypoScript/Tokenizer/Line/LineInterface.php

@@ -0,0 +1,42 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenStreamInterface;
+
+/**
+ * The TypoScript tokenizers deliver streams of lines. This is the main line interface.
+ *
+ * Each line is represented by a specific line type. For instance, "foo.bar {" creates
+ * an IdentifierBlockOpenLine and has the additional method getIdentifierTokenStream()
+ * to retrieve the "foo" and "bar" identifier tokens.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+interface LineInterface
+{
+    /**
+     * Set and get the token stream that represents the full line. This is mostly used
+     * in backend to for instance create a TypoScript string back from tokenized lines.
+     *
+     * Note: Only the LosslessTokenizer fills this 'full line' stream, LossyTokenizer
+     * does not for performance reasons.
+     */
+    public function setTokenStream(TokenStreamInterface $tokenStream): static;
+    public function getTokenStream(): TokenStreamInterface;
+}

Classes/TypoScript/Tokenizer/Line/LineStream.php

@@ -0,0 +1,124 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+/**
+ * Each TypoScript snippet is turned by the tokenizers into a
+ * stream of lines. Tokenizers return instances of this class.
+ *
+ * Iterate line streams in a foreach loop using getNextLine().
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class LineStream
+{
+    /**
+     * @var LineInterface[]
+     */
+    private array $lines = [];
+    private int $currentIndex = -1;
+
+    /**
+     * Create a source string from given token lines. This is used in backend
+     * to turn the "full" token streams of lines into strings for output.
+     */
+    public function __toString(): string
+    {
+        $source = '';
+        foreach ($this->getNextLine() as $line) {
+            // We do *not* implement __toString() on lines since this is a
+            // backend thing only, and we do not want to accidentally stringify
+            // lines based on the full stream anywhere.
+            $source .= $line->getTokenStream()->reset();
+        }
+        return $source;
+    }
+
+    /**
+     * When storing to cache, we only store FE relevant properties and skip
+     * irrelevant things. In particular, $currentIndex should always initialize
+     * to -1 and does not need to be stored.
+     */
+    final public function __serialize(): array
+    {
+        return [
+            'lines' => $this->lines,
+        ];
+    }
+
+    /**
+     * Stream creation.
+     */
+    public function append(LineInterface $line): self
+    {
+        $this->lines[] = $line;
+        return $this;
+    }
+
+    /**
+     * We sometimes create a line stream but don't add lines.
+     * This method returns true if lines have been added.
+     */
+    public function isEmpty(): bool
+    {
+        return empty($this->lines);
+    }
+
+    /**
+     * @return iterable<LineInterface>
+     */
+    public function getNextLine(): iterable
+    {
+        foreach ($this->lines as $child) {
+            yield $child;
+        }
+    }
+
+    /**
+     * Reset current pointer. Typically, call this before iterating with getNext().
+     */
+    public function reset(): self
+    {
+        $this->currentIndex = -1;
+        return $this;
+    }
+
+    /**
+     * Get next line and raise pointer.
+     *
+     * Methods getNext(), peekNext() and reset() are an alternative to
+     * getNextLine() which allow peek of the next line, which getNextLine()
+     * does not. The disadvantage is that these methods create internal
+     * state in $this->currentIndex, which getNextLine() does not. Use
+     * getNext() iteration only if peekNext() is needed to avoid creating
+     * useless state.
+     */
+    public function getNext(): ?LineInterface
+    {
+        $this->currentIndex++;
+        return $this->lines[$this->currentIndex] ?? null;
+    }
+
+    /**
+     * Get next line but do not raise pointer.
+     */
+    public function peekNext(): ?LineInterface
+    {
+        return $this->lines[$this->currentIndex + 1] ?? null;
+    }
+}

Classes/TypoScript/Tokenizer/LosslessTokenizer.php

@@ -0,0 +1,879 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\BlockCloseLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\CommentLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ConditionElseLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ConditionLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ConditionStopLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\EmptyLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierAssignmentLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierBlockOpenLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierCopyLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierFunctionLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierReferenceLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierUnsetLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ImportLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\InvalidLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\LineStream;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\ConstantAwareTokenStream;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\IdentifierToken;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\IdentifierTokenStream;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\Token;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenStream;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenStreamInterface;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenType;
+
+/**
+ * A lossless tokenizer for TypoScript syntax.
+ *
+ * tokenize() creates a flat stream of tokens from a TypoScript string. It is lossless
+ * and never "looses" characters to allow syntax linting and creating linter-fixed source
+ * strings: tokenize() to create a TokenStream and using string cast (__toString()) on
+ * that stream creates *the same* source string again.
+ *
+ * The tokenizer *does not* parse conditions or includes itself (no file / db lookups),
+ * this is part of the IncludeTree parser.
+ *
+ * This class is unit test covered by TokenizerInterfaceTest and paired with LossyTokenizer.
+ * Never change anything in this class without additional test coverage!
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class LosslessTokenizer implements TokenizerInterface
+{
+    private LineStream $lineStream;
+
+    private TokenStreamInterface $tokenStream;
+    private IdentifierTokenStream $identifierStream;
+    private TokenStreamInterface $valueStream;
+
+    private array $lines;
+    private int $currentLineNumber;
+    private string $currentLineString;
+    private \closure $currentLinebreakCallback;
+    private int $currentColumnInLine = 0;
+
+    public function tokenize(string $source): LineStream
+    {
+        $this->lineStream = new LineStream();
+        $this->currentLineNumber = -1;
+        $this->lines = $this->splitLines($source);
+
+        while (true) {
+            $this->tokenStream = new TokenStream();
+            $this->currentLineNumber++;
+            if (!array_key_exists($this->currentLineNumber, $this->lines)) {
+                break;
+            }
+            $this->currentColumnInLine = 0;
+            $this->currentLineString = $this->lines[$this->currentLineNumber]['line'];
+            $this->currentLinebreakCallback = $this->lines[$this->currentLineNumber]['linebreakCallback'];
+            $this->parseTabsAndWhitespaces();
+            $nextChar = substr($this->currentLineString, 0, 1);
+            if ($nextChar === '') {
+                ($this->currentLinebreakCallback)();
+                if (!$this->tokenStream->isEmpty()) {
+                    $this->createEmptyLine();
+                }
+                continue;
+            }
+            $nextTwoChars = substr($this->currentLineString, 0, 2);
+            if ($nextChar === '#') {
+                $this->createHashCommentLine();
+            } elseif ($nextTwoChars === '//') {
+                $this->createDoubleSlashCommentLine();
+            } elseif ($nextTwoChars === '/*') {
+                $this->createMultilineCommentLine();
+            } elseif ($nextChar === '[') {
+                $this->createConditionLine();
+            } elseif ($nextChar === '}') {
+                $this->createBlockStopLine();
+            } elseif (str_starts_with($this->currentLineString, '@import')) {
+                $this->parseImportLine();
+            } elseif (str_starts_with($this->currentLineString, '<INCLUDE_TYPOSCRIPT:')) {
+                // @todo: Could be relocated elsewhere. This is just to make sure this
+                //        old language construct is detected as InvalidLine.
+                $this->tokenStream->append(new Token(TokenType::T_VALUE, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine));
+                ($this->currentLinebreakCallback)();
+                $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+            } else {
+                $this->parseIdentifier();
+            }
+        }
+
+        return $this->lineStream;
+    }
+
+    private function splitLines($source): array
+    {
+        $vanillaLines = explode(chr(10), $source);
+        $lines = array_map(
+            fn(int $lineNumber, string $vanillaLine): array => [
+                'line' => rtrim($vanillaLine, "\r"),
+                'linebreakCallback' => str_ends_with($vanillaLine, "\r")
+                    ? fn() => $this->tokenStream->append(new Token(TokenType::T_NEWLINE, "\r\n", $lineNumber, mb_strlen($vanillaLine) - 1))
+                    : fn() => $this->tokenStream->append(new Token(TokenType::T_NEWLINE, "\n", $lineNumber, mb_strlen($vanillaLine))),
+            ],
+            array_keys($vanillaLines),
+            $vanillaLines
+        );
+        // Set the linebreak callback of last line to empty to suppress dangling linebreak tokens
+        $lines[count($vanillaLines) - 1]['linebreakCallback'] = function () {};
+        return $lines;
+    }
+
+    private function createEmptyLine(): void
+    {
+        $this->lineStream->append((new EmptyLine())->setTokenStream($this->tokenStream));
+    }
+
+    /**
+     * Add tabs and whitespaces until some different char appears.
+     */
+    private function parseTabsAndWhitespaces(): void
+    {
+        $matches = [];
+        if (preg_match('#^(\s+)(.*)$#', $this->currentLineString, $matches)) {
+            $this->tokenStream->append(new Token(TokenType::T_BLANK, $matches[1], $this->currentLineNumber, $this->currentColumnInLine));
+            $this->currentLineString = $matches[2];
+            $this->currentColumnInLine = $this->currentColumnInLine + strlen($matches[1]);
+        }
+    }
+
+    private function makeComment(): void
+    {
+        $nextChar = substr($this->currentLineString, 0, 1);
+        if ($nextChar === '') {
+            ($this->currentLinebreakCallback)();
+            return;
+        }
+        $nextTwoChars = substr($this->currentLineString, 0, 2);
+        if ($nextChar === '#') {
+            $this->parseHashComment();
+        } elseif ($nextTwoChars === '//') {
+            $this->parseDoubleSlashComment();
+        } elseif ($nextTwoChars === '/*') {
+            $this->parseMultilineComment();
+        } else {
+            $this->parseHashComment();
+        }
+    }
+
+    private function createHashCommentLine(): void
+    {
+        $this->parseHashComment();
+        $this->lineStream->append((new CommentLine())->setTokenStream($this->tokenStream));
+    }
+
+    private function parseHashComment(): void
+    {
+        $this->tokenStream->append(new Token(TokenType::T_COMMENT_ONELINE_HASH, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine));
+        ($this->currentLinebreakCallback)();
+    }
+
+    private function createDoubleSlashCommentLine(): void
+    {
+        $this->parseDoubleSlashComment();
+        $this->lineStream->append((new CommentLine())->setTokenStream($this->tokenStream));
+    }
+
+    private function parseDoubleSlashComment(): void
+    {
+        $this->tokenStream->append(new Token(TokenType::T_COMMENT_ONELINE_DOUBLESLASH, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine));
+        ($this->currentLinebreakCallback)();
+    }
+
+    private function createMultilineCommentLine(): void
+    {
+        $this->parseMultilineComment();
+        $this->lineStream->append((new CommentLine())->setTokenStream($this->tokenStream));
+    }
+
+    private function parseMultilineComment(): void
+    {
+        $this->tokenStream->append(new Token(TokenType::T_COMMENT_MULTILINE_START, '/*', $this->currentLineNumber, $this->currentColumnInLine));
+        $this->currentColumnInLine += 2;
+        $this->currentLineString = substr($this->currentLineString, 2);
+        while (true) {
+            if (str_ends_with($this->currentLineString, '*/')) {
+                if (strlen($this->currentLineString) > 2) {
+                    $this->tokenStream->append(new Token(TokenType::T_VALUE, substr($this->currentLineString, 0, -2), $this->currentLineNumber, $this->currentColumnInLine));
+                }
+                $this->tokenStream->append(new Token(TokenType::T_COMMENT_MULTILINE_STOP, '*/', $this->currentLineNumber, $this->currentColumnInLine + mb_strlen($this->currentLineString) - 2));
+                ($this->currentLinebreakCallback)();
+                return;
+            }
+            if (strlen($this->currentLineString)) {
+                $this->tokenStream->append(new Token(TokenType::T_VALUE, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine));
+            }
+            ($this->currentLinebreakCallback)();
+            if (!array_key_exists($this->currentLineNumber + 1, $this->lines)) {
+                return;
+            }
+            $this->currentLineNumber++;
+            $this->currentColumnInLine = 0;
+            $this->currentLineString = $this->lines[$this->currentLineNumber]['line'];
+            $this->currentLinebreakCallback = $this->lines[$this->currentLineNumber]['linebreakCallback'];
+        }
+    }
+
+    /**
+     * Create a condition line from token stream of this line.
+     */
+    private function createConditionLine(): void
+    {
+        $upperCaseLine = strtoupper($this->currentLineString);
+        $this->tokenStream->append(new Token(TokenType::T_CONDITION_START, '[', $this->currentLineNumber, $this->currentColumnInLine));
+        if (str_starts_with($upperCaseLine, '[ELSE]')) {
+            $this->tokenStream->append(new Token(TokenType::T_CONDITION_ELSE, substr($this->currentLineString, 1, 4), $this->currentLineNumber, $this->currentColumnInLine + 1));
+            $this->tokenStream->append(new Token(TokenType::T_CONDITION_STOP, ']', $this->currentLineNumber, $this->currentColumnInLine + 5));
+            $this->currentLineString = substr($this->currentLineString, 6);
+            $this->currentColumnInLine += 6;
+            $this->parseTabsAndWhitespaces();
+            $this->makeComment();
+            $this->lineStream->append((new ConditionElseLine())->setTokenStream($this->tokenStream));
+            return;
+        }
+        if (str_starts_with($upperCaseLine, '[END]')) {
+            $this->tokenStream->append(new Token(TokenType::T_CONDITION_END, substr($this->currentLineString, 1, 3), $this->currentLineNumber, $this->currentColumnInLine + 1));
+            $this->tokenStream->append(new Token(TokenType::T_CONDITION_STOP, ']', $this->currentLineNumber, $this->currentColumnInLine + 4));
+            $this->currentLineString = substr($this->currentLineString, 5);
+            $this->currentColumnInLine += 5;
+            $this->parseTabsAndWhitespaces();
+            $this->makeComment();
+            $this->lineStream->append((new ConditionStopLine())->setTokenStream($this->tokenStream));
+            return;
+        }
+        if (str_starts_with($upperCaseLine, '[GLOBAL]')) {
+            $this->tokenStream->append(new Token(TokenType::T_CONDITION_GLOBAL, substr($this->currentLineString, 1, 6), $this->currentLineNumber, $this->currentColumnInLine + 1));
+            $this->tokenStream->append(new Token(TokenType::T_CONDITION_STOP, ']', $this->currentLineNumber, $this->currentColumnInLine + 7));
+            $this->currentLineString = substr($this->currentLineString, 8);
+            $this->currentColumnInLine += 8;
+            $this->parseTabsAndWhitespaces();
+            $this->makeComment();
+            $this->lineStream->append((new ConditionStopLine())->setTokenStream($this->tokenStream));
+            return;
+        }
+        $conditionBody = '';
+        $conditionBodyStartPosition = $this->currentColumnInLine + 1;
+        $conditionBodyCharCount = 0;
+        $conditionBodyChars = mb_str_split(substr($this->currentLineString, 1), 1, 'UTF-8');
+        $bracketCount = 1;
+        while (true) {
+            $nextChar = $conditionBodyChars[$conditionBodyCharCount] ?? null;
+            if ($nextChar === null) {
+                // end of chars
+                if ($conditionBodyCharCount) {
+                    $this->tokenStream->append(new Token(TokenType::T_VALUE, $conditionBody, $this->currentLineNumber, $conditionBodyStartPosition));
+                }
+                ($this->currentLinebreakCallback)();
+                $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+                return;
+            }
+            if ($nextChar === '[') {
+                $bracketCount++;
+                $conditionBody .= $nextChar;
+                $conditionBodyCharCount++;
+                continue;
+            }
+            if ($nextChar === ']') {
+                $bracketCount--;
+                if ($bracketCount === 0) {
+                    if ($conditionBodyCharCount) {
+                        $conditionBodyToken = new Token(TokenType::T_VALUE, $conditionBody, $this->currentLineNumber, $conditionBodyStartPosition);
+                        $this->tokenStream->append($conditionBodyToken);
+                        $this->tokenStream->append(new Token(TokenType::T_CONDITION_STOP, ']', $this->currentLineNumber, $this->currentColumnInLine + $conditionBodyCharCount + 1));
+                        $this->currentLineString = mb_substr($this->currentLineString, $conditionBodyCharCount + 2);
+                        $this->currentColumnInLine = $this->currentColumnInLine + $conditionBodyCharCount + 2;
+                        $this->parseTabsAndWhitespaces();
+                        $this->makeComment();
+                        $this->lineStream->append((new ConditionLine())->setTokenStream($this->tokenStream)->setValueToken($conditionBodyToken));
+                        return;
+                    }
+                    $this->tokenStream->append(new Token(TokenType::T_CONDITION_STOP, ']', $this->currentLineNumber, $this->currentColumnInLine + $conditionBodyCharCount + 1));
+                    $this->currentLineString = mb_substr($this->currentLineString, $conditionBodyCharCount + 2);
+                    $this->currentColumnInLine = $this->currentColumnInLine + $conditionBodyCharCount + 2;
+                    $this->parseTabsAndWhitespaces();
+                    $this->makeComment();
+                    $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+                    return;
+                }
+                $conditionBody .= $nextChar;
+                $conditionBodyCharCount++;
+                continue;
+            }
+            $conditionBody .= $nextChar;
+            $conditionBodyCharCount++;
+        }
+    }
+
+    private function createBlockStopLine(): void
+    {
+        $this->tokenStream->append(new Token(TokenType::T_BLOCK_STOP, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine));
+        $this->currentColumnInLine++;
+        $this->currentLineString = substr($this->currentLineString, 1);
+        $this->makeComment();
+        $this->lineStream->append((new BlockCloseLine())->setTokenStream($this->tokenStream));
+    }
+
+    private function parseBlockStart(): void
+    {
+        $this->tokenStream->append(new Token(TokenType::T_BLOCK_START, '{', $this->currentLineNumber, $this->currentColumnInLine));
+        $this->currentColumnInLine++;
+        $this->currentLineString = substr($this->currentLineString, 1);
+        $this->parseTabsAndWhitespaces();
+        if (str_starts_with($this->currentLineString, '}')) {
+            // Edge case: foo = { } in one line. Note content within {} is not parsed, everything behind { ends up as comment.
+            $this->lineStream->append((new IdentifierBlockOpenLine())->setIdentifierTokenStream($this->identifierStream)->setTokenStream($this->tokenStream));
+            $this->tokenStream = new TokenStream();
+            $this->tokenStream->append(new Token(TokenType::T_BLOCK_STOP, '}', $this->currentLineNumber, $this->currentColumnInLine));
+            $this->currentLineString = substr($this->currentLineString, 1);
+            $this->currentColumnInLine++;
+            $this->makeComment();
+            $this->lineStream->append((new BlockCloseLine())->setTokenStream($this->tokenStream));
+            return;
+        }
+        $this->makeComment();
+        $this->lineStream->append((new IdentifierBlockOpenLine())->setIdentifierTokenStream($this->identifierStream)->setTokenStream($this->tokenStream));
+    }
+
+    private function parseImportLine(): void
+    {
+        $this->tokenStream->append(new Token(TokenType::T_IMPORT_KEYWORD, '@import', $this->currentLineNumber, $this->currentColumnInLine));
+        $this->currentColumnInLine += 7;
+        $this->currentLineString = substr($this->currentLineString, 7);
+        $this->parseTabsAndWhitespaces();
+
+        // Next char should be the opening tick or doubletick, otherwise we create a comment until end of line
+        $nextChar = substr($this->currentLineString, 0, 1);
+        if ($nextChar !== '\'' && $nextChar !== '"') {
+            $this->makeComment();
+            $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+            return;
+        }
+        $this->tokenStream->append(new Token(TokenType::T_IMPORT_START, $nextChar, $this->currentLineNumber, $this->currentColumnInLine));
+
+        $importBody = '';
+        $importBodyStartPosition = $this->currentColumnInLine + 1;
+        $importBodyCharCount = 0;
+        $importBodyChars = mb_str_split(substr($this->currentLineString, 1), 1, 'UTF-8');
+        while (true) {
+            $nextChar = $importBodyChars[$importBodyCharCount] ?? null;
+            if ($nextChar === null) {
+                // end of chars
+                if ($importBodyCharCount) {
+                    $importBodyToken = (new Token(TokenType::T_VALUE, $importBody, $this->currentLineNumber, $importBodyStartPosition));
+                    $this->tokenStream->append($importBodyToken);
+                    ($this->currentLinebreakCallback)();
+                    $this->lineStream->append((new ImportLine())->setTokenStream($this->tokenStream)->setValueToken($importBodyToken));
+                    return;
+                }
+                ($this->currentLinebreakCallback)();
+                $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+                return;
+            }
+            if ($nextChar === '\'' || $nextChar === '"') {
+                if ($importBodyCharCount) {
+                    $importBodyToken = new Token(TokenType::T_VALUE, $importBody, $this->currentLineNumber, $importBodyStartPosition);
+                    $this->tokenStream->append($importBodyToken);
+                    $this->tokenStream->append(new Token(TokenType::T_IMPORT_STOP, $nextChar, $this->currentLineNumber, $this->currentColumnInLine + $importBodyCharCount + 1));
+                    $this->currentLineString = mb_substr($this->currentLineString, $importBodyCharCount + 2);
+                    $this->currentColumnInLine = $this->currentColumnInLine + $importBodyCharCount + 2;
+                    $this->parseTabsAndWhitespaces();
+                    $this->makeComment();
+                    $this->lineStream->append((new ImportLine())->setTokenStream($this->tokenStream)->setValueToken($importBodyToken));
+                    return;
+                }
+                $this->tokenStream->append(new Token(TokenType::T_IMPORT_STOP, $nextChar, $this->currentLineNumber, $this->currentColumnInLine + $importBodyCharCount + 1));
+                $this->currentLineString = mb_substr($this->currentLineString, $importBodyCharCount + 2);
+                $this->currentColumnInLine = $this->currentColumnInLine + $importBodyCharCount + 2;
+                $this->parseTabsAndWhitespaces();
+                $this->makeComment();
+                $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+                return;
+            }
+            $importBody .= $nextChar;
+            $importBodyCharCount++;
+        }
+    }
+
+    private function parseIdentifier(): void
+    {
+        $splitLine = mb_str_split($this->currentLineString, 1, 'UTF-8');
+        $currentPosition = $this->parseIdentifierUntilStopChar($splitLine);
+        if (!$currentPosition) {
+            $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+            return;
+        }
+        $this->currentLineString = mb_substr($this->currentLineString, $currentPosition);
+        $this->currentColumnInLine = $this->currentColumnInLine + $currentPosition;
+        $currentColumnInLineBefore = $this->currentColumnInLine;
+        $this->parseTabsAndWhitespaces();
+        $currentPosition = $currentPosition + $this->currentColumnInLine - $currentColumnInLineBefore;
+        $nextChar = $splitLine[$currentPosition] ?? null;
+        $nextTwoChars = $nextChar . ($splitLine[$currentPosition + 1] ?? '');
+        if ($nextTwoChars === '=<') {
+            $this->parseOperatorReference();
+            return;
+        }
+        if ($nextChar === '=') {
+            $this->parseOperatorAssignment();
+            return;
+        }
+        if ($nextChar === '{') {
+            $this->parseBlockStart();
+            return;
+        }
+        if ($nextChar === '>') {
+            $this->parseOperatorUnset();
+            return;
+        }
+        if ($nextChar === '<') {
+            $this->parseOperatorCopy();
+            return;
+        }
+        if ($nextChar === '(') {
+            $this->parseOperatorMultilineAssignment();
+            return;
+        }
+        if ($nextTwoChars === ':=') {
+            $this->parseOperatorFunction();
+            return;
+        }
+        if ($nextChar === '#') {
+            $this->parseHashComment();
+            $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+            return;
+        }
+        if ($nextTwoChars === '//') {
+            $this->parseDoubleSlashComment();
+            $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+            return;
+        }
+        if ($nextTwoChars === '/*') {
+            $this->parseMultilineComment();
+            $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+            return;
+        }
+        if ($nextChar === null) {
+            ($this->currentLinebreakCallback)();
+            $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+        }
+    }
+
+    private function parseOperatorAssignment(): void
+    {
+        $this->tokenStream->append(new Token(TokenType::T_OPERATOR_ASSIGNMENT, '=', $this->currentLineNumber, $this->currentColumnInLine));
+        $this->currentColumnInLine++;
+        $this->currentLineString = substr($this->currentLineString, 1);
+        $this->parseTabsAndWhitespaces();
+        $this->valueStream = new TokenStream();
+        [$this->valueStream, $this->tokenStream] = $this->parseValueForConstants($this->valueStream, $this->tokenStream, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine);
+        ($this->currentLinebreakCallback)();
+        $this->lineStream->append((new IdentifierAssignmentLine())->setTokenStream($this->tokenStream)->setIdentifierTokenStream($this->identifierStream)->setValueTokenStream($this->valueStream));
+    }
+
+    private function parseOperatorMultilineAssignment(): void
+    {
+        $this->valueStream = new TokenStream();
+        $this->tokenStream->append(new Token(TokenType::T_OPERATOR_ASSIGNMENT_MULTILINE_START, '(', $this->currentLineNumber, $this->currentColumnInLine));
+        $this->currentColumnInLine++;
+        $this->currentLineString = substr($this->currentLineString, 1);
+        // True if we're currently in the line with the opening '('
+        $isFirstLine = true;
+        // True if the first line has a first value token: "foo ( thisIsTheFirstValueToken"
+        $valueOnFirstLine = false;
+        // True if the line after '(' is parsed
+        $isSecondLine = false;
+        $previousLineCallback = function () {};
+        while (true) {
+            if (str_starts_with(ltrim($this->currentLineString), ')')) {
+                $this->parseTabsAndWhitespaces();
+                $this->tokenStream->append(new Token(TokenType::T_OPERATOR_ASSIGNMENT_MULTILINE_STOP, ')', $this->currentLineNumber, $this->currentColumnInLine));
+                $this->currentLineString = substr($this->currentLineString, 1);
+                $this->currentColumnInLine++;
+                $this->parseTabsAndWhitespaces();
+                $this->makeComment();
+                if ($this->valueStream->isEmpty()) {
+                    $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+                } else {
+                    $this->lineStream->append((new IdentifierAssignmentLine())->setIdentifierTokenStream($this->identifierStream)->setValueTokenStream($this->valueStream)->setTokenStream($this->tokenStream));
+                }
+                return;
+            }
+            if ($isFirstLine && str_ends_with($this->currentLineString, ')')) {
+                // Special case if the ')' is on same line as the opening '('
+                $this->currentLineString = substr($this->currentLineString, 0, -1);
+                if (strlen($this->currentLineString) > 1) {
+                    [$this->valueStream, $this->tokenStream] = $this->parseValueForConstants($this->valueStream, $this->tokenStream, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine);
+                    $this->tokenStream->append(new Token(TokenType::T_OPERATOR_ASSIGNMENT_MULTILINE_STOP, ')', $this->currentLineNumber, $this->currentColumnInLine + mb_strlen($this->currentLineString)));
+                    // Tricky to swap the streams here, but that's the most effective solution I could come up with for the line endings here.
+                    ($this->currentLinebreakCallback)();
+                    $tempStream = $this->tokenStream;
+                    $this->tokenStream = $this->valueStream;
+                    ($this->currentLinebreakCallback)();
+                    $this->tokenStream = $tempStream;
+                    $this->lineStream->append((new IdentifierAssignmentLine())->setIdentifierTokenStream($this->identifierStream)->setValueTokenStream($this->valueStream)->setTokenStream($this->tokenStream));
+                    return;
+                }
+                $this->tokenStream->append(new Token(TokenType::T_OPERATOR_ASSIGNMENT_MULTILINE_STOP, ')', $this->currentLineNumber, $this->currentColumnInLine + strlen($this->currentLineString)));
+                ($this->currentLinebreakCallback)();
+                $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+                return;
+            }
+            if ($isFirstLine && strlen($this->currentLineString)) {
+                [$this->valueStream, $this->tokenStream] = $this->parseValueForConstants($this->valueStream, $this->tokenStream, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine);
+                $valueOnFirstLine = true;
+                $previousLineCallback = $this->currentLinebreakCallback;
+            }
+            if (($isFirstLine && $valueOnFirstLine)
+                || (!$isFirstLine && !$isSecondLine)
+            ) {
+                $tempStream = $this->tokenStream;
+                $this->tokenStream = $this->valueStream;
+                $previousLineCallback();
+                $this->tokenStream = $tempStream;
+            }
+            if (!$isFirstLine && strlen($this->currentLineString)) {
+                [$this->valueStream, $this->tokenStream] = $this->parseValueForConstants($this->valueStream, $this->tokenStream, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine);
+            }
+            $previousLineCallback = $this->currentLinebreakCallback;
+            ($this->currentLinebreakCallback)();
+            if (!array_key_exists($this->currentLineNumber + 1, $this->lines)) {
+                $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+                return;
+            }
+            if ($isFirstLine) {
+                $isSecondLine = true;
+            } else {
+                $isSecondLine = false;
+            }
+            $isFirstLine = false;
+            $valueOnFirstLine = false;
+            $this->currentLineNumber++;
+            $this->currentColumnInLine = 0;
+            $this->currentLineString = $this->lines[$this->currentLineNumber]['line'];
+            $this->currentLinebreakCallback = $this->lines[$this->currentLineNumber]['linebreakCallback'];
+        }
+    }
+
+    private function parseOperatorUnset(): void
+    {
+        $this->tokenStream->append(new Token(TokenType::T_OPERATOR_UNSET, '>', $this->currentLineNumber, $this->currentColumnInLine));
+        $this->currentColumnInLine++;
+        $this->currentLineString = substr($this->currentLineString, 1);
+        $this->parseTabsAndWhitespaces();
+        $this->makeComment();
+        $this->lineStream->append((new IdentifierUnsetLine())->setTokenStream($this->tokenStream)->setIdentifierTokenStream($this->identifierStream));
+    }
+
+    private function parseOperatorCopy(): void
+    {
+        $this->tokenStream->append(new Token(TokenType::T_OPERATOR_COPY, '<', $this->currentLineNumber, $this->currentColumnInLine));
+        $this->currentColumnInLine++;
+        $this->currentLineString = substr($this->currentLineString, 1);
+        $this->parseTabsAndWhitespaces();
+        $identifierStream = $this->identifierStream;
+        $this->parseIdentifierAtEndOfLine();
+        $referenceStream = $this->identifierStream;
+        if ($referenceStream->isEmpty()) {
+            // @todo: ($this->currentLinebreakCallback)(); is missing here?!
+            $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+            return;
+        }
+        $this->lineStream->append(
+            (new IdentifierCopyLine())
+                ->setIdentifierTokenStream($identifierStream)
+                ->setValueTokenStream($referenceStream)
+                ->setTokenStream($this->tokenStream)
+        );
+    }
+
+    private function parseOperatorReference(): void
+    {
+        $this->tokenStream->append(new Token(TokenType::T_OPERATOR_REFERENCE, '=<', $this->currentLineNumber, $this->currentColumnInLine));
+        $this->currentColumnInLine += 2;
+        $this->currentLineString = substr($this->currentLineString, 2);
+        $this->parseTabsAndWhitespaces();
+        $identifierStream = $this->identifierStream;
+        $this->parseIdentifierAtEndOfLine();
+        $referenceStream = $this->identifierStream;
+        if ($referenceStream->isEmpty()) {
+            $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+            return;
+        }
+        $this->lineStream->append(
+            (new IdentifierReferenceLine())
+                ->setIdentifierTokenStream($identifierStream)
+                ->setValueTokenStream($referenceStream)
+                ->setTokenStream($this->tokenStream)
+        );
+    }
+
+    private function parseIdentifierAtEndOfLine(): void
+    {
+        $this->identifierStream = new IdentifierTokenStream();
+        $isRelative = false;
+        $splitLine = mb_str_split($this->currentLineString, 1, 'UTF-8');
+        $char = $splitLine[0] ?? null;
+        if ($char === null) {
+            return;
+        }
+        $nextTwoChars = $char . ($splitLine[1] ?? '');
+        if ($char === '.') {
+            // A relative right side: foo.bar < .foo (note the dot!). we identifierStream->setRelative() and
+            // get rid of the dot for the rest of the processing.
+            $isRelative = true;
+            $this->tokenStream->append((new Token(TokenType::T_DOT, '.', 0, $this->currentColumnInLine)));
+            array_shift($splitLine);
+            $this->currentColumnInLine++;
+            $this->currentLineString = substr($this->currentLineString, 1);
+        }
+        if ($char === '#') {
+            $this->parseHashComment();
+            return;
+        }
+        if ($nextTwoChars === '//') {
+            $this->parseDoubleSlashComment();
+            return;
+        }
+        if ($nextTwoChars === '/*') {
+            $this->parseMultilineComment();
+            return;
+        }
+        $currentPosition = $this->parseIdentifierUntilStopChar($splitLine, $isRelative);
+        if (!$currentPosition) {
+            return;
+        }
+        $this->currentLineString = mb_substr($this->currentLineString, $currentPosition);
+        $this->currentColumnInLine = $this->currentColumnInLine + $currentPosition;
+        $this->parseTabsAndWhitespaces();
+        $this->makeComment();
+    }
+
+    private function parseIdentifierUntilStopChar(array $splitLine, bool $isRelative = false): ?int
+    {
+        $this->identifierStream = new IdentifierTokenStream();
+        if ($isRelative) {
+            $this->identifierStream->setRelative();
+        }
+        $currentPosition = 0;
+        $currentIdentifierStartPosition = $this->currentColumnInLine;
+        $currentIdentifierBody = '';
+        $currentIdentifierCharCount = 0;
+        while (true) {
+            $nextChar = $splitLine[$currentPosition] ?? null;
+            if ($nextChar === null) {
+                if ($currentIdentifierCharCount) {
+                    $identifierToken = new IdentifierToken(TokenType::T_IDENTIFIER, $currentIdentifierBody, $this->currentLineNumber, $currentIdentifierStartPosition);
+                    $this->tokenStream->append($identifierToken);
+                    $this->identifierStream->append($identifierToken);
+                }
+                ($this->currentLinebreakCallback)();
+                return null;
+            }
+            $nextTwoChars = $nextChar . ($splitLine[$currentPosition + 1] ?? null);
+            if ($currentPosition > 0
+                && ($nextChar === ' ' || $nextChar === "\t" || $nextChar === '=' || $nextChar === '<' || $nextChar === '>' || $nextChar === '{' || $nextTwoChars === ':=' || $nextChar === '(')
+            ) {
+                if ($currentIdentifierCharCount) {
+                    $identifierToken = new IdentifierToken(TokenType::T_IDENTIFIER, $currentIdentifierBody, $this->currentLineNumber, $currentIdentifierStartPosition);
+                    $this->tokenStream->append($identifierToken);
+                    $this->identifierStream->append($identifierToken);
+                }
+                break;
+            }
+            if ($nextTwoChars === '\\.') {
+                // A quoted dot is part of *this* identifier
+                $currentIdentifierBody .= '.';
+                $currentPosition += 2;
+                $currentIdentifierCharCount++;
+            } elseif ($nextChar === '.') {
+                if ($currentIdentifierCharCount) {
+                    $identifierToken = new IdentifierToken(TokenType::T_IDENTIFIER, $currentIdentifierBody, $this->currentLineNumber, $currentIdentifierStartPosition);
+                    $this->tokenStream->append($identifierToken);
+                    $this->identifierStream->append($identifierToken);
+                    $currentIdentifierCharCount = 0;
+                    $currentIdentifierBody = '';
+                }
+                $this->tokenStream->append(new Token(TokenType::T_DOT, '.', $this->currentLineNumber, $this->currentColumnInLine + $currentPosition));
+                $currentPosition++;
+                $currentIdentifierStartPosition = $this->currentColumnInLine + $currentPosition;
+            } else {
+                $currentIdentifierBody .= $nextChar;
+                $currentIdentifierCharCount++;
+                $currentPosition++;
+            }
+        }
+        return $currentPosition;
+    }
+
+    private function parseOperatorFunction(): void
+    {
+        $this->tokenStream->append(new Token(TokenType::T_OPERATOR_FUNCTION, ':=', $this->currentLineNumber, $this->currentColumnInLine));
+        $this->currentColumnInLine += 2;
+        $this->currentLineString = substr($this->currentLineString, 2);
+        $this->parseTabsAndWhitespaces();
+        if ($this->currentLineString === '') {
+            ($this->currentLinebreakCallback)();
+            $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+            return;
+        }
+        $functionName = '';
+        $functionNameStartPosition = $this->currentColumnInLine;
+        $functionNameCharCount = 0;
+        $functionChars = mb_str_split($this->currentLineString, 1, 'UTF-8');
+        while (true) {
+            $nextChar = $functionChars[$functionNameCharCount] ?? null;
+            if ($nextChar === null) {
+                // end of chars
+                if ($functionNameCharCount) {
+                    $this->tokenStream->append(new Token(TokenType::T_FUNCTION_NAME, $functionName, $this->currentLineNumber, $functionNameStartPosition));
+                }
+                ($this->currentLinebreakCallback)();
+                $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+                return;
+            }
+            if ($nextChar === '(') {
+                if ($functionNameCharCount) {
+                    $functionNameToken = new Token(TokenType::T_FUNCTION_NAME, $functionName, $this->currentLineNumber, $functionNameStartPosition);
+                    $this->tokenStream->append($functionNameToken);
+                    $this->tokenStream->append(new Token(TokenType::T_FUNCTION_VALUE_START, '(', $this->currentLineNumber, $this->currentColumnInLine + $functionNameCharCount));
+                    $functionNameCharCount++;
+                    break;
+                }
+                $this->makeComment();
+                $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+                return;
+            }
+            $functionName .= $nextChar;
+            $functionNameCharCount++;
+        }
+        $functionBodyStartPosition = $functionNameCharCount;
+        $functionBodyPart = '';
+        $functionBodyCharCount = 0;
+        $functionValueStream = new TokenStream();
+        $parenthesesLevel = 0;
+        while (true) {
+            $nextChar = $functionChars[$functionBodyStartPosition + $functionBodyCharCount] ?? null;
+            if ($nextChar === null) {
+                if ($functionBodyCharCount) {
+                    $this->tokenStream->append(new Token(TokenType::T_VALUE, $functionBodyPart, $this->currentLineNumber, $functionBodyStartPosition));
+                }
+                ($this->currentLinebreakCallback)();
+                $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+                return;
+            }
+            if ($nextChar === '(') {
+                // In case of a function call like "appendString(something(somethingelse))"
+                // we shall only stop processing when the last bracket was evaluated.
+                $parenthesesLevel++;
+            }
+            if ($nextChar === ')') {
+                if ($parenthesesLevel > 0) {
+                    $parenthesesLevel--;
+                    // Continue collecting characters from the (...) argument stream.
+                    // Also, ")" will be appended, thus intentionally no "break" occurs.
+                } else {
+                    if ($functionBodyCharCount) {
+                        [$functionValueStream, $this->tokenStream] = $this->parseValueForConstants($functionValueStream, $this->tokenStream, $functionBodyPart, $this->currentLineNumber, $this->currentColumnInLine, $functionBodyStartPosition);
+                    }
+                    $this->tokenStream->append(new Token(TokenType::T_FUNCTION_VALUE_STOP, ')', $this->currentLineNumber, $this->currentColumnInLine + $functionNameCharCount + $functionBodyCharCount));
+                    $functionBodyCharCount++;
+                    break;
+                }
+            }
+            $functionBodyPart .= $nextChar;
+            $functionBodyCharCount++;
+        }
+        $this->currentColumnInLine = $this->currentColumnInLine + $functionNameCharCount + $functionBodyCharCount;
+        $this->currentLineString = mb_substr($this->currentLineString, $functionNameCharCount + $functionBodyCharCount);
+        $this->parseTabsAndWhitespaces();
+        $this->makeComment();
+        $this->lineStream->append(
+            (new IdentifierFunctionLine())
+                ->setIdentifierTokenStream($this->identifierStream)
+                ->setFunctionNameToken($functionNameToken)
+                ->setTokenStream($this->tokenStream)
+                ->setFunctionValueTokenStream($functionValueStream)
+        );
+    }
+
+    /**
+     * @return array{0: TokenStreamInterface, 1: TokenStreamInterface}
+     */
+    private function parseValueForConstants(TokenStreamInterface $valueStream, TokenStreamInterface $tokenStream, string $value, int $line, int $column, int $tokenOffsetPosition = 0): array
+    {
+        if (!str_contains($value, '{$')) {
+            $valueToken = new Token(TokenType::T_VALUE, $value, $line, $column + $tokenOffsetPosition);
+            $valueStream->append($valueToken);
+            $tokenStream->append($valueToken);
+            return [$valueStream, $tokenStream];
+        }
+        $splitLine = mb_str_split($value, 1, 'UTF-8');
+        $isInConstant = false;
+        $currentPosition = 0;
+        $currentString = '';
+        $currentStringLength = 0;
+        $lastTokenEndPosition = 0;
+        while (true) {
+            $char = $splitLine[$currentPosition] ?? null;
+            if ($char === null) {
+                if ($currentStringLength) {
+                    $valueToken = new Token(TokenType::T_VALUE, $currentString, $line, $column + $lastTokenEndPosition + $tokenOffsetPosition);
+                    $valueStream->append($valueToken);
+                    $tokenStream->append($valueToken);
+                }
+                break;
+            }
+            $nextTwoChars = $char . ($splitLine[$currentPosition + 1] ?? '');
+            if ($nextTwoChars === '{$') {
+                $isInConstant = true;
+                if ($currentStringLength) {
+                    $valueToken = new Token(TokenType::T_VALUE, $currentString, $line, $column + $lastTokenEndPosition + $tokenOffsetPosition);
+                    $valueStream->append($valueToken);
+                    $tokenStream->append($valueToken);
+                    $lastTokenEndPosition = $currentPosition;
+                }
+                $currentString = '{$';
+                $currentPosition += 2;
+                continue;
+            }
+            if ($isInConstant && $char === '}') {
+                $valueToken = new Token(TokenType::T_CONSTANT, $currentString . '}', $line, $column + $lastTokenEndPosition + $tokenOffsetPosition);
+                if (!$valueStream instanceof ConstantAwareTokenStream) {
+                    $valueStream = (new ConstantAwareTokenStream())->setAll($valueStream->getAll());
+                }
+                $valueStream->append($valueToken);
+                $tokenStream->append($valueToken);
+                $currentPosition++;
+                $currentString = '';
+                $currentStringLength = 0;
+                $lastTokenEndPosition = $currentPosition;
+                $isInConstant = false;
+                continue;
+            }
+            $currentPosition++;
+            $currentStringLength++;
+            $currentString .= $char;
+        }
+        return [$valueStream, $tokenStream];
+    }
+}

Classes/TypoScript/Tokenizer/LossyTokenizer.php

@@ -0,0 +1,624 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer;
+
+use Symfony\Component\DependencyInjection\Attribute\AsAlias;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\BlockCloseLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ConditionElseLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ConditionLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ConditionStopLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierAssignmentLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierBlockOpenLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierCopyLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierFunctionLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierReferenceLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierUnsetLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ImportLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\LineStream;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\ConstantAwareTokenStream;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\IdentifierToken;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\IdentifierTokenStream;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\Token;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenStream;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenStreamInterface;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenType;
+
+/**
+ * A lossy tokenizer implementation: Do not create invalid lines, do not create empty lines,
+ * do not create token line and column positions.
+ *
+ * This tokenizer creates a much smaller streams of only relevant lines. All information
+ * not essential for the AstBuilder is skipped. This tokenizer is used in frontend rendering
+ * for quicker AST building.
+ *
+ * An instance of this tokenizer is injected by DI when injecting TokenizerInterface.
+ *
+ * This class is unit test covered by TokenizerInterfaceTest and paired with LossyTokenizer.
+ * Never change anything in this class without additional test coverage!
+ *
+ * @internal: Internal tokenizer structure.
+ */
+#[AsAlias(TokenizerInterface::class)]
+final class LossyTokenizer implements TokenizerInterface
+{
+    private LineStream $lineStream;
+
+    private IdentifierTokenStream $identifierStream;
+    private TokenStreamInterface $valueStream;
+
+    private array $lines;
+    private int $currentLineNumber;
+    private string $currentLineString;
+
+    public function tokenize(string $source): LineStream
+    {
+        $this->lineStream = new LineStream();
+        $this->currentLineNumber = -1;
+        $this->lines = $this->splitLines($source);
+
+        while (true) {
+            $this->currentLineNumber++;
+            if (!array_key_exists($this->currentLineNumber, $this->lines)) {
+                break;
+            }
+            $this->currentLineString = trim($this->lines[$this->currentLineNumber]['line']);
+            $nextChar = substr($this->currentLineString, 0, 1);
+            if ($nextChar === '') {
+                continue;
+            }
+            $nextTwoChars = substr($this->currentLineString, 0, 2);
+            if ($nextChar === '#' || $nextTwoChars === '//') {
+                continue;
+            }
+            if ($nextTwoChars === '/*') {
+                // @todo: This is one of multiple places where multiline "/*" comments are parsed in this tokenizer. Other
+                //        places are cluttered in detail methods. It might be more straight to have an early scanning
+                //        phase through all lines to remove comments up front, to not wire especially the multiline comment
+                //        parsing to single places, and throw away commented lines early. This isn't trivial though, since
+                //        for instance "foo = bar /* not a comment */" then needs to be sorted out, too. Having an early
+                //        "kick comments" loop however might be quicker in the end and would make the main parsing
+                //        methods more concise and probably more bullet proof.
+                //        Also note there are currently not-unit-tested edge cases, that will currently not parse as
+                //        (maybe) expected. In the example below, "foo2 = bar2" is ignored. This is an issue with the
+                //        LosslessTokenizer as well, probably, and we may rather want to declare this as invalid syntax?!
+                //          foo = bar /* comment start
+                //          comment end */ foo2 = bar2
+                $this->ignoreUntilEndOfMultilineComment();
+                continue;
+            }
+            if ($nextChar === '[') {
+                $this->createConditionLine();
+            } elseif ($nextChar === '}') {
+                $this->lineStream->append((new BlockCloseLine()));
+            } elseif (str_starts_with($this->currentLineString, '@import')) {
+                $this->parseImportLine();
+            } elseif (str_starts_with($this->currentLineString, '<INCLUDE_TYPOSCRIPT:')) {
+                // @todo: Do nothing. This creates an InvalidLine in LossyTokenizer.
+            } else {
+                $this->parseIdentifier();
+            }
+        }
+
+        return $this->lineStream;
+    }
+
+    private function splitLines($source): array
+    {
+        $vanillaLines = explode(chr(10), $source);
+        return array_map(
+            fn(int $lineNumber, string $vanillaLine): array => [
+                'line' => rtrim($vanillaLine, "\r"),
+            ],
+            array_keys($vanillaLines),
+            $vanillaLines
+        );
+    }
+
+    private function ignoreUntilEndOfMultilineComment(): void
+    {
+        while (true) {
+            if (str_contains($this->currentLineString, '*/')) {
+                return;
+            }
+            if (!array_key_exists($this->currentLineNumber + 1, $this->lines)) {
+                return;
+            }
+            $this->currentLineNumber++;
+            $this->currentLineString = trim($this->lines[$this->currentLineNumber]['line']);
+        }
+    }
+
+    /**
+     * Create a condition line from token stream of this line.
+     */
+    private function createConditionLine(): void
+    {
+        $upperCaseLine = strtoupper($this->currentLineString);
+        if (str_starts_with($upperCaseLine, '[ELSE]')) {
+            $this->lineStream->append((new ConditionElseLine()));
+            $this->currentLineString = trim(substr($this->currentLineString, 6));
+            if (str_starts_with($this->currentLineString, '/*')) {
+                $this->ignoreUntilEndOfMultilineComment();
+            }
+            return;
+        }
+        if (str_starts_with($upperCaseLine, '[END]')) {
+            $this->lineStream->append((new ConditionStopLine()));
+            $this->currentLineString = trim(substr($this->currentLineString, 5));
+            if (str_starts_with($this->currentLineString, '/*')) {
+                $this->ignoreUntilEndOfMultilineComment();
+            }
+            return;
+        }
+        if (str_starts_with($upperCaseLine, '[GLOBAL]')) {
+            $this->lineStream->append((new ConditionStopLine()));
+            $this->currentLineString = trim(substr($this->currentLineString, 8));
+            if (str_starts_with($this->currentLineString, '/*')) {
+                $this->ignoreUntilEndOfMultilineComment();
+            }
+            return;
+        }
+        $conditionBody = '';
+        $conditionBodyCharCount = 0;
+        $conditionBodyChars = mb_str_split(substr($this->currentLineString, 1), 1, 'UTF-8');
+        $bracketCount = 1;
+        while (true) {
+            $nextChar = $conditionBodyChars[$conditionBodyCharCount] ?? null;
+            if ($nextChar === null) {
+                // end of chars
+                return;
+            }
+            if ($nextChar === '[') {
+                $bracketCount++;
+                $conditionBody .= $nextChar;
+                $conditionBodyCharCount++;
+                continue;
+            }
+            if ($nextChar === ']') {
+                $bracketCount--;
+                if ($bracketCount === 0) {
+                    if ($conditionBodyCharCount) {
+                        $conditionBodyToken = new Token(TokenType::T_VALUE, $conditionBody);
+                        $this->lineStream->append((new ConditionLine())->setValueToken($conditionBodyToken));
+                        $conditionBodyCharCount++;
+                        break;
+                    }
+                    $conditionBodyCharCount++;
+                    break;
+                }
+                $conditionBody .= $nextChar;
+                $conditionBodyCharCount++;
+                continue;
+            }
+            $conditionBody .= $nextChar;
+            $conditionBodyCharCount++;
+        }
+        $this->currentLineString = trim(mb_substr($this->currentLineString, $conditionBodyCharCount + 1));
+        if (str_starts_with($this->currentLineString, '/*')) {
+            $this->ignoreUntilEndOfMultilineComment();
+        }
+    }
+
+    private function parseBlockStart(): void
+    {
+        $this->currentLineString = trim(substr($this->currentLineString, 1));
+        if (str_starts_with($this->currentLineString, '}')) {
+            // Edge case: foo = { } in one line. Note content within {} is not parsed, everything behind { ends up as comment.
+            $this->lineStream->append((new IdentifierBlockOpenLine())->setIdentifierTokenStream($this->identifierStream));
+            $this->lineStream->append((new BlockCloseLine()));
+            return;
+        }
+        $this->lineStream->append((new IdentifierBlockOpenLine())->setIdentifierTokenStream($this->identifierStream));
+    }
+
+    private function parseImportLine(): void
+    {
+        $this->currentLineString = trim(substr($this->currentLineString, 7));
+
+        // Next char should be the opening tick or doubletick, otherwise treat it as ignored comment
+        $nextChar = substr($this->currentLineString, 0, 1);
+        if ($nextChar !== '\'' && $nextChar !== '"') {
+            return;
+        }
+
+        $importBody = '';
+        $importBodyCharCount = 0;
+        $importBodyChars = mb_str_split(substr($this->currentLineString, 1), 1, 'UTF-8');
+        while (true) {
+            $nextChar = $importBodyChars[$importBodyCharCount] ?? null;
+            if ($nextChar === null) {
+                // end of chars
+                if ($importBodyCharCount) {
+                    $importBodyToken = (new Token(TokenType::T_VALUE, $importBody));
+                    $this->lineStream->append((new ImportLine())->setValueToken($importBodyToken));
+                    return;
+                }
+                return;
+            }
+            if ($nextChar === '\'' || $nextChar === '"') {
+                if ($importBodyCharCount) {
+                    $importBodyToken = new Token(TokenType::T_VALUE, $importBody);
+                    $this->lineStream->append((new ImportLine())->setValueToken($importBodyToken));
+                    break;
+                }
+                break;
+            }
+            $importBody .= $nextChar;
+            $importBodyCharCount++;
+        }
+        $this->currentLineString = trim(mb_substr($this->currentLineString, $importBodyCharCount + 2));
+        if (str_starts_with($this->currentLineString, '/*')) {
+            $this->ignoreUntilEndOfMultilineComment();
+        }
+    }
+
+    private function parseIdentifier(): void
+    {
+        $splitLine = mb_str_split($this->currentLineString, 1, 'UTF-8');
+        $currentPosition = $this->parseIdentifierUntilStopChar($splitLine);
+        if (!$currentPosition) {
+            return;
+        }
+        $this->currentLineString = trim(mb_substr($this->currentLineString, $currentPosition));
+        $nextChar = substr($this->currentLineString, 0, 1);
+        $nextTwoChars = $nextChar . substr($this->currentLineString, 1, 1);
+        if ($nextTwoChars === '=<') {
+            $this->parseOperatorReference();
+            return;
+        }
+        if ($nextChar === '=') {
+            $this->parseOperatorAssignment();
+            return;
+        }
+        if ($nextChar === '{') {
+            $this->parseBlockStart();
+            return;
+        }
+        if ($nextChar === '>') {
+            $this->parseOperatorUnset();
+            return;
+        }
+        if ($nextChar === '<') {
+            $this->parseOperatorCopy();
+            return;
+        }
+        if ($nextChar === '(') {
+            $this->parseOperatorMultilineAssignment();
+            return;
+        }
+        if ($nextTwoChars === ':=') {
+            $this->parseOperatorFunction();
+        }
+        if ($nextTwoChars === '/*') {
+            $this->ignoreUntilEndOfMultilineComment();
+        }
+    }
+
+    private function parseOperatorUnset(): void
+    {
+        $this->lineStream->append((new IdentifierUnsetLine())->setIdentifierTokenStream($this->identifierStream));
+        $this->currentLineString = trim(trim(trim($this->currentLineString), '>'));
+        if (str_starts_with($this->currentLineString, '/*')) {
+            $this->ignoreUntilEndOfMultilineComment();
+        }
+    }
+
+    private function parseOperatorAssignment(): void
+    {
+        $this->currentLineString = trim(substr($this->currentLineString, 1));
+        $this->valueStream = $this->parseValueForConstants(new TokenStream(), $this->currentLineString);
+        $this->lineStream->append((new IdentifierAssignmentLine())->setIdentifierTokenStream($this->identifierStream)->setValueTokenStream($this->valueStream));
+    }
+
+    private function parseOperatorMultilineAssignment(): void
+    {
+        $this->valueStream = new TokenStream();
+        $this->currentLineString = substr($this->currentLineString, 1);
+        // True if we're currently in the line with the opening '('
+        $isFirstLine = true;
+        // True if the first line has a first value token: "foo ( thisIsTheFirstValueToken"
+        $valueOnFirstLine = false;
+        // True if the line after '(' is parsed
+        $isSecondLine = false;
+        while (true) {
+            if (str_starts_with(ltrim($this->currentLineString), ')')) {
+                $this->currentLineString = trim(substr($this->currentLineString, 1));
+                if (!$this->valueStream->isEmpty()) {
+                    $this->lineStream->append((new IdentifierAssignmentLine())->setIdentifierTokenStream($this->identifierStream)->setValueTokenStream($this->valueStream));
+                }
+                if (str_starts_with($this->currentLineString, '/*')) {
+                    $this->ignoreUntilEndOfMultilineComment();
+                }
+                return;
+            }
+            if ($isFirstLine && str_ends_with($this->currentLineString, ')')) {
+                $this->currentLineString = substr($this->currentLineString, 0, -1);
+                if (strlen($this->currentLineString) > 1) {
+                    $this->valueStream = $this->parseValueForConstants($this->valueStream, $this->currentLineString);
+                    $this->lineStream->append((new IdentifierAssignmentLine())->setIdentifierTokenStream($this->identifierStream)->setValueTokenStream($this->valueStream));
+                    return;
+                }
+                return;
+            }
+            if ($isFirstLine && strlen($this->currentLineString)) {
+                $this->valueStream = $this->parseValueForConstants($this->valueStream, $this->currentLineString);
+                $valueOnFirstLine = true;
+            }
+            if (($isFirstLine && $valueOnFirstLine)
+                || (!$isFirstLine && !$isSecondLine)
+            ) {
+                $this->valueStream->append(new Token(TokenType::T_NEWLINE, "\n"));
+            }
+            if (!$isFirstLine && strlen($this->currentLineString)) {
+                $this->valueStream = $this->parseValueForConstants($this->valueStream, $this->currentLineString);
+            }
+            if (!array_key_exists($this->currentLineNumber + 1, $this->lines)) {
+                return;
+            }
+            if ($isFirstLine) {
+                $isSecondLine = true;
+            } else {
+                $isSecondLine = false;
+            }
+            $isFirstLine = false;
+            $valueOnFirstLine = false;
+            $this->currentLineNumber++;
+            $this->currentLineString = $this->lines[$this->currentLineNumber]['line'];
+        }
+    }
+
+    private function parseOperatorCopy(): void
+    {
+        $this->currentLineString = trim(substr($this->currentLineString, 1));
+        $identifierStream = $this->identifierStream;
+        $charsHandled = $this->parseIdentifierAtEndOfLine();
+        $referenceStream = $this->identifierStream;
+        if ($referenceStream->isEmpty()) {
+            return;
+        }
+        $this->lineStream->append(
+            (new IdentifierCopyLine())
+                ->setIdentifierTokenStream($identifierStream)
+                ->setValueTokenStream($referenceStream)
+        );
+        $this->currentLineString = trim(mb_substr($this->currentLineString, $charsHandled));
+        if (str_starts_with($this->currentLineString, '/*')) {
+            $this->ignoreUntilEndOfMultilineComment();
+        }
+    }
+
+    private function parseOperatorReference(): void
+    {
+        $this->currentLineString = trim(substr($this->currentLineString, 2));
+        $identifierStream = $this->identifierStream;
+        $charsHandled = $this->parseIdentifierAtEndOfLine();
+        $referenceStream = $this->identifierStream;
+        if ($referenceStream->isEmpty()) {
+            return;
+        }
+        $this->lineStream->append(
+            (new IdentifierReferenceLine())
+                ->setIdentifierTokenStream($identifierStream)
+                ->setValueTokenStream($referenceStream)
+        );
+        $this->currentLineString = trim(mb_substr($this->currentLineString, $charsHandled));
+        if (str_starts_with($this->currentLineString, '/*')) {
+            $this->ignoreUntilEndOfMultilineComment();
+        }
+    }
+
+    private function parseIdentifierAtEndOfLine(): int
+    {
+        $this->identifierStream = new IdentifierTokenStream();
+        $isRelative = false;
+        $splitLine = mb_str_split($this->currentLineString, 1, 'UTF-8');
+        $char = $splitLine[0] ?? null;
+        if ($char === null) {
+            return 0;
+        }
+        $nextTwoChars = $char . ($splitLine[1] ?? '');
+        if ($char === '.') {
+            // A relative right side: foo.bar < .foo (note the dot!). we identifierStream->setRelative() and
+            // get rid of the dot for the rest of the processing.
+            $isRelative = true;
+            array_shift($splitLine);
+            $this->currentLineString = substr($this->currentLineString, 1);
+        }
+        if ($char === '#') {
+            return 1;
+        }
+        if ($nextTwoChars === '//') {
+            return 2;
+        }
+        if ($nextTwoChars === '/*') {
+            $this->ignoreUntilEndOfMultilineComment();
+            return 0;
+        }
+        return $this->parseIdentifierUntilStopChar($splitLine, $isRelative);
+    }
+
+    private function parseIdentifierUntilStopChar(array $splitLine, bool $isRelative = false): int
+    {
+        $this->identifierStream = new IdentifierTokenStream();
+        if ($isRelative) {
+            $this->identifierStream->setRelative();
+        }
+        $currentPosition = 0;
+        $currentIdentifierBody = '';
+        $currentIdentifierCharCount = 0;
+        while (true) {
+            $nextChar = $splitLine[$currentPosition] ?? null;
+            if ($nextChar === null) {
+                if ($currentIdentifierCharCount) {
+                    $identifierToken = new IdentifierToken(TokenType::T_IDENTIFIER, $currentIdentifierBody);
+                    $this->identifierStream->append($identifierToken);
+                }
+                return $currentPosition;
+            }
+            $nextTwoChars = $nextChar . ($splitLine[$currentPosition + 1] ?? null);
+            if ($currentPosition > 0
+                && ($nextChar === ' ' || $nextChar === "\t" || $nextChar === '=' || $nextChar === '<' || $nextChar === '>' || $nextChar === '{' || $nextTwoChars === ':=' || $nextChar === '(')
+            ) {
+                if ($currentIdentifierCharCount) {
+                    $identifierToken = new IdentifierToken(TokenType::T_IDENTIFIER, $currentIdentifierBody);
+                    $this->identifierStream->append($identifierToken);
+                }
+                break;
+            }
+            if ($nextTwoChars === '\\.') {
+                // A quoted dot is part of *this* identifier
+                $currentIdentifierBody .= '.';
+                $currentPosition += 2;
+                $currentIdentifierCharCount++;
+            } elseif ($nextChar === '.') {
+                if ($currentIdentifierCharCount) {
+                    $identifierToken = new IdentifierToken(TokenType::T_IDENTIFIER, $currentIdentifierBody);
+                    $this->identifierStream->append($identifierToken);
+                    $currentIdentifierCharCount = 0;
+                    $currentIdentifierBody = '';
+                }
+                $currentPosition++;
+            } else {
+                $currentIdentifierBody .= $nextChar;
+                $currentIdentifierCharCount++;
+                $currentPosition++;
+            }
+        }
+        return $currentPosition;
+    }
+
+    private function parseOperatorFunction(): void
+    {
+        $this->currentLineString = trim(substr($this->currentLineString, 2));
+        if ($this->currentLineString === '') {
+            return;
+        }
+        $functionName = '';
+        $functionNameCharCount = 0;
+        $functionChars = mb_str_split($this->currentLineString, 1, 'UTF-8');
+        while (true) {
+            $nextChar = $functionChars[$functionNameCharCount] ?? null;
+            if ($nextChar === null) {
+                // end of chars
+                return;
+            }
+            if ($nextChar === '(') {
+                if ($functionNameCharCount) {
+                    $functionNameToken = new Token(TokenType::T_FUNCTION_NAME, $functionName);
+                    $functionNameCharCount++;
+                    break;
+                }
+                return;
+            }
+            $functionName .= $nextChar;
+            $functionNameCharCount++;
+        }
+        $functionBodyStartPosition = $functionNameCharCount;
+        $functionBodyPart = '';
+        $functionBodyCharCount = 0;
+        $functionValueStream = new TokenStream();
+        $parenthesesLevel = 0;
+        while (true) {
+            $nextChar = $functionChars[$functionBodyStartPosition + $functionBodyCharCount] ?? null;
+            if ($nextChar === null) {
+                return;
+            }
+            if ($nextChar === '(') {
+                // In case of a function call like "appendString(something(somethingelse))"
+                // we shall only stop processing when the last bracket was evaluated.
+                $parenthesesLevel++;
+            }
+            if ($nextChar === ')') {
+                if ($parenthesesLevel > 0) {
+                    $parenthesesLevel--;
+                    // Continue collecting characters from the (...) argument stream.
+                    // Also, ")" will be appended, thus intentionally no "break" occurs.
+                } else {
+                    if ($functionBodyCharCount) {
+                        $functionValueStream = $this->parseValueForConstants($functionValueStream, $functionBodyPart);
+                        $functionBodyCharCount++;
+                    }
+                    break;
+                }
+            }
+            $functionBodyPart .= $nextChar;
+            $functionBodyCharCount++;
+        }
+        $this->lineStream->append(
+            (new IdentifierFunctionLine())
+                ->setIdentifierTokenStream($this->identifierStream)
+                ->setFunctionNameToken($functionNameToken)
+                ->setFunctionValueTokenStream($functionValueStream)
+        );
+        // Check for multiline comment
+        $this->currentLineString = implode('', array_slice($functionChars, $functionBodyStartPosition + $functionBodyCharCount + 1));
+        if (mb_strlen($this->currentLineString) >= 1 && str_starts_with($this->currentLineString, '/*')) {
+            $this->ignoreUntilEndOfMultilineComment();
+        }
+    }
+
+    private function parseValueForConstants(TokenStreamInterface $valueStream, string $value): TokenStreamInterface
+    {
+        if (!str_contains($value, '{$')) {
+            $valueStream->append(new Token(TokenType::T_VALUE, $value));
+            return $valueStream;
+        }
+        $splitLine = mb_str_split($value, 1, 'UTF-8');
+        $isInConstant = false;
+        $currentPosition = 0;
+        $currentString = '';
+        $currentStringLength = 0;
+        while (true) {
+            $char = $splitLine[$currentPosition] ?? null;
+            if ($char === null) {
+                if ($currentStringLength) {
+                    $valueToken = new Token(TokenType::T_VALUE, $currentString);
+                    $valueStream->append($valueToken);
+                }
+                break;
+            }
+            $nextTwoChars = $char . ($splitLine[$currentPosition + 1] ?? '');
+            if ($nextTwoChars === '{$') {
+                $isInConstant = true;
+                if ($currentStringLength) {
+                    $valueToken = new Token(TokenType::T_VALUE, $currentString);
+                    $valueStream->append($valueToken);
+                }
+                $currentString = '{$';
+                $currentPosition += 2;
+                continue;
+            }
+            if ($isInConstant && $char === '}') {
+                $valueToken = new Token(TokenType::T_CONSTANT, $currentString . '}');
+                if (!$valueStream instanceof ConstantAwareTokenStream) {
+                    $valueStream = (new ConstantAwareTokenStream())->setAll($valueStream->getAll());
+                }
+                $valueStream->append($valueToken);
+                $currentPosition++;
+                $currentString = '';
+                $currentStringLength = 0;
+                $isInConstant = false;
+                continue;
+            }
+            $currentPosition++;
+            $currentStringLength++;
+            $currentString .= $char;
+        }
+        return $valueStream;
+    }
+}

Classes/TypoScript/Tokenizer/Token/AbstractToken.php

@@ -0,0 +1,79 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Token;
+
+/**
+ * Main implementation of a TokenInterface.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+abstract class AbstractToken implements TokenInterface
+{
+    protected int $line = 0;
+    protected int $column = 0;
+
+    public function __construct(
+        private readonly TokenType $type,
+        protected readonly string $value,
+        int $line = 0,
+        int $column = 0
+    ) {
+        // No constructor property promotion for $line and $column: We don't serialize
+        // these two and want to still default them to 0 (zero) when unserialized.
+        $this->line = $line;
+        $this->column = $column;
+    }
+
+    public function __toString(): string
+    {
+        return $this->value;
+    }
+
+    /**
+     * Do not store line and column when structure is serialized to cache.
+     * Not storing $line and $column reduces the cache size by about 1/3 since
+     * we're typically storing *a lot* of tokens.
+     */
+    public function __serialize(): array
+    {
+        return [
+            'type' => $this->type,
+            'value' => $this->value,
+        ];
+    }
+
+    public function getType(): TokenType
+    {
+        return $this->type;
+    }
+
+    public function getValue(): string
+    {
+        return $this->value;
+    }
+
+    public function getLine(): int
+    {
+        return $this->line;
+    }
+
+    public function getColumn(): int
+    {
+        return $this->column;
+    }
+}

Classes/TypoScript/Tokenizer/Token/AbstractTokenStream.php

@@ -0,0 +1,113 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Token;
+
+/**
+ * A generic implementation of TokenStreamInterface.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+abstract class AbstractTokenStream implements TokenStreamInterface
+{
+    /**
+     * @var TokenInterface[]
+     */
+    protected array $tokens = [];
+    protected int $currentIndex = -1;
+
+    /**
+     * Create a source string from given tokens.
+     */
+    public function __toString(): string
+    {
+        $source = '';
+        $this->reset();
+        while ($token = $this->getNext()) {
+            $source .= $token;
+        }
+        return $source;
+    }
+
+    /**
+     * When storing to cache, we only store FE relevant properties and skip
+     * irrelevant things. For instance $currentIndex should always initialize
+     * to -1 and does not need to be stored.
+     */
+    final public function __serialize(): array
+    {
+        return $this->serialize();
+    }
+
+    protected function serialize(): array
+    {
+        $result['tokens'] = $this->tokens;
+        return $result;
+    }
+
+    /**
+     * Stream creation.
+     */
+    public function append(TokenInterface $token): self
+    {
+        $this->tokens[] = $token;
+        return $this;
+    }
+
+    /**
+     * We sometimes create a stream but don't add tokens.
+     * This method returns true if tokens have been added.
+     */
+    public function isEmpty(): bool
+    {
+        return empty($this->tokens);
+    }
+
+    /**
+     * Reset current pointer. Typically, call this before iterating with getNext().
+     */
+    public function reset(): static
+    {
+        $this->currentIndex = -1;
+        return $this;
+    }
+
+    /**
+     * Get next token and raise pointer.
+     */
+    public function getNext(): ?TokenInterface
+    {
+        $this->currentIndex++;
+        return $this->tokens[$this->currentIndex] ?? null;
+    }
+
+    public function peekNext(): ?TokenInterface
+    {
+        return $this->tokens[$this->currentIndex + 1] ?? null;
+    }
+
+    public function getAll(): array
+    {
+        return $this->tokens;
+    }
+
+    public function setAll(array $tokens): self
+    {
+        $this->tokens = $tokens;
+        return $this;
+    }
+}

Classes/TypoScript/Tokenizer/Token/ConstantAwareTokenStream.php

@@ -0,0 +1,106 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Token;
+
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+/**
+ * A list of single T_VALUE, T_NEWLINE and T_CONSTANT tokens. This is only created for
+ * LineIdentifierAssignment lines if there is at least one T_CONSTANT token
+ * in the assignment that needs to be evaluated when string'ified by the
+ * AST-builder.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class ConstantAwareTokenStream extends AbstractTokenStream
+{
+    private ?array $flatConstants = null;
+
+    /**
+     * Set by the AstBuilder to resolve constant values. Never cached.
+     */
+    public function setFlatConstants(array $flatConstants): void
+    {
+        $this->flatConstants = $flatConstants;
+    }
+
+    /**
+     * Create a source string from given tokens.
+     * This resolves T_CONSTANT tokens to their value if they exist in $this->flatConstants.
+     */
+    public function __toString(): string
+    {
+        $source = '';
+        $this->reset();
+        while ($token = $this->getNext()) {
+            if ($token->getType() === TokenType::T_CONSTANT) {
+                $token = $this->getConstantValue($this->parseConstantExpression($token->getValue())) ?? $token;
+            }
+            $source .= $token;
+        }
+        $this->reset();
+        return $source;
+    }
+
+    private function getConstantValue(?array $constantNames): ?string
+    {
+        if ($this->flatConstants === null || $constantNames === null) {
+            return null;
+        }
+        foreach ($constantNames as $constantName) {
+            $value = $this->flatConstants[$constantName] ?? null;
+            if ($value !== null) {
+                return (string)$value;
+            }
+        }
+        return null;
+    }
+
+    /**
+     * Parse constant expression, including null coalescing operator into an
+     * array of constant names to look up in order.
+     *
+     * @todo: The tokenization of this constant expression should ideally be moved
+     *        into the TypoScript Tokenizer in order to produce a list of multiple tokens
+     *        instead of just a T_CONSTANT for the entire body.
+     *        This would allow early static syntax analysis of the construct and maybe
+     *        detection of invalid and fallback to T_CONSTANT_INVALID that is treated
+     *        like T_VALUE and can be detected. Maybe something like this:
+     *          TokenType::T_CONSTANT_START "{"
+     *          TokenType::T_CONSTANT_END "}"
+     *          TokenType::T_CONSTANT_NAME "$foo.bar"
+     *          TokenType::T_CONSTANT_OPERATOR_NULL_COALESCE " ?? "
+     *          TokenType::T_CONSTANT_INVALID "{$foo ?? bar}" (missing $ before bar)
+     */
+    private function parseConstantExpression(string $constantExpression): ?array
+    {
+        $innerExpression = ltrim(rtrim($constantExpression, '}'), '{');
+        $tokenValues = GeneralUtility::trimExplode(' ?? ', $innerExpression, true);
+        if ($tokenValues === []) {
+            return null;
+        }
+        $tokenValueNames = [];
+        foreach ($tokenValues as $tokenValue) {
+            if (!str_starts_with($tokenValue, '$')) {
+                return null;
+            }
+            $tokenValueNames[] = substr($tokenValue, 1);
+        }
+        return $tokenValueNames;
+    }
+}

Classes/TypoScript/Tokenizer/Token/IdentifierToken.php

@@ -0,0 +1,39 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Token;
+
+/**
+ * A special token if this token is a T_IDENTIFIER token:
+ * With a line like "foo = bar", "foo" is created as TokenIdentifier TokenInterface
+ * (as opposed to Token) having a TokenType::T_IDENTIFIER token.
+ * The only difference to all other tokens is that TokenIdentifier tokens
+ * quote any "." (dots) in their value with a backslash when output. This is
+ * mostly used in backend when rendering source of TokenLine's.
+ *
+ * Note we do *not* explicitly check if TokenType::T_IDENTIFIER is given in
+ * __construct() at the moment for performance reasons and inheritance considerations.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class IdentifierToken extends AbstractToken
+{
+    public function __toString(): string
+    {
+        return str_replace('.', '\.', $this->value);
+    }
+}

Classes/TypoScript/Tokenizer/Token/IdentifierTokenStream.php

@@ -0,0 +1,106 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Token;
+
+/**
+ * A list of single identifier (!) tokens: TokenType::T_IDENTIFIER, and only of those.
+ *
+ * This is used in TS lines that know certain parts have to be lists of identifier tokens only.
+ * For instance a LineIdentifierAssignment "foo.bar = barValue" return this stream for getIdentifierTokenStream():
+ * The left side of an assignment line is a list of identifier tokens.
+ *
+ * Identifiers can be "relative" on the right side for "<" (LineIdentifierCopy) and "=<" (LineIdentifierReference).
+ * Examples are "foo.bar < .baz" and "foo.bar =< .baz". These are identified by having a "." (dot) at the beginning
+ * on the right side. For these places, the toggle "relative" is set to true for the AST-builder to look for relative
+ * copy and copy-reference. The generic example are "relative" references in TS menus: 'RO < .NO'
+ *
+ * For example, with "foo.bar < baz", the Tokenizer creates a LineIdentifierCopy line, having a TokenStreamIdentifier
+ * list of the T_IDENTIFIER tokens for 'foo' and 'bar' for getIdentifierTokenStream(), plus a TokenStreamIdentifier list
+ * of T_IDENTIFIER tokens for 'baz' for getValueTokenStream().
+ *
+ * Note identifier streams on the left side (foo.bar = ...) are never relative, this toggle is true for "<" and "=<" only.
+ *
+ * Lines that know they can only return TokenStreamIdentifier's - they are more specific than just TokenStream, are
+ * type-hinted as such. For instance getIdentifierTokenStream() type hints TokenStreamIdentifier.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class IdentifierTokenStream extends AbstractTokenStream
+{
+    private bool $relative = false;
+
+    /**
+     * When rendering a source string from multiple identifiers, dots between single identifiers need to be added again.
+     * This is used in RootNode->toArray() to create that insane '< lib.whatever' as value when using the
+     * reference operator: "foo =< lib.whatever". See ContentObjectRenderer cObjGetSingle() and mergeTSRef().
+     */
+    public function __toString(): string
+    {
+        $source = [];
+        $this->reset();
+        while ($token = $this->getNext()) {
+            $source[] = (string)$token;
+        }
+        $source = implode('.', $source);
+        if ($this->relative) {
+            $source = '.' . $source;
+        }
+        return $source;
+    }
+
+    protected function serialize(): array
+    {
+        $result = parent::serialize();
+        if ($this->isRelative()) {
+            $result['relative'] = true;
+        }
+        return $result;
+    }
+
+    /**
+     * Append a token to the stream.
+     */
+    public function append(TokenInterface $token): self
+    {
+        if ($token->getType() !== TokenType::T_IDENTIFIER) {
+            throw new \LogicException(
+                'Trying to add a token of type TokenType::' . $token->getType()->name . ' to class TokenStreamIdentifier, but only TokenType::T_IDENTIFIERS are allowed.',
+                1655138907
+            );
+        }
+        $this->tokens[] = $token;
+        return $this;
+    }
+
+    /**
+     * This identifier token stream is relative! There is a dot on the right side of something like "foo.bar < .baz"
+     */
+    public function setRelative(): self
+    {
+        $this->relative = true;
+        return $this;
+    }
+
+    /**
+     * True if this identifier stream is relative to given context.
+     */
+    public function isRelative(): bool
+    {
+        return $this->relative;
+    }
+}

Classes/TypoScript/Tokenizer/Token/Token.php

@@ -0,0 +1,29 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Token;
+
+/**
+ * A casual token created from TypoScript source:
+ * When having a TypoScript line like "# a comment", then a LineComment
+ * is created having a token "T_COMMENT_ONELINE_HASH" and value "# a comment" as
+ * assigned TokenStream.
+ * See TokenType for on overview on which TokenTypes can exist.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class Token extends AbstractToken {}

Classes/TypoScript/Tokenizer/Token/TokenInterface.php

@@ -0,0 +1,68 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Token;
+
+/**
+ * A readonly token: Each line of TypoScript is split into a list of lines consisting of
+ * tokens by the tokenizers.
+ *
+ * As example, a "foo.bar = baz" line creates a LineIdentifierAssignment line, having
+ * TokenType::T_IDENTIFIER 'foo', plus TokenType::T_IDENTIFIER 'bar' as TokenStream for
+ * LineIdentifierAssignment->getIdentifierTokenStream(), plus a TokenType::T_VALUE 'baz'
+ * as LineIdentifierAssignment->getValueTokenStream().
+ *
+ * We have two different Token implementations: The casual "Token" class for everything, plus
+ * the "TokenIdentifier" class for identifier tokens. Identifier tokens are those "left" of
+ * for instance an assignment like "foo.bar = baz" ("foo" and "bar" are TokenIdentifier instances),
+ * and also on the right side when using expression with "<" and "=<" operator: Example "foo.bar < baz":
+ * "baz" is an instance of a TokenIdentifier ("foo" and "bar" as well).
+ *
+ * The reason to have two implementations is that TokenIdentifier needs to be handled slightly
+ * different when cast to string: For identifiers, all "." (dots) within a single identifier token
+ * need to be quoted with "\" (backslash), to not confuse the parser. The classic use-case is having dots in
+ * FlexForm identifiers for PageTS:
+ * "foo.bar\.baz.foobar = value" - three identifier tokens (not four!): "foo", "bar.baz" and "foobar".
+ * So the difference between "TokenIdentifier" and "Token" is just that "TokenIdentifier" quotes dots
+ * in its value when string'ified, while Token does not and __toString() on Token simply says ->getValue().
+ *
+ * Multiple tokens are encapsulated in TokenStreamInterface. TokenStreamInterface has a __toString()
+ * method as well, which calls __toString() on all assigned tokens. This way, a TokenIdentifier will
+ * do its quoting magic, and casual Token instances return their value.
+ *
+ * The idea is here that TokenStreams are cast to string quite often. For instance, an assignment line
+ * like "foo = bar" creates a token stream of one token for the right side (things after "="):
+ * A T_VALUE Token instance with value "bar". The AST builder then at some point needs to resolve this
+ * TokenStream to string. This will directly call __toString on token "bar", and does not deal with quoting,
+ * since its no TokenIdentifier and just a Token.
+ *
+ * Note on getLine() and getColumn(): These two represent the position of a token in the source file:
+ * We start counting at 0 (zero): The first token on the first line is line 0, column 0.
+ * Only the LosslessTokenizer sets these, it's too expensive and of no relevance for the LossyTokenizer
+ * that is used for instance in FE TS tokenizing. That's why these two properties are optional
+ * and 0 (zero) by default.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+interface TokenInterface
+{
+    public function __toString(): string;
+    public function getType(): TokenType;
+    public function getValue(): string;
+    public function getLine(): int;
+    public function getColumn(): int;
+}

Classes/TypoScript/Tokenizer/Token/TokenStream.php

@@ -0,0 +1,26 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Token;
+
+/**
+ * A list of single tokens. These are typically used in TokenLines: A TypoScript
+ * line consists of one or more streams of tokens, depending on the line type.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class TokenStream extends AbstractTokenStream {}

Classes/TypoScript/Tokenizer/Token/TokenStreamInterface.php

@@ -0,0 +1,77 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Token;
+
+/**
+ * A generic stream of tokens used in single LineInterface lines.
+ *
+ * The tokenizers create these streams for various lists of tokens, the generic
+ * implementation is class TokenStream. For lists of identifier tokens the special
+ * class TokenStreamIdentifier is created.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+interface TokenStreamInterface
+{
+    /**
+     * Create a source string from given tokens.
+     */
+    public function __toString(): string;
+
+    /**
+     * Stream creation.
+     */
+    public function append(TokenInterface $token): self;
+
+    /**
+     * We sometimes create a stream but don't add tokens.
+     * This method returns true if tokens have been added.
+     */
+    public function isEmpty(): bool;
+
+    /**
+     * Reset current pointer. Typically, call this before iterating with getNext().
+     */
+    public function reset(): self;
+
+    /**
+     * Get next token and raise pointer.
+     */
+    public function getNext(): ?TokenInterface;
+
+    /**
+     * Get next token but do not raise pointer.
+     */
+    public function peekNext(): ?TokenInterface;
+
+    /**
+     * Only used internally when one Stream is transferred to another,
+     * in particular when a TokenStream is turned into TokenStreamConstantAware.
+     *
+     * @return TokenInterface[]
+     */
+    public function getAll(): array;
+
+    /**
+     * Only used internally when one Stream is transferred to another,
+     * in particular when a TokenStream is turned into TokenStreamConstantAware.
+     *
+     * @param TokenInterface[] $tokens
+     */
+    public function setAll(array $tokens): self;
+}

Classes/TypoScript/Tokenizer/Token/TokenType.php

@@ -0,0 +1,69 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Token;
+
+/**
+ * Each TokenInterface instance is a type of this Enum.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+enum TokenType: int
+{
+    case T_NONE = 0; // tokenizer internal handling
+
+    case T_IDENTIFIER = 100; // single word left of an operator. 'foo.bar' are two identifiers
+    case T_VALUE = 200; // right side of an assignment, does not contain line breaks, also used as 'comment' body
+
+    case T_OPERATOR_ASSIGNMENT = 300; // '='
+    case T_OPERATOR_REFERENCE = 301; // '=<'
+    case T_OPERATOR_COPY = 302; // '<'
+    case T_OPERATOR_UNSET = 303; // '>'
+    case T_OPERATOR_FUNCTION = 304; // ':='
+    case T_OPERATOR_ASSIGNMENT_MULTILINE_START = 310; // '('
+    case T_OPERATOR_ASSIGNMENT_MULTILINE_STOP = 311; // ')'
+
+    case T_BLOCK_START = 400; // '{'
+    case T_BLOCK_STOP = 401; // '}'
+
+    case T_DOT = 500; // '.' identifier separator
+
+    case T_BLANK = 600; // list of ' ' and "\t"
+
+    case T_NEWLINE = 700; // "\n" or "\r\n"
+
+    case T_COMMENT_ONELINE_HASH = 800; // '#...'
+    case T_COMMENT_ONELINE_DOUBLESLASH = 801; // '//'
+    case T_COMMENT_MULTILINE_START = 802; // '/*'
+    case T_COMMENT_MULTILINE_STOP = 803; // '*/'
+
+    case T_FUNCTION_NAME = 900; // 'addToList' and others
+    case T_FUNCTION_VALUE_START = 901; // '(' after T_FUNCTION_NAME
+    case T_FUNCTION_VALUE_STOP = 902; // ')' after T_FUNCTION_NAME
+
+    case T_CONDITION_START = 1000; // '[' at start of line
+    case T_CONDITION_STOP = 1001; // ']' after '[' in same line, body is a T_VALUE
+    case T_CONDITION_ELSE = 1002; // 'ELSE' surrounded by '[' and ']'
+    case T_CONDITION_END = 1003; // 'END' surrounded by '[' and ']'
+    case T_CONDITION_GLOBAL = 1004; // 'GLOBAL' surrounded by '[' and ']'
+
+    case T_CONSTANT = 1100; // '{$...}'
+
+    case T_IMPORT_KEYWORD = 1200; // '@import'
+    case T_IMPORT_START = 1201; // ''' (tick) or '"' (doubletick) after @import
+    case T_IMPORT_STOP = 1202; // ''' (tick) or '"' (doubletick) after T_IMPORT_START
+}

Classes/TypoScript/Tokenizer/TokenizerInterface.php

@@ -0,0 +1,42 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\LineStream;
+
+/**
+ * A lossless tokenizer for TypoScript syntax.
+ *
+ * tokenize() creates a stream of LineInterface objects from a TypoScript string, each line
+ * contains the important streams or tokens of a single line.
+ *
+ * There are two tokenizer implementations:
+ * - LossyTokenizer: This one skip all invalid lines and comments and everything that is
+ *                   not needed for AST building.
+ * - LosslessTokenizer: This one creates a stream of lines useful for backend template module
+ *                      to elaborate on details and failures in TypoScript.
+ *
+ * The tokenizer *does not* parse conditions or includes itself (no file / db lookups),
+ * this is part of the IncludeTree parser.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+interface TokenizerInterface
+{
+    public function tokenize(string $source): LineStream;
+}

Classes/TypoScript/TypoScriptService.php

@@ -0,0 +1,193 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript;
+
+/**
+ * Helper class to manage and convert TypoScript into differently shaped arrays.
+ * Also contains the functionality in TypoScript called "optionSplit".
+ *
+ * @internal
+ */
+readonly class TypoScriptService
+{
+    /**
+     * Removes all trailing dots recursively from TS settings array
+     *
+     * Extbase converts the "classical" TypoScript (with trailing dot) to a format without trailing dot,
+     * to be more future-proof and not to have any conflicts with Fluid object accessor syntax.
+     *
+     * @param array<string|int, mixed> $typoScriptArray for example `['foo' => 'TEXT', 'foo.' => ['bar' => 'baz']]`
+     * @return array<string|int, mixed> for example `['foo' => ['_typoScriptNodeValue' => 'TEXT', 'bar' => 'baz']]`
+     * @internal Avoid using this method. This has been invented for Extbase, which decided to move TypoScript
+     *           arrays around in just another different way.
+     */
+    public function convertTypoScriptArrayToPlainArray(array $typoScriptArray): array
+    {
+        foreach ($typoScriptArray as $key => $value) {
+            if (str_ends_with((string)$key, '.')) {
+                $keyWithoutDot = substr((string)$key, 0, -1);
+                $typoScriptNodeValue = $typoScriptArray[$keyWithoutDot] ?? null;
+                if (is_array($value)) {
+                    $typoScriptArray[$keyWithoutDot] = $this->convertTypoScriptArrayToPlainArray($value);
+                    if ($typoScriptNodeValue !== null) {
+                        $typoScriptArray[$keyWithoutDot]['_typoScriptNodeValue'] = $typoScriptNodeValue;
+                    }
+                    unset($typoScriptArray[$key]);
+                } else {
+                    $typoScriptArray[$keyWithoutDot] = null;
+                }
+            }
+        }
+        return $typoScriptArray;
+    }
+
+    /**
+     * Returns an array with Typoscript the old way (with dot).
+     *
+     * Extbase converts the "classical" TypoScript (with trailing dot) to a format without trailing dot,
+     * to be more future-proof and not to have any conflicts with Fluid object accessor syntax.
+     * However, if you want to call legacy TypoScript objects, you somehow need the "old" syntax (because this is what TYPO3 is used to).
+     * With this method, you can convert the extbase TypoScript to classical TYPO3 TypoScript which is understood by the rest of TYPO3.
+     *
+     * @param array $plainArray A TypoScript Array with Extbase Syntax (without dot but with _typoScriptNodeValue)
+     * @return array Array with TypoScript as usual (with dot)
+     * @internal Avoid using this method. This has been invented for Extbase, which decided to move TypoScript
+     *           arrays around in just another different way.
+     */
+    public function convertPlainArrayToTypoScriptArray(array $plainArray): array
+    {
+        $typoScriptArray = [];
+        foreach ($plainArray as $key => $value) {
+            if (is_array($value)) {
+                if (isset($value['_typoScriptNodeValue'])) {
+                    $typoScriptArray[$key] = $value['_typoScriptNodeValue'];
+                    unset($value['_typoScriptNodeValue']);
+                }
+                $typoScriptArray[$key . '.'] = $this->convertPlainArrayToTypoScriptArray($value);
+            } else {
+                $typoScriptArray[$key] = $value ?? '';
+            }
+        }
+        return $typoScriptArray;
+    }
+
+    /**
+     * Implementation of the "optionSplit" feature in TypoScript (used eg. for MENU objects)
+     * What it does is to split the incoming TypoScript array so that the values are exploded by certain
+     * strings ("||" and "|*|") and each part distributed into individual TypoScript arrays with a similar structure,
+     * but individualized values.
+     * The concept is known as "optionSplit" and is rather advanced to handle but quite powerful, in particular
+     * for creating menus in TYPO3.
+     *
+     * @param array $originalConfiguration A TypoScript array
+     * @param int $splitCount The number of items for which to generate individual TypoScript arrays
+     * @return array The individualized TypoScript array.
+     */
+    public function explodeConfigurationForOptionSplit(array $originalConfiguration, int $splitCount): array
+    {
+        $finalConfiguration = [];
+        if (!$splitCount) {
+            return $finalConfiguration;
+        }
+        // Initialize output to carry at least the keys
+        for ($aKey = 0; $aKey < $splitCount; $aKey++) {
+            $finalConfiguration[$aKey] = [];
+        }
+        // Recursive processing of array keys
+        foreach ($originalConfiguration as $cKey => $val) {
+            if (is_array($val)) {
+                $tempConf = $this->explodeConfigurationForOptionSplit($val, $splitCount);
+                foreach ($tempConf as $aKey => $val2) {
+                    $finalConfiguration[$aKey][$cKey] = $val2;
+                }
+            } elseif (is_string($val)) {
+                // Splitting of all values on this level of the TypoScript object tree:
+                if ($cKey === 'noTrimWrap' || (!str_contains($val, '|*|') && !str_contains($val, '||'))) {
+                    for ($aKey = 0; $aKey < $splitCount; $aKey++) {
+                        $finalConfiguration[$aKey][$cKey] = $val;
+                    }
+                } else {
+                    $main = explode('|*|', $val);
+                    $lastC = 0;
+                    $middleC = 0;
+                    $firstC = 0;
+                    if ($main[0]) {
+                        $first = explode('||', $main[0]);
+                        $firstC = count($first);
+                    }
+                    $middle = [];
+                    if (!empty($main[1])) {
+                        $middle = explode('||', $main[1]);
+                        $middleC = count($middle);
+                    }
+                    $last = [];
+                    $value = '';
+                    if (!empty($main[2])) {
+                        $last = explode('||', $main[2]);
+                        $lastC = count($last);
+                        $value = $last[0];
+                    }
+                    for ($aKey = 0; $aKey < $splitCount; $aKey++) {
+                        if ($firstC && isset($first[$aKey])) {
+                            $value = $first[$aKey];
+                        } elseif ($middleC) {
+                            $value = $middle[($aKey - $firstC) % $middleC];
+                        }
+                        if ($lastC && $lastC >= $splitCount - $aKey) {
+                            $value = $last[$lastC - ($splitCount - $aKey)];
+                        }
+                        $finalConfiguration[$aKey][$cKey] = trim($value);
+                    }
+                }
+            }
+        }
+        return $finalConfiguration;
+    }
+
+    /**
+     * Flatten TypoScript label array; converting a hierarchical array into a flat
+     * array with the keys separated by dots.
+     *
+     * Example Input:  array('k1' => array('subkey1' => 'val1'))
+     * Example Output: array('k1.subkey1' => 'val1')
+     *
+     * @param array $labelValues Hierarchical array of labels
+     * @param string $parentKey the name of the parent key in the recursion; is only needed for recursion.
+     * @return array flattened array of labels.
+     */
+    public function flattenTypoScriptLabelArray(array $labelValues, string $parentKey = ''): array
+    {
+        $result = [];
+        foreach ($labelValues as $key => $labelValue) {
+            if (!empty($parentKey)) {
+                if ($key === '_typoScriptNodeValue') {
+                    $key = $parentKey;
+                } else {
+                    $key = $parentKey . '.' . $key;
+                }
+            }
+            if (is_array($labelValue)) {
+                $labelValue = $this->flattenTypoScriptLabelArray($labelValue, $key);
+                $result = array_merge($result, $labelValue);
+            } else {
+                $result[$key] = $labelValue;
+            }
+        }
+        return $result;
+    }
+}

Classes/TypoScript/TypoScriptStringFactory.php

@@ -0,0 +1,72 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript;
+
+use Psr\Container\ContainerInterface;
+use TYPO3\CMS\Core\Cache\CacheManager;
+use TYPO3\CMS\Core\Cache\Frontend\PhpFrontend;
+use TYPO3\CMS\Core\TypoScript\AST\AstBuilderInterface;
+use TYPO3\CMS\Core\TypoScript\AST\Node\RootNode;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\StringTreeBuilder;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\Traverser\IncludeTreeTraverser;
+use TYPO3\CMS\Core\TypoScript\IncludeTree\Visitor\IncludeTreeAstBuilderVisitor;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\TokenizerInterface;
+
+/**
+ * A factory to create the AST object tree for a given TypoScript snippet.
+ *
+ * This is used by some consumers in the core that parse a TypoScript a-like
+ * syntax that is not Frontend TypoScript and TsConfig directly.
+ */
+final readonly class TypoScriptStringFactory
+{
+    public function __construct(
+        private ContainerInterface $container,
+        private TokenizerInterface $tokenizer,
+    ) {}
+
+    /**
+     * Parse a single string and support imports and conditions, cache optionally.
+     *
+     * @param non-empty-string $name A name used as cache identifier, [a-z,A-Z,-] only
+     */
+    public function parseFromStringWithIncludes(string $name, string $typoScript): RootNode
+    {
+        $cacheManager = $this->container->get(CacheManager::class);
+        /** @var PhpFrontend $cache */
+        $cache = $cacheManager->getCache('typoscript');
+        $stringTreeBuilder = $this->container->get(StringTreeBuilder::class);
+        $includeTree = $stringTreeBuilder->getTreeFromString($name, $typoScript, $this->tokenizer, $cache);
+        $includeTreeTraverserConditionVerdictAware = new IncludeTreeTraverser();
+        $astBuilderVisitor = $this->container->get(IncludeTreeAstBuilderVisitor::class);
+        $includeTreeTraverserConditionVerdictAware->traverse($includeTree, [$astBuilderVisitor]);
+        return $astBuilderVisitor->getAst();
+    }
+
+    /**
+     * Parse a single string *not* supporting imports, conditions and caching.
+     * Detail method used in install tool and in a couple of other special cases.
+     *
+     * @internal
+     */
+    public function parseFromString(string $typoScript, AstBuilderInterface $astBuilder): RootNode
+    {
+        $lineStream = $this->tokenizer->tokenize($typoScript);
+        return $astBuilder->build($lineStream, new RootNode());
+    }
+}