TYPO3 v15 dev-main snapshot ()

Classes/TypoScript/Tokenizer/Line/AbstractLine.php

@@ -0,0 +1,41 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenStreamInterface;
+
+/**
+ * Implement main LineInterface methods.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+abstract class AbstractLine implements LineInterface
+{
+    protected TokenStreamInterface $tokenStream;
+
+    public function setTokenStream(TokenStreamInterface $tokenStream): static
+    {
+        $this->tokenStream = $tokenStream;
+        return $this;
+    }
+
+    public function getTokenStream(): TokenStreamInterface
+    {
+        return $this->tokenStream;
+    }
+}

Classes/TypoScript/Tokenizer/Line/BlockCloseLine.php

@@ -0,0 +1,25 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+/**
+ * A block close line, essentially "}".
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class BlockCloseLine extends AbstractLine {}

Classes/TypoScript/Tokenizer/Line/CommentLine.php

@@ -0,0 +1,29 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+/**
+ * A commented TypoScript line: Lines that start with "#", "//" and multiline comments "/* ... *\/"
+ *
+ * Note multiline comments often represent multiple source lines: An opening "/*" as
+ * first source line, then the comment body with one or more source lines, then finally
+ * the closing "*\/". These still create only one "CommentLine".
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class CommentLine extends AbstractLine {}

Classes/TypoScript/Tokenizer/Line/ConditionElseLine.php

@@ -0,0 +1,25 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+/**
+ * "[ELSE]" / "[else]": An else block after a starting ConditionLine.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class ConditionElseLine extends AbstractLine {}

Classes/TypoScript/Tokenizer/Line/ConditionLine.php

@@ -0,0 +1,45 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\Token;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenType;
+
+/**
+ * A condition line: "[foo == bar]".
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class ConditionLine extends AbstractLine
+{
+    private Token $valueToken;
+
+    public function setValueToken(Token $token): static
+    {
+        if ($token->getType() !== TokenType::T_VALUE) {
+            throw new \LogicException('Token must be of type T_VALUE', 1655823705);
+        }
+        $this->valueToken = $token;
+        return $this;
+    }
+
+    public function getTokenValue(): Token
+    {
+        return $this->valueToken;
+    }
+}

Classes/TypoScript/Tokenizer/Line/ConditionStopLine.php

@@ -0,0 +1,26 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+/**
+ * A line stopping current ConditionLine context:
+ * "[END]" / "[end]" / "[GLOBAL]" / "[global]".
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class ConditionStopLine extends AbstractLine {}

Classes/TypoScript/Tokenizer/Line/EmptyLine.php

@@ -0,0 +1,32 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+/**
+ * A completely empty line, or a line consisting of tabs or whitespaces only.
+ *
+ * This is not created when the TypoScript source line is within multiline "("
+ * assignments and multiline "/*" comments: The T_BLANK and T_NEWLINE tokens
+ * are part of the value steram in these contexts.
+ *
+ * Note the LossyTokenizers does not create these and just skips them since
+ * they have no semantic meaning for the resulting TypoScript tree.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class EmptyLine extends AbstractLine {}

Classes/TypoScript/Tokenizer/Line/IdentifierAssignmentLine.php

@@ -0,0 +1,68 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\IdentifierTokenStream;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenStreamInterface;
+
+/**
+ * Simple "=" assignments and multiline "(" assignments: "foo.bar = barValue".
+ *
+ * Each line has two additional token streams: $identifierTokenStream for the
+ * left side ("foo" and "bar" tokens) and $valueTokenStream for the right side
+ * ("barValue" token). Right side is often a single token only, but can be many
+ * tokens when constants and multiline assignments are involved.
+ *
+ * Neither the left, nor the right side streams can be empty: Even with "foo.bar ="
+ * a T_VALUE token with empty value is created for the right side.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class IdentifierAssignmentLine extends AbstractLine
+{
+    private IdentifierTokenStream $identifierTokenStream;
+    private TokenStreamInterface $valueTokenStream;
+
+    public function setIdentifierTokenStream(IdentifierTokenStream $tokenStream): static
+    {
+        if ($tokenStream->isEmpty()) {
+            throw new \LogicException('Identifier token stream must not be empty', 1655824257);
+        }
+        $this->identifierTokenStream = $tokenStream;
+        return $this;
+    }
+
+    public function getIdentifierTokenStream(): IdentifierTokenStream
+    {
+        return $this->identifierTokenStream;
+    }
+
+    public function setValueTokenStream(TokenStreamInterface $tokenStream): static
+    {
+        if ($tokenStream->isEmpty()) {
+            throw new \LogicException('Value token stream must not be empty', 1655824258);
+        }
+        $this->valueTokenStream = $tokenStream;
+        return $this;
+    }
+
+    public function getValueTokenStream(): TokenStreamInterface
+    {
+        return $this->valueTokenStream;
+    }
+}

Classes/TypoScript/Tokenizer/Line/IdentifierBlockOpenLine.php

@@ -0,0 +1,47 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\IdentifierTokenStream;
+
+/**
+ * A block open line: "foo.bar {".
+ *
+ * $identifierTokenStream is a stream of tokens on the left side, "foo"
+ * and "bar" token in the example above. That stream must not be empty.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class IdentifierBlockOpenLine extends AbstractLine
+{
+    private IdentifierTokenStream $identifierTokenStream;
+
+    public function setIdentifierTokenStream(IdentifierTokenStream $tokenStream): static
+    {
+        if ($tokenStream->isEmpty()) {
+            throw new \LogicException('Identifier token stream must not be empty', 1655824621);
+        }
+        $this->identifierTokenStream = $tokenStream;
+        return $this;
+    }
+
+    public function getIdentifierTokenStream(): IdentifierTokenStream
+    {
+        return $this->identifierTokenStream;
+    }
+}

Classes/TypoScript/Tokenizer/Line/IdentifierCopyLine.php

@@ -0,0 +1,68 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\IdentifierTokenStream;
+
+/**
+ * A line using the copy operator: "foo.bar < lib.myLib".
+ *
+ * Contains a stream of tokens for the left side ("foo" and "bar" tokens) and
+ * a stream of tokens for the right side ("lib" and "myLib"). None of these
+ * token streams can be empty, it's an InvalidLine otherwise.
+ *
+ * Note the right side TokenStreamIdentifier can be relative: "foo.bar < .baz".
+ * Flag $relative in TokenStreamIdentifier represents this start dot on the right side.
+ *
+ * None of the two streams can be empty.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class IdentifierCopyLine extends AbstractLine
+{
+    private IdentifierTokenStream $identifierTokenStream;
+    private IdentifierTokenStream $valueTokenStream;
+
+    public function setIdentifierTokenStream(IdentifierTokenStream $tokenStream): static
+    {
+        if ($tokenStream->isEmpty()) {
+            throw new \LogicException('Identifier token stream must not be empty', 1655824946);
+        }
+        $this->identifierTokenStream = $tokenStream;
+        return $this;
+    }
+
+    public function getIdentifierTokenStream(): IdentifierTokenStream
+    {
+        return $this->identifierTokenStream;
+    }
+
+    public function setValueTokenStream(IdentifierTokenStream $tokenStream): static
+    {
+        if ($tokenStream->isEmpty()) {
+            throw new \LogicException('Value token stream must not be empty', 1655824947);
+        }
+        $this->valueTokenStream = $tokenStream;
+        return $this;
+    }
+
+    public function getValueTokenStream(): IdentifierTokenStream
+    {
+        return $this->valueTokenStream;
+    }
+}

Classes/TypoScript/Tokenizer/Line/IdentifierFunctionLine.php

@@ -0,0 +1,87 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\IdentifierTokenStream;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\Token;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenStreamInterface;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenType;
+
+/**
+ * A line with a function assignment using the ":=" operator: "foo.bar := addToList(42)".
+ *
+ * Contains $identifierTokenStream for the left side ("foo" and "bar" token), a single
+ * token for the function name ("addToList"), and an optional token for the value ("42").
+ * Note the value token is optional since there are functions without values (eg. "uniqueList()").
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class IdentifierFunctionLine extends AbstractLine
+{
+    private ?IdentifierTokenStream $identifierTokenStream = null;
+    private ?Token $functionNameToken = null;
+    private ?TokenStreamInterface $functionValueTokenStream = null;
+
+    public function setIdentifierTokenStream(IdentifierTokenStream $tokenStream): IdentifierFunctionLine
+    {
+        if ($tokenStream->isEmpty()) {
+            throw new \LogicException('Identifier token stream must not be empty', 1655825120);
+        }
+        $this->identifierTokenStream = $tokenStream;
+        return $this;
+    }
+
+    public function getIdentifierTokenStream(): IdentifierTokenStream
+    {
+        if ($this->identifierTokenStream === null) {
+            throw new \RuntimeException('Identifier token stream has not been set', 1717495444);
+        }
+        return $this->identifierTokenStream;
+    }
+
+    public function setFunctionNameToken(Token $token): IdentifierFunctionLine
+    {
+        if ($token->getType() !== TokenType::T_FUNCTION_NAME) {
+            throw new \LogicException('Function name token must be of type T_FUNCTION_NAME', 1655825121);
+        }
+        $this->functionNameToken = $token;
+        return $this;
+    }
+
+    public function getFunctionNameToken(): Token
+    {
+        if ($this->functionNameToken === null) {
+            throw new \RuntimeException('Function name token has not been set', 1717495576);
+        }
+        return $this->functionNameToken;
+    }
+
+    public function setFunctionValueTokenStream(TokenStreamInterface $tokenStream): IdentifierFunctionLine
+    {
+        $this->functionValueTokenStream = $tokenStream;
+        return $this;
+    }
+
+    public function getFunctionValueTokenStream(): TokenStreamInterface
+    {
+        if ($this->functionValueTokenStream === null) {
+            throw new \RuntimeException('Function value token stream has not been set', 1717495996);
+        }
+        return $this->functionValueTokenStream;
+    }
+}

Classes/TypoScript/Tokenizer/Line/IdentifierReferenceLine.php

@@ -0,0 +1,66 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\IdentifierTokenStream;
+
+/**
+ * A line using the reference ("=<") operator: "foo.bar =< lib.myLib".
+ *
+ * Contains two non-empty token streams: One for the left side ("foo" and "bar" tokens),
+ * and one for the right side ("lib" and "myLib"). Both streams must not be empty.
+ *
+ * Note the AstBuilder does not directly resolve "=<" operators. This is
+ * not a language construct itself and is only resolved in some special cases
+ * in frontend. See ContentObjectRenderer->cObjGetSingle() for more details.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class IdentifierReferenceLine extends AbstractLine
+{
+    private IdentifierTokenStream $identifierTokenStream;
+    private IdentifierTokenStream $valueTokenStream;
+
+    public function setIdentifierTokenStream(IdentifierTokenStream $tokenStream): static
+    {
+        if ($tokenStream->isEmpty()) {
+            throw new \LogicException('Identifier token stream must not be empty', 1655825891);
+        }
+        $this->identifierTokenStream = $tokenStream;
+        return $this;
+    }
+
+    public function getIdentifierTokenStream(): IdentifierTokenStream
+    {
+        return $this->identifierTokenStream;
+    }
+
+    public function setValueTokenStream(IdentifierTokenStream $tokenStream): static
+    {
+        if ($tokenStream->isEmpty()) {
+            throw new \LogicException('Value token stream must not be empty', 1655825892);
+        }
+        $this->valueTokenStream = $tokenStream;
+        return $this;
+    }
+
+    public function getValueTokenStream(): IdentifierTokenStream
+    {
+        return $this->valueTokenStream;
+    }
+}

Classes/TypoScript/Tokenizer/Line/IdentifierUnsetLine.php

@@ -0,0 +1,47 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\IdentifierTokenStream;
+
+/**
+ * A line using the unset (">") operator: "foo.bar >".
+ *
+ * Has $identifierTokenStream for the stream of tokens on the left
+ * side ("foo" and "bar" tokens).
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class IdentifierUnsetLine extends AbstractLine
+{
+    private IdentifierTokenStream $identifierTokenStream;
+
+    public function setIdentifierTokenStream(IdentifierTokenStream $tokenStream): static
+    {
+        if ($tokenStream->isEmpty()) {
+            throw new \LogicException('Identifier token stream must not be empty', 1655826025);
+        }
+        $this->identifierTokenStream = $tokenStream;
+        return $this;
+    }
+
+    public function getIdentifierTokenStream(): IdentifierTokenStream
+    {
+        return $this->identifierTokenStream;
+    }
+}

Classes/TypoScript/Tokenizer/Line/ImportLine.php

@@ -0,0 +1,49 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\Token;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenType;
+
+/**
+ * A line using the "@import" keyword: "@import 'EXT:my_extension/Configuration/TypoScript/randomfile.typoscript'"
+ *
+ * Contains the $valueToken ("EXT:my_extension/Configuration/TypoScript/randomfile.typoscript"), without the
+ * surrounding tick (') or doubletick ("). The value itself is not parsed further at this point, this
+ * is done by the IncludeTree classes.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class ImportLine extends AbstractLine
+{
+    private Token $valueToken;
+
+    public function setValueToken(Token $token): static
+    {
+        if ($token->getType() !== TokenType::T_VALUE) {
+            throw new \LogicException('Value token must be of type T_VALUE', 1655826193);
+        }
+        $this->valueToken = $token;
+        return $this;
+    }
+
+    public function getValueToken(): Token
+    {
+        return $this->valueToken;
+    }
+}

Classes/TypoScript/Tokenizer/Line/InvalidLine.php

@@ -0,0 +1,33 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+/**
+ * A line that is syntactically invalid.
+ *
+ * This is created by LosslessTokenizer whenever a line does not make sense.
+ * Examples:
+ * "foo.bar" - no operator
+ * "foo.bar <" - right side empty
+ * "@import ''" - no import value
+ *
+ * Note only LosslessTokenizer creates these lines, LossyTokenizer just skips them.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class InvalidLine extends AbstractLine {}

Classes/TypoScript/Tokenizer/Line/LineInterface.php

@@ -0,0 +1,42 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenStreamInterface;
+
+/**
+ * The TypoScript tokenizers deliver streams of lines. This is the main line interface.
+ *
+ * Each line is represented by a specific line type. For instance, "foo.bar {" creates
+ * an IdentifierBlockOpenLine and has the additional method getIdentifierTokenStream()
+ * to retrieve the "foo" and "bar" identifier tokens.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+interface LineInterface
+{
+    /**
+     * Set and get the token stream that represents the full line. This is mostly used
+     * in backend to for instance create a TypoScript string back from tokenized lines.
+     *
+     * Note: Only the LosslessTokenizer fills this 'full line' stream, LossyTokenizer
+     * does not for performance reasons.
+     */
+    public function setTokenStream(TokenStreamInterface $tokenStream): static;
+    public function getTokenStream(): TokenStreamInterface;
+}

Classes/TypoScript/Tokenizer/Line/LineStream.php

@@ -0,0 +1,124 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Line;
+
+/**
+ * Each TypoScript snippet is turned by the tokenizers into a
+ * stream of lines. Tokenizers return instances of this class.
+ *
+ * Iterate line streams in a foreach loop using getNextLine().
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class LineStream
+{
+    /**
+     * @var LineInterface[]
+     */
+    private array $lines = [];
+    private int $currentIndex = -1;
+
+    /**
+     * Create a source string from given token lines. This is used in backend
+     * to turn the "full" token streams of lines into strings for output.
+     */
+    public function __toString(): string
+    {
+        $source = '';
+        foreach ($this->getNextLine() as $line) {
+            // We do *not* implement __toString() on lines since this is a
+            // backend thing only, and we do not want to accidentally stringify
+            // lines based on the full stream anywhere.
+            $source .= $line->getTokenStream()->reset();
+        }
+        return $source;
+    }
+
+    /**
+     * When storing to cache, we only store FE relevant properties and skip
+     * irrelevant things. In particular, $currentIndex should always initialize
+     * to -1 and does not need to be stored.
+     */
+    final public function __serialize(): array
+    {
+        return [
+            'lines' => $this->lines,
+        ];
+    }
+
+    /**
+     * Stream creation.
+     */
+    public function append(LineInterface $line): self
+    {
+        $this->lines[] = $line;
+        return $this;
+    }
+
+    /**
+     * We sometimes create a line stream but don't add lines.
+     * This method returns true if lines have been added.
+     */
+    public function isEmpty(): bool
+    {
+        return empty($this->lines);
+    }
+
+    /**
+     * @return iterable<LineInterface>
+     */
+    public function getNextLine(): iterable
+    {
+        foreach ($this->lines as $child) {
+            yield $child;
+        }
+    }
+
+    /**
+     * Reset current pointer. Typically, call this before iterating with getNext().
+     */
+    public function reset(): self
+    {
+        $this->currentIndex = -1;
+        return $this;
+    }
+
+    /**
+     * Get next line and raise pointer.
+     *
+     * Methods getNext(), peekNext() and reset() are an alternative to
+     * getNextLine() which allow peek of the next line, which getNextLine()
+     * does not. The disadvantage is that these methods create internal
+     * state in $this->currentIndex, which getNextLine() does not. Use
+     * getNext() iteration only if peekNext() is needed to avoid creating
+     * useless state.
+     */
+    public function getNext(): ?LineInterface
+    {
+        $this->currentIndex++;
+        return $this->lines[$this->currentIndex] ?? null;
+    }
+
+    /**
+     * Get next line but do not raise pointer.
+     */
+    public function peekNext(): ?LineInterface
+    {
+        return $this->lines[$this->currentIndex + 1] ?? null;
+    }
+}

Classes/TypoScript/Tokenizer/LosslessTokenizer.php

@@ -0,0 +1,879 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\BlockCloseLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\CommentLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ConditionElseLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ConditionLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ConditionStopLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\EmptyLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierAssignmentLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierBlockOpenLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierCopyLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierFunctionLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierReferenceLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierUnsetLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ImportLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\InvalidLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\LineStream;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\ConstantAwareTokenStream;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\IdentifierToken;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\IdentifierTokenStream;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\Token;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenStream;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenStreamInterface;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenType;
+
+/**
+ * A lossless tokenizer for TypoScript syntax.
+ *
+ * tokenize() creates a flat stream of tokens from a TypoScript string. It is lossless
+ * and never "looses" characters to allow syntax linting and creating linter-fixed source
+ * strings: tokenize() to create a TokenStream and using string cast (__toString()) on
+ * that stream creates *the same* source string again.
+ *
+ * The tokenizer *does not* parse conditions or includes itself (no file / db lookups),
+ * this is part of the IncludeTree parser.
+ *
+ * This class is unit test covered by TokenizerInterfaceTest and paired with LossyTokenizer.
+ * Never change anything in this class without additional test coverage!
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class LosslessTokenizer implements TokenizerInterface
+{
+    private LineStream $lineStream;
+
+    private TokenStreamInterface $tokenStream;
+    private IdentifierTokenStream $identifierStream;
+    private TokenStreamInterface $valueStream;
+
+    private array $lines;
+    private int $currentLineNumber;
+    private string $currentLineString;
+    private \closure $currentLinebreakCallback;
+    private int $currentColumnInLine = 0;
+
+    public function tokenize(string $source): LineStream
+    {
+        $this->lineStream = new LineStream();
+        $this->currentLineNumber = -1;
+        $this->lines = $this->splitLines($source);
+
+        while (true) {
+            $this->tokenStream = new TokenStream();
+            $this->currentLineNumber++;
+            if (!array_key_exists($this->currentLineNumber, $this->lines)) {
+                break;
+            }
+            $this->currentColumnInLine = 0;
+            $this->currentLineString = $this->lines[$this->currentLineNumber]['line'];
+            $this->currentLinebreakCallback = $this->lines[$this->currentLineNumber]['linebreakCallback'];
+            $this->parseTabsAndWhitespaces();
+            $nextChar = substr($this->currentLineString, 0, 1);
+            if ($nextChar === '') {
+                ($this->currentLinebreakCallback)();
+                if (!$this->tokenStream->isEmpty()) {
+                    $this->createEmptyLine();
+                }
+                continue;
+            }
+            $nextTwoChars = substr($this->currentLineString, 0, 2);
+            if ($nextChar === '#') {
+                $this->createHashCommentLine();
+            } elseif ($nextTwoChars === '//') {
+                $this->createDoubleSlashCommentLine();
+            } elseif ($nextTwoChars === '/*') {
+                $this->createMultilineCommentLine();
+            } elseif ($nextChar === '[') {
+                $this->createConditionLine();
+            } elseif ($nextChar === '}') {
+                $this->createBlockStopLine();
+            } elseif (str_starts_with($this->currentLineString, '@import')) {
+                $this->parseImportLine();
+            } elseif (str_starts_with($this->currentLineString, '<INCLUDE_TYPOSCRIPT:')) {
+                // @todo: Could be relocated elsewhere. This is just to make sure this
+                //        old language construct is detected as InvalidLine.
+                $this->tokenStream->append(new Token(TokenType::T_VALUE, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine));
+                ($this->currentLinebreakCallback)();
+                $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+            } else {
+                $this->parseIdentifier();
+            }
+        }
+
+        return $this->lineStream;
+    }
+
+    private function splitLines($source): array
+    {
+        $vanillaLines = explode(chr(10), $source);
+        $lines = array_map(
+            fn(int $lineNumber, string $vanillaLine): array => [
+                'line' => rtrim($vanillaLine, "\r"),
+                'linebreakCallback' => str_ends_with($vanillaLine, "\r")
+                    ? fn() => $this->tokenStream->append(new Token(TokenType::T_NEWLINE, "\r\n", $lineNumber, mb_strlen($vanillaLine) - 1))
+                    : fn() => $this->tokenStream->append(new Token(TokenType::T_NEWLINE, "\n", $lineNumber, mb_strlen($vanillaLine))),
+            ],
+            array_keys($vanillaLines),
+            $vanillaLines
+        );
+        // Set the linebreak callback of last line to empty to suppress dangling linebreak tokens
+        $lines[count($vanillaLines) - 1]['linebreakCallback'] = function () {};
+        return $lines;
+    }
+
+    private function createEmptyLine(): void
+    {
+        $this->lineStream->append((new EmptyLine())->setTokenStream($this->tokenStream));
+    }
+
+    /**
+     * Add tabs and whitespaces until some different char appears.
+     */
+    private function parseTabsAndWhitespaces(): void
+    {
+        $matches = [];
+        if (preg_match('#^(\s+)(.*)$#', $this->currentLineString, $matches)) {
+            $this->tokenStream->append(new Token(TokenType::T_BLANK, $matches[1], $this->currentLineNumber, $this->currentColumnInLine));
+            $this->currentLineString = $matches[2];
+            $this->currentColumnInLine = $this->currentColumnInLine + strlen($matches[1]);
+        }
+    }
+
+    private function makeComment(): void
+    {
+        $nextChar = substr($this->currentLineString, 0, 1);
+        if ($nextChar === '') {
+            ($this->currentLinebreakCallback)();
+            return;
+        }
+        $nextTwoChars = substr($this->currentLineString, 0, 2);
+        if ($nextChar === '#') {
+            $this->parseHashComment();
+        } elseif ($nextTwoChars === '//') {
+            $this->parseDoubleSlashComment();
+        } elseif ($nextTwoChars === '/*') {
+            $this->parseMultilineComment();
+        } else {
+            $this->parseHashComment();
+        }
+    }
+
+    private function createHashCommentLine(): void
+    {
+        $this->parseHashComment();
+        $this->lineStream->append((new CommentLine())->setTokenStream($this->tokenStream));
+    }
+
+    private function parseHashComment(): void
+    {
+        $this->tokenStream->append(new Token(TokenType::T_COMMENT_ONELINE_HASH, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine));
+        ($this->currentLinebreakCallback)();
+    }
+
+    private function createDoubleSlashCommentLine(): void
+    {
+        $this->parseDoubleSlashComment();
+        $this->lineStream->append((new CommentLine())->setTokenStream($this->tokenStream));
+    }
+
+    private function parseDoubleSlashComment(): void
+    {
+        $this->tokenStream->append(new Token(TokenType::T_COMMENT_ONELINE_DOUBLESLASH, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine));
+        ($this->currentLinebreakCallback)();
+    }
+
+    private function createMultilineCommentLine(): void
+    {
+        $this->parseMultilineComment();
+        $this->lineStream->append((new CommentLine())->setTokenStream($this->tokenStream));
+    }
+
+    private function parseMultilineComment(): void
+    {
+        $this->tokenStream->append(new Token(TokenType::T_COMMENT_MULTILINE_START, '/*', $this->currentLineNumber, $this->currentColumnInLine));
+        $this->currentColumnInLine += 2;
+        $this->currentLineString = substr($this->currentLineString, 2);
+        while (true) {
+            if (str_ends_with($this->currentLineString, '*/')) {
+                if (strlen($this->currentLineString) > 2) {
+                    $this->tokenStream->append(new Token(TokenType::T_VALUE, substr($this->currentLineString, 0, -2), $this->currentLineNumber, $this->currentColumnInLine));
+                }
+                $this->tokenStream->append(new Token(TokenType::T_COMMENT_MULTILINE_STOP, '*/', $this->currentLineNumber, $this->currentColumnInLine + mb_strlen($this->currentLineString) - 2));
+                ($this->currentLinebreakCallback)();
+                return;
+            }
+            if (strlen($this->currentLineString)) {
+                $this->tokenStream->append(new Token(TokenType::T_VALUE, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine));
+            }
+            ($this->currentLinebreakCallback)();
+            if (!array_key_exists($this->currentLineNumber + 1, $this->lines)) {
+                return;
+            }
+            $this->currentLineNumber++;
+            $this->currentColumnInLine = 0;
+            $this->currentLineString = $this->lines[$this->currentLineNumber]['line'];
+            $this->currentLinebreakCallback = $this->lines[$this->currentLineNumber]['linebreakCallback'];
+        }
+    }
+
+    /**
+     * Create a condition line from token stream of this line.
+     */
+    private function createConditionLine(): void
+    {
+        $upperCaseLine = strtoupper($this->currentLineString);
+        $this->tokenStream->append(new Token(TokenType::T_CONDITION_START, '[', $this->currentLineNumber, $this->currentColumnInLine));
+        if (str_starts_with($upperCaseLine, '[ELSE]')) {
+            $this->tokenStream->append(new Token(TokenType::T_CONDITION_ELSE, substr($this->currentLineString, 1, 4), $this->currentLineNumber, $this->currentColumnInLine + 1));
+            $this->tokenStream->append(new Token(TokenType::T_CONDITION_STOP, ']', $this->currentLineNumber, $this->currentColumnInLine + 5));
+            $this->currentLineString = substr($this->currentLineString, 6);
+            $this->currentColumnInLine += 6;
+            $this->parseTabsAndWhitespaces();
+            $this->makeComment();
+            $this->lineStream->append((new ConditionElseLine())->setTokenStream($this->tokenStream));
+            return;
+        }
+        if (str_starts_with($upperCaseLine, '[END]')) {
+            $this->tokenStream->append(new Token(TokenType::T_CONDITION_END, substr($this->currentLineString, 1, 3), $this->currentLineNumber, $this->currentColumnInLine + 1));
+            $this->tokenStream->append(new Token(TokenType::T_CONDITION_STOP, ']', $this->currentLineNumber, $this->currentColumnInLine + 4));
+            $this->currentLineString = substr($this->currentLineString, 5);
+            $this->currentColumnInLine += 5;
+            $this->parseTabsAndWhitespaces();
+            $this->makeComment();
+            $this->lineStream->append((new ConditionStopLine())->setTokenStream($this->tokenStream));
+            return;
+        }
+        if (str_starts_with($upperCaseLine, '[GLOBAL]')) {
+            $this->tokenStream->append(new Token(TokenType::T_CONDITION_GLOBAL, substr($this->currentLineString, 1, 6), $this->currentLineNumber, $this->currentColumnInLine + 1));
+            $this->tokenStream->append(new Token(TokenType::T_CONDITION_STOP, ']', $this->currentLineNumber, $this->currentColumnInLine + 7));
+            $this->currentLineString = substr($this->currentLineString, 8);
+            $this->currentColumnInLine += 8;
+            $this->parseTabsAndWhitespaces();
+            $this->makeComment();
+            $this->lineStream->append((new ConditionStopLine())->setTokenStream($this->tokenStream));
+            return;
+        }
+        $conditionBody = '';
+        $conditionBodyStartPosition = $this->currentColumnInLine + 1;
+        $conditionBodyCharCount = 0;
+        $conditionBodyChars = mb_str_split(substr($this->currentLineString, 1), 1, 'UTF-8');
+        $bracketCount = 1;
+        while (true) {
+            $nextChar = $conditionBodyChars[$conditionBodyCharCount] ?? null;
+            if ($nextChar === null) {
+                // end of chars
+                if ($conditionBodyCharCount) {
+                    $this->tokenStream->append(new Token(TokenType::T_VALUE, $conditionBody, $this->currentLineNumber, $conditionBodyStartPosition));
+                }
+                ($this->currentLinebreakCallback)();
+                $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+                return;
+            }
+            if ($nextChar === '[') {
+                $bracketCount++;
+                $conditionBody .= $nextChar;
+                $conditionBodyCharCount++;
+                continue;
+            }
+            if ($nextChar === ']') {
+                $bracketCount--;
+                if ($bracketCount === 0) {
+                    if ($conditionBodyCharCount) {
+                        $conditionBodyToken = new Token(TokenType::T_VALUE, $conditionBody, $this->currentLineNumber, $conditionBodyStartPosition);
+                        $this->tokenStream->append($conditionBodyToken);
+                        $this->tokenStream->append(new Token(TokenType::T_CONDITION_STOP, ']', $this->currentLineNumber, $this->currentColumnInLine + $conditionBodyCharCount + 1));
+                        $this->currentLineString = mb_substr($this->currentLineString, $conditionBodyCharCount + 2);
+                        $this->currentColumnInLine = $this->currentColumnInLine + $conditionBodyCharCount + 2;
+                        $this->parseTabsAndWhitespaces();
+                        $this->makeComment();
+                        $this->lineStream->append((new ConditionLine())->setTokenStream($this->tokenStream)->setValueToken($conditionBodyToken));
+                        return;
+                    }
+                    $this->tokenStream->append(new Token(TokenType::T_CONDITION_STOP, ']', $this->currentLineNumber, $this->currentColumnInLine + $conditionBodyCharCount + 1));
+                    $this->currentLineString = mb_substr($this->currentLineString, $conditionBodyCharCount + 2);
+                    $this->currentColumnInLine = $this->currentColumnInLine + $conditionBodyCharCount + 2;
+                    $this->parseTabsAndWhitespaces();
+                    $this->makeComment();
+                    $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+                    return;
+                }
+                $conditionBody .= $nextChar;
+                $conditionBodyCharCount++;
+                continue;
+            }
+            $conditionBody .= $nextChar;
+            $conditionBodyCharCount++;
+        }
+    }
+
+    private function createBlockStopLine(): void
+    {
+        $this->tokenStream->append(new Token(TokenType::T_BLOCK_STOP, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine));
+        $this->currentColumnInLine++;
+        $this->currentLineString = substr($this->currentLineString, 1);
+        $this->makeComment();
+        $this->lineStream->append((new BlockCloseLine())->setTokenStream($this->tokenStream));
+    }
+
+    private function parseBlockStart(): void
+    {
+        $this->tokenStream->append(new Token(TokenType::T_BLOCK_START, '{', $this->currentLineNumber, $this->currentColumnInLine));
+        $this->currentColumnInLine++;
+        $this->currentLineString = substr($this->currentLineString, 1);
+        $this->parseTabsAndWhitespaces();
+        if (str_starts_with($this->currentLineString, '}')) {
+            // Edge case: foo = { } in one line. Note content within {} is not parsed, everything behind { ends up as comment.
+            $this->lineStream->append((new IdentifierBlockOpenLine())->setIdentifierTokenStream($this->identifierStream)->setTokenStream($this->tokenStream));
+            $this->tokenStream = new TokenStream();
+            $this->tokenStream->append(new Token(TokenType::T_BLOCK_STOP, '}', $this->currentLineNumber, $this->currentColumnInLine));
+            $this->currentLineString = substr($this->currentLineString, 1);
+            $this->currentColumnInLine++;
+            $this->makeComment();
+            $this->lineStream->append((new BlockCloseLine())->setTokenStream($this->tokenStream));
+            return;
+        }
+        $this->makeComment();
+        $this->lineStream->append((new IdentifierBlockOpenLine())->setIdentifierTokenStream($this->identifierStream)->setTokenStream($this->tokenStream));
+    }
+
+    private function parseImportLine(): void
+    {
+        $this->tokenStream->append(new Token(TokenType::T_IMPORT_KEYWORD, '@import', $this->currentLineNumber, $this->currentColumnInLine));
+        $this->currentColumnInLine += 7;
+        $this->currentLineString = substr($this->currentLineString, 7);
+        $this->parseTabsAndWhitespaces();
+
+        // Next char should be the opening tick or doubletick, otherwise we create a comment until end of line
+        $nextChar = substr($this->currentLineString, 0, 1);
+        if ($nextChar !== '\'' && $nextChar !== '"') {
+            $this->makeComment();
+            $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+            return;
+        }
+        $this->tokenStream->append(new Token(TokenType::T_IMPORT_START, $nextChar, $this->currentLineNumber, $this->currentColumnInLine));
+
+        $importBody = '';
+        $importBodyStartPosition = $this->currentColumnInLine + 1;
+        $importBodyCharCount = 0;
+        $importBodyChars = mb_str_split(substr($this->currentLineString, 1), 1, 'UTF-8');
+        while (true) {
+            $nextChar = $importBodyChars[$importBodyCharCount] ?? null;
+            if ($nextChar === null) {
+                // end of chars
+                if ($importBodyCharCount) {
+                    $importBodyToken = (new Token(TokenType::T_VALUE, $importBody, $this->currentLineNumber, $importBodyStartPosition));
+                    $this->tokenStream->append($importBodyToken);
+                    ($this->currentLinebreakCallback)();
+                    $this->lineStream->append((new ImportLine())->setTokenStream($this->tokenStream)->setValueToken($importBodyToken));
+                    return;
+                }
+                ($this->currentLinebreakCallback)();
+                $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+                return;
+            }
+            if ($nextChar === '\'' || $nextChar === '"') {
+                if ($importBodyCharCount) {
+                    $importBodyToken = new Token(TokenType::T_VALUE, $importBody, $this->currentLineNumber, $importBodyStartPosition);
+                    $this->tokenStream->append($importBodyToken);
+                    $this->tokenStream->append(new Token(TokenType::T_IMPORT_STOP, $nextChar, $this->currentLineNumber, $this->currentColumnInLine + $importBodyCharCount + 1));
+                    $this->currentLineString = mb_substr($this->currentLineString, $importBodyCharCount + 2);
+                    $this->currentColumnInLine = $this->currentColumnInLine + $importBodyCharCount + 2;
+                    $this->parseTabsAndWhitespaces();
+                    $this->makeComment();
+                    $this->lineStream->append((new ImportLine())->setTokenStream($this->tokenStream)->setValueToken($importBodyToken));
+                    return;
+                }
+                $this->tokenStream->append(new Token(TokenType::T_IMPORT_STOP, $nextChar, $this->currentLineNumber, $this->currentColumnInLine + $importBodyCharCount + 1));
+                $this->currentLineString = mb_substr($this->currentLineString, $importBodyCharCount + 2);
+                $this->currentColumnInLine = $this->currentColumnInLine + $importBodyCharCount + 2;
+                $this->parseTabsAndWhitespaces();
+                $this->makeComment();
+                $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+                return;
+            }
+            $importBody .= $nextChar;
+            $importBodyCharCount++;
+        }
+    }
+
+    private function parseIdentifier(): void
+    {
+        $splitLine = mb_str_split($this->currentLineString, 1, 'UTF-8');
+        $currentPosition = $this->parseIdentifierUntilStopChar($splitLine);
+        if (!$currentPosition) {
+            $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+            return;
+        }
+        $this->currentLineString = mb_substr($this->currentLineString, $currentPosition);
+        $this->currentColumnInLine = $this->currentColumnInLine + $currentPosition;
+        $currentColumnInLineBefore = $this->currentColumnInLine;
+        $this->parseTabsAndWhitespaces();
+        $currentPosition = $currentPosition + $this->currentColumnInLine - $currentColumnInLineBefore;
+        $nextChar = $splitLine[$currentPosition] ?? null;
+        $nextTwoChars = $nextChar . ($splitLine[$currentPosition + 1] ?? '');
+        if ($nextTwoChars === '=<') {
+            $this->parseOperatorReference();
+            return;
+        }
+        if ($nextChar === '=') {
+            $this->parseOperatorAssignment();
+            return;
+        }
+        if ($nextChar === '{') {
+            $this->parseBlockStart();
+            return;
+        }
+        if ($nextChar === '>') {
+            $this->parseOperatorUnset();
+            return;
+        }
+        if ($nextChar === '<') {
+            $this->parseOperatorCopy();
+            return;
+        }
+        if ($nextChar === '(') {
+            $this->parseOperatorMultilineAssignment();
+            return;
+        }
+        if ($nextTwoChars === ':=') {
+            $this->parseOperatorFunction();
+            return;
+        }
+        if ($nextChar === '#') {
+            $this->parseHashComment();
+            $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+            return;
+        }
+        if ($nextTwoChars === '//') {
+            $this->parseDoubleSlashComment();
+            $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+            return;
+        }
+        if ($nextTwoChars === '/*') {
+            $this->parseMultilineComment();
+            $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+            return;
+        }
+        if ($nextChar === null) {
+            ($this->currentLinebreakCallback)();
+            $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+        }
+    }
+
+    private function parseOperatorAssignment(): void
+    {
+        $this->tokenStream->append(new Token(TokenType::T_OPERATOR_ASSIGNMENT, '=', $this->currentLineNumber, $this->currentColumnInLine));
+        $this->currentColumnInLine++;
+        $this->currentLineString = substr($this->currentLineString, 1);
+        $this->parseTabsAndWhitespaces();
+        $this->valueStream = new TokenStream();
+        [$this->valueStream, $this->tokenStream] = $this->parseValueForConstants($this->valueStream, $this->tokenStream, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine);
+        ($this->currentLinebreakCallback)();
+        $this->lineStream->append((new IdentifierAssignmentLine())->setTokenStream($this->tokenStream)->setIdentifierTokenStream($this->identifierStream)->setValueTokenStream($this->valueStream));
+    }
+
+    private function parseOperatorMultilineAssignment(): void
+    {
+        $this->valueStream = new TokenStream();
+        $this->tokenStream->append(new Token(TokenType::T_OPERATOR_ASSIGNMENT_MULTILINE_START, '(', $this->currentLineNumber, $this->currentColumnInLine));
+        $this->currentColumnInLine++;
+        $this->currentLineString = substr($this->currentLineString, 1);
+        // True if we're currently in the line with the opening '('
+        $isFirstLine = true;
+        // True if the first line has a first value token: "foo ( thisIsTheFirstValueToken"
+        $valueOnFirstLine = false;
+        // True if the line after '(' is parsed
+        $isSecondLine = false;
+        $previousLineCallback = function () {};
+        while (true) {
+            if (str_starts_with(ltrim($this->currentLineString), ')')) {
+                $this->parseTabsAndWhitespaces();
+                $this->tokenStream->append(new Token(TokenType::T_OPERATOR_ASSIGNMENT_MULTILINE_STOP, ')', $this->currentLineNumber, $this->currentColumnInLine));
+                $this->currentLineString = substr($this->currentLineString, 1);
+                $this->currentColumnInLine++;
+                $this->parseTabsAndWhitespaces();
+                $this->makeComment();
+                if ($this->valueStream->isEmpty()) {
+                    $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+                } else {
+                    $this->lineStream->append((new IdentifierAssignmentLine())->setIdentifierTokenStream($this->identifierStream)->setValueTokenStream($this->valueStream)->setTokenStream($this->tokenStream));
+                }
+                return;
+            }
+            if ($isFirstLine && str_ends_with($this->currentLineString, ')')) {
+                // Special case if the ')' is on same line as the opening '('
+                $this->currentLineString = substr($this->currentLineString, 0, -1);
+                if (strlen($this->currentLineString) > 1) {
+                    [$this->valueStream, $this->tokenStream] = $this->parseValueForConstants($this->valueStream, $this->tokenStream, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine);
+                    $this->tokenStream->append(new Token(TokenType::T_OPERATOR_ASSIGNMENT_MULTILINE_STOP, ')', $this->currentLineNumber, $this->currentColumnInLine + mb_strlen($this->currentLineString)));
+                    // Tricky to swap the streams here, but that's the most effective solution I could come up with for the line endings here.
+                    ($this->currentLinebreakCallback)();
+                    $tempStream = $this->tokenStream;
+                    $this->tokenStream = $this->valueStream;
+                    ($this->currentLinebreakCallback)();
+                    $this->tokenStream = $tempStream;
+                    $this->lineStream->append((new IdentifierAssignmentLine())->setIdentifierTokenStream($this->identifierStream)->setValueTokenStream($this->valueStream)->setTokenStream($this->tokenStream));
+                    return;
+                }
+                $this->tokenStream->append(new Token(TokenType::T_OPERATOR_ASSIGNMENT_MULTILINE_STOP, ')', $this->currentLineNumber, $this->currentColumnInLine + strlen($this->currentLineString)));
+                ($this->currentLinebreakCallback)();
+                $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+                return;
+            }
+            if ($isFirstLine && strlen($this->currentLineString)) {
+                [$this->valueStream, $this->tokenStream] = $this->parseValueForConstants($this->valueStream, $this->tokenStream, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine);
+                $valueOnFirstLine = true;
+                $previousLineCallback = $this->currentLinebreakCallback;
+            }
+            if (($isFirstLine && $valueOnFirstLine)
+                || (!$isFirstLine && !$isSecondLine)
+            ) {
+                $tempStream = $this->tokenStream;
+                $this->tokenStream = $this->valueStream;
+                $previousLineCallback();
+                $this->tokenStream = $tempStream;
+            }
+            if (!$isFirstLine && strlen($this->currentLineString)) {
+                [$this->valueStream, $this->tokenStream] = $this->parseValueForConstants($this->valueStream, $this->tokenStream, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine);
+            }
+            $previousLineCallback = $this->currentLinebreakCallback;
+            ($this->currentLinebreakCallback)();
+            if (!array_key_exists($this->currentLineNumber + 1, $this->lines)) {
+                $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+                return;
+            }
+            if ($isFirstLine) {
+                $isSecondLine = true;
+            } else {
+                $isSecondLine = false;
+            }
+            $isFirstLine = false;
+            $valueOnFirstLine = false;
+            $this->currentLineNumber++;
+            $this->currentColumnInLine = 0;
+            $this->currentLineString = $this->lines[$this->currentLineNumber]['line'];
+            $this->currentLinebreakCallback = $this->lines[$this->currentLineNumber]['linebreakCallback'];
+        }
+    }
+
+    private function parseOperatorUnset(): void
+    {
+        $this->tokenStream->append(new Token(TokenType::T_OPERATOR_UNSET, '>', $this->currentLineNumber, $this->currentColumnInLine));
+        $this->currentColumnInLine++;
+        $this->currentLineString = substr($this->currentLineString, 1);
+        $this->parseTabsAndWhitespaces();
+        $this->makeComment();
+        $this->lineStream->append((new IdentifierUnsetLine())->setTokenStream($this->tokenStream)->setIdentifierTokenStream($this->identifierStream));
+    }
+
+    private function parseOperatorCopy(): void
+    {
+        $this->tokenStream->append(new Token(TokenType::T_OPERATOR_COPY, '<', $this->currentLineNumber, $this->currentColumnInLine));
+        $this->currentColumnInLine++;
+        $this->currentLineString = substr($this->currentLineString, 1);
+        $this->parseTabsAndWhitespaces();
+        $identifierStream = $this->identifierStream;
+        $this->parseIdentifierAtEndOfLine();
+        $referenceStream = $this->identifierStream;
+        if ($referenceStream->isEmpty()) {
+            // @todo: ($this->currentLinebreakCallback)(); is missing here?!
+            $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+            return;
+        }
+        $this->lineStream->append(
+            (new IdentifierCopyLine())
+                ->setIdentifierTokenStream($identifierStream)
+                ->setValueTokenStream($referenceStream)
+                ->setTokenStream($this->tokenStream)
+        );
+    }
+
+    private function parseOperatorReference(): void
+    {
+        $this->tokenStream->append(new Token(TokenType::T_OPERATOR_REFERENCE, '=<', $this->currentLineNumber, $this->currentColumnInLine));
+        $this->currentColumnInLine += 2;
+        $this->currentLineString = substr($this->currentLineString, 2);
+        $this->parseTabsAndWhitespaces();
+        $identifierStream = $this->identifierStream;
+        $this->parseIdentifierAtEndOfLine();
+        $referenceStream = $this->identifierStream;
+        if ($referenceStream->isEmpty()) {
+            $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+            return;
+        }
+        $this->lineStream->append(
+            (new IdentifierReferenceLine())
+                ->setIdentifierTokenStream($identifierStream)
+                ->setValueTokenStream($referenceStream)
+                ->setTokenStream($this->tokenStream)
+        );
+    }
+
+    private function parseIdentifierAtEndOfLine(): void
+    {
+        $this->identifierStream = new IdentifierTokenStream();
+        $isRelative = false;
+        $splitLine = mb_str_split($this->currentLineString, 1, 'UTF-8');
+        $char = $splitLine[0] ?? null;
+        if ($char === null) {
+            return;
+        }
+        $nextTwoChars = $char . ($splitLine[1] ?? '');
+        if ($char === '.') {
+            // A relative right side: foo.bar < .foo (note the dot!). we identifierStream->setRelative() and
+            // get rid of the dot for the rest of the processing.
+            $isRelative = true;
+            $this->tokenStream->append((new Token(TokenType::T_DOT, '.', 0, $this->currentColumnInLine)));
+            array_shift($splitLine);
+            $this->currentColumnInLine++;
+            $this->currentLineString = substr($this->currentLineString, 1);
+        }
+        if ($char === '#') {
+            $this->parseHashComment();
+            return;
+        }
+        if ($nextTwoChars === '//') {
+            $this->parseDoubleSlashComment();
+            return;
+        }
+        if ($nextTwoChars === '/*') {
+            $this->parseMultilineComment();
+            return;
+        }
+        $currentPosition = $this->parseIdentifierUntilStopChar($splitLine, $isRelative);
+        if (!$currentPosition) {
+            return;
+        }
+        $this->currentLineString = mb_substr($this->currentLineString, $currentPosition);
+        $this->currentColumnInLine = $this->currentColumnInLine + $currentPosition;
+        $this->parseTabsAndWhitespaces();
+        $this->makeComment();
+    }
+
+    private function parseIdentifierUntilStopChar(array $splitLine, bool $isRelative = false): ?int
+    {
+        $this->identifierStream = new IdentifierTokenStream();
+        if ($isRelative) {
+            $this->identifierStream->setRelative();
+        }
+        $currentPosition = 0;
+        $currentIdentifierStartPosition = $this->currentColumnInLine;
+        $currentIdentifierBody = '';
+        $currentIdentifierCharCount = 0;
+        while (true) {
+            $nextChar = $splitLine[$currentPosition] ?? null;
+            if ($nextChar === null) {
+                if ($currentIdentifierCharCount) {
+                    $identifierToken = new IdentifierToken(TokenType::T_IDENTIFIER, $currentIdentifierBody, $this->currentLineNumber, $currentIdentifierStartPosition);
+                    $this->tokenStream->append($identifierToken);
+                    $this->identifierStream->append($identifierToken);
+                }
+                ($this->currentLinebreakCallback)();
+                return null;
+            }
+            $nextTwoChars = $nextChar . ($splitLine[$currentPosition + 1] ?? null);
+            if ($currentPosition > 0
+                && ($nextChar === ' ' || $nextChar === "\t" || $nextChar === '=' || $nextChar === '<' || $nextChar === '>' || $nextChar === '{' || $nextTwoChars === ':=' || $nextChar === '(')
+            ) {
+                if ($currentIdentifierCharCount) {
+                    $identifierToken = new IdentifierToken(TokenType::T_IDENTIFIER, $currentIdentifierBody, $this->currentLineNumber, $currentIdentifierStartPosition);
+                    $this->tokenStream->append($identifierToken);
+                    $this->identifierStream->append($identifierToken);
+                }
+                break;
+            }
+            if ($nextTwoChars === '\\.') {
+                // A quoted dot is part of *this* identifier
+                $currentIdentifierBody .= '.';
+                $currentPosition += 2;
+                $currentIdentifierCharCount++;
+            } elseif ($nextChar === '.') {
+                if ($currentIdentifierCharCount) {
+                    $identifierToken = new IdentifierToken(TokenType::T_IDENTIFIER, $currentIdentifierBody, $this->currentLineNumber, $currentIdentifierStartPosition);
+                    $this->tokenStream->append($identifierToken);
+                    $this->identifierStream->append($identifierToken);
+                    $currentIdentifierCharCount = 0;
+                    $currentIdentifierBody = '';
+                }
+                $this->tokenStream->append(new Token(TokenType::T_DOT, '.', $this->currentLineNumber, $this->currentColumnInLine + $currentPosition));
+                $currentPosition++;
+                $currentIdentifierStartPosition = $this->currentColumnInLine + $currentPosition;
+            } else {
+                $currentIdentifierBody .= $nextChar;
+                $currentIdentifierCharCount++;
+                $currentPosition++;
+            }
+        }
+        return $currentPosition;
+    }
+
+    private function parseOperatorFunction(): void
+    {
+        $this->tokenStream->append(new Token(TokenType::T_OPERATOR_FUNCTION, ':=', $this->currentLineNumber, $this->currentColumnInLine));
+        $this->currentColumnInLine += 2;
+        $this->currentLineString = substr($this->currentLineString, 2);
+        $this->parseTabsAndWhitespaces();
+        if ($this->currentLineString === '') {
+            ($this->currentLinebreakCallback)();
+            $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+            return;
+        }
+        $functionName = '';
+        $functionNameStartPosition = $this->currentColumnInLine;
+        $functionNameCharCount = 0;
+        $functionChars = mb_str_split($this->currentLineString, 1, 'UTF-8');
+        while (true) {
+            $nextChar = $functionChars[$functionNameCharCount] ?? null;
+            if ($nextChar === null) {
+                // end of chars
+                if ($functionNameCharCount) {
+                    $this->tokenStream->append(new Token(TokenType::T_FUNCTION_NAME, $functionName, $this->currentLineNumber, $functionNameStartPosition));
+                }
+                ($this->currentLinebreakCallback)();
+                $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+                return;
+            }
+            if ($nextChar === '(') {
+                if ($functionNameCharCount) {
+                    $functionNameToken = new Token(TokenType::T_FUNCTION_NAME, $functionName, $this->currentLineNumber, $functionNameStartPosition);
+                    $this->tokenStream->append($functionNameToken);
+                    $this->tokenStream->append(new Token(TokenType::T_FUNCTION_VALUE_START, '(', $this->currentLineNumber, $this->currentColumnInLine + $functionNameCharCount));
+                    $functionNameCharCount++;
+                    break;
+                }
+                $this->makeComment();
+                $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+                return;
+            }
+            $functionName .= $nextChar;
+            $functionNameCharCount++;
+        }
+        $functionBodyStartPosition = $functionNameCharCount;
+        $functionBodyPart = '';
+        $functionBodyCharCount = 0;
+        $functionValueStream = new TokenStream();
+        $parenthesesLevel = 0;
+        while (true) {
+            $nextChar = $functionChars[$functionBodyStartPosition + $functionBodyCharCount] ?? null;
+            if ($nextChar === null) {
+                if ($functionBodyCharCount) {
+                    $this->tokenStream->append(new Token(TokenType::T_VALUE, $functionBodyPart, $this->currentLineNumber, $functionBodyStartPosition));
+                }
+                ($this->currentLinebreakCallback)();
+                $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
+                return;
+            }
+            if ($nextChar === '(') {
+                // In case of a function call like "appendString(something(somethingelse))"
+                // we shall only stop processing when the last bracket was evaluated.
+                $parenthesesLevel++;
+            }
+            if ($nextChar === ')') {
+                if ($parenthesesLevel > 0) {
+                    $parenthesesLevel--;
+                    // Continue collecting characters from the (...) argument stream.
+                    // Also, ")" will be appended, thus intentionally no "break" occurs.
+                } else {
+                    if ($functionBodyCharCount) {
+                        [$functionValueStream, $this->tokenStream] = $this->parseValueForConstants($functionValueStream, $this->tokenStream, $functionBodyPart, $this->currentLineNumber, $this->currentColumnInLine, $functionBodyStartPosition);
+                    }
+                    $this->tokenStream->append(new Token(TokenType::T_FUNCTION_VALUE_STOP, ')', $this->currentLineNumber, $this->currentColumnInLine + $functionNameCharCount + $functionBodyCharCount));
+                    $functionBodyCharCount++;
+                    break;
+                }
+            }
+            $functionBodyPart .= $nextChar;
+            $functionBodyCharCount++;
+        }
+        $this->currentColumnInLine = $this->currentColumnInLine + $functionNameCharCount + $functionBodyCharCount;
+        $this->currentLineString = mb_substr($this->currentLineString, $functionNameCharCount + $functionBodyCharCount);
+        $this->parseTabsAndWhitespaces();
+        $this->makeComment();
+        $this->lineStream->append(
+            (new IdentifierFunctionLine())
+                ->setIdentifierTokenStream($this->identifierStream)
+                ->setFunctionNameToken($functionNameToken)
+                ->setTokenStream($this->tokenStream)
+                ->setFunctionValueTokenStream($functionValueStream)
+        );
+    }
+
+    /**
+     * @return array{0: TokenStreamInterface, 1: TokenStreamInterface}
+     */
+    private function parseValueForConstants(TokenStreamInterface $valueStream, TokenStreamInterface $tokenStream, string $value, int $line, int $column, int $tokenOffsetPosition = 0): array
+    {
+        if (!str_contains($value, '{$')) {
+            $valueToken = new Token(TokenType::T_VALUE, $value, $line, $column + $tokenOffsetPosition);
+            $valueStream->append($valueToken);
+            $tokenStream->append($valueToken);
+            return [$valueStream, $tokenStream];
+        }
+        $splitLine = mb_str_split($value, 1, 'UTF-8');
+        $isInConstant = false;
+        $currentPosition = 0;
+        $currentString = '';
+        $currentStringLength = 0;
+        $lastTokenEndPosition = 0;
+        while (true) {
+            $char = $splitLine[$currentPosition] ?? null;
+            if ($char === null) {
+                if ($currentStringLength) {
+                    $valueToken = new Token(TokenType::T_VALUE, $currentString, $line, $column + $lastTokenEndPosition + $tokenOffsetPosition);
+                    $valueStream->append($valueToken);
+                    $tokenStream->append($valueToken);
+                }
+                break;
+            }
+            $nextTwoChars = $char . ($splitLine[$currentPosition + 1] ?? '');
+            if ($nextTwoChars === '{$') {
+                $isInConstant = true;
+                if ($currentStringLength) {
+                    $valueToken = new Token(TokenType::T_VALUE, $currentString, $line, $column + $lastTokenEndPosition + $tokenOffsetPosition);
+                    $valueStream->append($valueToken);
+                    $tokenStream->append($valueToken);
+                    $lastTokenEndPosition = $currentPosition;
+                }
+                $currentString = '{$';
+                $currentPosition += 2;
+                continue;
+            }
+            if ($isInConstant && $char === '}') {
+                $valueToken = new Token(TokenType::T_CONSTANT, $currentString . '}', $line, $column + $lastTokenEndPosition + $tokenOffsetPosition);
+                if (!$valueStream instanceof ConstantAwareTokenStream) {
+                    $valueStream = (new ConstantAwareTokenStream())->setAll($valueStream->getAll());
+                }
+                $valueStream->append($valueToken);
+                $tokenStream->append($valueToken);
+                $currentPosition++;
+                $currentString = '';
+                $currentStringLength = 0;
+                $lastTokenEndPosition = $currentPosition;
+                $isInConstant = false;
+                continue;
+            }
+            $currentPosition++;
+            $currentStringLength++;
+            $currentString .= $char;
+        }
+        return [$valueStream, $tokenStream];
+    }
+}

Classes/TypoScript/Tokenizer/LossyTokenizer.php

@@ -0,0 +1,624 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer;
+
+use Symfony\Component\DependencyInjection\Attribute\AsAlias;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\BlockCloseLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ConditionElseLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ConditionLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ConditionStopLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierAssignmentLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierBlockOpenLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierCopyLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierFunctionLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierReferenceLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierUnsetLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ImportLine;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\LineStream;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\ConstantAwareTokenStream;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\IdentifierToken;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\IdentifierTokenStream;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\Token;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenStream;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenStreamInterface;
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenType;
+
+/**
+ * A lossy tokenizer implementation: Do not create invalid lines, do not create empty lines,
+ * do not create token line and column positions.
+ *
+ * This tokenizer creates a much smaller streams of only relevant lines. All information
+ * not essential for the AstBuilder is skipped. This tokenizer is used in frontend rendering
+ * for quicker AST building.
+ *
+ * An instance of this tokenizer is injected by DI when injecting TokenizerInterface.
+ *
+ * This class is unit test covered by TokenizerInterfaceTest and paired with LossyTokenizer.
+ * Never change anything in this class without additional test coverage!
+ *
+ * @internal: Internal tokenizer structure.
+ */
+#[AsAlias(TokenizerInterface::class)]
+final class LossyTokenizer implements TokenizerInterface
+{
+    private LineStream $lineStream;
+
+    private IdentifierTokenStream $identifierStream;
+    private TokenStreamInterface $valueStream;
+
+    private array $lines;
+    private int $currentLineNumber;
+    private string $currentLineString;
+
+    public function tokenize(string $source): LineStream
+    {
+        $this->lineStream = new LineStream();
+        $this->currentLineNumber = -1;
+        $this->lines = $this->splitLines($source);
+
+        while (true) {
+            $this->currentLineNumber++;
+            if (!array_key_exists($this->currentLineNumber, $this->lines)) {
+                break;
+            }
+            $this->currentLineString = trim($this->lines[$this->currentLineNumber]['line']);
+            $nextChar = substr($this->currentLineString, 0, 1);
+            if ($nextChar === '') {
+                continue;
+            }
+            $nextTwoChars = substr($this->currentLineString, 0, 2);
+            if ($nextChar === '#' || $nextTwoChars === '//') {
+                continue;
+            }
+            if ($nextTwoChars === '/*') {
+                // @todo: This is one of multiple places where multiline "/*" comments are parsed in this tokenizer. Other
+                //        places are cluttered in detail methods. It might be more straight to have an early scanning
+                //        phase through all lines to remove comments up front, to not wire especially the multiline comment
+                //        parsing to single places, and throw away commented lines early. This isn't trivial though, since
+                //        for instance "foo = bar /* not a comment */" then needs to be sorted out, too. Having an early
+                //        "kick comments" loop however might be quicker in the end and would make the main parsing
+                //        methods more concise and probably more bullet proof.
+                //        Also note there are currently not-unit-tested edge cases, that will currently not parse as
+                //        (maybe) expected. In the example below, "foo2 = bar2" is ignored. This is an issue with the
+                //        LosslessTokenizer as well, probably, and we may rather want to declare this as invalid syntax?!
+                //          foo = bar /* comment start
+                //          comment end */ foo2 = bar2
+                $this->ignoreUntilEndOfMultilineComment();
+                continue;
+            }
+            if ($nextChar === '[') {
+                $this->createConditionLine();
+            } elseif ($nextChar === '}') {
+                $this->lineStream->append((new BlockCloseLine()));
+            } elseif (str_starts_with($this->currentLineString, '@import')) {
+                $this->parseImportLine();
+            } elseif (str_starts_with($this->currentLineString, '<INCLUDE_TYPOSCRIPT:')) {
+                // @todo: Do nothing. This creates an InvalidLine in LossyTokenizer.
+            } else {
+                $this->parseIdentifier();
+            }
+        }
+
+        return $this->lineStream;
+    }
+
+    private function splitLines($source): array
+    {
+        $vanillaLines = explode(chr(10), $source);
+        return array_map(
+            fn(int $lineNumber, string $vanillaLine): array => [
+                'line' => rtrim($vanillaLine, "\r"),
+            ],
+            array_keys($vanillaLines),
+            $vanillaLines
+        );
+    }
+
+    private function ignoreUntilEndOfMultilineComment(): void
+    {
+        while (true) {
+            if (str_contains($this->currentLineString, '*/')) {
+                return;
+            }
+            if (!array_key_exists($this->currentLineNumber + 1, $this->lines)) {
+                return;
+            }
+            $this->currentLineNumber++;
+            $this->currentLineString = trim($this->lines[$this->currentLineNumber]['line']);
+        }
+    }
+
+    /**
+     * Create a condition line from token stream of this line.
+     */
+    private function createConditionLine(): void
+    {
+        $upperCaseLine = strtoupper($this->currentLineString);
+        if (str_starts_with($upperCaseLine, '[ELSE]')) {
+            $this->lineStream->append((new ConditionElseLine()));
+            $this->currentLineString = trim(substr($this->currentLineString, 6));
+            if (str_starts_with($this->currentLineString, '/*')) {
+                $this->ignoreUntilEndOfMultilineComment();
+            }
+            return;
+        }
+        if (str_starts_with($upperCaseLine, '[END]')) {
+            $this->lineStream->append((new ConditionStopLine()));
+            $this->currentLineString = trim(substr($this->currentLineString, 5));
+            if (str_starts_with($this->currentLineString, '/*')) {
+                $this->ignoreUntilEndOfMultilineComment();
+            }
+            return;
+        }
+        if (str_starts_with($upperCaseLine, '[GLOBAL]')) {
+            $this->lineStream->append((new ConditionStopLine()));
+            $this->currentLineString = trim(substr($this->currentLineString, 8));
+            if (str_starts_with($this->currentLineString, '/*')) {
+                $this->ignoreUntilEndOfMultilineComment();
+            }
+            return;
+        }
+        $conditionBody = '';
+        $conditionBodyCharCount = 0;
+        $conditionBodyChars = mb_str_split(substr($this->currentLineString, 1), 1, 'UTF-8');
+        $bracketCount = 1;
+        while (true) {
+            $nextChar = $conditionBodyChars[$conditionBodyCharCount] ?? null;
+            if ($nextChar === null) {
+                // end of chars
+                return;
+            }
+            if ($nextChar === '[') {
+                $bracketCount++;
+                $conditionBody .= $nextChar;
+                $conditionBodyCharCount++;
+                continue;
+            }
+            if ($nextChar === ']') {
+                $bracketCount--;
+                if ($bracketCount === 0) {
+                    if ($conditionBodyCharCount) {
+                        $conditionBodyToken = new Token(TokenType::T_VALUE, $conditionBody);
+                        $this->lineStream->append((new ConditionLine())->setValueToken($conditionBodyToken));
+                        $conditionBodyCharCount++;
+                        break;
+                    }
+                    $conditionBodyCharCount++;
+                    break;
+                }
+                $conditionBody .= $nextChar;
+                $conditionBodyCharCount++;
+                continue;
+            }
+            $conditionBody .= $nextChar;
+            $conditionBodyCharCount++;
+        }
+        $this->currentLineString = trim(mb_substr($this->currentLineString, $conditionBodyCharCount + 1));
+        if (str_starts_with($this->currentLineString, '/*')) {
+            $this->ignoreUntilEndOfMultilineComment();
+        }
+    }
+
+    private function parseBlockStart(): void
+    {
+        $this->currentLineString = trim(substr($this->currentLineString, 1));
+        if (str_starts_with($this->currentLineString, '}')) {
+            // Edge case: foo = { } in one line. Note content within {} is not parsed, everything behind { ends up as comment.
+            $this->lineStream->append((new IdentifierBlockOpenLine())->setIdentifierTokenStream($this->identifierStream));
+            $this->lineStream->append((new BlockCloseLine()));
+            return;
+        }
+        $this->lineStream->append((new IdentifierBlockOpenLine())->setIdentifierTokenStream($this->identifierStream));
+    }
+
+    private function parseImportLine(): void
+    {
+        $this->currentLineString = trim(substr($this->currentLineString, 7));
+
+        // Next char should be the opening tick or doubletick, otherwise treat it as ignored comment
+        $nextChar = substr($this->currentLineString, 0, 1);
+        if ($nextChar !== '\'' && $nextChar !== '"') {
+            return;
+        }
+
+        $importBody = '';
+        $importBodyCharCount = 0;
+        $importBodyChars = mb_str_split(substr($this->currentLineString, 1), 1, 'UTF-8');
+        while (true) {
+            $nextChar = $importBodyChars[$importBodyCharCount] ?? null;
+            if ($nextChar === null) {
+                // end of chars
+                if ($importBodyCharCount) {
+                    $importBodyToken = (new Token(TokenType::T_VALUE, $importBody));
+                    $this->lineStream->append((new ImportLine())->setValueToken($importBodyToken));
+                    return;
+                }
+                return;
+            }
+            if ($nextChar === '\'' || $nextChar === '"') {
+                if ($importBodyCharCount) {
+                    $importBodyToken = new Token(TokenType::T_VALUE, $importBody);
+                    $this->lineStream->append((new ImportLine())->setValueToken($importBodyToken));
+                    break;
+                }
+                break;
+            }
+            $importBody .= $nextChar;
+            $importBodyCharCount++;
+        }
+        $this->currentLineString = trim(mb_substr($this->currentLineString, $importBodyCharCount + 2));
+        if (str_starts_with($this->currentLineString, '/*')) {
+            $this->ignoreUntilEndOfMultilineComment();
+        }
+    }
+
+    private function parseIdentifier(): void
+    {
+        $splitLine = mb_str_split($this->currentLineString, 1, 'UTF-8');
+        $currentPosition = $this->parseIdentifierUntilStopChar($splitLine);
+        if (!$currentPosition) {
+            return;
+        }
+        $this->currentLineString = trim(mb_substr($this->currentLineString, $currentPosition));
+        $nextChar = substr($this->currentLineString, 0, 1);
+        $nextTwoChars = $nextChar . substr($this->currentLineString, 1, 1);
+        if ($nextTwoChars === '=<') {
+            $this->parseOperatorReference();
+            return;
+        }
+        if ($nextChar === '=') {
+            $this->parseOperatorAssignment();
+            return;
+        }
+        if ($nextChar === '{') {
+            $this->parseBlockStart();
+            return;
+        }
+        if ($nextChar === '>') {
+            $this->parseOperatorUnset();
+            return;
+        }
+        if ($nextChar === '<') {
+            $this->parseOperatorCopy();
+            return;
+        }
+        if ($nextChar === '(') {
+            $this->parseOperatorMultilineAssignment();
+            return;
+        }
+        if ($nextTwoChars === ':=') {
+            $this->parseOperatorFunction();
+        }
+        if ($nextTwoChars === '/*') {
+            $this->ignoreUntilEndOfMultilineComment();
+        }
+    }
+
+    private function parseOperatorUnset(): void
+    {
+        $this->lineStream->append((new IdentifierUnsetLine())->setIdentifierTokenStream($this->identifierStream));
+        $this->currentLineString = trim(trim(trim($this->currentLineString), '>'));
+        if (str_starts_with($this->currentLineString, '/*')) {
+            $this->ignoreUntilEndOfMultilineComment();
+        }
+    }
+
+    private function parseOperatorAssignment(): void
+    {
+        $this->currentLineString = trim(substr($this->currentLineString, 1));
+        $this->valueStream = $this->parseValueForConstants(new TokenStream(), $this->currentLineString);
+        $this->lineStream->append((new IdentifierAssignmentLine())->setIdentifierTokenStream($this->identifierStream)->setValueTokenStream($this->valueStream));
+    }
+
+    private function parseOperatorMultilineAssignment(): void
+    {
+        $this->valueStream = new TokenStream();
+        $this->currentLineString = substr($this->currentLineString, 1);
+        // True if we're currently in the line with the opening '('
+        $isFirstLine = true;
+        // True if the first line has a first value token: "foo ( thisIsTheFirstValueToken"
+        $valueOnFirstLine = false;
+        // True if the line after '(' is parsed
+        $isSecondLine = false;
+        while (true) {
+            if (str_starts_with(ltrim($this->currentLineString), ')')) {
+                $this->currentLineString = trim(substr($this->currentLineString, 1));
+                if (!$this->valueStream->isEmpty()) {
+                    $this->lineStream->append((new IdentifierAssignmentLine())->setIdentifierTokenStream($this->identifierStream)->setValueTokenStream($this->valueStream));
+                }
+                if (str_starts_with($this->currentLineString, '/*')) {
+                    $this->ignoreUntilEndOfMultilineComment();
+                }
+                return;
+            }
+            if ($isFirstLine && str_ends_with($this->currentLineString, ')')) {
+                $this->currentLineString = substr($this->currentLineString, 0, -1);
+                if (strlen($this->currentLineString) > 1) {
+                    $this->valueStream = $this->parseValueForConstants($this->valueStream, $this->currentLineString);
+                    $this->lineStream->append((new IdentifierAssignmentLine())->setIdentifierTokenStream($this->identifierStream)->setValueTokenStream($this->valueStream));
+                    return;
+                }
+                return;
+            }
+            if ($isFirstLine && strlen($this->currentLineString)) {
+                $this->valueStream = $this->parseValueForConstants($this->valueStream, $this->currentLineString);
+                $valueOnFirstLine = true;
+            }
+            if (($isFirstLine && $valueOnFirstLine)
+                || (!$isFirstLine && !$isSecondLine)
+            ) {
+                $this->valueStream->append(new Token(TokenType::T_NEWLINE, "\n"));
+            }
+            if (!$isFirstLine && strlen($this->currentLineString)) {
+                $this->valueStream = $this->parseValueForConstants($this->valueStream, $this->currentLineString);
+            }
+            if (!array_key_exists($this->currentLineNumber + 1, $this->lines)) {
+                return;
+            }
+            if ($isFirstLine) {
+                $isSecondLine = true;
+            } else {
+                $isSecondLine = false;
+            }
+            $isFirstLine = false;
+            $valueOnFirstLine = false;
+            $this->currentLineNumber++;
+            $this->currentLineString = $this->lines[$this->currentLineNumber]['line'];
+        }
+    }
+
+    private function parseOperatorCopy(): void
+    {
+        $this->currentLineString = trim(substr($this->currentLineString, 1));
+        $identifierStream = $this->identifierStream;
+        $charsHandled = $this->parseIdentifierAtEndOfLine();
+        $referenceStream = $this->identifierStream;
+        if ($referenceStream->isEmpty()) {
+            return;
+        }
+        $this->lineStream->append(
+            (new IdentifierCopyLine())
+                ->setIdentifierTokenStream($identifierStream)
+                ->setValueTokenStream($referenceStream)
+        );
+        $this->currentLineString = trim(mb_substr($this->currentLineString, $charsHandled));
+        if (str_starts_with($this->currentLineString, '/*')) {
+            $this->ignoreUntilEndOfMultilineComment();
+        }
+    }
+
+    private function parseOperatorReference(): void
+    {
+        $this->currentLineString = trim(substr($this->currentLineString, 2));
+        $identifierStream = $this->identifierStream;
+        $charsHandled = $this->parseIdentifierAtEndOfLine();
+        $referenceStream = $this->identifierStream;
+        if ($referenceStream->isEmpty()) {
+            return;
+        }
+        $this->lineStream->append(
+            (new IdentifierReferenceLine())
+                ->setIdentifierTokenStream($identifierStream)
+                ->setValueTokenStream($referenceStream)
+        );
+        $this->currentLineString = trim(mb_substr($this->currentLineString, $charsHandled));
+        if (str_starts_with($this->currentLineString, '/*')) {
+            $this->ignoreUntilEndOfMultilineComment();
+        }
+    }
+
+    private function parseIdentifierAtEndOfLine(): int
+    {
+        $this->identifierStream = new IdentifierTokenStream();
+        $isRelative = false;
+        $splitLine = mb_str_split($this->currentLineString, 1, 'UTF-8');
+        $char = $splitLine[0] ?? null;
+        if ($char === null) {
+            return 0;
+        }
+        $nextTwoChars = $char . ($splitLine[1] ?? '');
+        if ($char === '.') {
+            // A relative right side: foo.bar < .foo (note the dot!). we identifierStream->setRelative() and
+            // get rid of the dot for the rest of the processing.
+            $isRelative = true;
+            array_shift($splitLine);
+            $this->currentLineString = substr($this->currentLineString, 1);
+        }
+        if ($char === '#') {
+            return 1;
+        }
+        if ($nextTwoChars === '//') {
+            return 2;
+        }
+        if ($nextTwoChars === '/*') {
+            $this->ignoreUntilEndOfMultilineComment();
+            return 0;
+        }
+        return $this->parseIdentifierUntilStopChar($splitLine, $isRelative);
+    }
+
+    private function parseIdentifierUntilStopChar(array $splitLine, bool $isRelative = false): int
+    {
+        $this->identifierStream = new IdentifierTokenStream();
+        if ($isRelative) {
+            $this->identifierStream->setRelative();
+        }
+        $currentPosition = 0;
+        $currentIdentifierBody = '';
+        $currentIdentifierCharCount = 0;
+        while (true) {
+            $nextChar = $splitLine[$currentPosition] ?? null;
+            if ($nextChar === null) {
+                if ($currentIdentifierCharCount) {
+                    $identifierToken = new IdentifierToken(TokenType::T_IDENTIFIER, $currentIdentifierBody);
+                    $this->identifierStream->append($identifierToken);
+                }
+                return $currentPosition;
+            }
+            $nextTwoChars = $nextChar . ($splitLine[$currentPosition + 1] ?? null);
+            if ($currentPosition > 0
+                && ($nextChar === ' ' || $nextChar === "\t" || $nextChar === '=' || $nextChar === '<' || $nextChar === '>' || $nextChar === '{' || $nextTwoChars === ':=' || $nextChar === '(')
+            ) {
+                if ($currentIdentifierCharCount) {
+                    $identifierToken = new IdentifierToken(TokenType::T_IDENTIFIER, $currentIdentifierBody);
+                    $this->identifierStream->append($identifierToken);
+                }
+                break;
+            }
+            if ($nextTwoChars === '\\.') {
+                // A quoted dot is part of *this* identifier
+                $currentIdentifierBody .= '.';
+                $currentPosition += 2;
+                $currentIdentifierCharCount++;
+            } elseif ($nextChar === '.') {
+                if ($currentIdentifierCharCount) {
+                    $identifierToken = new IdentifierToken(TokenType::T_IDENTIFIER, $currentIdentifierBody);
+                    $this->identifierStream->append($identifierToken);
+                    $currentIdentifierCharCount = 0;
+                    $currentIdentifierBody = '';
+                }
+                $currentPosition++;
+            } else {
+                $currentIdentifierBody .= $nextChar;
+                $currentIdentifierCharCount++;
+                $currentPosition++;
+            }
+        }
+        return $currentPosition;
+    }
+
+    private function parseOperatorFunction(): void
+    {
+        $this->currentLineString = trim(substr($this->currentLineString, 2));
+        if ($this->currentLineString === '') {
+            return;
+        }
+        $functionName = '';
+        $functionNameCharCount = 0;
+        $functionChars = mb_str_split($this->currentLineString, 1, 'UTF-8');
+        while (true) {
+            $nextChar = $functionChars[$functionNameCharCount] ?? null;
+            if ($nextChar === null) {
+                // end of chars
+                return;
+            }
+            if ($nextChar === '(') {
+                if ($functionNameCharCount) {
+                    $functionNameToken = new Token(TokenType::T_FUNCTION_NAME, $functionName);
+                    $functionNameCharCount++;
+                    break;
+                }
+                return;
+            }
+            $functionName .= $nextChar;
+            $functionNameCharCount++;
+        }
+        $functionBodyStartPosition = $functionNameCharCount;
+        $functionBodyPart = '';
+        $functionBodyCharCount = 0;
+        $functionValueStream = new TokenStream();
+        $parenthesesLevel = 0;
+        while (true) {
+            $nextChar = $functionChars[$functionBodyStartPosition + $functionBodyCharCount] ?? null;
+            if ($nextChar === null) {
+                return;
+            }
+            if ($nextChar === '(') {
+                // In case of a function call like "appendString(something(somethingelse))"
+                // we shall only stop processing when the last bracket was evaluated.
+                $parenthesesLevel++;
+            }
+            if ($nextChar === ')') {
+                if ($parenthesesLevel > 0) {
+                    $parenthesesLevel--;
+                    // Continue collecting characters from the (...) argument stream.
+                    // Also, ")" will be appended, thus intentionally no "break" occurs.
+                } else {
+                    if ($functionBodyCharCount) {
+                        $functionValueStream = $this->parseValueForConstants($functionValueStream, $functionBodyPart);
+                        $functionBodyCharCount++;
+                    }
+                    break;
+                }
+            }
+            $functionBodyPart .= $nextChar;
+            $functionBodyCharCount++;
+        }
+        $this->lineStream->append(
+            (new IdentifierFunctionLine())
+                ->setIdentifierTokenStream($this->identifierStream)
+                ->setFunctionNameToken($functionNameToken)
+                ->setFunctionValueTokenStream($functionValueStream)
+        );
+        // Check for multiline comment
+        $this->currentLineString = implode('', array_slice($functionChars, $functionBodyStartPosition + $functionBodyCharCount + 1));
+        if (mb_strlen($this->currentLineString) >= 1 && str_starts_with($this->currentLineString, '/*')) {
+            $this->ignoreUntilEndOfMultilineComment();
+        }
+    }
+
+    private function parseValueForConstants(TokenStreamInterface $valueStream, string $value): TokenStreamInterface
+    {
+        if (!str_contains($value, '{$')) {
+            $valueStream->append(new Token(TokenType::T_VALUE, $value));
+            return $valueStream;
+        }
+        $splitLine = mb_str_split($value, 1, 'UTF-8');
+        $isInConstant = false;
+        $currentPosition = 0;
+        $currentString = '';
+        $currentStringLength = 0;
+        while (true) {
+            $char = $splitLine[$currentPosition] ?? null;
+            if ($char === null) {
+                if ($currentStringLength) {
+                    $valueToken = new Token(TokenType::T_VALUE, $currentString);
+                    $valueStream->append($valueToken);
+                }
+                break;
+            }
+            $nextTwoChars = $char . ($splitLine[$currentPosition + 1] ?? '');
+            if ($nextTwoChars === '{$') {
+                $isInConstant = true;
+                if ($currentStringLength) {
+                    $valueToken = new Token(TokenType::T_VALUE, $currentString);
+                    $valueStream->append($valueToken);
+                }
+                $currentString = '{$';
+                $currentPosition += 2;
+                continue;
+            }
+            if ($isInConstant && $char === '}') {
+                $valueToken = new Token(TokenType::T_CONSTANT, $currentString . '}');
+                if (!$valueStream instanceof ConstantAwareTokenStream) {
+                    $valueStream = (new ConstantAwareTokenStream())->setAll($valueStream->getAll());
+                }
+                $valueStream->append($valueToken);
+                $currentPosition++;
+                $currentString = '';
+                $currentStringLength = 0;
+                $isInConstant = false;
+                continue;
+            }
+            $currentPosition++;
+            $currentStringLength++;
+            $currentString .= $char;
+        }
+        return $valueStream;
+    }
+}

Classes/TypoScript/Tokenizer/Token/AbstractToken.php

@@ -0,0 +1,79 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Token;
+
+/**
+ * Main implementation of a TokenInterface.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+abstract class AbstractToken implements TokenInterface
+{
+    protected int $line = 0;
+    protected int $column = 0;
+
+    public function __construct(
+        private readonly TokenType $type,
+        protected readonly string $value,
+        int $line = 0,
+        int $column = 0
+    ) {
+        // No constructor property promotion for $line and $column: We don't serialize
+        // these two and want to still default them to 0 (zero) when unserialized.
+        $this->line = $line;
+        $this->column = $column;
+    }
+
+    public function __toString(): string
+    {
+        return $this->value;
+    }
+
+    /**
+     * Do not store line and column when structure is serialized to cache.
+     * Not storing $line and $column reduces the cache size by about 1/3 since
+     * we're typically storing *a lot* of tokens.
+     */
+    public function __serialize(): array
+    {
+        return [
+            'type' => $this->type,
+            'value' => $this->value,
+        ];
+    }
+
+    public function getType(): TokenType
+    {
+        return $this->type;
+    }
+
+    public function getValue(): string
+    {
+        return $this->value;
+    }
+
+    public function getLine(): int
+    {
+        return $this->line;
+    }
+
+    public function getColumn(): int
+    {
+        return $this->column;
+    }
+}

Classes/TypoScript/Tokenizer/Token/AbstractTokenStream.php

@@ -0,0 +1,113 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Token;
+
+/**
+ * A generic implementation of TokenStreamInterface.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+abstract class AbstractTokenStream implements TokenStreamInterface
+{
+    /**
+     * @var TokenInterface[]
+     */
+    protected array $tokens = [];
+    protected int $currentIndex = -1;
+
+    /**
+     * Create a source string from given tokens.
+     */
+    public function __toString(): string
+    {
+        $source = '';
+        $this->reset();
+        while ($token = $this->getNext()) {
+            $source .= $token;
+        }
+        return $source;
+    }
+
+    /**
+     * When storing to cache, we only store FE relevant properties and skip
+     * irrelevant things. For instance $currentIndex should always initialize
+     * to -1 and does not need to be stored.
+     */
+    final public function __serialize(): array
+    {
+        return $this->serialize();
+    }
+
+    protected function serialize(): array
+    {
+        $result['tokens'] = $this->tokens;
+        return $result;
+    }
+
+    /**
+     * Stream creation.
+     */
+    public function append(TokenInterface $token): self
+    {
+        $this->tokens[] = $token;
+        return $this;
+    }
+
+    /**
+     * We sometimes create a stream but don't add tokens.
+     * This method returns true if tokens have been added.
+     */
+    public function isEmpty(): bool
+    {
+        return empty($this->tokens);
+    }
+
+    /**
+     * Reset current pointer. Typically, call this before iterating with getNext().
+     */
+    public function reset(): static
+    {
+        $this->currentIndex = -1;
+        return $this;
+    }
+
+    /**
+     * Get next token and raise pointer.
+     */
+    public function getNext(): ?TokenInterface
+    {
+        $this->currentIndex++;
+        return $this->tokens[$this->currentIndex] ?? null;
+    }
+
+    public function peekNext(): ?TokenInterface
+    {
+        return $this->tokens[$this->currentIndex + 1] ?? null;
+    }
+
+    public function getAll(): array
+    {
+        return $this->tokens;
+    }
+
+    public function setAll(array $tokens): self
+    {
+        $this->tokens = $tokens;
+        return $this;
+    }
+}

Classes/TypoScript/Tokenizer/Token/ConstantAwareTokenStream.php

@@ -0,0 +1,106 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Token;
+
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+/**
+ * A list of single T_VALUE, T_NEWLINE and T_CONSTANT tokens. This is only created for
+ * LineIdentifierAssignment lines if there is at least one T_CONSTANT token
+ * in the assignment that needs to be evaluated when string'ified by the
+ * AST-builder.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class ConstantAwareTokenStream extends AbstractTokenStream
+{
+    private ?array $flatConstants = null;
+
+    /**
+     * Set by the AstBuilder to resolve constant values. Never cached.
+     */
+    public function setFlatConstants(array $flatConstants): void
+    {
+        $this->flatConstants = $flatConstants;
+    }
+
+    /**
+     * Create a source string from given tokens.
+     * This resolves T_CONSTANT tokens to their value if they exist in $this->flatConstants.
+     */
+    public function __toString(): string
+    {
+        $source = '';
+        $this->reset();
+        while ($token = $this->getNext()) {
+            if ($token->getType() === TokenType::T_CONSTANT) {
+                $token = $this->getConstantValue($this->parseConstantExpression($token->getValue())) ?? $token;
+            }
+            $source .= $token;
+        }
+        $this->reset();
+        return $source;
+    }
+
+    private function getConstantValue(?array $constantNames): ?string
+    {
+        if ($this->flatConstants === null || $constantNames === null) {
+            return null;
+        }
+        foreach ($constantNames as $constantName) {
+            $value = $this->flatConstants[$constantName] ?? null;
+            if ($value !== null) {
+                return (string)$value;
+            }
+        }
+        return null;
+    }
+
+    /**
+     * Parse constant expression, including null coalescing operator into an
+     * array of constant names to look up in order.
+     *
+     * @todo: The tokenization of this constant expression should ideally be moved
+     *        into the TypoScript Tokenizer in order to produce a list of multiple tokens
+     *        instead of just a T_CONSTANT for the entire body.
+     *        This would allow early static syntax analysis of the construct and maybe
+     *        detection of invalid and fallback to T_CONSTANT_INVALID that is treated
+     *        like T_VALUE and can be detected. Maybe something like this:
+     *          TokenType::T_CONSTANT_START "{"
+     *          TokenType::T_CONSTANT_END "}"
+     *          TokenType::T_CONSTANT_NAME "$foo.bar"
+     *          TokenType::T_CONSTANT_OPERATOR_NULL_COALESCE " ?? "
+     *          TokenType::T_CONSTANT_INVALID "{$foo ?? bar}" (missing $ before bar)
+     */
+    private function parseConstantExpression(string $constantExpression): ?array
+    {
+        $innerExpression = ltrim(rtrim($constantExpression, '}'), '{');
+        $tokenValues = GeneralUtility::trimExplode(' ?? ', $innerExpression, true);
+        if ($tokenValues === []) {
+            return null;
+        }
+        $tokenValueNames = [];
+        foreach ($tokenValues as $tokenValue) {
+            if (!str_starts_with($tokenValue, '$')) {
+                return null;
+            }
+            $tokenValueNames[] = substr($tokenValue, 1);
+        }
+        return $tokenValueNames;
+    }
+}

Classes/TypoScript/Tokenizer/Token/IdentifierToken.php

@@ -0,0 +1,39 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Token;
+
+/**
+ * A special token if this token is a T_IDENTIFIER token:
+ * With a line like "foo = bar", "foo" is created as TokenIdentifier TokenInterface
+ * (as opposed to Token) having a TokenType::T_IDENTIFIER token.
+ * The only difference to all other tokens is that TokenIdentifier tokens
+ * quote any "." (dots) in their value with a backslash when output. This is
+ * mostly used in backend when rendering source of TokenLine's.
+ *
+ * Note we do *not* explicitly check if TokenType::T_IDENTIFIER is given in
+ * __construct() at the moment for performance reasons and inheritance considerations.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class IdentifierToken extends AbstractToken
+{
+    public function __toString(): string
+    {
+        return str_replace('.', '\.', $this->value);
+    }
+}

Classes/TypoScript/Tokenizer/Token/IdentifierTokenStream.php

@@ -0,0 +1,106 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Token;
+
+/**
+ * A list of single identifier (!) tokens: TokenType::T_IDENTIFIER, and only of those.
+ *
+ * This is used in TS lines that know certain parts have to be lists of identifier tokens only.
+ * For instance a LineIdentifierAssignment "foo.bar = barValue" return this stream for getIdentifierTokenStream():
+ * The left side of an assignment line is a list of identifier tokens.
+ *
+ * Identifiers can be "relative" on the right side for "<" (LineIdentifierCopy) and "=<" (LineIdentifierReference).
+ * Examples are "foo.bar < .baz" and "foo.bar =< .baz". These are identified by having a "." (dot) at the beginning
+ * on the right side. For these places, the toggle "relative" is set to true for the AST-builder to look for relative
+ * copy and copy-reference. The generic example are "relative" references in TS menus: 'RO < .NO'
+ *
+ * For example, with "foo.bar < baz", the Tokenizer creates a LineIdentifierCopy line, having a TokenStreamIdentifier
+ * list of the T_IDENTIFIER tokens for 'foo' and 'bar' for getIdentifierTokenStream(), plus a TokenStreamIdentifier list
+ * of T_IDENTIFIER tokens for 'baz' for getValueTokenStream().
+ *
+ * Note identifier streams on the left side (foo.bar = ...) are never relative, this toggle is true for "<" and "=<" only.
+ *
+ * Lines that know they can only return TokenStreamIdentifier's - they are more specific than just TokenStream, are
+ * type-hinted as such. For instance getIdentifierTokenStream() type hints TokenStreamIdentifier.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class IdentifierTokenStream extends AbstractTokenStream
+{
+    private bool $relative = false;
+
+    /**
+     * When rendering a source string from multiple identifiers, dots between single identifiers need to be added again.
+     * This is used in RootNode->toArray() to create that insane '< lib.whatever' as value when using the
+     * reference operator: "foo =< lib.whatever". See ContentObjectRenderer cObjGetSingle() and mergeTSRef().
+     */
+    public function __toString(): string
+    {
+        $source = [];
+        $this->reset();
+        while ($token = $this->getNext()) {
+            $source[] = (string)$token;
+        }
+        $source = implode('.', $source);
+        if ($this->relative) {
+            $source = '.' . $source;
+        }
+        return $source;
+    }
+
+    protected function serialize(): array
+    {
+        $result = parent::serialize();
+        if ($this->isRelative()) {
+            $result['relative'] = true;
+        }
+        return $result;
+    }
+
+    /**
+     * Append a token to the stream.
+     */
+    public function append(TokenInterface $token): self
+    {
+        if ($token->getType() !== TokenType::T_IDENTIFIER) {
+            throw new \LogicException(
+                'Trying to add a token of type TokenType::' . $token->getType()->name . ' to class TokenStreamIdentifier, but only TokenType::T_IDENTIFIERS are allowed.',
+                1655138907
+            );
+        }
+        $this->tokens[] = $token;
+        return $this;
+    }
+
+    /**
+     * This identifier token stream is relative! There is a dot on the right side of something like "foo.bar < .baz"
+     */
+    public function setRelative(): self
+    {
+        $this->relative = true;
+        return $this;
+    }
+
+    /**
+     * True if this identifier stream is relative to given context.
+     */
+    public function isRelative(): bool
+    {
+        return $this->relative;
+    }
+}

Classes/TypoScript/Tokenizer/Token/Token.php

@@ -0,0 +1,29 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Token;
+
+/**
+ * A casual token created from TypoScript source:
+ * When having a TypoScript line like "# a comment", then a LineComment
+ * is created having a token "T_COMMENT_ONELINE_HASH" and value "# a comment" as
+ * assigned TokenStream.
+ * See TokenType for on overview on which TokenTypes can exist.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class Token extends AbstractToken {}

Classes/TypoScript/Tokenizer/Token/TokenInterface.php

@@ -0,0 +1,68 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Token;
+
+/**
+ * A readonly token: Each line of TypoScript is split into a list of lines consisting of
+ * tokens by the tokenizers.
+ *
+ * As example, a "foo.bar = baz" line creates a LineIdentifierAssignment line, having
+ * TokenType::T_IDENTIFIER 'foo', plus TokenType::T_IDENTIFIER 'bar' as TokenStream for
+ * LineIdentifierAssignment->getIdentifierTokenStream(), plus a TokenType::T_VALUE 'baz'
+ * as LineIdentifierAssignment->getValueTokenStream().
+ *
+ * We have two different Token implementations: The casual "Token" class for everything, plus
+ * the "TokenIdentifier" class for identifier tokens. Identifier tokens are those "left" of
+ * for instance an assignment like "foo.bar = baz" ("foo" and "bar" are TokenIdentifier instances),
+ * and also on the right side when using expression with "<" and "=<" operator: Example "foo.bar < baz":
+ * "baz" is an instance of a TokenIdentifier ("foo" and "bar" as well).
+ *
+ * The reason to have two implementations is that TokenIdentifier needs to be handled slightly
+ * different when cast to string: For identifiers, all "." (dots) within a single identifier token
+ * need to be quoted with "\" (backslash), to not confuse the parser. The classic use-case is having dots in
+ * FlexForm identifiers for PageTS:
+ * "foo.bar\.baz.foobar = value" - three identifier tokens (not four!): "foo", "bar.baz" and "foobar".
+ * So the difference between "TokenIdentifier" and "Token" is just that "TokenIdentifier" quotes dots
+ * in its value when string'ified, while Token does not and __toString() on Token simply says ->getValue().
+ *
+ * Multiple tokens are encapsulated in TokenStreamInterface. TokenStreamInterface has a __toString()
+ * method as well, which calls __toString() on all assigned tokens. This way, a TokenIdentifier will
+ * do its quoting magic, and casual Token instances return their value.
+ *
+ * The idea is here that TokenStreams are cast to string quite often. For instance, an assignment line
+ * like "foo = bar" creates a token stream of one token for the right side (things after "="):
+ * A T_VALUE Token instance with value "bar". The AST builder then at some point needs to resolve this
+ * TokenStream to string. This will directly call __toString on token "bar", and does not deal with quoting,
+ * since its no TokenIdentifier and just a Token.
+ *
+ * Note on getLine() and getColumn(): These two represent the position of a token in the source file:
+ * We start counting at 0 (zero): The first token on the first line is line 0, column 0.
+ * Only the LosslessTokenizer sets these, it's too expensive and of no relevance for the LossyTokenizer
+ * that is used for instance in FE TS tokenizing. That's why these two properties are optional
+ * and 0 (zero) by default.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+interface TokenInterface
+{
+    public function __toString(): string;
+    public function getType(): TokenType;
+    public function getValue(): string;
+    public function getLine(): int;
+    public function getColumn(): int;
+}

Classes/TypoScript/Tokenizer/Token/TokenStream.php

@@ -0,0 +1,26 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Token;
+
+/**
+ * A list of single tokens. These are typically used in TokenLines: A TypoScript
+ * line consists of one or more streams of tokens, depending on the line type.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+final class TokenStream extends AbstractTokenStream {}

Classes/TypoScript/Tokenizer/Token/TokenStreamInterface.php

@@ -0,0 +1,77 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Token;
+
+/**
+ * A generic stream of tokens used in single LineInterface lines.
+ *
+ * The tokenizers create these streams for various lists of tokens, the generic
+ * implementation is class TokenStream. For lists of identifier tokens the special
+ * class TokenStreamIdentifier is created.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+interface TokenStreamInterface
+{
+    /**
+     * Create a source string from given tokens.
+     */
+    public function __toString(): string;
+
+    /**
+     * Stream creation.
+     */
+    public function append(TokenInterface $token): self;
+
+    /**
+     * We sometimes create a stream but don't add tokens.
+     * This method returns true if tokens have been added.
+     */
+    public function isEmpty(): bool;
+
+    /**
+     * Reset current pointer. Typically, call this before iterating with getNext().
+     */
+    public function reset(): self;
+
+    /**
+     * Get next token and raise pointer.
+     */
+    public function getNext(): ?TokenInterface;
+
+    /**
+     * Get next token but do not raise pointer.
+     */
+    public function peekNext(): ?TokenInterface;
+
+    /**
+     * Only used internally when one Stream is transferred to another,
+     * in particular when a TokenStream is turned into TokenStreamConstantAware.
+     *
+     * @return TokenInterface[]
+     */
+    public function getAll(): array;
+
+    /**
+     * Only used internally when one Stream is transferred to another,
+     * in particular when a TokenStream is turned into TokenStreamConstantAware.
+     *
+     * @param TokenInterface[] $tokens
+     */
+    public function setAll(array $tokens): self;
+}

Classes/TypoScript/Tokenizer/Token/TokenType.php

@@ -0,0 +1,69 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Token;
+
+/**
+ * Each TokenInterface instance is a type of this Enum.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+enum TokenType: int
+{
+    case T_NONE = 0; // tokenizer internal handling
+
+    case T_IDENTIFIER = 100; // single word left of an operator. 'foo.bar' are two identifiers
+    case T_VALUE = 200; // right side of an assignment, does not contain line breaks, also used as 'comment' body
+
+    case T_OPERATOR_ASSIGNMENT = 300; // '='
+    case T_OPERATOR_REFERENCE = 301; // '=<'
+    case T_OPERATOR_COPY = 302; // '<'
+    case T_OPERATOR_UNSET = 303; // '>'
+    case T_OPERATOR_FUNCTION = 304; // ':='
+    case T_OPERATOR_ASSIGNMENT_MULTILINE_START = 310; // '('
+    case T_OPERATOR_ASSIGNMENT_MULTILINE_STOP = 311; // ')'
+
+    case T_BLOCK_START = 400; // '{'
+    case T_BLOCK_STOP = 401; // '}'
+
+    case T_DOT = 500; // '.' identifier separator
+
+    case T_BLANK = 600; // list of ' ' and "\t"
+
+    case T_NEWLINE = 700; // "\n" or "\r\n"
+
+    case T_COMMENT_ONELINE_HASH = 800; // '#...'
+    case T_COMMENT_ONELINE_DOUBLESLASH = 801; // '//'
+    case T_COMMENT_MULTILINE_START = 802; // '/*'
+    case T_COMMENT_MULTILINE_STOP = 803; // '*/'
+
+    case T_FUNCTION_NAME = 900; // 'addToList' and others
+    case T_FUNCTION_VALUE_START = 901; // '(' after T_FUNCTION_NAME
+    case T_FUNCTION_VALUE_STOP = 902; // ')' after T_FUNCTION_NAME
+
+    case T_CONDITION_START = 1000; // '[' at start of line
+    case T_CONDITION_STOP = 1001; // ']' after '[' in same line, body is a T_VALUE
+    case T_CONDITION_ELSE = 1002; // 'ELSE' surrounded by '[' and ']'
+    case T_CONDITION_END = 1003; // 'END' surrounded by '[' and ']'
+    case T_CONDITION_GLOBAL = 1004; // 'GLOBAL' surrounded by '[' and ']'
+
+    case T_CONSTANT = 1100; // '{$...}'
+
+    case T_IMPORT_KEYWORD = 1200; // '@import'
+    case T_IMPORT_START = 1201; // ''' (tick) or '"' (doubletick) after @import
+    case T_IMPORT_STOP = 1202; // ''' (tick) or '"' (doubletick) after T_IMPORT_START
+}

Classes/TypoScript/Tokenizer/TokenizerInterface.php

@@ -0,0 +1,42 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\Core\TypoScript\Tokenizer;
+
+use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\LineStream;
+
+/**
+ * A lossless tokenizer for TypoScript syntax.
+ *
+ * tokenize() creates a stream of LineInterface objects from a TypoScript string, each line
+ * contains the important streams or tokens of a single line.
+ *
+ * There are two tokenizer implementations:
+ * - LossyTokenizer: This one skip all invalid lines and comments and everything that is
+ *                   not needed for AST building.
+ * - LosslessTokenizer: This one creates a stream of lines useful for backend template module
+ *                      to elaborate on details and failures in TypoScript.
+ *
+ * The tokenizer *does not* parse conditions or includes itself (no file / db lookups),
+ * this is part of the IncludeTree parser.
+ *
+ * @internal: Internal tokenizer structure.
+ */
+interface TokenizerInterface
+{
+    public function tokenize(string $source): LineStream;
+}