<?php

declare(strict_types=1);

/*
 * This file is part of the TYPO3 CMS project.
 *
 * It is free software; you can redistribute it and/or modify it under
 * the terms of the GNU General Public License, either version 2
 * of the License, or any later version.
 *
 * For the full copyright and license information, please read the
 * LICENSE.txt file that was distributed with this source code.
 *
 * The TYPO3 project - inspiring people to share!
 */

namespace TYPO3\CMS\Core\TypoScript\Tokenizer;

use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\BlockCloseLine;
use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\CommentLine;
use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ConditionElseLine;
use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ConditionLine;
use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ConditionStopLine;
use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\EmptyLine;
use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierAssignmentLine;
use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierBlockOpenLine;
use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierCopyLine;
use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierFunctionLine;
use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierReferenceLine;
use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierUnsetLine;
use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ImportLine;
use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\InvalidLine;
use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\LineStream;
use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\ConstantAwareTokenStream;
use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\IdentifierToken;
use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\IdentifierTokenStream;
use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\Token;
use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenStream;
use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenStreamInterface;
use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenType;

/**
 * A lossless tokenizer for TypoScript syntax.
 *
 * tokenize() creates a flat stream of tokens from a TypoScript string. It is lossless
 * and never "looses" characters to allow syntax linting and creating linter-fixed source
 * strings: tokenize() to create a TokenStream and using string cast (__toString()) on
 * that stream creates *the same* source string again.
 *
 * The tokenizer *does not* parse conditions or includes itself (no file / db lookups),
 * this is part of the IncludeTree parser.
 *
 * This class is unit test covered by TokenizerInterfaceTest and paired with LossyTokenizer.
 * Never change anything in this class without additional test coverage!
 *
 * @internal: Internal tokenizer structure.
 */
final class LosslessTokenizer implements TokenizerInterface
{
    private LineStream $lineStream;

    private TokenStreamInterface $tokenStream;
    private IdentifierTokenStream $identifierStream;
    private TokenStreamInterface $valueStream;

    private array $lines;
    private int $currentLineNumber;
    private string $currentLineString;
    private \closure $currentLinebreakCallback;
    private int $currentColumnInLine = 0;

    public function tokenize(string $source): LineStream
    {
        $this->lineStream = new LineStream();
        $this->currentLineNumber = -1;
        $this->lines = $this->splitLines($source);

        while (true) {
            $this->tokenStream = new TokenStream();
            $this->currentLineNumber++;
            if (!array_key_exists($this->currentLineNumber, $this->lines)) {
                break;
            }
            $this->currentColumnInLine = 0;
            $this->currentLineString = $this->lines[$this->currentLineNumber]['line'];
            $this->currentLinebreakCallback = $this->lines[$this->currentLineNumber]['linebreakCallback'];
            $this->parseTabsAndWhitespaces();
            $nextChar = substr($this->currentLineString, 0, 1);
            if ($nextChar === '') {
                ($this->currentLinebreakCallback)();
                if (!$this->tokenStream->isEmpty()) {
                    $this->createEmptyLine();
                }
                continue;
            }
            $nextTwoChars = substr($this->currentLineString, 0, 2);
            if ($nextChar === '#') {
                $this->createHashCommentLine();
            } elseif ($nextTwoChars === '//') {
                $this->createDoubleSlashCommentLine();
            } elseif ($nextTwoChars === '/*') {
                $this->createMultilineCommentLine();
            } elseif ($nextChar === '[') {
                $this->createConditionLine();
            } elseif ($nextChar === '}') {
                $this->createBlockStopLine();
            } elseif (str_starts_with($this->currentLineString, '@import')) {
                $this->parseImportLine();
            } elseif (str_starts_with($this->currentLineString, '<INCLUDE_TYPOSCRIPT:')) {
                // @todo: Could be relocated elsewhere. This is just to make sure this
                //        old language construct is detected as InvalidLine.
                $this->tokenStream->append(new Token(TokenType::T_VALUE, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine));
                ($this->currentLinebreakCallback)();
                $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
            } else {
                $this->parseIdentifier();
            }
        }

        return $this->lineStream;
    }

    private function splitLines($source): array
    {
        $vanillaLines = explode(chr(10), $source);
        $lines = array_map(
            fn(int $lineNumber, string $vanillaLine): array => [
                'line' => rtrim($vanillaLine, "\r"),
                'linebreakCallback' => str_ends_with($vanillaLine, "\r")
                    ? fn() => $this->tokenStream->append(new Token(TokenType::T_NEWLINE, "\r\n", $lineNumber, mb_strlen($vanillaLine) - 1))
                    : fn() => $this->tokenStream->append(new Token(TokenType::T_NEWLINE, "\n", $lineNumber, mb_strlen($vanillaLine))),
            ],
            array_keys($vanillaLines),
            $vanillaLines
        );
        // Set the linebreak callback of last line to empty to suppress dangling linebreak tokens
        $lines[count($vanillaLines) - 1]['linebreakCallback'] = function () {};
        return $lines;
    }

    private function createEmptyLine(): void
    {
        $this->lineStream->append((new EmptyLine())->setTokenStream($this->tokenStream));
    }

    /**
     * Add tabs and whitespaces until some different char appears.
     */
    private function parseTabsAndWhitespaces(): void
    {
        $matches = [];
        if (preg_match('#^(\s+)(.*)$#', $this->currentLineString, $matches)) {
            $this->tokenStream->append(new Token(TokenType::T_BLANK, $matches[1], $this->currentLineNumber, $this->currentColumnInLine));
            $this->currentLineString = $matches[2];
            $this->currentColumnInLine = $this->currentColumnInLine + strlen($matches[1]);
        }
    }

    private function makeComment(): void
    {
        $nextChar = substr($this->currentLineString, 0, 1);
        if ($nextChar === '') {
            ($this->currentLinebreakCallback)();
            return;
        }
        $nextTwoChars = substr($this->currentLineString, 0, 2);
        if ($nextChar === '#') {
            $this->parseHashComment();
        } elseif ($nextTwoChars === '//') {
            $this->parseDoubleSlashComment();
        } elseif ($nextTwoChars === '/*') {
            $this->parseMultilineComment();
        } else {
            $this->parseHashComment();
        }
    }

    private function createHashCommentLine(): void
    {
        $this->parseHashComment();
        $this->lineStream->append((new CommentLine())->setTokenStream($this->tokenStream));
    }

    private function parseHashComment(): void
    {
        $this->tokenStream->append(new Token(TokenType::T_COMMENT_ONELINE_HASH, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine));
        ($this->currentLinebreakCallback)();
    }

    private function createDoubleSlashCommentLine(): void
    {
        $this->parseDoubleSlashComment();
        $this->lineStream->append((new CommentLine())->setTokenStream($this->tokenStream));
    }

    private function parseDoubleSlashComment(): void
    {
        $this->tokenStream->append(new Token(TokenType::T_COMMENT_ONELINE_DOUBLESLASH, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine));
        ($this->currentLinebreakCallback)();
    }

    private function createMultilineCommentLine(): void
    {
        $this->parseMultilineComment();
        $this->lineStream->append((new CommentLine())->setTokenStream($this->tokenStream));
    }

    private function parseMultilineComment(): void
    {
        $this->tokenStream->append(new Token(TokenType::T_COMMENT_MULTILINE_START, '/*', $this->currentLineNumber, $this->currentColumnInLine));
        $this->currentColumnInLine += 2;
        $this->currentLineString = substr($this->currentLineString, 2);
        while (true) {
            if (str_ends_with($this->currentLineString, '*/')) {
                if (strlen($this->currentLineString) > 2) {
                    $this->tokenStream->append(new Token(TokenType::T_VALUE, substr($this->currentLineString, 0, -2), $this->currentLineNumber, $this->currentColumnInLine));
                }
                $this->tokenStream->append(new Token(TokenType::T_COMMENT_MULTILINE_STOP, '*/', $this->currentLineNumber, $this->currentColumnInLine + mb_strlen($this->currentLineString) - 2));
                ($this->currentLinebreakCallback)();
                return;
            }
            if (strlen($this->currentLineString)) {
                $this->tokenStream->append(new Token(TokenType::T_VALUE, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine));
            }
            ($this->currentLinebreakCallback)();
            if (!array_key_exists($this->currentLineNumber + 1, $this->lines)) {
                return;
            }
            $this->currentLineNumber++;
            $this->currentColumnInLine = 0;
            $this->currentLineString = $this->lines[$this->currentLineNumber]['line'];
            $this->currentLinebreakCallback = $this->lines[$this->currentLineNumber]['linebreakCallback'];
        }
    }

    /**
     * Create a condition line from token stream of this line.
     */
    private function createConditionLine(): void
    {
        $upperCaseLine = strtoupper($this->currentLineString);
        $this->tokenStream->append(new Token(TokenType::T_CONDITION_START, '[', $this->currentLineNumber, $this->currentColumnInLine));
        if (str_starts_with($upperCaseLine, '[ELSE]')) {
            $this->tokenStream->append(new Token(TokenType::T_CONDITION_ELSE, substr($this->currentLineString, 1, 4), $this->currentLineNumber, $this->currentColumnInLine + 1));
            $this->tokenStream->append(new Token(TokenType::T_CONDITION_STOP, ']', $this->currentLineNumber, $this->currentColumnInLine + 5));
            $this->currentLineString = substr($this->currentLineString, 6);
            $this->currentColumnInLine += 6;
            $this->parseTabsAndWhitespaces();
            $this->makeComment();
            $this->lineStream->append((new ConditionElseLine())->setTokenStream($this->tokenStream));
            return;
        }
        if (str_starts_with($upperCaseLine, '[END]')) {
            $this->tokenStream->append(new Token(TokenType::T_CONDITION_END, substr($this->currentLineString, 1, 3), $this->currentLineNumber, $this->currentColumnInLine + 1));
            $this->tokenStream->append(new Token(TokenType::T_CONDITION_STOP, ']', $this->currentLineNumber, $this->currentColumnInLine + 4));
            $this->currentLineString = substr($this->currentLineString, 5);
            $this->currentColumnInLine += 5;
            $this->parseTabsAndWhitespaces();
            $this->makeComment();
            $this->lineStream->append((new ConditionStopLine())->setTokenStream($this->tokenStream));
            return;
        }
        if (str_starts_with($upperCaseLine, '[GLOBAL]')) {
            $this->tokenStream->append(new Token(TokenType::T_CONDITION_GLOBAL, substr($this->currentLineString, 1, 6), $this->currentLineNumber, $this->currentColumnInLine + 1));
            $this->tokenStream->append(new Token(TokenType::T_CONDITION_STOP, ']', $this->currentLineNumber, $this->currentColumnInLine + 7));
            $this->currentLineString = substr($this->currentLineString, 8);
            $this->currentColumnInLine += 8;
            $this->parseTabsAndWhitespaces();
            $this->makeComment();
            $this->lineStream->append((new ConditionStopLine())->setTokenStream($this->tokenStream));
            return;
        }
        $conditionBody = '';
        $conditionBodyStartPosition = $this->currentColumnInLine + 1;
        $conditionBodyCharCount = 0;
        $conditionBodyChars = mb_str_split(substr($this->currentLineString, 1), 1, 'UTF-8');
        $bracketCount = 1;
        while (true) {
            $nextChar = $conditionBodyChars[$conditionBodyCharCount] ?? null;
            if ($nextChar === null) {
                // end of chars
                if ($conditionBodyCharCount) {
                    $this->tokenStream->append(new Token(TokenType::T_VALUE, $conditionBody, $this->currentLineNumber, $conditionBodyStartPosition));
                }
                ($this->currentLinebreakCallback)();
                $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
                return;
            }
            if ($nextChar === '[') {
                $bracketCount++;
                $conditionBody .= $nextChar;
                $conditionBodyCharCount++;
                continue;
            }
            if ($nextChar === ']') {
                $bracketCount--;
                if ($bracketCount === 0) {
                    if ($conditionBodyCharCount) {
                        $conditionBodyToken = new Token(TokenType::T_VALUE, $conditionBody, $this->currentLineNumber, $conditionBodyStartPosition);
                        $this->tokenStream->append($conditionBodyToken);
                        $this->tokenStream->append(new Token(TokenType::T_CONDITION_STOP, ']', $this->currentLineNumber, $this->currentColumnInLine + $conditionBodyCharCount + 1));
                        $this->currentLineString = mb_substr($this->currentLineString, $conditionBodyCharCount + 2);
                        $this->currentColumnInLine = $this->currentColumnInLine + $conditionBodyCharCount + 2;
                        $this->parseTabsAndWhitespaces();
                        $this->makeComment();
                        $this->lineStream->append((new ConditionLine())->setTokenStream($this->tokenStream)->setValueToken($conditionBodyToken));
                        return;
                    }
                    $this->tokenStream->append(new Token(TokenType::T_CONDITION_STOP, ']', $this->currentLineNumber, $this->currentColumnInLine + $conditionBodyCharCount + 1));
                    $this->currentLineString = mb_substr($this->currentLineString, $conditionBodyCharCount + 2);
                    $this->currentColumnInLine = $this->currentColumnInLine + $conditionBodyCharCount + 2;
                    $this->parseTabsAndWhitespaces();
                    $this->makeComment();
                    $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
                    return;
                }
                $conditionBody .= $nextChar;
                $conditionBodyCharCount++;
                continue;
            }
            $conditionBody .= $nextChar;
            $conditionBodyCharCount++;
        }
    }

    private function createBlockStopLine(): void
    {
        $this->tokenStream->append(new Token(TokenType::T_BLOCK_STOP, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine));
        $this->currentColumnInLine++;
        $this->currentLineString = substr($this->currentLineString, 1);
        $this->makeComment();
        $this->lineStream->append((new BlockCloseLine())->setTokenStream($this->tokenStream));
    }

    private function parseBlockStart(): void
    {
        $this->tokenStream->append(new Token(TokenType::T_BLOCK_START, '{', $this->currentLineNumber, $this->currentColumnInLine));
        $this->currentColumnInLine++;
        $this->currentLineString = substr($this->currentLineString, 1);
        $this->parseTabsAndWhitespaces();
        if (str_starts_with($this->currentLineString, '}')) {
            // Edge case: foo = { } in one line. Note content within {} is not parsed, everything behind { ends up as comment.
            $this->lineStream->append((new IdentifierBlockOpenLine())->setIdentifierTokenStream($this->identifierStream)->setTokenStream($this->tokenStream));
            $this->tokenStream = new TokenStream();
            $this->tokenStream->append(new Token(TokenType::T_BLOCK_STOP, '}', $this->currentLineNumber, $this->currentColumnInLine));
            $this->currentLineString = substr($this->currentLineString, 1);
            $this->currentColumnInLine++;
            $this->makeComment();
            $this->lineStream->append((new BlockCloseLine())->setTokenStream($this->tokenStream));
            return;
        }
        $this->makeComment();
        $this->lineStream->append((new IdentifierBlockOpenLine())->setIdentifierTokenStream($this->identifierStream)->setTokenStream($this->tokenStream));
    }

    private function parseImportLine(): void
    {
        $this->tokenStream->append(new Token(TokenType::T_IMPORT_KEYWORD, '@import', $this->currentLineNumber, $this->currentColumnInLine));
        $this->currentColumnInLine += 7;
        $this->currentLineString = substr($this->currentLineString, 7);
        $this->parseTabsAndWhitespaces();

        // Next char should be the opening tick or doubletick, otherwise we create a comment until end of line
        $nextChar = substr($this->currentLineString, 0, 1);
        if ($nextChar !== '\'' && $nextChar !== '"') {
            $this->makeComment();
            $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
            return;
        }
        $this->tokenStream->append(new Token(TokenType::T_IMPORT_START, $nextChar, $this->currentLineNumber, $this->currentColumnInLine));

        $importBody = '';
        $importBodyStartPosition = $this->currentColumnInLine + 1;
        $importBodyCharCount = 0;
        $importBodyChars = mb_str_split(substr($this->currentLineString, 1), 1, 'UTF-8');
        while (true) {
            $nextChar = $importBodyChars[$importBodyCharCount] ?? null;
            if ($nextChar === null) {
                // end of chars
                if ($importBodyCharCount) {
                    $importBodyToken = (new Token(TokenType::T_VALUE, $importBody, $this->currentLineNumber, $importBodyStartPosition));
                    $this->tokenStream->append($importBodyToken);
                    ($this->currentLinebreakCallback)();
                    $this->lineStream->append((new ImportLine())->setTokenStream($this->tokenStream)->setValueToken($importBodyToken));
                    return;
                }
                ($this->currentLinebreakCallback)();
                $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
                return;
            }
            if ($nextChar === '\'' || $nextChar === '"') {
                if ($importBodyCharCount) {
                    $importBodyToken = new Token(TokenType::T_VALUE, $importBody, $this->currentLineNumber, $importBodyStartPosition);
                    $this->tokenStream->append($importBodyToken);
                    $this->tokenStream->append(new Token(TokenType::T_IMPORT_STOP, $nextChar, $this->currentLineNumber, $this->currentColumnInLine + $importBodyCharCount + 1));
                    $this->currentLineString = mb_substr($this->currentLineString, $importBodyCharCount + 2);
                    $this->currentColumnInLine = $this->currentColumnInLine + $importBodyCharCount + 2;
                    $this->parseTabsAndWhitespaces();
                    $this->makeComment();
                    $this->lineStream->append((new ImportLine())->setTokenStream($this->tokenStream)->setValueToken($importBodyToken));
                    return;
                }
                $this->tokenStream->append(new Token(TokenType::T_IMPORT_STOP, $nextChar, $this->currentLineNumber, $this->currentColumnInLine + $importBodyCharCount + 1));
                $this->currentLineString = mb_substr($this->currentLineString, $importBodyCharCount + 2);
                $this->currentColumnInLine = $this->currentColumnInLine + $importBodyCharCount + 2;
                $this->parseTabsAndWhitespaces();
                $this->makeComment();
                $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
                return;
            }
            $importBody .= $nextChar;
            $importBodyCharCount++;
        }
    }

    private function parseIdentifier(): void
    {
        $splitLine = mb_str_split($this->currentLineString, 1, 'UTF-8');
        $currentPosition = $this->parseIdentifierUntilStopChar($splitLine);
        if (!$currentPosition) {
            $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
            return;
        }
        $this->currentLineString = mb_substr($this->currentLineString, $currentPosition);
        $this->currentColumnInLine = $this->currentColumnInLine + $currentPosition;
        $currentColumnInLineBefore = $this->currentColumnInLine;
        $this->parseTabsAndWhitespaces();
        $currentPosition = $currentPosition + $this->currentColumnInLine - $currentColumnInLineBefore;
        $nextChar = $splitLine[$currentPosition] ?? null;
        $nextTwoChars = $nextChar . ($splitLine[$currentPosition + 1] ?? '');
        if ($nextTwoChars === '=<') {
            $this->parseOperatorReference();
            return;
        }
        if ($nextChar === '=') {
            $this->parseOperatorAssignment();
            return;
        }
        if ($nextChar === '{') {
            $this->parseBlockStart();
            return;
        }
        if ($nextChar === '>') {
            $this->parseOperatorUnset();
            return;
        }
        if ($nextChar === '<') {
            $this->parseOperatorCopy();
            return;
        }
        if ($nextChar === '(') {
            $this->parseOperatorMultilineAssignment();
            return;
        }
        if ($nextTwoChars === ':=') {
            $this->parseOperatorFunction();
            return;
        }
        if ($nextChar === '#') {
            $this->parseHashComment();
            $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
            return;
        }
        if ($nextTwoChars === '//') {
            $this->parseDoubleSlashComment();
            $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
            return;
        }
        if ($nextTwoChars === '/*') {
            $this->parseMultilineComment();
            $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
            return;
        }
        if ($nextChar === null) {
            ($this->currentLinebreakCallback)();
            $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
        }
    }

    private function parseOperatorAssignment(): void
    {
        $this->tokenStream->append(new Token(TokenType::T_OPERATOR_ASSIGNMENT, '=', $this->currentLineNumber, $this->currentColumnInLine));
        $this->currentColumnInLine++;
        $this->currentLineString = substr($this->currentLineString, 1);
        $this->parseTabsAndWhitespaces();
        $this->valueStream = new TokenStream();
        [$this->valueStream, $this->tokenStream] = $this->parseValueForConstants($this->valueStream, $this->tokenStream, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine);
        ($this->currentLinebreakCallback)();
        $this->lineStream->append((new IdentifierAssignmentLine())->setTokenStream($this->tokenStream)->setIdentifierTokenStream($this->identifierStream)->setValueTokenStream($this->valueStream));
    }

    private function parseOperatorMultilineAssignment(): void
    {
        $this->valueStream = new TokenStream();
        $this->tokenStream->append(new Token(TokenType::T_OPERATOR_ASSIGNMENT_MULTILINE_START, '(', $this->currentLineNumber, $this->currentColumnInLine));
        $this->currentColumnInLine++;
        $this->currentLineString = substr($this->currentLineString, 1);
        // True if we're currently in the line with the opening '('
        $isFirstLine = true;
        // True if the first line has a first value token: "foo ( thisIsTheFirstValueToken"
        $valueOnFirstLine = false;
        // True if the line after '(' is parsed
        $isSecondLine = false;
        $previousLineCallback = function () {};
        while (true) {
            if (str_starts_with(ltrim($this->currentLineString), ')')) {
                $this->parseTabsAndWhitespaces();
                $this->tokenStream->append(new Token(TokenType::T_OPERATOR_ASSIGNMENT_MULTILINE_STOP, ')', $this->currentLineNumber, $this->currentColumnInLine));
                $this->currentLineString = substr($this->currentLineString, 1);
                $this->currentColumnInLine++;
                $this->parseTabsAndWhitespaces();
                $this->makeComment();
                if ($this->valueStream->isEmpty()) {
                    $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
                } else {
                    $this->lineStream->append((new IdentifierAssignmentLine())->setIdentifierTokenStream($this->identifierStream)->setValueTokenStream($this->valueStream)->setTokenStream($this->tokenStream));
                }
                return;
            }
            if ($isFirstLine && str_ends_with($this->currentLineString, ')')) {
                // Special case if the ')' is on same line as the opening '('
                $this->currentLineString = substr($this->currentLineString, 0, -1);
                if (strlen($this->currentLineString) > 1) {
                    [$this->valueStream, $this->tokenStream] = $this->parseValueForConstants($this->valueStream, $this->tokenStream, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine);
                    $this->tokenStream->append(new Token(TokenType::T_OPERATOR_ASSIGNMENT_MULTILINE_STOP, ')', $this->currentLineNumber, $this->currentColumnInLine + mb_strlen($this->currentLineString)));
                    // Tricky to swap the streams here, but that's the most effective solution I could come up with for the line endings here.
                    ($this->currentLinebreakCallback)();
                    $tempStream = $this->tokenStream;
                    $this->tokenStream = $this->valueStream;
                    ($this->currentLinebreakCallback)();
                    $this->tokenStream = $tempStream;
                    $this->lineStream->append((new IdentifierAssignmentLine())->setIdentifierTokenStream($this->identifierStream)->setValueTokenStream($this->valueStream)->setTokenStream($this->tokenStream));
                    return;
                }
                $this->tokenStream->append(new Token(TokenType::T_OPERATOR_ASSIGNMENT_MULTILINE_STOP, ')', $this->currentLineNumber, $this->currentColumnInLine + strlen($this->currentLineString)));
                ($this->currentLinebreakCallback)();
                $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
                return;
            }
            if ($isFirstLine && strlen($this->currentLineString)) {
                [$this->valueStream, $this->tokenStream] = $this->parseValueForConstants($this->valueStream, $this->tokenStream, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine);
                $valueOnFirstLine = true;
                $previousLineCallback = $this->currentLinebreakCallback;
            }
            if (($isFirstLine && $valueOnFirstLine)
                || (!$isFirstLine && !$isSecondLine)
            ) {
                $tempStream = $this->tokenStream;
                $this->tokenStream = $this->valueStream;
                $previousLineCallback();
                $this->tokenStream = $tempStream;
            }
            if (!$isFirstLine && strlen($this->currentLineString)) {
                [$this->valueStream, $this->tokenStream] = $this->parseValueForConstants($this->valueStream, $this->tokenStream, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine);
            }
            $previousLineCallback = $this->currentLinebreakCallback;
            ($this->currentLinebreakCallback)();
            if (!array_key_exists($this->currentLineNumber + 1, $this->lines)) {
                $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
                return;
            }
            if ($isFirstLine) {
                $isSecondLine = true;
            } else {
                $isSecondLine = false;
            }
            $isFirstLine = false;
            $valueOnFirstLine = false;
            $this->currentLineNumber++;
            $this->currentColumnInLine = 0;
            $this->currentLineString = $this->lines[$this->currentLineNumber]['line'];
            $this->currentLinebreakCallback = $this->lines[$this->currentLineNumber]['linebreakCallback'];
        }
    }

    private function parseOperatorUnset(): void
    {
        $this->tokenStream->append(new Token(TokenType::T_OPERATOR_UNSET, '>', $this->currentLineNumber, $this->currentColumnInLine));
        $this->currentColumnInLine++;
        $this->currentLineString = substr($this->currentLineString, 1);
        $this->parseTabsAndWhitespaces();
        $this->makeComment();
        $this->lineStream->append((new IdentifierUnsetLine())->setTokenStream($this->tokenStream)->setIdentifierTokenStream($this->identifierStream));
    }

    private function parseOperatorCopy(): void
    {
        $this->tokenStream->append(new Token(TokenType::T_OPERATOR_COPY, '<', $this->currentLineNumber, $this->currentColumnInLine));
        $this->currentColumnInLine++;
        $this->currentLineString = substr($this->currentLineString, 1);
        $this->parseTabsAndWhitespaces();
        $identifierStream = $this->identifierStream;
        $this->parseIdentifierAtEndOfLine();
        $referenceStream = $this->identifierStream;
        if ($referenceStream->isEmpty()) {
            // @todo: ($this->currentLinebreakCallback)(); is missing here?!
            $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
            return;
        }
        $this->lineStream->append(
            (new IdentifierCopyLine())
                ->setIdentifierTokenStream($identifierStream)
                ->setValueTokenStream($referenceStream)
                ->setTokenStream($this->tokenStream)
        );
    }

    private function parseOperatorReference(): void
    {
        $this->tokenStream->append(new Token(TokenType::T_OPERATOR_REFERENCE, '=<', $this->currentLineNumber, $this->currentColumnInLine));
        $this->currentColumnInLine += 2;
        $this->currentLineString = substr($this->currentLineString, 2);
        $this->parseTabsAndWhitespaces();
        $identifierStream = $this->identifierStream;
        $this->parseIdentifierAtEndOfLine();
        $referenceStream = $this->identifierStream;
        if ($referenceStream->isEmpty()) {
            $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
            return;
        }
        $this->lineStream->append(
            (new IdentifierReferenceLine())
                ->setIdentifierTokenStream($identifierStream)
                ->setValueTokenStream($referenceStream)
                ->setTokenStream($this->tokenStream)
        );
    }

    private function parseIdentifierAtEndOfLine(): void
    {
        $this->identifierStream = new IdentifierTokenStream();
        $isRelative = false;
        $splitLine = mb_str_split($this->currentLineString, 1, 'UTF-8');
        $char = $splitLine[0] ?? null;
        if ($char === null) {
            return;
        }
        $nextTwoChars = $char . ($splitLine[1] ?? '');
        if ($char === '.') {
            // A relative right side: foo.bar < .foo (note the dot!). we identifierStream->setRelative() and
            // get rid of the dot for the rest of the processing.
            $isRelative = true;
            $this->tokenStream->append((new Token(TokenType::T_DOT, '.', 0, $this->currentColumnInLine)));
            array_shift($splitLine);
            $this->currentColumnInLine++;
            $this->currentLineString = substr($this->currentLineString, 1);
        }
        if ($char === '#') {
            $this->parseHashComment();
            return;
        }
        if ($nextTwoChars === '//') {
            $this->parseDoubleSlashComment();
            return;
        }
        if ($nextTwoChars === '/*') {
            $this->parseMultilineComment();
            return;
        }
        $currentPosition = $this->parseIdentifierUntilStopChar($splitLine, $isRelative);
        if (!$currentPosition) {
            return;
        }
        $this->currentLineString = mb_substr($this->currentLineString, $currentPosition);
        $this->currentColumnInLine = $this->currentColumnInLine + $currentPosition;
        $this->parseTabsAndWhitespaces();
        $this->makeComment();
    }

    private function parseIdentifierUntilStopChar(array $splitLine, bool $isRelative = false): ?int
    {
        $this->identifierStream = new IdentifierTokenStream();
        if ($isRelative) {
            $this->identifierStream->setRelative();
        }
        $currentPosition = 0;
        $currentIdentifierStartPosition = $this->currentColumnInLine;
        $currentIdentifierBody = '';
        $currentIdentifierCharCount = 0;
        while (true) {
            $nextChar = $splitLine[$currentPosition] ?? null;
            if ($nextChar === null) {
                if ($currentIdentifierCharCount) {
                    $identifierToken = new IdentifierToken(TokenType::T_IDENTIFIER, $currentIdentifierBody, $this->currentLineNumber, $currentIdentifierStartPosition);
                    $this->tokenStream->append($identifierToken);
                    $this->identifierStream->append($identifierToken);
                }
                ($this->currentLinebreakCallback)();
                return null;
            }
            $nextTwoChars = $nextChar . ($splitLine[$currentPosition + 1] ?? null);
            if ($currentPosition > 0
                && ($nextChar === ' ' || $nextChar === "\t" || $nextChar === '=' || $nextChar === '<' || $nextChar === '>' || $nextChar === '{' || $nextTwoChars === ':=' || $nextChar === '(')
            ) {
                if ($currentIdentifierCharCount) {
                    $identifierToken = new IdentifierToken(TokenType::T_IDENTIFIER, $currentIdentifierBody, $this->currentLineNumber, $currentIdentifierStartPosition);
                    $this->tokenStream->append($identifierToken);
                    $this->identifierStream->append($identifierToken);
                }
                break;
            }
            if ($nextTwoChars === '\\.') {
                // A quoted dot is part of *this* identifier
                $currentIdentifierBody .= '.';
                $currentPosition += 2;
                $currentIdentifierCharCount++;
            } elseif ($nextChar === '.') {
                if ($currentIdentifierCharCount) {
                    $identifierToken = new IdentifierToken(TokenType::T_IDENTIFIER, $currentIdentifierBody, $this->currentLineNumber, $currentIdentifierStartPosition);
                    $this->tokenStream->append($identifierToken);
                    $this->identifierStream->append($identifierToken);
                    $currentIdentifierCharCount = 0;
                    $currentIdentifierBody = '';
                }
                $this->tokenStream->append(new Token(TokenType::T_DOT, '.', $this->currentLineNumber, $this->currentColumnInLine + $currentPosition));
                $currentPosition++;
                $currentIdentifierStartPosition = $this->currentColumnInLine + $currentPosition;
            } else {
                $currentIdentifierBody .= $nextChar;
                $currentIdentifierCharCount++;
                $currentPosition++;
            }
        }
        return $currentPosition;
    }

    private function parseOperatorFunction(): void
    {
        $this->tokenStream->append(new Token(TokenType::T_OPERATOR_FUNCTION, ':=', $this->currentLineNumber, $this->currentColumnInLine));
        $this->currentColumnInLine += 2;
        $this->currentLineString = substr($this->currentLineString, 2);
        $this->parseTabsAndWhitespaces();
        if ($this->currentLineString === '') {
            ($this->currentLinebreakCallback)();
            $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
            return;
        }
        $functionName = '';
        $functionNameStartPosition = $this->currentColumnInLine;
        $functionNameCharCount = 0;
        $functionChars = mb_str_split($this->currentLineString, 1, 'UTF-8');
        while (true) {
            $nextChar = $functionChars[$functionNameCharCount] ?? null;
            if ($nextChar === null) {
                // end of chars
                if ($functionNameCharCount) {
                    $this->tokenStream->append(new Token(TokenType::T_FUNCTION_NAME, $functionName, $this->currentLineNumber, $functionNameStartPosition));
                }
                ($this->currentLinebreakCallback)();
                $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
                return;
            }
            if ($nextChar === '(') {
                if ($functionNameCharCount) {
                    $functionNameToken = new Token(TokenType::T_FUNCTION_NAME, $functionName, $this->currentLineNumber, $functionNameStartPosition);
                    $this->tokenStream->append($functionNameToken);
                    $this->tokenStream->append(new Token(TokenType::T_FUNCTION_VALUE_START, '(', $this->currentLineNumber, $this->currentColumnInLine + $functionNameCharCount));
                    $functionNameCharCount++;
                    break;
                }
                $this->makeComment();
                $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
                return;
            }
            $functionName .= $nextChar;
            $functionNameCharCount++;
        }
        $functionBodyStartPosition = $functionNameCharCount;
        $functionBodyPart = '';
        $functionBodyCharCount = 0;
        $functionValueStream = new TokenStream();
        $parenthesesLevel = 0;
        while (true) {
            $nextChar = $functionChars[$functionBodyStartPosition + $functionBodyCharCount] ?? null;
            if ($nextChar === null) {
                if ($functionBodyCharCount) {
                    $this->tokenStream->append(new Token(TokenType::T_VALUE, $functionBodyPart, $this->currentLineNumber, $functionBodyStartPosition));
                }
                ($this->currentLinebreakCallback)();
                $this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
                return;
            }
            if ($nextChar === '(') {
                // In case of a function call like "appendString(something(somethingelse))"
                // we shall only stop processing when the last bracket was evaluated.
                $parenthesesLevel++;
            }
            if ($nextChar === ')') {
                if ($parenthesesLevel > 0) {
                    $parenthesesLevel--;
                    // Continue collecting characters from the (...) argument stream.
                    // Also, ")" will be appended, thus intentionally no "break" occurs.
                } else {
                    if ($functionBodyCharCount) {
                        [$functionValueStream, $this->tokenStream] = $this->parseValueForConstants($functionValueStream, $this->tokenStream, $functionBodyPart, $this->currentLineNumber, $this->currentColumnInLine, $functionBodyStartPosition);
                    }
                    $this->tokenStream->append(new Token(TokenType::T_FUNCTION_VALUE_STOP, ')', $this->currentLineNumber, $this->currentColumnInLine + $functionNameCharCount + $functionBodyCharCount));
                    $functionBodyCharCount++;
                    break;
                }
            }
            $functionBodyPart .= $nextChar;
            $functionBodyCharCount++;
        }
        $this->currentColumnInLine = $this->currentColumnInLine + $functionNameCharCount + $functionBodyCharCount;
        $this->currentLineString = mb_substr($this->currentLineString, $functionNameCharCount + $functionBodyCharCount);
        $this->parseTabsAndWhitespaces();
        $this->makeComment();
        $this->lineStream->append(
            (new IdentifierFunctionLine())
                ->setIdentifierTokenStream($this->identifierStream)
                ->setFunctionNameToken($functionNameToken)
                ->setTokenStream($this->tokenStream)
                ->setFunctionValueTokenStream($functionValueStream)
        );
    }

    /**
     * @return array{0: TokenStreamInterface, 1: TokenStreamInterface}
     */
    private function parseValueForConstants(TokenStreamInterface $valueStream, TokenStreamInterface $tokenStream, string $value, int $line, int $column, int $tokenOffsetPosition = 0): array
    {
        if (!str_contains($value, '{$')) {
            $valueToken = new Token(TokenType::T_VALUE, $value, $line, $column + $tokenOffsetPosition);
            $valueStream->append($valueToken);
            $tokenStream->append($valueToken);
            return [$valueStream, $tokenStream];
        }
        $splitLine = mb_str_split($value, 1, 'UTF-8');
        $isInConstant = false;
        $currentPosition = 0;
        $currentString = '';
        $currentStringLength = 0;
        $lastTokenEndPosition = 0;
        while (true) {
            $char = $splitLine[$currentPosition] ?? null;
            if ($char === null) {
                if ($currentStringLength) {
                    $valueToken = new Token(TokenType::T_VALUE, $currentString, $line, $column + $lastTokenEndPosition + $tokenOffsetPosition);
                    $valueStream->append($valueToken);
                    $tokenStream->append($valueToken);
                }
                break;
            }
            $nextTwoChars = $char . ($splitLine[$currentPosition + 1] ?? '');
            if ($nextTwoChars === '{$') {
                $isInConstant = true;
                if ($currentStringLength) {
                    $valueToken = new Token(TokenType::T_VALUE, $currentString, $line, $column + $lastTokenEndPosition + $tokenOffsetPosition);
                    $valueStream->append($valueToken);
                    $tokenStream->append($valueToken);
                    $lastTokenEndPosition = $currentPosition;
                }
                $currentString = '{$';
                $currentPosition += 2;
                continue;
            }
            if ($isInConstant && $char === '}') {
                $valueToken = new Token(TokenType::T_CONSTANT, $currentString . '}', $line, $column + $lastTokenEndPosition + $tokenOffsetPosition);
                if (!$valueStream instanceof ConstantAwareTokenStream) {
                    $valueStream = (new ConstantAwareTokenStream())->setAll($valueStream->getAll());
                }
                $valueStream->append($valueToken);
                $tokenStream->append($valueToken);
                $currentPosition++;
                $currentString = '';
                $currentStringLength = 0;
                $lastTokenEndPosition = $currentPosition;
                $isInConstant = false;
                continue;
            }
            $currentPosition++;
            $currentStringLength++;
            $currentString .= $char;
        }
        return [$valueStream, $tokenStream];
    }
}