880 lines
44 KiB
PHP
880 lines
44 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
/*
|
|
* This file is part of the TYPO3 CMS project.
|
|
*
|
|
* It is free software; you can redistribute it and/or modify it under
|
|
* the terms of the GNU General Public License, either version 2
|
|
* of the License, or any later version.
|
|
*
|
|
* For the full copyright and license information, please read the
|
|
* LICENSE.txt file that was distributed with this source code.
|
|
*
|
|
* The TYPO3 project - inspiring people to share!
|
|
*/
|
|
|
|
namespace TYPO3\CMS\Core\TypoScript\Tokenizer;
|
|
|
|
use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\BlockCloseLine;
|
|
use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\CommentLine;
|
|
use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ConditionElseLine;
|
|
use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ConditionLine;
|
|
use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ConditionStopLine;
|
|
use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\EmptyLine;
|
|
use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierAssignmentLine;
|
|
use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierBlockOpenLine;
|
|
use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierCopyLine;
|
|
use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierFunctionLine;
|
|
use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierReferenceLine;
|
|
use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\IdentifierUnsetLine;
|
|
use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\ImportLine;
|
|
use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\InvalidLine;
|
|
use TYPO3\CMS\Core\TypoScript\Tokenizer\Line\LineStream;
|
|
use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\ConstantAwareTokenStream;
|
|
use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\IdentifierToken;
|
|
use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\IdentifierTokenStream;
|
|
use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\Token;
|
|
use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenStream;
|
|
use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenStreamInterface;
|
|
use TYPO3\CMS\Core\TypoScript\Tokenizer\Token\TokenType;
|
|
|
|
/**
|
|
* A lossless tokenizer for TypoScript syntax.
|
|
*
|
|
* tokenize() creates a flat stream of tokens from a TypoScript string. It is lossless
|
|
* and never "looses" characters to allow syntax linting and creating linter-fixed source
|
|
* strings: tokenize() to create a TokenStream and using string cast (__toString()) on
|
|
* that stream creates *the same* source string again.
|
|
*
|
|
* The tokenizer *does not* parse conditions or includes itself (no file / db lookups),
|
|
* this is part of the IncludeTree parser.
|
|
*
|
|
* This class is unit test covered by TokenizerInterfaceTest and paired with LossyTokenizer.
|
|
* Never change anything in this class without additional test coverage!
|
|
*
|
|
* @internal: Internal tokenizer structure.
|
|
*/
|
|
final class LosslessTokenizer implements TokenizerInterface
|
|
{
|
|
private LineStream $lineStream;
|
|
|
|
private TokenStreamInterface $tokenStream;
|
|
private IdentifierTokenStream $identifierStream;
|
|
private TokenStreamInterface $valueStream;
|
|
|
|
private array $lines;
|
|
private int $currentLineNumber;
|
|
private string $currentLineString;
|
|
private \closure $currentLinebreakCallback;
|
|
private int $currentColumnInLine = 0;
|
|
|
|
public function tokenize(string $source): LineStream
|
|
{
|
|
$this->lineStream = new LineStream();
|
|
$this->currentLineNumber = -1;
|
|
$this->lines = $this->splitLines($source);
|
|
|
|
while (true) {
|
|
$this->tokenStream = new TokenStream();
|
|
$this->currentLineNumber++;
|
|
if (!array_key_exists($this->currentLineNumber, $this->lines)) {
|
|
break;
|
|
}
|
|
$this->currentColumnInLine = 0;
|
|
$this->currentLineString = $this->lines[$this->currentLineNumber]['line'];
|
|
$this->currentLinebreakCallback = $this->lines[$this->currentLineNumber]['linebreakCallback'];
|
|
$this->parseTabsAndWhitespaces();
|
|
$nextChar = substr($this->currentLineString, 0, 1);
|
|
if ($nextChar === '') {
|
|
($this->currentLinebreakCallback)();
|
|
if (!$this->tokenStream->isEmpty()) {
|
|
$this->createEmptyLine();
|
|
}
|
|
continue;
|
|
}
|
|
$nextTwoChars = substr($this->currentLineString, 0, 2);
|
|
if ($nextChar === '#') {
|
|
$this->createHashCommentLine();
|
|
} elseif ($nextTwoChars === '//') {
|
|
$this->createDoubleSlashCommentLine();
|
|
} elseif ($nextTwoChars === '/*') {
|
|
$this->createMultilineCommentLine();
|
|
} elseif ($nextChar === '[') {
|
|
$this->createConditionLine();
|
|
} elseif ($nextChar === '}') {
|
|
$this->createBlockStopLine();
|
|
} elseif (str_starts_with($this->currentLineString, '@import')) {
|
|
$this->parseImportLine();
|
|
} elseif (str_starts_with($this->currentLineString, '<INCLUDE_TYPOSCRIPT:')) {
|
|
// @todo: Could be relocated elsewhere. This is just to make sure this
|
|
// old language construct is detected as InvalidLine.
|
|
$this->tokenStream->append(new Token(TokenType::T_VALUE, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine));
|
|
($this->currentLinebreakCallback)();
|
|
$this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
|
|
} else {
|
|
$this->parseIdentifier();
|
|
}
|
|
}
|
|
|
|
return $this->lineStream;
|
|
}
|
|
|
|
private function splitLines($source): array
|
|
{
|
|
$vanillaLines = explode(chr(10), $source);
|
|
$lines = array_map(
|
|
fn(int $lineNumber, string $vanillaLine): array => [
|
|
'line' => rtrim($vanillaLine, "\r"),
|
|
'linebreakCallback' => str_ends_with($vanillaLine, "\r")
|
|
? fn() => $this->tokenStream->append(new Token(TokenType::T_NEWLINE, "\r\n", $lineNumber, mb_strlen($vanillaLine) - 1))
|
|
: fn() => $this->tokenStream->append(new Token(TokenType::T_NEWLINE, "\n", $lineNumber, mb_strlen($vanillaLine))),
|
|
],
|
|
array_keys($vanillaLines),
|
|
$vanillaLines
|
|
);
|
|
// Set the linebreak callback of last line to empty to suppress dangling linebreak tokens
|
|
$lines[count($vanillaLines) - 1]['linebreakCallback'] = function () {};
|
|
return $lines;
|
|
}
|
|
|
|
private function createEmptyLine(): void
|
|
{
|
|
$this->lineStream->append((new EmptyLine())->setTokenStream($this->tokenStream));
|
|
}
|
|
|
|
/**
|
|
* Add tabs and whitespaces until some different char appears.
|
|
*/
|
|
private function parseTabsAndWhitespaces(): void
|
|
{
|
|
$matches = [];
|
|
if (preg_match('#^(\s+)(.*)$#', $this->currentLineString, $matches)) {
|
|
$this->tokenStream->append(new Token(TokenType::T_BLANK, $matches[1], $this->currentLineNumber, $this->currentColumnInLine));
|
|
$this->currentLineString = $matches[2];
|
|
$this->currentColumnInLine = $this->currentColumnInLine + strlen($matches[1]);
|
|
}
|
|
}
|
|
|
|
private function makeComment(): void
|
|
{
|
|
$nextChar = substr($this->currentLineString, 0, 1);
|
|
if ($nextChar === '') {
|
|
($this->currentLinebreakCallback)();
|
|
return;
|
|
}
|
|
$nextTwoChars = substr($this->currentLineString, 0, 2);
|
|
if ($nextChar === '#') {
|
|
$this->parseHashComment();
|
|
} elseif ($nextTwoChars === '//') {
|
|
$this->parseDoubleSlashComment();
|
|
} elseif ($nextTwoChars === '/*') {
|
|
$this->parseMultilineComment();
|
|
} else {
|
|
$this->parseHashComment();
|
|
}
|
|
}
|
|
|
|
private function createHashCommentLine(): void
|
|
{
|
|
$this->parseHashComment();
|
|
$this->lineStream->append((new CommentLine())->setTokenStream($this->tokenStream));
|
|
}
|
|
|
|
private function parseHashComment(): void
|
|
{
|
|
$this->tokenStream->append(new Token(TokenType::T_COMMENT_ONELINE_HASH, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine));
|
|
($this->currentLinebreakCallback)();
|
|
}
|
|
|
|
private function createDoubleSlashCommentLine(): void
|
|
{
|
|
$this->parseDoubleSlashComment();
|
|
$this->lineStream->append((new CommentLine())->setTokenStream($this->tokenStream));
|
|
}
|
|
|
|
private function parseDoubleSlashComment(): void
|
|
{
|
|
$this->tokenStream->append(new Token(TokenType::T_COMMENT_ONELINE_DOUBLESLASH, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine));
|
|
($this->currentLinebreakCallback)();
|
|
}
|
|
|
|
private function createMultilineCommentLine(): void
|
|
{
|
|
$this->parseMultilineComment();
|
|
$this->lineStream->append((new CommentLine())->setTokenStream($this->tokenStream));
|
|
}
|
|
|
|
private function parseMultilineComment(): void
|
|
{
|
|
$this->tokenStream->append(new Token(TokenType::T_COMMENT_MULTILINE_START, '/*', $this->currentLineNumber, $this->currentColumnInLine));
|
|
$this->currentColumnInLine += 2;
|
|
$this->currentLineString = substr($this->currentLineString, 2);
|
|
while (true) {
|
|
if (str_ends_with($this->currentLineString, '*/')) {
|
|
if (strlen($this->currentLineString) > 2) {
|
|
$this->tokenStream->append(new Token(TokenType::T_VALUE, substr($this->currentLineString, 0, -2), $this->currentLineNumber, $this->currentColumnInLine));
|
|
}
|
|
$this->tokenStream->append(new Token(TokenType::T_COMMENT_MULTILINE_STOP, '*/', $this->currentLineNumber, $this->currentColumnInLine + mb_strlen($this->currentLineString) - 2));
|
|
($this->currentLinebreakCallback)();
|
|
return;
|
|
}
|
|
if (strlen($this->currentLineString)) {
|
|
$this->tokenStream->append(new Token(TokenType::T_VALUE, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine));
|
|
}
|
|
($this->currentLinebreakCallback)();
|
|
if (!array_key_exists($this->currentLineNumber + 1, $this->lines)) {
|
|
return;
|
|
}
|
|
$this->currentLineNumber++;
|
|
$this->currentColumnInLine = 0;
|
|
$this->currentLineString = $this->lines[$this->currentLineNumber]['line'];
|
|
$this->currentLinebreakCallback = $this->lines[$this->currentLineNumber]['linebreakCallback'];
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Create a condition line from token stream of this line.
|
|
*/
|
|
private function createConditionLine(): void
|
|
{
|
|
$upperCaseLine = strtoupper($this->currentLineString);
|
|
$this->tokenStream->append(new Token(TokenType::T_CONDITION_START, '[', $this->currentLineNumber, $this->currentColumnInLine));
|
|
if (str_starts_with($upperCaseLine, '[ELSE]')) {
|
|
$this->tokenStream->append(new Token(TokenType::T_CONDITION_ELSE, substr($this->currentLineString, 1, 4), $this->currentLineNumber, $this->currentColumnInLine + 1));
|
|
$this->tokenStream->append(new Token(TokenType::T_CONDITION_STOP, ']', $this->currentLineNumber, $this->currentColumnInLine + 5));
|
|
$this->currentLineString = substr($this->currentLineString, 6);
|
|
$this->currentColumnInLine += 6;
|
|
$this->parseTabsAndWhitespaces();
|
|
$this->makeComment();
|
|
$this->lineStream->append((new ConditionElseLine())->setTokenStream($this->tokenStream));
|
|
return;
|
|
}
|
|
if (str_starts_with($upperCaseLine, '[END]')) {
|
|
$this->tokenStream->append(new Token(TokenType::T_CONDITION_END, substr($this->currentLineString, 1, 3), $this->currentLineNumber, $this->currentColumnInLine + 1));
|
|
$this->tokenStream->append(new Token(TokenType::T_CONDITION_STOP, ']', $this->currentLineNumber, $this->currentColumnInLine + 4));
|
|
$this->currentLineString = substr($this->currentLineString, 5);
|
|
$this->currentColumnInLine += 5;
|
|
$this->parseTabsAndWhitespaces();
|
|
$this->makeComment();
|
|
$this->lineStream->append((new ConditionStopLine())->setTokenStream($this->tokenStream));
|
|
return;
|
|
}
|
|
if (str_starts_with($upperCaseLine, '[GLOBAL]')) {
|
|
$this->tokenStream->append(new Token(TokenType::T_CONDITION_GLOBAL, substr($this->currentLineString, 1, 6), $this->currentLineNumber, $this->currentColumnInLine + 1));
|
|
$this->tokenStream->append(new Token(TokenType::T_CONDITION_STOP, ']', $this->currentLineNumber, $this->currentColumnInLine + 7));
|
|
$this->currentLineString = substr($this->currentLineString, 8);
|
|
$this->currentColumnInLine += 8;
|
|
$this->parseTabsAndWhitespaces();
|
|
$this->makeComment();
|
|
$this->lineStream->append((new ConditionStopLine())->setTokenStream($this->tokenStream));
|
|
return;
|
|
}
|
|
$conditionBody = '';
|
|
$conditionBodyStartPosition = $this->currentColumnInLine + 1;
|
|
$conditionBodyCharCount = 0;
|
|
$conditionBodyChars = mb_str_split(substr($this->currentLineString, 1), 1, 'UTF-8');
|
|
$bracketCount = 1;
|
|
while (true) {
|
|
$nextChar = $conditionBodyChars[$conditionBodyCharCount] ?? null;
|
|
if ($nextChar === null) {
|
|
// end of chars
|
|
if ($conditionBodyCharCount) {
|
|
$this->tokenStream->append(new Token(TokenType::T_VALUE, $conditionBody, $this->currentLineNumber, $conditionBodyStartPosition));
|
|
}
|
|
($this->currentLinebreakCallback)();
|
|
$this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
|
|
return;
|
|
}
|
|
if ($nextChar === '[') {
|
|
$bracketCount++;
|
|
$conditionBody .= $nextChar;
|
|
$conditionBodyCharCount++;
|
|
continue;
|
|
}
|
|
if ($nextChar === ']') {
|
|
$bracketCount--;
|
|
if ($bracketCount === 0) {
|
|
if ($conditionBodyCharCount) {
|
|
$conditionBodyToken = new Token(TokenType::T_VALUE, $conditionBody, $this->currentLineNumber, $conditionBodyStartPosition);
|
|
$this->tokenStream->append($conditionBodyToken);
|
|
$this->tokenStream->append(new Token(TokenType::T_CONDITION_STOP, ']', $this->currentLineNumber, $this->currentColumnInLine + $conditionBodyCharCount + 1));
|
|
$this->currentLineString = mb_substr($this->currentLineString, $conditionBodyCharCount + 2);
|
|
$this->currentColumnInLine = $this->currentColumnInLine + $conditionBodyCharCount + 2;
|
|
$this->parseTabsAndWhitespaces();
|
|
$this->makeComment();
|
|
$this->lineStream->append((new ConditionLine())->setTokenStream($this->tokenStream)->setValueToken($conditionBodyToken));
|
|
return;
|
|
}
|
|
$this->tokenStream->append(new Token(TokenType::T_CONDITION_STOP, ']', $this->currentLineNumber, $this->currentColumnInLine + $conditionBodyCharCount + 1));
|
|
$this->currentLineString = mb_substr($this->currentLineString, $conditionBodyCharCount + 2);
|
|
$this->currentColumnInLine = $this->currentColumnInLine + $conditionBodyCharCount + 2;
|
|
$this->parseTabsAndWhitespaces();
|
|
$this->makeComment();
|
|
$this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
|
|
return;
|
|
}
|
|
$conditionBody .= $nextChar;
|
|
$conditionBodyCharCount++;
|
|
continue;
|
|
}
|
|
$conditionBody .= $nextChar;
|
|
$conditionBodyCharCount++;
|
|
}
|
|
}
|
|
|
|
private function createBlockStopLine(): void
|
|
{
|
|
$this->tokenStream->append(new Token(TokenType::T_BLOCK_STOP, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine));
|
|
$this->currentColumnInLine++;
|
|
$this->currentLineString = substr($this->currentLineString, 1);
|
|
$this->makeComment();
|
|
$this->lineStream->append((new BlockCloseLine())->setTokenStream($this->tokenStream));
|
|
}
|
|
|
|
private function parseBlockStart(): void
|
|
{
|
|
$this->tokenStream->append(new Token(TokenType::T_BLOCK_START, '{', $this->currentLineNumber, $this->currentColumnInLine));
|
|
$this->currentColumnInLine++;
|
|
$this->currentLineString = substr($this->currentLineString, 1);
|
|
$this->parseTabsAndWhitespaces();
|
|
if (str_starts_with($this->currentLineString, '}')) {
|
|
// Edge case: foo = { } in one line. Note content within {} is not parsed, everything behind { ends up as comment.
|
|
$this->lineStream->append((new IdentifierBlockOpenLine())->setIdentifierTokenStream($this->identifierStream)->setTokenStream($this->tokenStream));
|
|
$this->tokenStream = new TokenStream();
|
|
$this->tokenStream->append(new Token(TokenType::T_BLOCK_STOP, '}', $this->currentLineNumber, $this->currentColumnInLine));
|
|
$this->currentLineString = substr($this->currentLineString, 1);
|
|
$this->currentColumnInLine++;
|
|
$this->makeComment();
|
|
$this->lineStream->append((new BlockCloseLine())->setTokenStream($this->tokenStream));
|
|
return;
|
|
}
|
|
$this->makeComment();
|
|
$this->lineStream->append((new IdentifierBlockOpenLine())->setIdentifierTokenStream($this->identifierStream)->setTokenStream($this->tokenStream));
|
|
}
|
|
|
|
private function parseImportLine(): void
|
|
{
|
|
$this->tokenStream->append(new Token(TokenType::T_IMPORT_KEYWORD, '@import', $this->currentLineNumber, $this->currentColumnInLine));
|
|
$this->currentColumnInLine += 7;
|
|
$this->currentLineString = substr($this->currentLineString, 7);
|
|
$this->parseTabsAndWhitespaces();
|
|
|
|
// Next char should be the opening tick or doubletick, otherwise we create a comment until end of line
|
|
$nextChar = substr($this->currentLineString, 0, 1);
|
|
if ($nextChar !== '\'' && $nextChar !== '"') {
|
|
$this->makeComment();
|
|
$this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
|
|
return;
|
|
}
|
|
$this->tokenStream->append(new Token(TokenType::T_IMPORT_START, $nextChar, $this->currentLineNumber, $this->currentColumnInLine));
|
|
|
|
$importBody = '';
|
|
$importBodyStartPosition = $this->currentColumnInLine + 1;
|
|
$importBodyCharCount = 0;
|
|
$importBodyChars = mb_str_split(substr($this->currentLineString, 1), 1, 'UTF-8');
|
|
while (true) {
|
|
$nextChar = $importBodyChars[$importBodyCharCount] ?? null;
|
|
if ($nextChar === null) {
|
|
// end of chars
|
|
if ($importBodyCharCount) {
|
|
$importBodyToken = (new Token(TokenType::T_VALUE, $importBody, $this->currentLineNumber, $importBodyStartPosition));
|
|
$this->tokenStream->append($importBodyToken);
|
|
($this->currentLinebreakCallback)();
|
|
$this->lineStream->append((new ImportLine())->setTokenStream($this->tokenStream)->setValueToken($importBodyToken));
|
|
return;
|
|
}
|
|
($this->currentLinebreakCallback)();
|
|
$this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
|
|
return;
|
|
}
|
|
if ($nextChar === '\'' || $nextChar === '"') {
|
|
if ($importBodyCharCount) {
|
|
$importBodyToken = new Token(TokenType::T_VALUE, $importBody, $this->currentLineNumber, $importBodyStartPosition);
|
|
$this->tokenStream->append($importBodyToken);
|
|
$this->tokenStream->append(new Token(TokenType::T_IMPORT_STOP, $nextChar, $this->currentLineNumber, $this->currentColumnInLine + $importBodyCharCount + 1));
|
|
$this->currentLineString = mb_substr($this->currentLineString, $importBodyCharCount + 2);
|
|
$this->currentColumnInLine = $this->currentColumnInLine + $importBodyCharCount + 2;
|
|
$this->parseTabsAndWhitespaces();
|
|
$this->makeComment();
|
|
$this->lineStream->append((new ImportLine())->setTokenStream($this->tokenStream)->setValueToken($importBodyToken));
|
|
return;
|
|
}
|
|
$this->tokenStream->append(new Token(TokenType::T_IMPORT_STOP, $nextChar, $this->currentLineNumber, $this->currentColumnInLine + $importBodyCharCount + 1));
|
|
$this->currentLineString = mb_substr($this->currentLineString, $importBodyCharCount + 2);
|
|
$this->currentColumnInLine = $this->currentColumnInLine + $importBodyCharCount + 2;
|
|
$this->parseTabsAndWhitespaces();
|
|
$this->makeComment();
|
|
$this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
|
|
return;
|
|
}
|
|
$importBody .= $nextChar;
|
|
$importBodyCharCount++;
|
|
}
|
|
}
|
|
|
|
private function parseIdentifier(): void
|
|
{
|
|
$splitLine = mb_str_split($this->currentLineString, 1, 'UTF-8');
|
|
$currentPosition = $this->parseIdentifierUntilStopChar($splitLine);
|
|
if (!$currentPosition) {
|
|
$this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
|
|
return;
|
|
}
|
|
$this->currentLineString = mb_substr($this->currentLineString, $currentPosition);
|
|
$this->currentColumnInLine = $this->currentColumnInLine + $currentPosition;
|
|
$currentColumnInLineBefore = $this->currentColumnInLine;
|
|
$this->parseTabsAndWhitespaces();
|
|
$currentPosition = $currentPosition + $this->currentColumnInLine - $currentColumnInLineBefore;
|
|
$nextChar = $splitLine[$currentPosition] ?? null;
|
|
$nextTwoChars = $nextChar . ($splitLine[$currentPosition + 1] ?? '');
|
|
if ($nextTwoChars === '=<') {
|
|
$this->parseOperatorReference();
|
|
return;
|
|
}
|
|
if ($nextChar === '=') {
|
|
$this->parseOperatorAssignment();
|
|
return;
|
|
}
|
|
if ($nextChar === '{') {
|
|
$this->parseBlockStart();
|
|
return;
|
|
}
|
|
if ($nextChar === '>') {
|
|
$this->parseOperatorUnset();
|
|
return;
|
|
}
|
|
if ($nextChar === '<') {
|
|
$this->parseOperatorCopy();
|
|
return;
|
|
}
|
|
if ($nextChar === '(') {
|
|
$this->parseOperatorMultilineAssignment();
|
|
return;
|
|
}
|
|
if ($nextTwoChars === ':=') {
|
|
$this->parseOperatorFunction();
|
|
return;
|
|
}
|
|
if ($nextChar === '#') {
|
|
$this->parseHashComment();
|
|
$this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
|
|
return;
|
|
}
|
|
if ($nextTwoChars === '//') {
|
|
$this->parseDoubleSlashComment();
|
|
$this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
|
|
return;
|
|
}
|
|
if ($nextTwoChars === '/*') {
|
|
$this->parseMultilineComment();
|
|
$this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
|
|
return;
|
|
}
|
|
if ($nextChar === null) {
|
|
($this->currentLinebreakCallback)();
|
|
$this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
|
|
}
|
|
}
|
|
|
|
private function parseOperatorAssignment(): void
|
|
{
|
|
$this->tokenStream->append(new Token(TokenType::T_OPERATOR_ASSIGNMENT, '=', $this->currentLineNumber, $this->currentColumnInLine));
|
|
$this->currentColumnInLine++;
|
|
$this->currentLineString = substr($this->currentLineString, 1);
|
|
$this->parseTabsAndWhitespaces();
|
|
$this->valueStream = new TokenStream();
|
|
[$this->valueStream, $this->tokenStream] = $this->parseValueForConstants($this->valueStream, $this->tokenStream, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine);
|
|
($this->currentLinebreakCallback)();
|
|
$this->lineStream->append((new IdentifierAssignmentLine())->setTokenStream($this->tokenStream)->setIdentifierTokenStream($this->identifierStream)->setValueTokenStream($this->valueStream));
|
|
}
|
|
|
|
private function parseOperatorMultilineAssignment(): void
|
|
{
|
|
$this->valueStream = new TokenStream();
|
|
$this->tokenStream->append(new Token(TokenType::T_OPERATOR_ASSIGNMENT_MULTILINE_START, '(', $this->currentLineNumber, $this->currentColumnInLine));
|
|
$this->currentColumnInLine++;
|
|
$this->currentLineString = substr($this->currentLineString, 1);
|
|
// True if we're currently in the line with the opening '('
|
|
$isFirstLine = true;
|
|
// True if the first line has a first value token: "foo ( thisIsTheFirstValueToken"
|
|
$valueOnFirstLine = false;
|
|
// True if the line after '(' is parsed
|
|
$isSecondLine = false;
|
|
$previousLineCallback = function () {};
|
|
while (true) {
|
|
if (str_starts_with(ltrim($this->currentLineString), ')')) {
|
|
$this->parseTabsAndWhitespaces();
|
|
$this->tokenStream->append(new Token(TokenType::T_OPERATOR_ASSIGNMENT_MULTILINE_STOP, ')', $this->currentLineNumber, $this->currentColumnInLine));
|
|
$this->currentLineString = substr($this->currentLineString, 1);
|
|
$this->currentColumnInLine++;
|
|
$this->parseTabsAndWhitespaces();
|
|
$this->makeComment();
|
|
if ($this->valueStream->isEmpty()) {
|
|
$this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
|
|
} else {
|
|
$this->lineStream->append((new IdentifierAssignmentLine())->setIdentifierTokenStream($this->identifierStream)->setValueTokenStream($this->valueStream)->setTokenStream($this->tokenStream));
|
|
}
|
|
return;
|
|
}
|
|
if ($isFirstLine && str_ends_with($this->currentLineString, ')')) {
|
|
// Special case if the ')' is on same line as the opening '('
|
|
$this->currentLineString = substr($this->currentLineString, 0, -1);
|
|
if (strlen($this->currentLineString) > 1) {
|
|
[$this->valueStream, $this->tokenStream] = $this->parseValueForConstants($this->valueStream, $this->tokenStream, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine);
|
|
$this->tokenStream->append(new Token(TokenType::T_OPERATOR_ASSIGNMENT_MULTILINE_STOP, ')', $this->currentLineNumber, $this->currentColumnInLine + mb_strlen($this->currentLineString)));
|
|
// Tricky to swap the streams here, but that's the most effective solution I could come up with for the line endings here.
|
|
($this->currentLinebreakCallback)();
|
|
$tempStream = $this->tokenStream;
|
|
$this->tokenStream = $this->valueStream;
|
|
($this->currentLinebreakCallback)();
|
|
$this->tokenStream = $tempStream;
|
|
$this->lineStream->append((new IdentifierAssignmentLine())->setIdentifierTokenStream($this->identifierStream)->setValueTokenStream($this->valueStream)->setTokenStream($this->tokenStream));
|
|
return;
|
|
}
|
|
$this->tokenStream->append(new Token(TokenType::T_OPERATOR_ASSIGNMENT_MULTILINE_STOP, ')', $this->currentLineNumber, $this->currentColumnInLine + strlen($this->currentLineString)));
|
|
($this->currentLinebreakCallback)();
|
|
$this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
|
|
return;
|
|
}
|
|
if ($isFirstLine && strlen($this->currentLineString)) {
|
|
[$this->valueStream, $this->tokenStream] = $this->parseValueForConstants($this->valueStream, $this->tokenStream, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine);
|
|
$valueOnFirstLine = true;
|
|
$previousLineCallback = $this->currentLinebreakCallback;
|
|
}
|
|
if (($isFirstLine && $valueOnFirstLine)
|
|
|| (!$isFirstLine && !$isSecondLine)
|
|
) {
|
|
$tempStream = $this->tokenStream;
|
|
$this->tokenStream = $this->valueStream;
|
|
$previousLineCallback();
|
|
$this->tokenStream = $tempStream;
|
|
}
|
|
if (!$isFirstLine && strlen($this->currentLineString)) {
|
|
[$this->valueStream, $this->tokenStream] = $this->parseValueForConstants($this->valueStream, $this->tokenStream, $this->currentLineString, $this->currentLineNumber, $this->currentColumnInLine);
|
|
}
|
|
$previousLineCallback = $this->currentLinebreakCallback;
|
|
($this->currentLinebreakCallback)();
|
|
if (!array_key_exists($this->currentLineNumber + 1, $this->lines)) {
|
|
$this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
|
|
return;
|
|
}
|
|
if ($isFirstLine) {
|
|
$isSecondLine = true;
|
|
} else {
|
|
$isSecondLine = false;
|
|
}
|
|
$isFirstLine = false;
|
|
$valueOnFirstLine = false;
|
|
$this->currentLineNumber++;
|
|
$this->currentColumnInLine = 0;
|
|
$this->currentLineString = $this->lines[$this->currentLineNumber]['line'];
|
|
$this->currentLinebreakCallback = $this->lines[$this->currentLineNumber]['linebreakCallback'];
|
|
}
|
|
}
|
|
|
|
private function parseOperatorUnset(): void
|
|
{
|
|
$this->tokenStream->append(new Token(TokenType::T_OPERATOR_UNSET, '>', $this->currentLineNumber, $this->currentColumnInLine));
|
|
$this->currentColumnInLine++;
|
|
$this->currentLineString = substr($this->currentLineString, 1);
|
|
$this->parseTabsAndWhitespaces();
|
|
$this->makeComment();
|
|
$this->lineStream->append((new IdentifierUnsetLine())->setTokenStream($this->tokenStream)->setIdentifierTokenStream($this->identifierStream));
|
|
}
|
|
|
|
private function parseOperatorCopy(): void
|
|
{
|
|
$this->tokenStream->append(new Token(TokenType::T_OPERATOR_COPY, '<', $this->currentLineNumber, $this->currentColumnInLine));
|
|
$this->currentColumnInLine++;
|
|
$this->currentLineString = substr($this->currentLineString, 1);
|
|
$this->parseTabsAndWhitespaces();
|
|
$identifierStream = $this->identifierStream;
|
|
$this->parseIdentifierAtEndOfLine();
|
|
$referenceStream = $this->identifierStream;
|
|
if ($referenceStream->isEmpty()) {
|
|
// @todo: ($this->currentLinebreakCallback)(); is missing here?!
|
|
$this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
|
|
return;
|
|
}
|
|
$this->lineStream->append(
|
|
(new IdentifierCopyLine())
|
|
->setIdentifierTokenStream($identifierStream)
|
|
->setValueTokenStream($referenceStream)
|
|
->setTokenStream($this->tokenStream)
|
|
);
|
|
}
|
|
|
|
private function parseOperatorReference(): void
|
|
{
|
|
$this->tokenStream->append(new Token(TokenType::T_OPERATOR_REFERENCE, '=<', $this->currentLineNumber, $this->currentColumnInLine));
|
|
$this->currentColumnInLine += 2;
|
|
$this->currentLineString = substr($this->currentLineString, 2);
|
|
$this->parseTabsAndWhitespaces();
|
|
$identifierStream = $this->identifierStream;
|
|
$this->parseIdentifierAtEndOfLine();
|
|
$referenceStream = $this->identifierStream;
|
|
if ($referenceStream->isEmpty()) {
|
|
$this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
|
|
return;
|
|
}
|
|
$this->lineStream->append(
|
|
(new IdentifierReferenceLine())
|
|
->setIdentifierTokenStream($identifierStream)
|
|
->setValueTokenStream($referenceStream)
|
|
->setTokenStream($this->tokenStream)
|
|
);
|
|
}
|
|
|
|
private function parseIdentifierAtEndOfLine(): void
|
|
{
|
|
$this->identifierStream = new IdentifierTokenStream();
|
|
$isRelative = false;
|
|
$splitLine = mb_str_split($this->currentLineString, 1, 'UTF-8');
|
|
$char = $splitLine[0] ?? null;
|
|
if ($char === null) {
|
|
return;
|
|
}
|
|
$nextTwoChars = $char . ($splitLine[1] ?? '');
|
|
if ($char === '.') {
|
|
// A relative right side: foo.bar < .foo (note the dot!). we identifierStream->setRelative() and
|
|
// get rid of the dot for the rest of the processing.
|
|
$isRelative = true;
|
|
$this->tokenStream->append((new Token(TokenType::T_DOT, '.', 0, $this->currentColumnInLine)));
|
|
array_shift($splitLine);
|
|
$this->currentColumnInLine++;
|
|
$this->currentLineString = substr($this->currentLineString, 1);
|
|
}
|
|
if ($char === '#') {
|
|
$this->parseHashComment();
|
|
return;
|
|
}
|
|
if ($nextTwoChars === '//') {
|
|
$this->parseDoubleSlashComment();
|
|
return;
|
|
}
|
|
if ($nextTwoChars === '/*') {
|
|
$this->parseMultilineComment();
|
|
return;
|
|
}
|
|
$currentPosition = $this->parseIdentifierUntilStopChar($splitLine, $isRelative);
|
|
if (!$currentPosition) {
|
|
return;
|
|
}
|
|
$this->currentLineString = mb_substr($this->currentLineString, $currentPosition);
|
|
$this->currentColumnInLine = $this->currentColumnInLine + $currentPosition;
|
|
$this->parseTabsAndWhitespaces();
|
|
$this->makeComment();
|
|
}
|
|
|
|
private function parseIdentifierUntilStopChar(array $splitLine, bool $isRelative = false): ?int
|
|
{
|
|
$this->identifierStream = new IdentifierTokenStream();
|
|
if ($isRelative) {
|
|
$this->identifierStream->setRelative();
|
|
}
|
|
$currentPosition = 0;
|
|
$currentIdentifierStartPosition = $this->currentColumnInLine;
|
|
$currentIdentifierBody = '';
|
|
$currentIdentifierCharCount = 0;
|
|
while (true) {
|
|
$nextChar = $splitLine[$currentPosition] ?? null;
|
|
if ($nextChar === null) {
|
|
if ($currentIdentifierCharCount) {
|
|
$identifierToken = new IdentifierToken(TokenType::T_IDENTIFIER, $currentIdentifierBody, $this->currentLineNumber, $currentIdentifierStartPosition);
|
|
$this->tokenStream->append($identifierToken);
|
|
$this->identifierStream->append($identifierToken);
|
|
}
|
|
($this->currentLinebreakCallback)();
|
|
return null;
|
|
}
|
|
$nextTwoChars = $nextChar . ($splitLine[$currentPosition + 1] ?? null);
|
|
if ($currentPosition > 0
|
|
&& ($nextChar === ' ' || $nextChar === "\t" || $nextChar === '=' || $nextChar === '<' || $nextChar === '>' || $nextChar === '{' || $nextTwoChars === ':=' || $nextChar === '(')
|
|
) {
|
|
if ($currentIdentifierCharCount) {
|
|
$identifierToken = new IdentifierToken(TokenType::T_IDENTIFIER, $currentIdentifierBody, $this->currentLineNumber, $currentIdentifierStartPosition);
|
|
$this->tokenStream->append($identifierToken);
|
|
$this->identifierStream->append($identifierToken);
|
|
}
|
|
break;
|
|
}
|
|
if ($nextTwoChars === '\\.') {
|
|
// A quoted dot is part of *this* identifier
|
|
$currentIdentifierBody .= '.';
|
|
$currentPosition += 2;
|
|
$currentIdentifierCharCount++;
|
|
} elseif ($nextChar === '.') {
|
|
if ($currentIdentifierCharCount) {
|
|
$identifierToken = new IdentifierToken(TokenType::T_IDENTIFIER, $currentIdentifierBody, $this->currentLineNumber, $currentIdentifierStartPosition);
|
|
$this->tokenStream->append($identifierToken);
|
|
$this->identifierStream->append($identifierToken);
|
|
$currentIdentifierCharCount = 0;
|
|
$currentIdentifierBody = '';
|
|
}
|
|
$this->tokenStream->append(new Token(TokenType::T_DOT, '.', $this->currentLineNumber, $this->currentColumnInLine + $currentPosition));
|
|
$currentPosition++;
|
|
$currentIdentifierStartPosition = $this->currentColumnInLine + $currentPosition;
|
|
} else {
|
|
$currentIdentifierBody .= $nextChar;
|
|
$currentIdentifierCharCount++;
|
|
$currentPosition++;
|
|
}
|
|
}
|
|
return $currentPosition;
|
|
}
|
|
|
|
private function parseOperatorFunction(): void
|
|
{
|
|
$this->tokenStream->append(new Token(TokenType::T_OPERATOR_FUNCTION, ':=', $this->currentLineNumber, $this->currentColumnInLine));
|
|
$this->currentColumnInLine += 2;
|
|
$this->currentLineString = substr($this->currentLineString, 2);
|
|
$this->parseTabsAndWhitespaces();
|
|
if ($this->currentLineString === '') {
|
|
($this->currentLinebreakCallback)();
|
|
$this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
|
|
return;
|
|
}
|
|
$functionName = '';
|
|
$functionNameStartPosition = $this->currentColumnInLine;
|
|
$functionNameCharCount = 0;
|
|
$functionChars = mb_str_split($this->currentLineString, 1, 'UTF-8');
|
|
while (true) {
|
|
$nextChar = $functionChars[$functionNameCharCount] ?? null;
|
|
if ($nextChar === null) {
|
|
// end of chars
|
|
if ($functionNameCharCount) {
|
|
$this->tokenStream->append(new Token(TokenType::T_FUNCTION_NAME, $functionName, $this->currentLineNumber, $functionNameStartPosition));
|
|
}
|
|
($this->currentLinebreakCallback)();
|
|
$this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
|
|
return;
|
|
}
|
|
if ($nextChar === '(') {
|
|
if ($functionNameCharCount) {
|
|
$functionNameToken = new Token(TokenType::T_FUNCTION_NAME, $functionName, $this->currentLineNumber, $functionNameStartPosition);
|
|
$this->tokenStream->append($functionNameToken);
|
|
$this->tokenStream->append(new Token(TokenType::T_FUNCTION_VALUE_START, '(', $this->currentLineNumber, $this->currentColumnInLine + $functionNameCharCount));
|
|
$functionNameCharCount++;
|
|
break;
|
|
}
|
|
$this->makeComment();
|
|
$this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
|
|
return;
|
|
}
|
|
$functionName .= $nextChar;
|
|
$functionNameCharCount++;
|
|
}
|
|
$functionBodyStartPosition = $functionNameCharCount;
|
|
$functionBodyPart = '';
|
|
$functionBodyCharCount = 0;
|
|
$functionValueStream = new TokenStream();
|
|
$parenthesesLevel = 0;
|
|
while (true) {
|
|
$nextChar = $functionChars[$functionBodyStartPosition + $functionBodyCharCount] ?? null;
|
|
if ($nextChar === null) {
|
|
if ($functionBodyCharCount) {
|
|
$this->tokenStream->append(new Token(TokenType::T_VALUE, $functionBodyPart, $this->currentLineNumber, $functionBodyStartPosition));
|
|
}
|
|
($this->currentLinebreakCallback)();
|
|
$this->lineStream->append((new InvalidLine())->setTokenStream($this->tokenStream));
|
|
return;
|
|
}
|
|
if ($nextChar === '(') {
|
|
// In case of a function call like "appendString(something(somethingelse))"
|
|
// we shall only stop processing when the last bracket was evaluated.
|
|
$parenthesesLevel++;
|
|
}
|
|
if ($nextChar === ')') {
|
|
if ($parenthesesLevel > 0) {
|
|
$parenthesesLevel--;
|
|
// Continue collecting characters from the (...) argument stream.
|
|
// Also, ")" will be appended, thus intentionally no "break" occurs.
|
|
} else {
|
|
if ($functionBodyCharCount) {
|
|
[$functionValueStream, $this->tokenStream] = $this->parseValueForConstants($functionValueStream, $this->tokenStream, $functionBodyPart, $this->currentLineNumber, $this->currentColumnInLine, $functionBodyStartPosition);
|
|
}
|
|
$this->tokenStream->append(new Token(TokenType::T_FUNCTION_VALUE_STOP, ')', $this->currentLineNumber, $this->currentColumnInLine + $functionNameCharCount + $functionBodyCharCount));
|
|
$functionBodyCharCount++;
|
|
break;
|
|
}
|
|
}
|
|
$functionBodyPart .= $nextChar;
|
|
$functionBodyCharCount++;
|
|
}
|
|
$this->currentColumnInLine = $this->currentColumnInLine + $functionNameCharCount + $functionBodyCharCount;
|
|
$this->currentLineString = mb_substr($this->currentLineString, $functionNameCharCount + $functionBodyCharCount);
|
|
$this->parseTabsAndWhitespaces();
|
|
$this->makeComment();
|
|
$this->lineStream->append(
|
|
(new IdentifierFunctionLine())
|
|
->setIdentifierTokenStream($this->identifierStream)
|
|
->setFunctionNameToken($functionNameToken)
|
|
->setTokenStream($this->tokenStream)
|
|
->setFunctionValueTokenStream($functionValueStream)
|
|
);
|
|
}
|
|
|
|
/**
|
|
* @return array{0: TokenStreamInterface, 1: TokenStreamInterface}
|
|
*/
|
|
private function parseValueForConstants(TokenStreamInterface $valueStream, TokenStreamInterface $tokenStream, string $value, int $line, int $column, int $tokenOffsetPosition = 0): array
|
|
{
|
|
if (!str_contains($value, '{$')) {
|
|
$valueToken = new Token(TokenType::T_VALUE, $value, $line, $column + $tokenOffsetPosition);
|
|
$valueStream->append($valueToken);
|
|
$tokenStream->append($valueToken);
|
|
return [$valueStream, $tokenStream];
|
|
}
|
|
$splitLine = mb_str_split($value, 1, 'UTF-8');
|
|
$isInConstant = false;
|
|
$currentPosition = 0;
|
|
$currentString = '';
|
|
$currentStringLength = 0;
|
|
$lastTokenEndPosition = 0;
|
|
while (true) {
|
|
$char = $splitLine[$currentPosition] ?? null;
|
|
if ($char === null) {
|
|
if ($currentStringLength) {
|
|
$valueToken = new Token(TokenType::T_VALUE, $currentString, $line, $column + $lastTokenEndPosition + $tokenOffsetPosition);
|
|
$valueStream->append($valueToken);
|
|
$tokenStream->append($valueToken);
|
|
}
|
|
break;
|
|
}
|
|
$nextTwoChars = $char . ($splitLine[$currentPosition + 1] ?? '');
|
|
if ($nextTwoChars === '{$') {
|
|
$isInConstant = true;
|
|
if ($currentStringLength) {
|
|
$valueToken = new Token(TokenType::T_VALUE, $currentString, $line, $column + $lastTokenEndPosition + $tokenOffsetPosition);
|
|
$valueStream->append($valueToken);
|
|
$tokenStream->append($valueToken);
|
|
$lastTokenEndPosition = $currentPosition;
|
|
}
|
|
$currentString = '{$';
|
|
$currentPosition += 2;
|
|
continue;
|
|
}
|
|
if ($isInConstant && $char === '}') {
|
|
$valueToken = new Token(TokenType::T_CONSTANT, $currentString . '}', $line, $column + $lastTokenEndPosition + $tokenOffsetPosition);
|
|
if (!$valueStream instanceof ConstantAwareTokenStream) {
|
|
$valueStream = (new ConstantAwareTokenStream())->setAll($valueStream->getAll());
|
|
}
|
|
$valueStream->append($valueToken);
|
|
$tokenStream->append($valueToken);
|
|
$currentPosition++;
|
|
$currentString = '';
|
|
$currentStringLength = 0;
|
|
$lastTokenEndPosition = $currentPosition;
|
|
$isInConstant = false;
|
|
continue;
|
|
}
|
|
$currentPosition++;
|
|
$currentStringLength++;
|
|
$currentString .= $char;
|
|
}
|
|
return [$valueStream, $tokenStream];
|
|
}
|
|
}
|