TYPO3-translation/cms-core
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TYPO3 v15 dev-main snapshot ()

Browse Source
This commit is contained in:
wappler
2026-08-10 22:31:09 +02:00
commit af8cc155b5
6818 changed files with 642608 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Classes/TypoScript/Tokenizer/Token/AbstractToken.php
+79
View File
@@ -0,0 +1,79 @@
<?php
declare(strict_types=1);
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Token;
/**
* Main implementation of a TokenInterface.
*
* @internal: Internal tokenizer structure.
*/
abstract class AbstractToken implements TokenInterface
{
protected int $line = 0;
protected int $column = 0;
public function __construct(
private readonly TokenType $type,
protected readonly string $value,
int $line = 0,
int $column = 0
) {
// No constructor property promotion for $line and $column: We don't serialize
// these two and want to still default them to 0 (zero) when unserialized.
$this->line = $line;
$this->column = $column;
}
public function __toString(): string
{
return $this->value;
}
/**
* Do not store line and column when structure is serialized to cache.
* Not storing $line and $column reduces the cache size by about 1/3 since
* we're typically storing *a lot* of tokens.
*/
public function __serialize(): array
{
return [
'type' => $this->type,
'value' => $this->value,
];
}
public function getType(): TokenType
{
return $this->type;
}
public function getValue(): string
{
return $this->value;
}
public function getLine(): int
{
return $this->line;
}
public function getColumn(): int
{
return $this->column;
}
}
Classes/TypoScript/Tokenizer/Token/AbstractTokenStream.php
+113
View File
@@ -0,0 +1,113 @@
<?php
declare(strict_types=1);
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Token;
/**
* A generic implementation of TokenStreamInterface.
*
* @internal: Internal tokenizer structure.
*/
abstract class AbstractTokenStream implements TokenStreamInterface
{
/**
* @var TokenInterface[]
*/
protected array $tokens = [];
protected int $currentIndex = -1;
/**
* Create a source string from given tokens.
*/
public function __toString(): string
{
$source = '';
$this->reset();
while ($token = $this->getNext()) {
$source .= $token;
}
return $source;
}
/**
* When storing to cache, we only store FE relevant properties and skip
* irrelevant things. For instance $currentIndex should always initialize
* to -1 and does not need to be stored.
*/
final public function __serialize(): array
{
return $this->serialize();
}
protected function serialize(): array
{
$result['tokens'] = $this->tokens;
return $result;
}
/**
* Stream creation.
*/
public function append(TokenInterface $token): self
{
$this->tokens[] = $token;
return $this;
}
/**
* We sometimes create a stream but don't add tokens.
* This method returns true if tokens have been added.
*/
public function isEmpty(): bool
{
return empty($this->tokens);
}
/**
* Reset current pointer. Typically, call this before iterating with getNext().
*/
public function reset(): static
{
$this->currentIndex = -1;
return $this;
}
/**
* Get next token and raise pointer.
*/
public function getNext(): ?TokenInterface
{
$this->currentIndex++;
return $this->tokens[$this->currentIndex] ?? null;
}
public function peekNext(): ?TokenInterface
{
return $this->tokens[$this->currentIndex + 1] ?? null;
}
public function getAll(): array
{
return $this->tokens;
}
public function setAll(array $tokens): self
{
$this->tokens = $tokens;
return $this;
}
}
Classes/TypoScript/Tokenizer/Token/ConstantAwareTokenStream.php
+106
View File
@@ -0,0 +1,106 @@
<?php
declare(strict_types=1);
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Token;
use TYPO3\CMS\Core\Utility\GeneralUtility;
/**
* A list of single T_VALUE, T_NEWLINE and T_CONSTANT tokens. This is only created for
* LineIdentifierAssignment lines if there is at least one T_CONSTANT token
* in the assignment that needs to be evaluated when string'ified by the
* AST-builder.
*
* @internal: Internal tokenizer structure.
*/
final class ConstantAwareTokenStream extends AbstractTokenStream
{
private ?array $flatConstants = null;
/**
* Set by the AstBuilder to resolve constant values. Never cached.
*/
public function setFlatConstants(array $flatConstants): void
{
$this->flatConstants = $flatConstants;
}
/**
* Create a source string from given tokens.
* This resolves T_CONSTANT tokens to their value if they exist in $this->flatConstants.
*/
public function __toString(): string
{
$source = '';
$this->reset();
while ($token = $this->getNext()) {
if ($token->getType() === TokenType::T_CONSTANT) {
$token = $this->getConstantValue($this->parseConstantExpression($token->getValue())) ?? $token;
}
$source .= $token;
}
$this->reset();
return $source;
}
private function getConstantValue(?array $constantNames): ?string
{
if ($this->flatConstants === null || $constantNames === null) {
return null;
}
foreach ($constantNames as $constantName) {
$value = $this->flatConstants[$constantName] ?? null;
if ($value !== null) {
return (string)$value;
}
}
return null;
}
/**
* Parse constant expression, including null coalescing operator into an
* array of constant names to look up in order.
*
* @todo: The tokenization of this constant expression should ideally be moved
* into the TypoScript Tokenizer in order to produce a list of multiple tokens
* instead of just a T_CONSTANT for the entire body.
* This would allow early static syntax analysis of the construct and maybe
* detection of invalid and fallback to T_CONSTANT_INVALID that is treated
* like T_VALUE and can be detected. Maybe something like this:
* TokenType::T_CONSTANT_START "{"
* TokenType::T_CONSTANT_END "}"
* TokenType::T_CONSTANT_NAME "$foo.bar"
* TokenType::T_CONSTANT_OPERATOR_NULL_COALESCE " ?? "
* TokenType::T_CONSTANT_INVALID "{$foo ?? bar}" (missing $ before bar)
*/
private function parseConstantExpression(string $constantExpression): ?array
{
$innerExpression = ltrim(rtrim($constantExpression, '}'), '{');
$tokenValues = GeneralUtility::trimExplode(' ?? ', $innerExpression, true);
if ($tokenValues === []) {
return null;
}
$tokenValueNames = [];
foreach ($tokenValues as $tokenValue) {
if (!str_starts_with($tokenValue, '$')) {
return null;
}
$tokenValueNames[] = substr($tokenValue, 1);
}
return $tokenValueNames;
}
}
Classes/TypoScript/Tokenizer/Token/IdentifierToken.php
+39
View File
@@ -0,0 +1,39 @@
<?php
declare(strict_types=1);
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Token;
/**
* A special token if this token is a T_IDENTIFIER token:
* With a line like "foo = bar", "foo" is created as TokenIdentifier TokenInterface
* (as opposed to Token) having a TokenType::T_IDENTIFIER token.
* The only difference to all other tokens is that TokenIdentifier tokens
* quote any "." (dots) in their value with a backslash when output. This is
* mostly used in backend when rendering source of TokenLine's.
*
* Note we do *not* explicitly check if TokenType::T_IDENTIFIER is given in
* __construct() at the moment for performance reasons and inheritance considerations.
*
* @internal: Internal tokenizer structure.
*/
final class IdentifierToken extends AbstractToken
{
public function __toString(): string
{
return str_replace('.', '\.', $this->value);
}
}
Classes/TypoScript/Tokenizer/Token/IdentifierTokenStream.php
+106
View File
@@ -0,0 +1,106 @@
<?php
declare(strict_types=1);
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Token;
/**
* A list of single identifier (!) tokens: TokenType::T_IDENTIFIER, and only of those.
*
* This is used in TS lines that know certain parts have to be lists of identifier tokens only.
* For instance a LineIdentifierAssignment "foo.bar = barValue" return this stream for getIdentifierTokenStream():
* The left side of an assignment line is a list of identifier tokens.
*
* Identifiers can be "relative" on the right side for "<" (LineIdentifierCopy) and "=<" (LineIdentifierReference).
* Examples are "foo.bar < .baz" and "foo.bar =< .baz". These are identified by having a "." (dot) at the beginning
* on the right side. For these places, the toggle "relative" is set to true for the AST-builder to look for relative
* copy and copy-reference. The generic example are "relative" references in TS menus: 'RO < .NO'
*
* For example, with "foo.bar < baz", the Tokenizer creates a LineIdentifierCopy line, having a TokenStreamIdentifier
* list of the T_IDENTIFIER tokens for 'foo' and 'bar' for getIdentifierTokenStream(), plus a TokenStreamIdentifier list
* of T_IDENTIFIER tokens for 'baz' for getValueTokenStream().
*
* Note identifier streams on the left side (foo.bar = ...) are never relative, this toggle is true for "<" and "=<" only.
*
* Lines that know they can only return TokenStreamIdentifier's - they are more specific than just TokenStream, are
* type-hinted as such. For instance getIdentifierTokenStream() type hints TokenStreamIdentifier.
*
* @internal: Internal tokenizer structure.
*/
final class IdentifierTokenStream extends AbstractTokenStream
{
private bool $relative = false;
/**
* When rendering a source string from multiple identifiers, dots between single identifiers need to be added again.
* This is used in RootNode->toArray() to create that insane '< lib.whatever' as value when using the
* reference operator: "foo =< lib.whatever". See ContentObjectRenderer cObjGetSingle() and mergeTSRef().
*/
public function __toString(): string
{
$source = [];
$this->reset();
while ($token = $this->getNext()) {
$source[] = (string)$token;
}
$source = implode('.', $source);
if ($this->relative) {
$source = '.' . $source;
}
return $source;
}
protected function serialize(): array
{
$result = parent::serialize();
if ($this->isRelative()) {
$result['relative'] = true;
}
return $result;
}
/**
* Append a token to the stream.
*/
public function append(TokenInterface $token): self
{
if ($token->getType() !== TokenType::T_IDENTIFIER) {
throw new \LogicException(
'Trying to add a token of type TokenType::' . $token->getType()->name . ' to class TokenStreamIdentifier, but only TokenType::T_IDENTIFIERS are allowed.',
1655138907
);
}
$this->tokens[] = $token;
return $this;
}
/**
* This identifier token stream is relative! There is a dot on the right side of something like "foo.bar < .baz"
*/
public function setRelative(): self
{
$this->relative = true;
return $this;
}
/**
* True if this identifier stream is relative to given context.
*/
public function isRelative(): bool
{
return $this->relative;
}
}
Classes/TypoScript/Tokenizer/Token/Token.php
+29
View File
@@ -0,0 +1,29 @@
<?php
declare(strict_types=1);
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Token;
/**
* A casual token created from TypoScript source:
* When having a TypoScript line like "# a comment", then a LineComment
* is created having a token "T_COMMENT_ONELINE_HASH" and value "# a comment" as
* assigned TokenStream.
* See TokenType for on overview on which TokenTypes can exist.
*
* @internal: Internal tokenizer structure.
*/
final class Token extends AbstractToken {}
Classes/TypoScript/Tokenizer/Token/TokenInterface.php
+68
View File
@@ -0,0 +1,68 @@
<?php
declare(strict_types=1);
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Token;
/**
* A readonly token: Each line of TypoScript is split into a list of lines consisting of
* tokens by the tokenizers.
*
* As example, a "foo.bar = baz" line creates a LineIdentifierAssignment line, having
* TokenType::T_IDENTIFIER 'foo', plus TokenType::T_IDENTIFIER 'bar' as TokenStream for
* LineIdentifierAssignment->getIdentifierTokenStream(), plus a TokenType::T_VALUE 'baz'
* as LineIdentifierAssignment->getValueTokenStream().
*
* We have two different Token implementations: The casual "Token" class for everything, plus
* the "TokenIdentifier" class for identifier tokens. Identifier tokens are those "left" of
* for instance an assignment like "foo.bar = baz" ("foo" and "bar" are TokenIdentifier instances),
* and also on the right side when using expression with "<" and "=<" operator: Example "foo.bar < baz":
* "baz" is an instance of a TokenIdentifier ("foo" and "bar" as well).
*
* The reason to have two implementations is that TokenIdentifier needs to be handled slightly
* different when cast to string: For identifiers, all "." (dots) within a single identifier token
* need to be quoted with "\" (backslash), to not confuse the parser. The classic use-case is having dots in
* FlexForm identifiers for PageTS:
* "foo.bar\.baz.foobar = value" - three identifier tokens (not four!): "foo", "bar.baz" and "foobar".
* So the difference between "TokenIdentifier" and "Token" is just that "TokenIdentifier" quotes dots
* in its value when string'ified, while Token does not and __toString() on Token simply says ->getValue().
*
* Multiple tokens are encapsulated in TokenStreamInterface. TokenStreamInterface has a __toString()
* method as well, which calls __toString() on all assigned tokens. This way, a TokenIdentifier will
* do its quoting magic, and casual Token instances return their value.
*
* The idea is here that TokenStreams are cast to string quite often. For instance, an assignment line
* like "foo = bar" creates a token stream of one token for the right side (things after "="):
* A T_VALUE Token instance with value "bar". The AST builder then at some point needs to resolve this
* TokenStream to string. This will directly call __toString on token "bar", and does not deal with quoting,
* since its no TokenIdentifier and just a Token.
*
* Note on getLine() and getColumn(): These two represent the position of a token in the source file:
* We start counting at 0 (zero): The first token on the first line is line 0, column 0.
* Only the LosslessTokenizer sets these, it's too expensive and of no relevance for the LossyTokenizer
* that is used for instance in FE TS tokenizing. That's why these two properties are optional
* and 0 (zero) by default.
*
* @internal: Internal tokenizer structure.
*/
interface TokenInterface
{
public function __toString(): string;
public function getType(): TokenType;
public function getValue(): string;
public function getLine(): int;
public function getColumn(): int;
}
Classes/TypoScript/Tokenizer/Token/TokenStream.php
+26
View File
@@ -0,0 +1,26 @@
<?php
declare(strict_types=1);
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Token;
/**
* A list of single tokens. These are typically used in TokenLines: A TypoScript
* line consists of one or more streams of tokens, depending on the line type.
*
* @internal: Internal tokenizer structure.
*/
final class TokenStream extends AbstractTokenStream {}
Classes/TypoScript/Tokenizer/Token/TokenStreamInterface.php
+77
View File
@@ -0,0 +1,77 @@
<?php
declare(strict_types=1);
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Token;
/**
* A generic stream of tokens used in single LineInterface lines.
*
* The tokenizers create these streams for various lists of tokens, the generic
* implementation is class TokenStream. For lists of identifier tokens the special
* class TokenStreamIdentifier is created.
*
* @internal: Internal tokenizer structure.
*/
interface TokenStreamInterface
{
/**
* Create a source string from given tokens.
*/
public function __toString(): string;
/**
* Stream creation.
*/
public function append(TokenInterface $token): self;
/**
* We sometimes create a stream but don't add tokens.
* This method returns true if tokens have been added.
*/
public function isEmpty(): bool;
/**
* Reset current pointer. Typically, call this before iterating with getNext().
*/
public function reset(): self;
/**
* Get next token and raise pointer.
*/
public function getNext(): ?TokenInterface;
/**
* Get next token but do not raise pointer.
*/
public function peekNext(): ?TokenInterface;
/**
* Only used internally when one Stream is transferred to another,
* in particular when a TokenStream is turned into TokenStreamConstantAware.
*
* @return TokenInterface[]
*/
public function getAll(): array;
/**
* Only used internally when one Stream is transferred to another,
* in particular when a TokenStream is turned into TokenStreamConstantAware.
*
* @param TokenInterface[] $tokens
*/
public function setAll(array $tokens): self;
}
Classes/TypoScript/Tokenizer/Token/TokenType.php
+69
View File
@@ -0,0 +1,69 @@
<?php
declare(strict_types=1);
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
namespace TYPO3\CMS\Core\TypoScript\Tokenizer\Token;
/**
* Each TokenInterface instance is a type of this Enum.
*
* @internal: Internal tokenizer structure.
*/
enum TokenType: int
{
case T_NONE = 0; // tokenizer internal handling
case T_IDENTIFIER = 100; // single word left of an operator. 'foo.bar' are two identifiers
case T_VALUE = 200; // right side of an assignment, does not contain line breaks, also used as 'comment' body
case T_OPERATOR_ASSIGNMENT = 300; // '='
case T_OPERATOR_REFERENCE = 301; // '=<'
case T_OPERATOR_COPY = 302; // '<'
case T_OPERATOR_UNSET = 303; // '>'
case T_OPERATOR_FUNCTION = 304; // ':='
case T_OPERATOR_ASSIGNMENT_MULTILINE_START = 310; // '('
case T_OPERATOR_ASSIGNMENT_MULTILINE_STOP = 311; // ')'
case T_BLOCK_START = 400; // '{'
case T_BLOCK_STOP = 401; // '}'
case T_DOT = 500; // '.' identifier separator
case T_BLANK = 600; // list of ' ' and "\t"
case T_NEWLINE = 700; // "\n" or "\r\n"
case T_COMMENT_ONELINE_HASH = 800; // '#...'
case T_COMMENT_ONELINE_DOUBLESLASH = 801; // '//'
case T_COMMENT_MULTILINE_START = 802; // '/*'
case T_COMMENT_MULTILINE_STOP = 803; // '*/'
case T_FUNCTION_NAME = 900; // 'addToList' and others
case T_FUNCTION_VALUE_START = 901; // '(' after T_FUNCTION_NAME
case T_FUNCTION_VALUE_STOP = 902; // ')' after T_FUNCTION_NAME
case T_CONDITION_START = 1000; // '[' at start of line
case T_CONDITION_STOP = 1001; // ']' after '[' in same line, body is a T_VALUE
case T_CONDITION_ELSE = 1002; // 'ELSE' surrounded by '[' and ']'
case T_CONDITION_END = 1003; // 'END' surrounded by '[' and ']'
case T_CONDITION_GLOBAL = 1004; // 'GLOBAL' surrounded by '[' and ']'
case T_CONSTANT = 1100; // '{$...}'
case T_IMPORT_KEYWORD = 1200; // '@import'
case T_IMPORT_START = 1201; // ''' (tick) or '"' (doubletick) after @import
case T_IMPORT_STOP = 1202; // ''' (tick) or '"' (doubletick) after T_IMPORT_START
}