TYPO3 v15 dev-main snapshot ()

This commit is contained in:
2026-08-10 22:31:09 +02:00
commit af8cc155b5
6818 changed files with 642608 additions and 0 deletions
@@ -0,0 +1,86 @@
<?php
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
namespace TYPO3\CMS\Core\Resource\Index;
use Symfony\Component\DependencyInjection\Attribute\AutoconfigureTag;
use TYPO3\CMS\Core\Resource;
use TYPO3\CMS\Core\Resource\File;
/**
* An Interface for MetaData extractors the FAL Indexer uses
*/
#[AutoconfigureTag('metadata.extractor')]
interface ExtractorInterface
{
/**
* Returns an array of supported file types;
* An empty array indicates all filetypes
*
* @return array
*/
public function getFileTypeRestrictions();
/**
* Get all supported DriverClasses
*
* Since some extractors may only work for local files, and other extractors
* are especially made for grabbing data from remote.
*
* Returns array of string with driver names of Drivers which are supported,
* If the driver did not register a name, it's the classname.
* empty array indicates no restrictions
*
* @return array
*/
public function getDriverRestrictions();
/**
* Returns the data priority of the extraction Service.
* Defines the precedence of Data if several extractors
* extracted the same property.
*
* Should be between 1 and 100, 100 is more important than 1
*
* @return int
*/
public function getPriority();
/**
* Returns the execution priority of the extraction Service
* Should be between 1 and 100, 100 means runs as first service, 1 runs at last service
*
* @return int
*/
public function getExecutionPriority();
/**
* Checks if the given file can be processed by this Extractor
*
* @return bool
*/
public function canProcess(File $file);
/**
* The actual processing TASK
*
* Should return an array with database properties for sys_file_metadata to write
*
* @param Resource\File $file
* @param array $previousExtractedData optional, contains the array of already extracted data
* @return array
*/
public function extractMetaData(File $file, array $previousExtractedData = []);
}
@@ -0,0 +1,80 @@
<?php
declare(strict_types=1);
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
namespace TYPO3\CMS\Core\Resource\Index;
use Symfony\Component\DependencyInjection\Attribute\Autoconfigure;
use Symfony\Component\DependencyInjection\Attribute\AutowireLocator;
use Symfony\Component\DependencyInjection\ServiceLocator;
/**
* Registry for MetaData extraction Services
*/
#[Autoconfigure(public: true)]
readonly class ExtractorRegistry
{
public function __construct(
#[AutowireLocator('metadata.extractor')]
private ServiceLocator $extractors
) {}
/**
* Get all registered extractor instances.
*
* @return ExtractorInterface[]
*/
public function getExtractors(): array
{
// @todo Isn't there an option to get the services ordered automatically by the ServiceLocator?
$extractors = [];
foreach ($this->extractors as $extractor) {
$extractors[] = $extractor;
}
usort($extractors, [$this, 'compareExtractorPriority']);
return $extractors;
}
/**
* Get Extractors which work for a specific driver.
*
* @return ExtractorInterface[]
*/
public function getExtractorsWithDriverSupport(string $driverType): array
{
return array_filter(
$this->getExtractors(),
function (ExtractorInterface $extractor) use ($driverType): bool {
return empty($extractor->getDriverRestrictions())
|| in_array($driverType, $extractor->getDriverRestrictions(), true);
}
);
}
/**
* Compare the priority of two Extractor classes.
* Is used for sorting array of Extractor instances by priority.
* We want the result to be ordered from high to low so a higher
* priority comes before a lower.
*
* @return int -1 a > b, 0 a == b, 1 a < b
*/
private function compareExtractorPriority(ExtractorInterface $extractorA, ExtractorInterface $extractorB): int
{
return $extractorB->getExecutionPriority() - $extractorA->getExecutionPriority();
}
}
@@ -0,0 +1,414 @@
<?php
declare(strict_types=1);
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
namespace TYPO3\CMS\Core\Resource\Index;
use Psr\EventDispatcher\EventDispatcherInterface;
use Symfony\Component\DependencyInjection\Attribute\Autoconfigure;
use TYPO3\CMS\Core\Database\Connection;
use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\Database\ReferenceIndex;
use TYPO3\CMS\Core\Resource\Event\AfterFileAddedToIndexEvent;
use TYPO3\CMS\Core\Resource\Event\AfterFileMarkedAsMissingEvent;
use TYPO3\CMS\Core\Resource\Event\AfterFileRemovedFromIndexEvent;
use TYPO3\CMS\Core\Resource\Event\AfterFileUpdatedInIndexEvent;
use TYPO3\CMS\Core\Resource\File;
use TYPO3\CMS\Core\Resource\FileInterface;
use TYPO3\CMS\Core\Resource\Folder;
use TYPO3\CMS\Core\Resource\ResourceStorage;
use TYPO3\CMS\Core\Utility\GeneralUtility;
/**
* Repository Class as an abstraction layer to sys_file
*
* Every access to table sys_file_metadata which is not handled by DataHandler
* has to use this Repository class.
*
* @internal This is meant for FAL internal use only!
*/
#[Autoconfigure(public: true)]
readonly class FileIndexRepository
{
/**
* A list of properties which are to be persisted
*/
protected const FIELDS = [
'uid', 'pid', 'missing', 'type', 'storage', 'identifier', 'identifier_hash', 'extension',
'mime_type', 'name', 'sha1', 'size', 'creation_date', 'modification_date', 'folder_hash',
];
public function __construct(
private EventDispatcherInterface $eventDispatcher,
private ConnectionPool $connectionPool,
) {}
/**
* Retrieves Index record for a given $fileUid
*/
public function findOneByUid(int $fileUid): array|false
{
$queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_file');
$row = $queryBuilder
->select(...self::FIELDS)
->from('sys_file')
->where(
$queryBuilder->expr()->eq('uid', $queryBuilder->createNamedParameter($fileUid, Connection::PARAM_INT))
)
->executeQuery()
->fetchAssociative();
return is_array($row) ? $row : false;
}
/**
* Retrieves Index record for a given $storageUid and $identifier
*
* @internal only for use from FileRepository
*/
public function findOneByStorageUidAndIdentifierHash(int $storageUid, string $identifierHash): array|false
{
$queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_file');
$row = $queryBuilder
->select(...self::FIELDS)
->from('sys_file')
->where(
$queryBuilder->expr()->eq('storage', $queryBuilder->createNamedParameter($storageUid, Connection::PARAM_INT)),
$queryBuilder->expr()->eq('identifier_hash', $queryBuilder->createNamedParameter($identifierHash))
)
->executeQuery()
->fetchAssociative();
return is_array($row) ? $row : false;
}
/**
* Retrieves Index record for a given $storageUid and $identifier
*
* @internal only for use from FileRepository
*/
public function findOneByStorageAndIdentifier(ResourceStorage $storage, string $identifier): array|false
{
$identifierHash = $storage->hashFileIdentifier($identifier);
return $this->findOneByStorageUidAndIdentifierHash($storage->getUid(), $identifierHash);
}
/**
* Retrieves Index record for a given $fileObject
*
* @internal only for use from FileRepository
*/
public function findOneByFileObject(FileInterface $fileObject): array|false
{
return $this->findOneByStorageAndIdentifier($fileObject->getStorage(), $fileObject->getIdentifier());
}
/**
* Returns all indexed files which match the content hash
* Used by the indexer to detect already present files
*/
public function findByContentHash(string $hash): array
{
if (!preg_match('/^[0-9a-f]{40}$/i', $hash)) {
return [];
}
$queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_file');
return $queryBuilder
->select(...self::FIELDS)
->from('sys_file')
->where(
$queryBuilder->expr()->eq('sha1', $queryBuilder->createNamedParameter($hash))
)
->executeQuery()
->fetchAllAssociative();
}
/**
* Find all records for files in a Folder
*/
public function findByFolder(Folder $folder): array
{
$queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_file');
$result = $queryBuilder
->select(...self::FIELDS)
->from('sys_file')
->where(
$queryBuilder->expr()->eq('folder_hash', $queryBuilder->createNamedParameter($folder->getHashedIdentifier())),
$queryBuilder->expr()->eq('storage', $queryBuilder->createNamedParameter($folder->getStorage()->getUid(), Connection::PARAM_INT))
)
->executeQuery();
$resultRows = [];
while ($row = $result->fetchAssociative()) {
$resultRows[$row['identifier']] = $row;
}
return $resultRows;
}
/**
* Find all records for files in an array of Folders
*
* @param Folder[] $folders
*/
public function findByFolders(array $folders, bool $includeMissing = true, ?string $fileName = null): array
{
$storageUids = [];
$folderIdentifiers = [];
foreach ($folders as $folder) {
$storageUids[] = $folder->getStorage()->getUid();
$folderIdentifiers[] = $folder->getHashedIdentifier();
}
$storageUids = array_unique($storageUids);
$folderIdentifiers = array_unique($folderIdentifiers);
$queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_file');
$queryBuilder
->select(...self::FIELDS)
->from('sys_file')
->where(
$queryBuilder->expr()->in('folder_hash', $queryBuilder->createNamedParameter($folderIdentifiers, Connection::PARAM_STR_ARRAY)),
$queryBuilder->expr()->in('storage', $queryBuilder->createNamedParameter($storageUids, Connection::PARAM_INT_ARRAY))
);
if (isset($fileName)) {
$nameParts = str_getcsv($fileName, ' ', '"', '\\');
foreach ($nameParts as $part) {
$part = trim($part);
if ($part !== '') {
$queryBuilder->andWhere(
$queryBuilder->expr()->like(
'name',
$queryBuilder->createNamedParameter(
'%' . $queryBuilder->escapeLikeWildcards($part) . '%'
)
)
);
}
}
}
if (!$includeMissing) {
$queryBuilder->andWhere($queryBuilder->expr()->eq('missing', $queryBuilder->createNamedParameter(0, Connection::PARAM_INT)));
}
$result = $queryBuilder->executeQuery();
$fileRecords = [];
while ($fileRecord = $result->fetchAssociative()) {
$fileRecords[$fileRecord['identifier']] = $fileRecord;
}
return $fileRecords;
}
/**
* Adds a file to the index
*/
public function add(File $file): void
{
if ($this->hasIndexRecord($file)) {
$this->update($file);
if ($file->_getPropertyRaw('uid') === null) {
$file->updateProperties($this->findOneByFileObject($file));
}
} else {
$file->updateProperties(['uid' => $this->insertRecord($file->getProperties())]);
}
}
/**
* Add data from record (at indexing time)
*/
public function addRaw(array $data): array
{
$data['uid'] = $this->insertRecord($data);
return $data;
}
/**
* Helper to reduce code duplication
*/
protected function insertRecord(array $data): int
{
$data = array_intersect_key($data, array_flip(self::FIELDS));
$data['tstamp'] = time();
$connection = $this->connectionPool->getConnectionForTable('sys_file');
$connection->insert(
'sys_file',
$data
);
$data['uid'] = (int)$connection->lastInsertId();
$this->updateRefIndex($data['uid']);
$this->eventDispatcher->dispatch(new AfterFileAddedToIndexEvent($data['uid'], $data));
return $data['uid'];
}
/**
* Checks if a file is indexed
*/
public function hasIndexRecord(File $file): bool
{
$queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_file');
if ((int)$file->_getPropertyRaw('uid') > 0) {
$constraints = [
$queryBuilder->expr()->eq('uid', $queryBuilder->createNamedParameter($file->getUid(), Connection::PARAM_INT)),
];
} else {
$constraints = [
$queryBuilder->expr()->eq('storage', $queryBuilder->createNamedParameter($file->getStorage()->getUid(), Connection::PARAM_INT)),
$queryBuilder->expr()->eq('identifier', $queryBuilder->createNamedParameter($file->_getPropertyRaw('identifier'))),
];
}
$count = $queryBuilder
->count('uid')
->from('sys_file')
->where(...$constraints)
->executeQuery()
->fetchOne();
return (bool)$count;
}
/**
* Updates the index record in the database
*/
public function update(File $file): void
{
$updatedProperties = array_intersect(self::FIELDS, $file->getUpdatedProperties());
$updateRow = [];
foreach ($updatedProperties as $key) {
$updateRow[$key] = $file->getProperty($key);
}
if (!empty($updateRow)) {
if ((int)$file->_getPropertyRaw('uid') > 0) {
$constraints = ['uid' => $file->getUid()];
} else {
$constraints = [
'storage' => $file->getStorage()->getUid(),
'identifier' => $file->_getPropertyRaw('identifier'),
];
}
$connection = $this->connectionPool->getConnectionForTable('sys_file');
$updateRow['tstamp'] = time();
$connection->update(
'sys_file',
$updateRow,
$constraints
);
$this->updateRefIndex($file->getUid());
$this->eventDispatcher->dispatch(new AfterFileUpdatedInIndexEvent($file, array_intersect_key($file->getProperties(), array_flip(self::FIELDS)), $updateRow));
}
}
/**
* Finds the files needed for second indexer step
*/
public function findInStorageWithIndexOutstanding(ResourceStorage $storage, int $limit = -1): array
{
$queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_file');
if ($limit > 0) {
$queryBuilder->setMaxResults($limit);
}
return $queryBuilder
->select(...self::FIELDS)
->from('sys_file')
->where(
$queryBuilder->expr()->gt('tstamp', $queryBuilder->quoteIdentifier('last_indexed')),
$queryBuilder->expr()->eq('storage', $queryBuilder->createNamedParameter($storage->getUid(), Connection::PARAM_INT)),
$queryBuilder->expr()->eq('missing', $queryBuilder->createNamedParameter(0, Connection::PARAM_INT))
)
->orderBy('tstamp', 'ASC')
->executeQuery()
->fetchAllAssociative();
}
/**
* Helper function for the Indexer to detect missing files
*
* @param int[] $uidList
*/
public function findInStorageAndNotInUidList(ResourceStorage $storage, array $uidList): array
{
$queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_file');
$queryBuilder
->select(...self::FIELDS)
->from('sys_file')
->where(
$queryBuilder->expr()->eq('storage', $queryBuilder->createNamedParameter($storage->getUid(), Connection::PARAM_INT))
);
if (!empty($uidList)) {
$queryBuilder->andWhere(
$queryBuilder->expr()->notIn('uid', array_map(intval(...), $uidList))
);
}
return $queryBuilder->executeQuery()->fetchAllAssociative();
}
/**
* Updates the timestamp when the file indexer extracted metadata
*/
public function updateIndexingTime(int $fileUid): void
{
$this->connectionPool
->getConnectionForTable('sys_file')
->update(
'sys_file',
[
'last_indexed' => time(),
],
[
'uid' => $fileUid,
]
);
}
/**
* Marks given file as missing in sys_file
*/
public function markFileAsMissing(int $fileUid): void
{
$this->connectionPool
->getConnectionForTable('sys_file')
->update(
'sys_file',
[
'missing' => 1,
],
[
'uid' => $fileUid,
]
);
$this->eventDispatcher->dispatch(new AfterFileMarkedAsMissingEvent($fileUid));
}
/**
* Remove a sys_file record from the database
*/
public function remove(int $fileUid): void
{
$this->connectionPool
->getConnectionForTable('sys_file')
->delete(
'sys_file',
[
'uid' => $fileUid,
]
);
$this->updateRefIndex($fileUid);
$this->eventDispatcher->dispatch(new AfterFileRemovedFromIndexEvent($fileUid));
}
/**
* Update Reference Index (sys_refindex) for a file
*/
protected function updateRefIndex(int $id): void
{
$refIndexObj = GeneralUtility::makeInstance(ReferenceIndex::class);
$refIndexObj->updateRefIndexTable('sys_file', $id);
}
}
+360
View File
@@ -0,0 +1,360 @@
<?php
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
namespace TYPO3\CMS\Core\Resource\Index;
use Psr\Log\LoggerAwareInterface;
use Psr\Log\LoggerAwareTrait;
use TYPO3\CMS\Core\Resource\Exception\IllegalFileExtensionException;
use TYPO3\CMS\Core\Resource\Exception\InsufficientFileAccessPermissionsException;
use TYPO3\CMS\Core\Resource\Exception\InvalidHashException;
use TYPO3\CMS\Core\Resource\File;
use TYPO3\CMS\Core\Resource\FileType;
use TYPO3\CMS\Core\Resource\ResourceFactory;
use TYPO3\CMS\Core\Resource\ResourceStorage;
use TYPO3\CMS\Core\Resource\Service\ExtractorService;
use TYPO3\CMS\Core\Type\File\ImageInfo;
use TYPO3\CMS\Core\Utility\GeneralUtility;
/**
* The FAL Indexer
*/
class Indexer implements LoggerAwareInterface
{
use LoggerAwareTrait;
protected array $filesToUpdate = [];
/**
* @var int[]
*/
protected array $identifiedFileUids = [];
protected ResourceStorage $storage;
protected ?ExtractorService $extractorService = null;
public function __construct(ResourceStorage $storage)
{
$this->storage = $storage;
}
/**
* Create index entry
*
* @throws \InvalidArgumentException
*/
public function createIndexEntry(string $identifier): File
{
if ($identifier === '') {
throw new \InvalidArgumentException(
'Invalid file identifier given. It must not empty.',
1401732565
);
}
$fileProperties = $this->gatherFileInformationArray($identifier);
$fileIndexRepository = $this->getFileIndexRepository();
$record = $fileIndexRepository->addRaw($fileProperties);
$fileObject = $this->getResourceFactory()->getFileObject($record['uid'], $record);
$fileIndexRepository->updateIndexingTime($fileObject->getUid());
$metaData = $this->extractRequiredMetaData($fileObject);
if ($this->storage->autoExtractMetadataEnabled()) {
$metaData = array_merge($metaData, $this->getExtractorService()->extractMetaData($fileObject));
}
$fileObject->getMetaData()->add($metaData)->save();
return $fileObject;
}
/**
* Update index entry
*/
public function updateIndexEntry(File $fileObject): File
{
$updatedInformation = $this->gatherFileInformationArray($fileObject->getIdentifier());
$fileObject->updateProperties($updatedInformation);
$fileIndexRepository = $this->getFileIndexRepository();
$fileIndexRepository->update($fileObject);
$fileIndexRepository->updateIndexingTime($fileObject->getUid());
$metaData = $this->extractRequiredMetaData($fileObject);
if ($this->storage->autoExtractMetadataEnabled()) {
$metaData = array_merge($metaData, $this->getExtractorService()->extractMetaData($fileObject));
}
$fileObject->getMetaData()->add($metaData)->save();
return $fileObject;
}
public function processChangesInStorages(): void
{
// get all file-identifiers from the storage
$availableFiles = $this->storage->getFileIdentifiersInFolder($this->storage->getRootLevelFolder(false)->getIdentifier(), true, true);
$this->detectChangedFilesInStorage($availableFiles);
$this->processChangedAndNewFiles();
$this->detectMissingFiles();
}
public function runMetaDataExtraction(int $maximumFileCount = -1): void
{
$fileIndexRecords = $this->getFileIndexRepository()->findInStorageWithIndexOutstanding($this->storage, $maximumFileCount);
foreach ($fileIndexRecords as $indexRecord) {
$fileObject = $this->getResourceFactory()->getFileObject($indexRecord['uid'], $indexRecord);
// Check for existence of file before extraction
if ($fileObject->exists()) {
try {
$this->extractMetaData($fileObject);
} catch (InsufficientFileAccessPermissionsException $e) {
// We skip files that are not accessible
} catch (IllegalFileExtensionException $e) {
// We skip files that have an extension that we don't allow
}
} else {
// Mark file as missing and continue with next record
$this->getFileIndexRepository()->markFileAsMissing($indexRecord['uid']);
}
}
}
/**
* Extract metadata for given fileObject
*/
public function extractMetaData(File $fileObject): void
{
$metaData = array_merge([
$fileObject->getMetaData()->get(),
], $this->getExtractorService()->extractMetaData($fileObject));
$fileObject->getMetaData()->add($metaData)->save();
$this->getFileIndexRepository()->updateIndexingTime($fileObject->getUid());
}
/**
* Since by now all files in filesystem have been looked at, it is safe to assume,
* that files that are indexed, but not touched in this run, are missing
*/
protected function detectMissingFiles(): void
{
$allCurrentFiles = $this->getFileIndexRepository()->findInStorageAndNotInUidList(
$this->storage,
[]
);
foreach ($allCurrentFiles as $record) {
// Check if the record retrieved from the database was associated
// with an existing file.
// If yes: All is good, file is in index and in database.
// If no: Database record may need to be marked as removed (extra check!)
if (in_array($record['uid'], $this->identifiedFileUids, true)) {
continue;
}
if (!$this->storage->hasFile($record['identifier'])) {
$this->getFileIndexRepository()->markFileAsMissing($record['uid']);
}
}
}
/**
* Check whether the extractor service supports this file according to file type restrictions.
*/
protected function isFileTypeSupportedByExtractor(File $file, ExtractorInterface $extractor): bool
{
$isSupported = true;
$fileTypeRestrictions = $extractor->getFileTypeRestrictions();
if (!empty($fileTypeRestrictions) && !in_array($file->getType(), $fileTypeRestrictions)) {
$isSupported = false;
}
return $isSupported;
}
/**
* Adds updated files to the processing queue
*/
protected function detectChangedFilesInStorage(array $fileIdentifierArray): void
{
foreach ($fileIdentifierArray as $fileIdentifier) {
// skip processed files
if ($this->storage->isWithinProcessingFolder($fileIdentifier)) {
continue;
}
// Get the modification time for file-identifier from the storage
$modificationTime = $this->storage->getFileInfoByIdentifier($fileIdentifier, ['mtime']);
// Look if the the modification time in FS is higher than the one in database (key needed on timestamps)
$indexRecord = $this->getFileIndexRepository()->findOneByStorageAndIdentifier($this->storage, $fileIdentifier);
if ($indexRecord !== false) {
$this->identifiedFileUids[] = $indexRecord['uid'];
if ((int)$indexRecord['modification_date'] !== $modificationTime['mtime'] || $indexRecord['missing']) {
$this->filesToUpdate[$fileIdentifier] = $indexRecord;
}
} else {
$this->filesToUpdate[$fileIdentifier] = null;
}
}
}
/**
* Processes the Files which have been detected as "changed or new"
* in the storage
*/
protected function processChangedAndNewFiles(): void
{
foreach ($this->filesToUpdate as $identifier => $data) {
try {
if ($data === null) {
// search for files with same content hash in indexed storage
$fileHash = $this->storage->hashFileByIdentifier($identifier, 'sha1');
$files = $this->getFileIndexRepository()->findByContentHash($fileHash);
$fileObject = null;
if (!empty($files)) {
foreach ($files as $fileIndexEntry) {
// check if file is missing then we assume it's moved/renamed
if (!$this->storage->hasFile($fileIndexEntry['identifier'])) {
$fileObject = $this->getResourceFactory()->getFileObject(
$fileIndexEntry['uid'],
$fileIndexEntry
);
$fileObject->updateProperties(
[
'identifier' => $identifier,
]
);
$this->updateIndexEntry($fileObject);
$this->identifiedFileUids[] = $fileObject->getUid();
break;
}
}
}
// create new index when no missing file with same content hash is found
if ($fileObject === null) {
$fileObject = $this->createIndexEntry($identifier);
$this->identifiedFileUids[] = $fileObject->getUid();
}
} else {
// update existing file
$fileObject = $this->getResourceFactory()->getFileObject($data['uid'], $data);
$this->updateIndexEntry($fileObject);
}
} catch (InvalidHashException $e) {
$this->logger->error('Unable to create hash for file: {identifier}', ['identifier' => $identifier]);
} catch (\Exception $e) {
$this->logger->error('Unable to index / update file with identifier {identifier}', [
'identifier' => $identifier,
'exception' => $e,
]);
}
}
}
/**
* Since the core desperately needs image sizes in metadata table put them there
* This should be called after every "content" update and "record" creation
*/
protected function extractRequiredMetaData(File $fileObject): array
{
$metaData = [];
// since the core desperately needs image sizes in metadata table do this manually
// prevent doing this for remote storages, remote storages must provide the data with extractors
if ($fileObject->isImage() && $this->storage->getDriverType() === 'Local') {
$rawFileLocation = $fileObject->getForLocalProcessing(false);
$imageInfo = GeneralUtility::makeInstance(ImageInfo::class, $rawFileLocation);
$metaData = [
'width' => $imageInfo->getWidth(),
'height' => $imageInfo->getHeight(),
];
}
return $metaData;
}
/****************************
* UTILITY
****************************/
/**
* Collects the information to be cached in sys_file
*/
protected function gatherFileInformationArray(string $identifier): array
{
$fileInfo = $this->storage->getFileInfoByIdentifier($identifier);
$fileInfo = $this->transformFromDriverFileInfoArrayToFileObjectFormat($fileInfo);
$fileInfo['type'] = $this->getFileType($fileInfo['mime_type'])->value;
$fileInfo['sha1'] = $this->storage->hashFileByIdentifier($identifier, 'sha1');
$fileInfo['missing'] = 0;
return $fileInfo;
}
/**
* Maps the mimetype to a sys_file table type
*/
protected function getFileType(string $mimeType): FileType
{
return FileType::tryFromMimeType($mimeType);
}
/**
* However it happened, the properties of a file object which
* are persisted to the database are named different than the
* properties the driver returns in getFileInfo.
* Therefore, a mapping must happen.
*/
protected function transformFromDriverFileInfoArrayToFileObjectFormat(array $fileInfo): array
{
$mappingInfo = [
// 'driverKey' => 'fileProperty' Key is from the driver, value is for the property in the file
'size' => 'size',
'atime' => null,
'mtime' => 'modification_date',
'ctime' => 'creation_date',
'mimetype' => 'mime_type',
];
$mappedFileInfo = [];
foreach ($fileInfo as $key => $value) {
if (array_key_exists($key, $mappingInfo)) {
if ($mappingInfo[$key] !== null) {
$mappedFileInfo[$mappingInfo[$key]] = $value;
}
} else {
$mappedFileInfo[$key] = $value;
}
}
return $mappedFileInfo;
}
protected function getFileIndexRepository(): FileIndexRepository
{
return GeneralUtility::makeInstance(FileIndexRepository::class);
}
protected function getResourceFactory(): ResourceFactory
{
return GeneralUtility::makeInstance(ResourceFactory::class);
}
protected function getExtractorService(): ExtractorService
{
if ($this->extractorService === null) {
$this->extractorService = GeneralUtility::makeInstance(ExtractorService::class);
}
return $this->extractorService;
}
}
@@ -0,0 +1,219 @@
<?php
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
namespace TYPO3\CMS\Core\Resource\Index;
use Psr\EventDispatcher\EventDispatcherInterface;
use Symfony\Component\DependencyInjection\Attribute\Autoconfigure;
use TYPO3\CMS\Core\Context\Context;
use TYPO3\CMS\Core\Database\Connection;
use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\Database\Query\Restriction\RootLevelRestriction;
use TYPO3\CMS\Core\Database\Query\Restriction\WorkspaceRestriction;
use TYPO3\CMS\Core\Database\Schema\Information\ColumnInfo;
use TYPO3\CMS\Core\Resource\Event\AfterFileMetaDataCreatedEvent;
use TYPO3\CMS\Core\Resource\Event\AfterFileMetaDataDeletedEvent;
use TYPO3\CMS\Core\Resource\Event\AfterFileMetaDataUpdatedEvent;
use TYPO3\CMS\Core\Resource\Event\EnrichFileMetaDataEvent;
use TYPO3\CMS\Core\Resource\Exception\InvalidUidException;
use TYPO3\CMS\Core\Resource\File;
use TYPO3\CMS\Core\Resource\FileType;
use TYPO3\CMS\Core\Type\File\ImageInfo;
use TYPO3\CMS\Core\Utility\GeneralUtility;
/**
* Repository Class as an abstraction layer to sys_file_metadata
*
* Every access to table sys_file_metadata which is not handled by DataHandler
* has to use this Repository class
*/
#[Autoconfigure(public: true)]
readonly class MetaDataRepository
{
public function __construct(
private EventDispatcherInterface $eventDispatcher,
private ConnectionPool $connectionPool,
private Context $context,
) {}
/**
* Returns array of meta-data properties
*/
public function findByFile(File $file): array
{
$record = $this->findByFileUid($file->getUid());
// It could be possible that the meta information is freshly
// created and inserted into the database. If this is the case
// we have to take care about correct meta information for width and
// height in case of an image.
// This logic can be transferred into a custom PSR-14 event listener in the future by just using
// the AfterMetaDataCreated event.
if (!empty($record['crdate']) && (int)$record['crdate'] === $GLOBALS['EXEC_TIME']) {
if ($file->isType(FileType::IMAGE) && $file->getStorage()->getDriverType() === 'Local') {
$fileNameAndPath = $file->getForLocalProcessing(false);
$imageInfo = GeneralUtility::makeInstance(ImageInfo::class, $fileNameAndPath);
$additionalMetaInformation = [
'width' => $imageInfo->getWidth(),
'height' => $imageInfo->getHeight(),
];
$this->update($file->getUid(), $additionalMetaInformation, $record);
}
$record = $this->findByFileUid($file->getUid());
}
return $record;
}
/**
* Retrieves metadata for file
*
* @param int $uid
* @return array<string, string> $metaData
* @throws InvalidUidException
*/
public function findByFileUid(int $uid): array
{
if ($uid <= 0) {
throw new InvalidUidException('Metadata can only be retrieved for indexed files. UID: "' . $uid . '"', 1381590731);
}
$queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_file_metadata');
$queryBuilder->getRestrictions()
->add(GeneralUtility::makeInstance(RootLevelRestriction::class))
->add(GeneralUtility::makeInstance(WorkspaceRestriction::class, $this->context->getAspect('workspace')->getId()));
$record = $queryBuilder
->select('*')
->from('sys_file_metadata')
->where(
$queryBuilder->expr()->eq('file', $queryBuilder->createNamedParameter($uid, Connection::PARAM_INT)),
$queryBuilder->expr()->in('language_tag', $queryBuilder->createNamedParameter([\Local\Multilanguage\Service\DefaultLanguageTagService::getTag(), ''], Connection::PARAM_STR_ARRAY))
)
// assure deterministic sorting across all databases
->orderBy('uid', 'ASC')
->setMaxResults(1)
->executeQuery()
->fetchAssociative();
if (empty($record)) {
return [];
}
return $this->eventDispatcher->dispatch(new EnrichFileMetaDataEvent($uid, (int)$record['uid'], $record))->getRecord();
}
/**
* Create empty
*/
public function createMetaDataRecord(int $fileUid, array $additionalFields = []): array
{
$emptyRecord = [
'file' => $fileUid,
'pid' => 0,
'crdate' => $GLOBALS['EXEC_TIME'],
'tstamp' => $GLOBALS['EXEC_TIME'],
'l10n_diffsource' => '',
];
$additionalFields = array_intersect_key($additionalFields, $this->getTableFields());
$emptyRecord = array_merge($emptyRecord, $additionalFields);
$connection = $this->connectionPool->getConnectionForTable('sys_file_metadata');
$connection->insert(
'sys_file_metadata',
$emptyRecord,
['l10n_diffsource' => Connection::PARAM_LOB]
);
$record = $emptyRecord;
$record['uid'] = $connection->lastInsertId();
return $this->eventDispatcher->dispatch(new AfterFileMetaDataCreatedEvent($fileUid, (int)$record['uid'], $record))->getRecord();
}
/**
* Updates the metadata record in the database
*
* @param int $fileUid the file uid to update
* @param array $updateData Data to update
* @param ?array $metaDataFromDatabase Current meta data from database
* @return array The updated database record - or just $metaDataFromDatabase if no update was done
* @internal
*/
public function update(int $fileUid, array $updateData, ?array $metaDataFromDatabase = null): array
{
// backwards compatibility layer
$metaDataFromDatabase ??= $this->findByFileUid($fileUid);
$updateRow = array_intersect_key($updateData, $this->getTableFields());
if ($updateRow === []) {
// No valid keys to update - return current database row
return $metaDataFromDatabase;
}
if (array_key_exists('uid', $updateRow)) {
unset($updateRow['uid']);
}
$updateRow = array_diff_assoc($updateRow, $metaDataFromDatabase);
if ($updateRow === []) {
// Nothing to update - return current database row
return $metaDataFromDatabase;
}
$updateRow['tstamp'] = time();
$this->connectionPool->getConnectionForTable('sys_file_metadata')->update(
'sys_file_metadata',
$updateRow,
[
'uid' => (int)$metaDataFromDatabase['uid'],
]
);
return $this->eventDispatcher->dispatch(
new AfterFileMetaDataUpdatedEvent($fileUid, (int)$metaDataFromDatabase['uid'], array_merge($metaDataFromDatabase, $updateRow))
)->getRecord();
}
/**
* Remove all metadata records for a certain file from the database
*
* @param int $fileUid
*/
public function removeByFileUid(int $fileUid): void
{
$this->connectionPool->getConnectionForTable('sys_file_metadata')->delete(
'sys_file_metadata',
[
'file' => $fileUid,
]
);
$this->eventDispatcher->dispatch(new AfterFileMetaDataDeletedEvent($fileUid));
}
/**
* Gets the fields that are available in the table
*
* @return array<string, ColumnInfo>
*/
protected function getTableFields(): array
{
return $this->connectionPool
->getConnectionForTable('sys_file_metadata')
->getSchemaInformation()
->listTableColumnInfos('sys_file_metadata');
}
}