<?php

declare(strict_types=1);

/*
 * This file is part of the TYPO3 CMS project.
 *
 * It is free software; you can redistribute it and/or modify it under
 * the terms of the GNU General Public License, either version 2
 * of the License, or any later version.
 *
 * For the full copyright and license information, please read the
 * LICENSE.txt file that was distributed with this source code.
 *
 * The TYPO3 project - inspiring people to share!
 */

namespace TYPO3\CMS\Impexp;

use Symfony\Component\DependencyInjection\Attribute\Autoconfigure;
use TYPO3\CMS\Backend\Utility\BackendUtility;
use TYPO3\CMS\Core\Configuration\Exception\SiteConfigurationWriteException;
use TYPO3\CMS\Core\Configuration\FlexForm\FlexFormTools;
use TYPO3\CMS\Core\Configuration\SiteWriter;
use TYPO3\CMS\Core\Database\Connection;
use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\DataHandling\DataHandler;
use TYPO3\CMS\Core\DataHandling\TableColumnType;
use TYPO3\CMS\Core\Exception;
use TYPO3\CMS\Core\Exception\SiteNotFoundException;
use TYPO3\CMS\Core\Resource\File;
use TYPO3\CMS\Core\Resource\ResourceInstructionTrait;
use TYPO3\CMS\Core\Resource\ResourceStorage;
use TYPO3\CMS\Core\Resource\StorageRepository;
use TYPO3\CMS\Core\Schema\Capability\RootLevelCapability;
use TYPO3\CMS\Core\Schema\Capability\TcaSchemaCapability;
use TYPO3\CMS\Core\Serializer\Typo3XmlParser;
use TYPO3\CMS\Core\Serializer\Typo3XmlSerializerOptions;
use TYPO3\CMS\Core\Site\SiteFinder;
use TYPO3\CMS\Core\Utility\ExtensionManagementUtility;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Core\Utility\MathUtility;
use TYPO3\CMS\Core\Utility\PathUtility;
use TYPO3\CMS\Core\Utility\StringUtility;
use TYPO3\CMS\Impexp\Exception\ImportFailedException;
use TYPO3\CMS\Impexp\Exception\LoadingFileFailedException;
use TYPO3\CMS\Impexp\Exception\PrerequisitesNotMetException;

/**
 * T3D file Import library (TYPO3 Record Document)
 *
 * @internal This class is not considered part of the public TYPO3 API.
 */
#[Autoconfigure(public: true, shared: false)]
class Import extends ImportExport
{
    use ResourceInstructionTrait;

    public const IMPORT_MODE_FORCE_UID = 'force_uid';
    public const IMPORT_MODE_AS_NEW = 'as_new';
    public const IMPORT_MODE_EXCLUDE = 'exclude';
    public const IMPORT_MODE_IGNORE_PID = 'ignore_pid';
    public const IMPORT_MODE_RESPECT_PID = 'respect_pid';

    public const SOFTREF_IMPORT_MODE_EXCLUDE = 'exclude';
    public const SOFTREF_IMPORT_MODE_EDITABLE = 'editable';

    protected string $mode = 'import';

    /**
     * Used to register the forced UID values for imported records that we want
     * to create with the same UIDs as in the import file. Admin-only feature.
     */
    protected array $suggestedInsertUids = [];

    /**
     * Disable logging when importing
     */
    protected bool $enableLogging = false;

    /**
     * Keys are [tablename]:[new NEWxxx ids (or when updating it is uids)]
     * while values are arrays with table/uid of the original record it is based on.
     * With the array keys the new ids can be looked up inside DataHandler
     */
    protected array $importNewId = [];

    /**
     * Page id map for page tree (import)
     */
    protected array $importNewIdPids = [];

    private array $supportedFileExtensions = [
        'xml',
        't3d',
    ];
    protected bool $isFilesSavedOutsideImportFile = false;

    /**
     * Array of currently registered storage objects
     *
     * @var ResourceStorage[]
     */
    protected array $storages = [];

    /**
     * Array of currently registered storage objects available for importing files to
     *
     * @var ResourceStorage[]
     */
    protected array $storagesAvailableForImport = [];

    /**
     * Currently registered default storage object
     */
    protected ?ResourceStorage $defaultStorage = null;

    /**
     * When false, processSiteConfigurations() is skipped. Set to false when the caller
     * (e.g. a distribution installer) handles site configuration import itself.
     */
    protected bool $importSiteConfigurations = true;

    public function __construct(
        protected readonly FlexFormTools $flexFormTools,
        protected readonly StorageRepository $storageRepository,
        protected readonly ConnectionPool $connectionPool,
        protected readonly SiteWriter $siteWriter,
        protected readonly SiteFinder $siteFinder,
    ) {
        $this->fetchStorages();
    }

    /**
     * Disable automatic site configuration import. Call this when the invoking
     * code (e.g. a distribution installer) handles site configuration itself.
     */
    public function disableSiteConfigurationImport(): void
    {
        $this->importSiteConfigurations = false;
    }

    /**
     * Fetch all available file storages and index by storage UID
     *
     * Note: It also creates a default storage record if the database table sys_file_storage is empty,
     * e.g. during tests.
     */
    protected function fetchStorages(): void
    {
        $this->storages = [];
        $this->storagesAvailableForImport = [];
        $this->defaultStorage = null;

        $this->storageRepository->flush();

        $storages = $this->storageRepository->findAll();
        foreach ($storages as $storage) {
            $this->storages[$storage->getUid()] = $storage;
            if ($storage->isOnline() && $storage->isWritable() && $storage->getDriverType() === 'Local') {
                $this->storagesAvailableForImport[$storage->getUid()] = $storage;
            }
            if ($this->defaultStorage === null && $storage->isDefault()) {
                $this->defaultStorage = $storage;
            }
        }
    }

    /**
     * Loads the TYPO3 import file $fileName into memory.
     *
     * @param string $fileName File path, has to be within the TYPO3's base folder
     * @throws LoadingFileFailedException
     */
    public function loadFile(string $fileName): void
    {
        $filePath = GeneralUtility::getFileAbsFileName($fileName);

        if (empty($filePath)) {
            $this->addError('File path is not valid: ' . $fileName);
        } elseif (!@is_file($filePath)) {
            $this->addError('File not found: ' . $filePath);
        }

        if ($this->hasErrors()) {
            throw new LoadingFileFailedException(
                sprintf('Loading of the import file "%s" failed.', $fileName),
                1484484619
            );
        }

        $pathInfo = pathinfo($filePath);
        $fileExtension = strtolower($pathInfo['extension']);

        if (!in_array($fileExtension, $this->supportedFileExtensions, true)) {
            $this->addError(
                sprintf(
                    'File extension "%s" is not valid. Supported file extensions are %s.',
                    $fileExtension,
                    implode(', ', array_map(static function ($supportedFileExtension) {
                        return '"' . $supportedFileExtension . '"';
                    }, $this->supportedFileExtensions))
                )
            );
        }

        if ($this->hasErrors() === false) {
            if (@is_dir($filePath . '.files')) {
                if (GeneralUtility::isAllowedAbsPath($filePath . '.files')) {
                    // copy the folder lowlevel to typo3temp, because the files would be deleted after import
                    GeneralUtility::copyDirectory($filePath . '.files', $this->getOrCreateTemporaryFolderName());
                } else {
                    $this->addError('External import files for the given import source is currently not supported.');
                }
                $this->isFilesSavedOutsideImportFile = true;
            } else {
                $this->isFilesSavedOutsideImportFile = false;
            }
            if ($fileExtension === 'xml') {
                $xmlContent = (string)file_get_contents($filePath);
                if (strlen($xmlContent)) {
                    try {
                        $dat = (new Typo3XmlParser())->decode(
                            $xmlContent,
                            new Typo3XmlSerializerOptions([
                                Typo3XmlSerializerOptions::RETURN_ROOT_NODE_NAME => true,
                                Typo3XmlSerializerOptions::LOAD_OPTIONS => \LIBXML_NONET | \LIBXML_NOBLANKS | \LIBXML_PARSEHUGE,
                                // @todo check if needed for imports/throw deprecation for invalid xml
                                Typo3XmlSerializerOptions::ALLOW_UNDEFINED_NAMESPACES,
                            ])
                        );
                        $this->dat = is_array($dat) ? $dat : [$dat];
                        if ($this->dat['_DOCUMENT_TAG'] === 'T3RecordDocument' && is_array($this->dat['header'] ?? null) && is_array($this->dat['records'] ?? null)) {
                            $this->loadInit();
                        } else {
                            $this->addError('XML file did not contain proper XML for TYPO3 Import');
                        }
                    } catch (\Throwable $e) {
                        $this->addError('XML could not be parsed: ' . $e->getMessage());
                    }
                } else {
                    $this->addError('Error opening file: ' . $filePath);
                }
            } elseif ($fileExtension === 't3d') {
                if ($fd = fopen($filePath, 'rb')) {
                    $this->dat['header'] = $this->getNextFilePart($fd, 'header');
                    $this->dat['records'] = $this->getNextFilePart($fd, 'records');
                    $this->dat['files'] = $this->getNextFilePart($fd, 'files');
                    $this->dat['files_fal'] = $this->getNextFilePart($fd, 'files_fal');
                    $this->loadInit();
                    fclose($fd);
                } else {
                    $this->addError('Error opening file: ' . $filePath);
                }
            }
        }

        if ($this->hasErrors()) {
            throw new LoadingFileFailedException(
                sprintf('Loading of the import file "%s" failed.', $fileName),
                1484484620
            );
        }
    }

    /**
     * Extracts the next content part of the T3D file
     *
     * @param resource $fd Import file pointer
     * @param string $name For error messages this indicates the section of the problem.
     * @return array|null Data array or NULL in case of an error
     */
    protected function getNextFilePart($fd, string $name): ?array
    {
        $headerLength = 32 + 1 + 1 + 1 + 10 + 1;
        $headerString = fread($fd, $headerLength);
        if (empty($headerString)) {
            $this->addError('File does not contain data for "' . $name . '"');
            return null;
        }

        $header = explode(':', $headerString);
        if (str_contains($header[0], 'Warning')) {
            $this->addError('File read error: Warning message in file. (' . $headerString . fgets($fd) . ')');
            return null;
        }
        if ((string)$header[3] !== '') {
            $this->addError('File read error: InitString had a wrong length. (' . $name . ')');
            return null;
        }

        $dataString = (string)fread($fd, (int)$header[2]);
        $isDataCompressed = $header[1] === '1';
        fread($fd, 1);
        if (!hash_equals($header[0], md5($dataString))) {
            $this->addError('MD5 check failed (' . $name . ')');
            return null;
        }

        if ($isDataCompressed) {
            if (!function_exists('gzuncompress')) {
                $this->addError('Content read error: This file requires decompression, '
                    . 'but this server does not offer gzcompress()/gzuncompress() functions.');
                return null;
            }
            $dataString = (string)gzuncompress($dataString);
        }

        return unserialize($dataString, ['allowed_classes' => false]) ?: null;
    }

    /**
     * Setting up the object based on the recently loaded ->dat array
     */
    protected function loadInit(): void
    {
        $this->relStaticTables = (array)($this->dat['header']['relStaticTables'] ?? []);
        $this->excludeMap = (array)($this->dat['header']['excludeMap'] ?? []);
        $this->softrefCfg = (array)($this->dat['header']['softrefCfg'] ?? []);
    }

    public function getMetaData(): array
    {
        return $this->dat['header']['meta'] ?? [];
    }

    public function getSiteConfigurations(): array
    {
        $siteConfigurations = $this->dat['header']['site_configurations'] ?? [];
        foreach ($siteConfigurations as $identifier => $config) {
            $rootPageId = (int)($config['rootPageId'] ?? 0);
            $siteConfigurations[$identifier]['_rootPageTitle'] = $this->dat['header']['records']['pages'][$rootPageId]['title'] ?? '';
        }
        return $siteConfigurations;
    }

    /**
     * Checks all requirements that must be met before import.
     *
     * @throws PrerequisitesNotMetException
     */
    public function checkImportPrerequisites(): void
    {
        // Check #1: Extension dependencies
        $extKeysToInstall = [];
        foreach ($this->dat['header']['extensionDependencies'] ?? [] as $extKey) {
            if (!empty($extKey) && !ExtensionManagementUtility::isLoaded($extKey)) {
                $extKeysToInstall[] = $extKey;
            }
        }
        if ($extKeysToInstall !== []) {
            $this->addError(
                sprintf(
                    'Before you can import this file you need to install the extensions "%s".',
                    implode('", "', $extKeysToInstall)
                )
            );
        }

        // Check #2: Presence of imported storage paths
        foreach ($this->dat['header']['records']['sys_file_storage'] ?? [] as $sysFileStorageUid => $_) {
            $storageRecord = &$this->dat['records']['sys_file_storage:' . $sysFileStorageUid]['data'];
            if ($storageRecord['driver'] === 'Local'
                && ($storageRecord['is_writable'] ?? 1)
                && ($storageRecord['is_online'] ?? 1)
            ) {
                $storageMapUid = -1;
                foreach ($this->storages as $storage) {
                    if ($this->isEquivalentStorage($storage, $storageRecord)) {
                        $storageMapUid = $storage->getUid();
                        break;
                    }
                }
                // The storage from the import does not have an equivalent storage
                // in the current instance (same driver, same path, etc.). Before
                // the storage record can get inserted later on take care the path
                // it points to really exists and is accessible.
                if ($storageMapUid === -1) {
                    // Unset the storage record UID when trying to create the storage object
                    // as the record does not already exist in database. The constructor of the
                    // storage object will check whether the target folder exists and set the
                    // isOnline flag depending on the outcome.
                    $storageRecordWithUid0 = $storageRecord;
                    $storageRecordWithUid0['uid'] = 0;
                    $storageObject = $this->storageRepository->createFromRecord($storageRecordWithUid0);
                    if (!$storageObject->isOnline()) {
                        $configuration = $storageObject->getConfiguration();
                        $this->addError(
                            sprintf(
                                'The file storage "%s" does not exist. '
                                . 'Please create the directory prior to starting the import!',
                                $storageObject->getName() . $configuration['basePath']
                            )
                        );
                    }
                }
            }
        }

        if ($this->hasErrors()) {
            throw new PrerequisitesNotMetException(
                'Prerequisites for file import are not met.',
                1484484612
            );
        }
    }

    /**
     * Imports the memory data into the TYPO3 database.
     *
     * @throws ImportFailedException
     */
    public function importData(): void
    {
        $this->initializeImport();

        // Write sys_file_storages first
        $this->writeSysFileStorageRecords();
        // Write sys_file records and write the binary file data
        $this->writeSysFileRecords();
        // Write records, first pages, then the rest
        // Fields with "hard" relations to database, files and flexform fields are kept empty during this run
        $this->writePages();
        $this->writeRecords();
        // Finally all the file and database record references must be fixed. This is done after all records have supposedly
        // been written to database. $this->importMapId will indicate two things:
        // 1) that a record WAS written to db and
        // 2) that it has got a new id-number.
        $this->setRelations();
        // And when all database relations are in place, we can fix file and database relations in flexform fields
        // - since data structures often depend on relations to a DS record:
        $this->setFlexFormRelations();
        // Finally, traverse all records and process soft references with substitution attributes.
        $this->processSoftReferences();
        // Write site configurations bundled in the export, with rootPageId remapped to imported UIDs.
        // Skipped when the caller handles site configuration import itself (e.g. distribution installers)
        // or when the current user is not an admin (site configurations are admin-only).
        if ($this->importSiteConfigurations && $this->getBackendUser()->isAdmin()) {
            $this->processSiteConfigurations();
        }
        // Cleanup
        $this->removeTemporaryFolderName();

        if ($this->hasErrors()) {
            throw new ImportFailedException('The import has failed.', 1484484613);
        }
    }

    /**
     * Write site configurations embedded in the import file.
     * Skips any site whose identifier already exists.
     * Remaps rootPageId from the export UID to the newly imported UID.
     */
    protected function processSiteConfigurations(): void
    {
        $siteConfigurations = $this->dat['header']['site_configurations'] ?? [];
        if (!is_array($siteConfigurations) || $siteConfigurations === []) {
            return;
        }
        $importedPageIds = $this->importMapId['pages'] ?? [];
        foreach ($siteConfigurations as $siteIdentifier => $configuration) {
            if (!is_string($siteIdentifier) || !is_array($configuration)) {
                continue;
            }
            $exportedRootPageId = (int)($configuration['rootPageId'] ?? 0);
            $importedRootPageId = $importedPageIds[$exportedRootPageId] ?? null;
            if ($importedRootPageId === null) {
                continue;
            }

            // Skip if a site configuration already exists for this root page.
            try {
                $this->siteFinder->getSiteByRootPageId((int)$importedRootPageId);
                continue;
            } catch (SiteNotFoundException) {
                // No site exists yet — proceed with creating one.
            }

            // Find a free identifier — never merge into an existing site config.
            $targetIdentifier = $siteIdentifier;
            $counter = 0;
            while (true) {
                try {
                    $this->siteFinder->getSiteByIdentifier($targetIdentifier);
                    $targetIdentifier = $siteIdentifier . '-' . (++$counter);
                } catch (SiteNotFoundException) {
                    break;
                }
            }

            $configuration['rootPageId'] = (int)$importedRootPageId;
            $configuration['base'] = '/' . $targetIdentifier . '/';
            // @TODO Add error handling / routes etc where page ids are used and configured

            try {
                $this->siteWriter->write($targetIdentifier, $configuration);
            } catch (SiteConfigurationWriteException) {
                // Site configuration write failures are non-fatal; the imported data remains intact.
            }
        }
    }

    /**
     * Initialize all settings for the import
     */
    protected function initializeImport(): void
    {
        $this->doesImport = true;
        $this->importMapId = [];
        $this->importNewId = [];
        $this->importNewIdPids = [];
    }

    /**
     * Imports the sys_file_storage records from memory data.
     */
    protected function writeSysFileStorageRecords(): void
    {
        if (!isset($this->dat['header']['records']['sys_file_storage'])) {
            return;
        }

        $importData = [];

        $storageUidsToBeResetToDefaultStorage = [];
        foreach ($this->dat['header']['records']['sys_file_storage'] as $sysFileStorageUid => $_) {
            $storageRecord = &$this->dat['records']['sys_file_storage:' . $sysFileStorageUid]['data'];
            if ($storageRecord['driver'] === 'Local'
                && ($storageRecord['is_writable'] ?? 1)
                && ($storageRecord['is_online'] ?? 1)
            ) {
                foreach ($this->storages as $storage) {
                    if ($this->isEquivalentStorage($storage, $storageRecord)) {
                        $this->importMapId['sys_file_storage'][$sysFileStorageUid] = $storage->getUid();
                        break;
                    }
                }

                if (!isset($this->importMapId['sys_file_storage'][$sysFileStorageUid])) {
                    // Local, writable and online storage. May be used later for writing files.
                    // Does not currently exist, mark the storage for import.
                    $this->addSingle($importData, 'sys_file_storage', $sysFileStorageUid, 0);
                }
            } else {
                // Storage with non-local drivers can be imported, but must not be used to save files as you cannot
                // be sure that this is supported. In this case the default storage is used. Non-writable and
                // non-online storage may be created as duplicates because you were unable to check the detailed
                // configuration options at that time.
                $this->addSingle($importData, 'sys_file_storage', $sysFileStorageUid, 0);
                $storageUidsToBeResetToDefaultStorage[] = $sysFileStorageUid;
            }
        }

        // Write new storages to the database
        $dataHandler = $this->createDataHandler();
        // Because all records are submitted in the correct order with positive pid numbers,
        // we should internally reverse the order of submission.
        $dataHandler->reverseOrder = true;
        $dataHandler->isImporting = true;
        $dataHandler->start($importData, []);
        $dataHandler->process_datamap();
        $this->addToMapId($importData, $dataHandler->substNEWwithIDs);

        // Refresh internal storage representation after potential storage import
        $this->fetchStorages();

        // Map references of non-local / non-writable / non-online storages to the default storage
        $defaultStorageUid = $this->defaultStorage?->getUid();
        foreach ($storageUidsToBeResetToDefaultStorage as $storageUidToBeResetToDefaultStorage) {
            $this->importMapId['sys_file_storage'][$storageUidToBeResetToDefaultStorage] = $defaultStorageUid;
        }

        // Unset the sys_file_storage records to prevent an import in writeRecords()
        unset($this->dat['header']['records']['sys_file_storage']);
    }

    /**
     * Determines whether the passed storage object and the storage record (sys_file_storage) can be considered
     * equivalent during the import.
     *
     * @param ResourceStorage $storageObject The storage object which should get compared
     * @param array $storageRecord The storage record which should get compared
     * @return bool Returns TRUE if both storage representations can be considered equal
     */
    protected function isEquivalentStorage(ResourceStorage $storageObject, array &$storageRecord): bool
    {
        if ($storageObject->getDriverType() === $storageRecord['driver']
            && $storageObject->isWritable() === (bool)($storageRecord['is_writable'] ?? 1)
            && $storageObject->isOnline() === (bool)($storageRecord['is_online'] ?? 1)
        ) {
            $storageRecordConfiguration = $this->flexFormTools->convertFlexFormContentToArray($storageRecord['configuration'] ?? '');
            $storageObjectConfiguration = $storageObject->getConfiguration();
            if ($storageRecordConfiguration['pathType'] === $storageObjectConfiguration['pathType']
                && $storageRecordConfiguration['basePath'] === $storageObjectConfiguration['basePath']
            ) {
                return true;
            }
        }
        return false;
    }

    /**
     * Imports the sys_file records and the binary files data from internal data array.
     */
    protected function writeSysFileRecords(): void
    {
        if (!isset($this->dat['header']['records']['sys_file'])) {
            return;
        }

        $this->addGeneralErrorsByTable('sys_file');

        $temporaryFolder = $this->getOrCreateTemporaryFolderName();
        $sanitizedFolderMappings = [];

        foreach ($this->dat['header']['records']['sys_file'] as $sysFileUid => $_) {
            $fileRecord = &$this->dat['records']['sys_file:' . $sysFileUid]['data'];

            $temporaryFile = null;
            $temporaryFilePath = $temporaryFolder . '/' . $fileRecord['sha1'];

            if ($this->isFilesSavedOutsideImportFile) {
                if (is_file($temporaryFilePath) && sha1_file($temporaryFilePath) === $fileRecord['sha1']) {
                    $temporaryFile = $temporaryFilePath;
                } else {
                    $this->addError(sprintf(
                        'Error: Temporary file %s could not be found or does not match the checksum!',
                        $temporaryFilePath
                    ));
                    continue;
                }
            } else {
                $fileId = md5($fileRecord['storage'] . ':' . $fileRecord['identifier_hash']);
                if (isset($this->dat['files_fal'][$fileId]['content'])) {
                    $fileInfo = &$this->dat['files_fal'][$fileId];
                    if (GeneralUtility::writeFile($temporaryFilePath, $fileInfo['content'], true)) {
                        clearstatcache();
                        $temporaryFile = $temporaryFilePath;
                    } else {
                        $this->addError(sprintf(
                            'Error: Temporary file %s was not written as it should have been!',
                            $temporaryFilePath
                        ));
                        continue;
                    }
                } else {
                    $this->addError(sprintf('Error: No file found for ID %s', $fileId));
                    continue;
                }
            }

            $storageUid = $this->importMapId['sys_file_storage'][$fileRecord['storage']] ?? $fileRecord['storage'];
            if (isset($this->storagesAvailableForImport[$storageUid])) {
                $storage = $this->storagesAvailableForImport[$storageUid];
            } elseif ($storageUid === 0 || $storageUid === '0') {
                $storage = $this->storageRepository->findByUid(0);
            } elseif ($this->defaultStorage !== null) {
                $storage = $this->defaultStorage;
            } else {
                $this->addError(sprintf(
                    'Error: No storage available for the file "%s" with storage uid "%s"',
                    $fileRecord['identifier'],
                    $fileRecord['storage']
                ));
                continue;
            }

            $file = null;
            try {
                if ($storage->hasFile($fileRecord['identifier'])) {
                    /** @var File $file */
                    $file = $storage->getFile($fileRecord['identifier']);
                    if ($file->getSha1() !== $fileRecord['sha1']) {
                        $file = null;
                    }
                }
            } catch (Exception $e) {
                // @todo: Can this exception be thrown anywhere?
                $file = null;
            }

            if ($file === null) {
                $folderName = PathUtility::dirname(ltrim($fileRecord['identifier'], '/'));
                if (in_array($folderName, $sanitizedFolderMappings, true)) {
                    $folderName = $sanitizedFolderMappings[$folderName];
                }
                if (!$storage->hasFolder($folderName)) {
                    try {
                        $importFolder = $storage->createFolder($folderName);
                        if ($importFolder->getIdentifier() !== $folderName && !in_array($folderName, $sanitizedFolderMappings, true)) {
                            $sanitizedFolderMappings[$folderName] = $importFolder->getIdentifier();
                        }
                    } catch (Exception $e) {
                        $this->addError(sprintf(
                            'Error: Folder "%s" could not be created for file "%s" with storage uid "%s"',
                            $folderName,
                            $fileRecord['identifier'],
                            $fileRecord['storage']
                        ));
                        continue;
                    }
                } else {
                    $importFolder = $storage->getFolder($folderName);
                }

                $this->callHook('before_addSysFileRecord', [
                    'fileRecord' => $fileRecord,
                    'importFolder' => $importFolder,
                    'temporaryFile' => $temporaryFile,
                ]);

                try {
                    $this->skipResourceConsistencyCheckForCommands($storage, $temporaryFile, $fileRecord['name']);
                    $file = $storage->addFile($temporaryFile, $importFolder, $fileRecord['name']);
                } catch (Exception $e) {
                    $this->addError(sprintf(
                        'Error: File could not be added to the storage: "%s" with storage uid "%s"',
                        $fileRecord['identifier'],
                        $fileRecord['storage']
                    ));
                    continue;
                }

                if ($file->getSha1() !== $fileRecord['sha1']) {
                    $this->addError(sprintf(
                        'Error: The hash of the written file is not identical to the import data! '
                        . 'File could be corrupted! File: "%s" with storage uid "%s"',
                        $fileRecord['identifier'],
                        $fileRecord['storage']
                    ));
                }
            }

            // save the new uid in the import id map
            $this->importMapId['sys_file'][$fileRecord['uid']] = $file->getUid();
            $this->fixUidLocalInSysFileReferenceRecords((int)$fileRecord['uid'], $file->getUid());
        }

        // unset the sys_file records to prevent an import in writeRecords()
        unset($this->dat['header']['records']['sys_file']);
        // remove all sys_file_reference records that point to file records which are unknown
        // in the system to prevent exceptions
        $this->removeSysFileReferenceRecordsWithRelationToMissingFile();
    }

    /**
     * Normally the importer works like the following:
     * Step 1: import the records with cleared field values of relation fields (see addSingle())
     * Step 2: update the records with the right relation ids (see setRelations())
     *
     * In step 2 the saving fields of type "relation to sys_file_reference" checks the related sys_file_reference
     * record (created in step 1) with the FileExtensionFilter for matching file extensions of the related file.
     * To make this work correct, the uid_local of sys_file_reference records has to be not empty AND has to
     * relate to the correct (imported) sys_file record uid!
     *
     * This is fixed here.
     *
     * @param int $oldFileUid
     * @param int $newFileUid
     */
    protected function fixUidLocalInSysFileReferenceRecords(int $oldFileUid, int $newFileUid): void
    {
        if (!isset($this->dat['header']['records']['sys_file_reference'])) {
            return;
        }

        foreach ($this->dat['header']['records']['sys_file_reference'] as $sysFileReferenceUid => $_) {
            if (!isset($this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['hasBeenMapped'])
                && $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data']['uid_local'] == $oldFileUid
            ) {
                $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['hasBeenMapped'] = true;
                $this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data']['uid_local'] = $newFileUid;
            }
        }
    }

    /**
     * Removes all sys_file_reference records from the import data array that are pointing to sys_file records which
     * are missing in the import data to prevent exceptions on checking the related file started by the DataHandler.
     */
    protected function removeSysFileReferenceRecordsWithRelationToMissingFile(): void
    {
        if (!isset($this->dat['header']['records']['sys_file_reference'])) {
            return;
        }

        foreach ($this->dat['header']['records']['sys_file_reference'] as $sysFileReferenceUid => $_) {
            $fileReferenceRecord = &$this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]['data'];
            if (!in_array($fileReferenceRecord['uid_local'], (array)($this->importMapId['sys_file'] ?? []))) {
                unset($this->dat['header']['records']['sys_file_reference'][$sysFileReferenceUid]);
                unset($this->dat['records']['sys_file_reference:' . $sysFileReferenceUid]);
                $this->addError(sprintf(
                    'Error: sys_file_reference record "%s" with relation to sys_file record "%s"'
                    . ', which is not part of the import data, was not imported.',
                    $sysFileReferenceUid,
                    $fileReferenceRecord['uid_local']
                ));
            }
        }
    }

    /**
     * Writing page tree / pages to database:
     * If the operation is an update operation, the root of the page tree inside will be moved to $this->pid
     * unless it is the same as the root page from the import.
     */
    protected function writePages(): void
    {
        if (!isset($this->dat['header']['records']['pages'])) {
            return;
        }

        $importData = [];

        // Add page tree
        $remainingPages = $this->dat['header']['records']['pages'];
        if (is_array($this->dat['header']['pagetree'] ?? null)) {
            $pageList = [];
            $this->flatInversePageTree($this->dat['header']['pagetree'], $pageList);
            foreach ($pageList as $pageUid => $_) {
                $pid = $this->dat['header']['records']['pages'][$pageUid]['pid'] ?? null;
                if ($pid !== null) {
                    $pid = (int)$pid;
                }
                $pid = $this->importNewIdPids[$pid ?? ''] ?? $this->pid;
                $this->addSingle($importData, 'pages', (int)$pageUid, $pid);
                unset($remainingPages[$pageUid]);
            }
        }

        // Add remaining pages on root level
        foreach ($remainingPages as $pageUid => $_) {
            $this->addSingle($importData, 'pages', (int)$pageUid, $this->pid);
        }

        // Write pages to the database
        $dataHandler = $this->createDataHandler();
        $dataHandler->isImporting = true;
        $this->callHook('before_writeRecordsPages', [
            'tce' => $dataHandler,
            'data' => &$importData,
        ]);
        $dataHandler->suggestedInsertUids = $this->suggestedInsertUids;
        $dataHandler->start($importData, []);
        $dataHandler->process_datamap();
        $this->callHook('after_writeRecordsPages', [
            'tce' => $dataHandler,
        ]);
        $this->addToMapId($importData, $dataHandler->substNEWwithIDs);

        // Sort pages
        $this->writePagesOrder();
    }

    /**
     * Organize all updated pages in page tree so they are related like in the import file.
     * Only used for updates.
     */
    protected function writePagesOrder(): void
    {
        if (!$this->update || !is_array($this->dat['header']['pagetree'] ?? null)) {
            return;
        }

        $importCmd = [];

        // Get uid-pid relations and traverse them in order to map to possible new IDs
        $pageList = [];
        $this->flatInversePageTree($this->dat['header']['pagetree'], $pageList);
        foreach ($pageList as $pageUid => $pagePid) {
            if ($pagePid >= 0 && $this->doRespectPid('pages', $pageUid)) {
                // If the page has been assigned a new ID (because it was created), use that instead!
                if (!MathUtility::canBeInterpretedAsInteger($this->importNewIdPids[$pageUid])) {
                    if ($this->importMapId['pages'][$pageUid]) {
                        $mappedUid = $this->importMapId['pages'][$pageUid];
                        $importCmd['pages'][$mappedUid]['move'] = $pagePid;
                    }
                } else {
                    $importCmd['pages'][$pageUid]['move'] = $pagePid;
                }
            }
        }

        // Move pages in the database
        if (!empty($importCmd)) {
            $dataHandler = $this->createDataHandler();
            $this->callHook('before_writeRecordsPagesOrder', [
                'tce' => &$dataHandler,
                'data' => &$importCmd,
            ]);
            $dataHandler->start([], $importCmd);
            $dataHandler->process_cmdmap();
            $this->callHook('after_writeRecordsPagesOrder', [
                'tce' => &$dataHandler,
            ]);
        }
    }

    /**
     * Checks if the position of an updated record is configured to be corrected.
     * This can be disabled globally and changed individually for elements.
     *
     * @param string $table Table name
     * @param int $uid Record UID
     * @return bool TRUE if the position of the record should be updated to match the one in the import structure
     */
    protected function doRespectPid(string $table, int $uid): bool
    {
        return ($this->importMode[$table . ':' . $uid] ?? '') !== self::IMPORT_MODE_IGNORE_PID
            && (!$this->globalIgnorePid || ($this->importMode[$table . ':' . $uid] ?? '') === self::IMPORT_MODE_RESPECT_PID);
    }

    /**
     * Write all database records except pages (written in writePages())
     */
    protected function writeRecords(): void
    {
        $importData = [];

        // Write the rest of the records
        if (is_array($this->dat['header']['records'] ?? null)) {
            foreach ($this->dat['header']['records'] as $table => $records) {
                $this->addGeneralErrorsByTable($table);
                if (!$this->tcaSchemaFactory->has($table)) {
                    continue;
                }
                if ($table !== 'pages') {
                    $schema = $this->tcaSchemaFactory->get($table);
                    $rootLevelCapability = $schema->getCapability(TcaSchemaCapability::RestrictionRootLevel);
                    foreach ($records as $uid => $record) {
                        // PID: Set the main $this->pid, unless a NEW-id is found
                        $pid = isset($this->importMapId['pages'][$record['pid']])
                            ? (int)$this->importMapId['pages'][$record['pid']]
                            : $this->pid;
                        if ($rootLevelCapability->getRootLevelType() === RootLevelCapability::TYPE_ONLY_ON_ROOTLEVEL) {
                            $pid = 0;
                        } elseif (!$rootLevelCapability->canExistOnRootLevel() && $pid === 0) {
                            $this->addError('Error: Record type ' . $table . ' is not allowed on pid 0');
                            continue;
                        }
                        // Add record
                        $this->addSingle($importData, $table, $uid, $pid);
                    }
                }
            }
        } else {
            $this->addError('Error: No records defined in internal data array.');
        }

        // Write records to the database
        $dataHandler = $this->createDataHandler();
        $this->callHook('before_writeRecordsRecords', [
            'tce' => $dataHandler,
            'data' => &$importData,
        ]);
        $dataHandler->suggestedInsertUids = $this->suggestedInsertUids;
        // Because all records are submitted in the correct order with positive pid numbers,
        // we should internally reverse the order of submission.
        $dataHandler->reverseOrder = true;
        $dataHandler->isImporting = true;
        $dataHandler->start($importData, []);
        $dataHandler->process_datamap();
        $this->callHook('after_writeRecordsRecords', [
            'tce' => $dataHandler,
        ]);
        $this->addToMapId($importData, $dataHandler->substNEWwithIDs);

        // Sort records
        $this->writeRecordsOrder();
    }

    /**
     * Organize all updated records so they are related like in the import file.
     * Only used for updates.
     */
    protected function writeRecordsOrder(): void
    {
        if (!$this->update) {
            return;
        }

        $importCmd = [];

        $pageList = [];
        if (is_array($this->dat['header']['pagetree'] ?? null)) {
            $this->flatInversePageTree($this->dat['header']['pagetree'], $pageList);
        }
        // @todo: drop by-reference and write final $this->dat at the end of method?!
        if (is_array($this->dat['header']['pid_lookup'] ?? null)) {
            foreach ($this->dat['header']['pid_lookup'] as $pid => &$recordsByPid) {
                $mappedPid = $this->importMapId['pages'][$pid] ?? $this->pid;
                if (MathUtility::canBeInterpretedAsInteger($mappedPid)) {
                    foreach ($recordsByPid as $table => &$records) {
                        // If $mappedPid === $this->pid then we are on root level and we can consider to move pages as well!
                        // (they will not be in the page tree!)
                        if ($table !== 'pages' || !isset($pageList[$pid])) {
                            foreach (array_reverse(array_keys($records)) as $uid) {
                                if ($this->doRespectPid($table, (int)$uid)) {
                                    if (isset($this->importMapId[$table][$uid])) {
                                        $mappedUid = $this->importMapId[$table][$uid];
                                        $importCmd[$table][$mappedUid]['move'] = $mappedPid;
                                    }
                                }
                            }
                        }
                    }
                }
            }
        }

        // Move records in the database
        if (!empty($importCmd)) {
            $dataHandler = $this->createDataHandler();
            $this->callHook('before_writeRecordsRecordsOrder', [
                'tce' => $dataHandler,
                'data' => &$importCmd,
            ]);
            $dataHandler->start([], $importCmd);
            $dataHandler->process_cmdmap();
            $this->callHook('after_writeRecordsRecordsOrder', [
                'tce' => $dataHandler,
            ]);
        }
    }

    /**
     * Adds a single record to the $importData array. Also copies files to the temporary folder.
     * However all file and database references and flexform fields are set to blank for now!
     * That is processed with setRelations() later.
     *
     * @param array $importData Data to be modified or inserted in the database during import
     * @param string $table Table name
     * @param int $uid Record UID
     * @param int|string $pid Page id or NEW-id, e.g. "NEW5fb3c2641281c885267727"
     */
    protected function addSingle(array &$importData, string $table, int $uid, $pid): void
    {
        // @todo return modified $importData instead of by-reference.
        if (($this->importMode[$table . ':' . $uid] ?? '') === self::IMPORT_MODE_EXCLUDE) {
            return;
        }

        $record = $this->dat['records'][$table . ':' . $uid]['data'] ?? null;

        if (!is_array($record)) {
            if (!($table === 'pages' && $uid === 0)) {
                // On root level we don't want this error message.
                $this->addError('Error: No record was found in data array!');
            }
            return;
        }

        // Generate record ID
        $ID = StringUtility::getUniqueId('NEW');
        if ($this->update
            && $this->getRecordFromDatabase($table, $uid) !== null
            && ($this->importMode[$table . ':' . $uid] ?? '') !== self::IMPORT_MODE_AS_NEW
        ) {
            $ID = $uid;
        } elseif ($table === 'sys_file_metadata'
            && $record['language_tag'] === '0'
            && isset($this->importMapId['sys_file'][$record['file']])
        ) {
            // On adding sys_file records the belonging sys_file_metadata record was also created:
            // If there is one, the record needs to be overwritten instead of a new one created.
            $databaseRecord = $this->getSysFileMetaDataFromDatabase(
                $this->importMapId['sys_file'][$record['file']],
                0
            );
            if (is_array($databaseRecord)) {
                $this->importMapId['sys_file_metadata'][$record['uid']] = $databaseRecord['uid'];
                $ID = $databaseRecord['uid'];
            }
        }

        // Mapping of generated record ID to original record UID
        $this->importNewId[$table . ':' . $ID] = ['table' => $table, 'uid' => $uid];
        if ($table === 'pages') {
            $this->importNewIdPids[$uid] = $ID;
        }

        // Record data
        $importData[$table][$ID] = $record;
        $importData[$table][$ID]['tx_impexp_origuid'] = $importData[$table][$ID]['uid'];

        // Record permissions
        if ($table === 'pages') {
            // Have to reset the user/group IDs so pages are owned by the importing user.
            // Otherwise strange things may happen for non-admins!
            unset($importData[$table][$ID]['perms_userid']);
            unset($importData[$table][$ID]['perms_groupid']);
        }

        // Record UID and PID
        unset($importData[$table][$ID]['uid']);
        // - for existing record
        if (MathUtility::canBeInterpretedAsInteger($ID)) {
            unset($importData[$table][$ID]['pid']);
        }
        // - for new record
        else {
            $importData[$table][$ID]['pid'] = $pid;
            if ((($this->importMode[$table . ':' . $uid] ?? '') === self::IMPORT_MODE_FORCE_UID && $this->update
                || $this->forceAllUids)
                && $this->getBackendUser()->isAdmin()
            ) {
                $importData[$table][$ID]['uid'] = $uid;
                $this->suggestedInsertUids[$table . ':' . $uid] = 'DELETE';
            }
        }

        // Record relations
        $schema = $this->tcaSchemaFactory->get($table);
        foreach ($this->dat['records'][$table . ':' . $uid]['rels'] ?? [] as $field => &$relation) {
            switch ($relation['type'] ?? '') {
                case 'db':
                case 'file':
                    // Set blank now, fix later in setRelations(),
                    // because we need to know ALL newly created IDs before we can map relations!
                    // In the meantime we set NO values for relations.
                    //
                    // BUT for field uid_local of table sys_file_reference the relation MUST not be cleared here,
                    // because the value is already the uid of the right imported sys_file record.
                    // @see fixUidLocalInSysFileReferenceRecords()
                    // If it's empty or a uid to another record the FileExtensionFilter will throw an exception or
                    // delete the reference record if the file extension of the related record doesn't match.
                    if (!($table === 'sys_file_reference' && $field === 'uid_local') && $schema->hasField($field)) {
                        $importData[$table][$ID][$field] = $this->getReferenceDefaultValue($schema->getField($field)->getConfiguration());
                    }
                    $translationSourceFieldName = null;
                    if ($schema->isLanguageAware()) {
                        $languageCapability = $schema->getCapability(TcaSchemaCapability::Language);
                        $translationSourceFieldName = $languageCapability->getTranslationSourceField()?->getName();
                    }

                    // Set to "0" for integer fields, or else we will get a db error in DataHandler persistence.
                    if ($translationSourceFieldName && $field === $translationSourceFieldName) {
                        $importData[$table][$ID][$field] = 0;
                    }
                    break;
                case 'flex':
                    // Set blank now, fix later in setFlexFormRelations().
                    // In the meantime we set NO values for flexforms - this is mainly because file references
                    // inside will not be processed properly. In fact references will point to no file
                    // or existing files (in which case there will be double-references which is a big problem of
                    // course!).
                    //
                    // BUT for the field "configuration" of the table "sys_file_storage" the relation MUST NOT be
                    // cleared, because the configuration array contains only string values, which are furthermore
                    // important for the further import, e.g. the base path.
                    if (!($table === 'sys_file_storage' && $field === 'configuration')) {
                        $importData[$table][$ID][$field] = $this->getReferenceDefaultValue($schema->getField($field)->getConfiguration());
                    }
                    break;
            }
        }
    }

    /**
     * Get the default value for a reference field.
     *
     * @param array $configuration The TCA configuration of the accordant field
     */
    protected function getReferenceDefaultValue(array $configuration): int|float|string
    {
        if (!empty($configuration['MM']) || !empty($configuration['foreign_field'])) {
            return 0;
        }
        if (array_key_exists('default', $configuration)) {
            return $configuration['default'];
        }
        return '';
    }

    /**
     * Selects sys_file_metadata database record.
     */
    protected function getSysFileMetaDataFromDatabase(int $file, int $sysLanguageUid): ?array
    {
        $queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_file_metadata');
        $databaseRecord = $queryBuilder->select('uid')
            ->from('sys_file_metadata')
            ->where(
                $queryBuilder->expr()->eq(
                    'file',
                    $queryBuilder->createNamedParameter($file, Connection::PARAM_INT)
                ),
                $queryBuilder->expr()->eq(
                    'sys_language_uid',
                    $queryBuilder->createNamedParameter($sysLanguageUid, Connection::PARAM_INT)
                )
            )
            ->executeQuery()
            ->fetchAssociative();
        return is_array($databaseRecord) ? $databaseRecord : null;
    }

    /**
     * Store the mapping between the import file record UIDs and the final record UIDs in the database after import.
     *
     * @param array $importData Data to be modified or inserted in the database during import
     * @param array $substNEWwithIDs A map between the "NEW..." string IDs and the eventual record UID in database
     */
    protected function addToMapId(array $importData, array $substNEWwithIDs): void
    {
        foreach ($importData as $table => $records) {
            foreach ($records as $ID => $_) {
                $uid = $this->importNewId[$table . ':' . $ID]['uid'];
                if (isset($substNEWwithIDs[$ID])) {
                    $this->importMapId[$table][$uid] = $substNEWwithIDs[$ID];
                } elseif ($this->update) {
                    // Map same ID to same ID....
                    $this->importMapId[$table][$uid] = $ID;
                } else {
                    // If $this->importMapId contains already the right mapping, skip the error message.
                    // See special handling of sys_file_metadata in addSingle() => nothing to do.
                    if (!($table === 'sys_file_metadata'
                        && isset($this->importMapId[$table][$uid])
                        && $this->importMapId[$table][$uid] == $ID)
                    ) {
                        $this->addError(
                            'Possible error: ' . $table . ':' . $uid . ' had no new id assigned to it. '
                            . 'This indicates that the record was not added to database during import. '
                            . 'Please check changelog!'
                        );
                    }
                }
            }
        }
    }

    protected function createDataHandler(): DataHandler
    {
        $dataHandler = GeneralUtility::makeInstance(DataHandler::class);
        $dataHandler->dontProcessTransformations = true;
        $dataHandler->enableLogging = $this->enableLogging;
        return $dataHandler;
    }

    /**
     * At the end of the import process all file and database relations should be set properly.
     * This means that the relations to imported records are all recreated so that the imported
     * records are correctly related again.
     * Relations in flexform fields are processed in setFlexFormRelations() after this function.
     */
    protected function setRelations(): void
    {
        $updateData = [];

        foreach ($this->importNewId as $original) {
            $table = $original['table'];
            $uid = $original['uid'];

            if (isset($this->importMapId[$table][$uid])) {
                if (!$this->tcaSchemaFactory->has($table)) {
                    $this->addError(sprintf('Error: This record does not have a TCA schema! (%s:%s)', $table, $uid));
                } else {
                    $schema = $this->tcaSchemaFactory->get($table);
                    $actualUid = BackendUtility::wsMapId($table, $this->importMapId[$table][$uid]);
                    foreach ($this->dat['records'][$table . ':' . $uid]['rels'] ?? [] as $field => $relation) {
                        // Field "uid_local" of sys_file_reference needs no update because the correct reference uid was already written.
                        // @see ImportExport::fixUidLocalInSysFileReferenceRecords()
                        if (isset($relation['type']) && !($table === 'sys_file_reference' && $field === 'uid_local') && $relation['type'] === 'db') {
                            if (!$schema->hasField($field)) {
                                $this->addError(sprintf('Error: Missing TCA "config" for field "%s:%s"', $table, $field));
                            } elseif (is_array($relation['itemArray'] ?? null) && !empty($relation['itemArray'])) {
                                $fieldInfo = $schema->getField($field);
                                $actualRelations = $this->remapRelationsOfField($relation['itemArray'], $fieldInfo->getConfiguration(), $field);
                                $updateData[$table][$actualUid][$field] = implode(',', $actualRelations);
                            }
                        }
                    }
                }
            } else {
                $this->addError(sprintf('Error: This record does not appear to have been created! (%s:%s)', $table, $uid));
            }
        }

        if (!empty($updateData)) {
            $dataHandler = $this->createDataHandler();
            $dataHandler->isImporting = true;
            $this->callHook('before_setRelation', [
                'tce' => $dataHandler,
                'data' => &$updateData,
            ]);
            $dataHandler->start($updateData, []);
            $dataHandler->process_datamap();
            $this->callHook('after_setRelations', [
                'tce' => $dataHandler,
            ]);
        }
    }

    /**
     * Maps the original record UIDs of the relations to the actual UIDs of the imported records and returns relations
     * as strings of type [table]_[uid] - or file:[uid] or [public url] for field of type "group" and internal_type
     * "file_reference". These strings have the regular DataHandler input group/select type format which means
     * they will automatically be processed into a list of UIDs or MM relations.
     *
     * @param array $fieldRelations Relations with original record UIDs
     * @param array $fieldConfig TCA configuration of the record field the relations belong to
     * @param string $field The TCA fieldname of the relation operated on
     * @return array Array of relation strings with actual record UIDs
     */
    protected function remapRelationsOfField(array $fieldRelations, array $fieldConfig, string $field = ''): array
    {
        $actualRelations = [];
        foreach ($fieldRelations as $relation) {
            if (!$this->tcaSchemaFactory->has($relation['table'])) {
                $this->addError('Lost relation due to missing TCA schema: ' . $relation['table'] . ':' . $relation['id']);
            } elseif (isset($this->importMapId[$relation['table']][$relation['id']])) {
                $schema = $this->tcaSchemaFactory->get($relation['table']);
                $translationSourceFieldName = null;
                if ($schema->isLanguageAware()) {
                    $languageCapability = $schema->getCapability(TcaSchemaCapability::Language);
                    $translationSourceFieldName = $languageCapability->getTranslationSourceField()?->getName();
                }
                $actualUid = $this->importMapId[$relation['table']][$relation['id']];
                if ($fieldConfig['type'] === 'input' && isset($fieldConfig['wizards']['link'])) {
                    // If an input field has a relation to a sys_file record this need to be converted back to
                    // the public path. But use getPublicUrl() here, because could normally only be a local file path.
                    try {
                        $file = $this->resourceFactory->retrieveFileOrFolderObject($actualUid);
                        $actualRelations[] = $file->getPublicUrl();
                    } catch (\Exception) {
                        $actualRelations[] = 'file:' . $actualUid;
                    }
                } elseif ($translationSourceFieldName && $field === $translationSourceFieldName) {
                    // "l10n_source" is of type "passthrough" so the "_" syntax won't be replaced.
                    $actualRelations[] = $actualUid;
                } else {
                    $actualRelations[] = $relation['table'] . '_' . $actualUid;
                }
            } elseif ($this->isTableStatic($relation['table']) || $this->isRecordExcluded($relation['table'], (int)$relation['id']) || $relation['id'] < 0) {
                // Some select types could contain negative values, e.g. fe_groups (-1, -2).
                // This must be handled on both export and import.
                $actualRelations[] = $relation['table'] . '_' . $relation['id'];
            } else {
                $this->addError('Lost relation: ' . $relation['table'] . ':' . $relation['id']);
            }
        }

        return $actualRelations;
    }

    /**
     * After all database relations have been set in the end of the import (see setRelations()) then it is time to
     * correct all relations inside FlexForm fields. The reason for doing this after is that the setting of relations
     * may affect (quite often!) which data structure is used for the FlexForm field!
     */
    protected function setFlexFormRelations(): void
    {
        $updateData = [];

        foreach ($this->importNewId as $original) {
            $table = $original['table'];
            $uid = $original['uid'];

            if (isset($this->importMapId[$table][$uid])) {
                if (!$this->tcaSchemaFactory->has($table)) {
                    $this->addError(sprintf('Error: This record does not appear to have a TCA schema! (%s:%s)', $table, $uid));
                } else {
                    $schema = $this->tcaSchemaFactory->get($table);
                    $actualUid = BackendUtility::wsMapId($table, $this->importMapId[$table][$uid]);
                    foreach ($this->dat['records'][$table . ':' . $uid]['rels'] ?? [] as $field => $relation) {
                        // Field "configuration" of sys_file_storage needs no update because it has not been removed
                        // and has no relations.
                        // @see Import::addSingle()
                        if (isset($relation['type']) && $relation['type'] == 'flex' && !($table === 'sys_file_storage' && $field === 'configuration')) {
                            // Re-insert temporarily removed original FlexForm data as fallback
                            // @see Import::addSingle()
                            $updateData[$table][$actualUid][$field] = $this->dat['records'][$table . ':' . $uid]['data'][$field];
                            if (!empty($relation['flexFormRels']['db']) && $schema->hasField($field)) {
                                $fieldInfo = $schema->getField($field);
                                if (!$fieldInfo->isType(TableColumnType::FLEX)) {
                                    continue;
                                }
                                $actualRecord = BackendUtility::getRecord($table, $actualUid);
                                if (is_array($actualRecord)) {
                                    $schema = $this->tcaSchemaFactory->get($table);
                                    $dataStructureIdentifier = $this->flexFormTools->getDataStructureIdentifier(
                                        ['config' => $fieldInfo->getConfiguration()],
                                        $table,
                                        $field,
                                        $actualRecord,
                                        $schema
                                    );
                                    $dataStructure = $this->flexFormTools->parseDataStructureByIdentifier($dataStructureIdentifier, $schema);
                                    $flexFormData = (new Typo3XmlParser())->decodeWithReturningExceptionAsString(
                                        (string)($this->dat['records'][$table . ':' . $uid]['data'][$field] ?? ''),
                                        new Typo3XmlSerializerOptions([
                                            Typo3XmlSerializerOptions::ALLOW_UNDEFINED_NAMESPACES,
                                        ])
                                    );
                                    if (is_array($flexFormData['data'] ?? null)) {
                                        $flexFormData['data'] = $this->remapFlexFormRelationsInData(
                                            $flexFormData['data'],
                                            $dataStructure,
                                            $relation
                                        );
                                    }
                                    if (is_array($flexFormData['data'] ?? null)) {
                                        $updateData[$table][$actualUid][$field] = $flexFormData;
                                    }
                                }
                            }
                        }
                    }
                }
            } else {
                $this->addError(sprintf('Error: This record does not appear to have been created! (%s:%s)', $table, $uid));
            }
        }

        if (!empty($updateData)) {
            $dataHandler = $this->createDataHandler();
            $dataHandler->isImporting = true;
            $this->callHook('before_setFlexFormRelations', [
                'tce' => $dataHandler,
                'data' => &$updateData,
            ]);
            $dataHandler->start($updateData, []);
            $dataHandler->process_datamap();
            $this->callHook('after_setFlexFormRelations', [
                'tce' => $dataHandler,
            ]);
        }
    }

    private function remapFlexFormRelationsInData(array $data, array $dataStructure, array $relation): array
    {
        foreach ($dataStructure['sheets'] as $sheetKey => $sheetData) {
            foreach (($sheetData['ROOT']['el'] ?? []) as $sheetElementKey => $sheetElementTca) {
                if (($sheetElementTca['type'] ?? '') === 'array') {
                    // Section element.
                    if (!is_array($sheetElementTca['el'] ?? false) || !is_array($data[$sheetKey]['lDEF'][$sheetElementKey]['el'] ?? false)) {
                        continue;
                    }
                    foreach ($data[$sheetKey]['lDEF'][$sheetElementKey]['el'] as $valueSectionContainerKey => $valueSectionContainers) {
                        if (!is_array($valueSectionContainers ?? false)) {
                            continue;
                        }
                        foreach ($valueSectionContainers as $valueContainerType => $valueContainerElements) {
                            if (!is_array($sheetElementTca['el'][$valueContainerType]['el'] ?? false)) {
                                continue;
                            }
                            foreach ($sheetElementTca['el'][$valueContainerType]['el'] as $containerElement => $containerElementTca) {
                                if (!isset($data[$sheetKey]['lDEF'][$sheetElementKey]['el'][$valueSectionContainerKey][$valueContainerType]['el'][$containerElement]['vDEF'])) {
                                    continue;
                                }
                                $structurePath = $sheetKey . '/lDEF/' . $sheetElementKey . '/el/' . $valueSectionContainerKey . '/' . $valueContainerType . '/el/' . $containerElement . '/vDEF/';
                                $fieldRelations = $relation['flexFormRels']['db'][$structurePath]
                                    ?? $relation['flexFormRels']['db'][rtrim($structurePath, '/')]
                                    ?? null;
                                if (is_array($fieldRelations)) {
                                    $data[$sheetKey]['lDEF'][$sheetElementKey]['el'][$valueSectionContainerKey][$valueContainerType]['el'][$containerElement]['vDEF']
                                        = implode(',', $this->remapRelationsOfField($fieldRelations, $containerElementTca['config'] ?? []));
                                }
                            }
                        }
                    }
                } elseif (isset($data[$sheetKey]['lDEF'][$sheetElementKey]['vDEF'])) {
                    // Simple field element.
                    $structurePath = $sheetKey . '/lDEF/' . $sheetElementKey . '/vDEF/';
                    $fieldRelations = $relation['flexFormRels']['db'][$structurePath]
                        ?? $relation['flexFormRels']['db'][rtrim($structurePath, '/')]
                        ?? null;
                    if (is_array($fieldRelations)) {
                        $data[$sheetKey]['lDEF'][$sheetElementKey]['vDEF']
                            = implode(',', $this->remapRelationsOfField($fieldRelations, $sheetElementTca['config'] ?? []));
                    }
                }
            }
        }
        return $data;
    }

    /**************************
     * Import soft references
     *************************/

    /**
     * Processing of soft references
     */
    protected function processSoftReferences(): void
    {
        $updateData = [];

        foreach ($this->dat['header']['records'] ?? [] as $table => $records) {
            if (!$this->tcaSchemaFactory->has($table)) {
                continue;
            }
            $schema = $this->tcaSchemaFactory->get($table);
            foreach ($records as $uid => $record) {
                if (is_array($record['softrefs'] ?? null)) {
                    $actualUid = BackendUtility::wsMapId($table, $this->importMapId[$table][$uid] ?? 0);
                    // First, group soft references by record field ...
                    // (this could probably also have been done with $this->dat['records'] instead of $this->dat['header'])
                    $softrefs = [];
                    foreach ($record['softrefs'] as $softref) {
                        if ($softref['field'] && is_array($softref['subst'] ?? null) && $softref['subst']['tokenID']) {
                            $softrefs[$softref['field']][$softref['subst']['tokenID']] = $softref;
                        }
                    }
                    // ... then process only fields which require substitution.
                    foreach ($softrefs as $field => $softrefsByField) {
                        if ($schema->hasField($field)) {
                            $fieldInfo = $schema->getField($field);
                            if ($fieldInfo->isType(TableColumnType::FLEX)) {
                                $actualRecord = BackendUtility::getRecord($table, $actualUid, '*');
                                if (is_array($actualRecord)) {
                                    $schema = $this->tcaSchemaFactory->get($table);
                                    $dataStructureIdentifier = $this->flexFormTools->getDataStructureIdentifier(
                                        ['config' => $fieldInfo->getConfiguration()],
                                        $table,
                                        $field,
                                        $actualRecord,
                                        $schema
                                    );
                                    $dataStructure = $this->flexFormTools->parseDataStructureByIdentifier($dataStructureIdentifier, $schema);
                                    $flexFormData = (new Typo3XmlParser())->decodeWithReturningExceptionAsString(
                                        (string)($actualRecord[$field] ?? ''),
                                        new Typo3XmlSerializerOptions([
                                            Typo3XmlSerializerOptions::ALLOW_UNDEFINED_NAMESPACES,
                                        ])
                                    );
                                    if (is_array($flexFormData['data'] ?? null)) {
                                        $flexFormData['data'] = $this->processFlexFormSoftRefsInData(
                                            $flexFormData['data'],
                                            $dataStructure,
                                            $table,
                                            $uid,
                                            $field,
                                            $softrefsByField
                                        );
                                    }
                                    if (is_array($flexFormData['data'] ?? null)) {
                                        $updateData[$table][$actualUid][$field] = $flexFormData;
                                    }
                                }
                            } else {
                                // Get tokenizedContent string and proceed only if that is not blank:
                                $tokenizedContent = $this->dat['records'][$table . ':' . $uid]['rels'][$field]['softrefs']['tokenizedContent'] ?? '';
                                if ($tokenizedContent !== '') {
                                    $updateData[$table][$actualUid][$field] = $this->processSoftReferencesSubstTokens($tokenizedContent, $softrefsByField, $table, (string)$uid);
                                }
                            }
                        }
                    }
                }
            }
        }

        // Update soft references in the database
        $dataHandler = $this->createDataHandler();
        $dataHandler->isImporting = true;
        $this->callHook('before_processSoftReferences', [
            'tce' => $dataHandler,
            'data' => &$updateData,
        ]);
        $dataHandler->enableLogging = true;
        $dataHandler->start($updateData, []);
        $dataHandler->process_datamap();
        $this->callHook('after_processSoftReferences', [
            'tce' => $dataHandler,
        ]);
    }

    private function processFlexFormSoftRefsInData(array $data, array $dataStructure, string $table, string|int $origUid, string $field, array $softrefs): array
    {
        foreach ($dataStructure['sheets'] as $sheetKey => $sheetData) {
            foreach (($sheetData['ROOT']['el'] ?? []) as $sheetElementKey => $sheetElementTca) {
                if (($sheetElementTca['type'] ?? '') === 'array') {
                    // Section element.
                    if (!is_array($sheetElementTca['el'] ?? false) || !is_array($data[$sheetKey]['lDEF'][$sheetElementKey]['el'] ?? false)) {
                        continue;
                    }
                    foreach ($data[$sheetKey]['lDEF'][$sheetElementKey]['el'] as $valueSectionContainerKey => $valueSectionContainers) {
                        if (!is_array($valueSectionContainers ?? false)) {
                            continue;
                        }
                        foreach ($valueSectionContainers as $valueContainerType => $valueContainerElements) {
                            if (!is_array($sheetElementTca['el'][$valueContainerType]['el'] ?? false)) {
                                continue;
                            }
                            foreach (array_keys($sheetElementTca['el'][$valueContainerType]['el']) as $containerElement) {
                                if (!isset($data[$sheetKey]['lDEF'][$sheetElementKey]['el'][$valueSectionContainerKey][$valueContainerType]['el'][$containerElement]['vDEF'])) {
                                    continue;
                                }
                                $structurePath = $sheetKey . '/lDEF/' . $sheetElementKey . '/el/' . $valueSectionContainerKey . '/' . $valueContainerType . '/el/' . $containerElement . '/vDEF/';
                                $softrefsByPath = array_filter($softrefs, static fn(array $softref): bool => $softref['structurePath'] === $structurePath);
                                if (!empty($softrefsByPath)) {
                                    $tokenizedContent = $this->dat['records'][$table . ':' . $origUid]['rels'][$field]['flexFormRels']['softrefs'][$structurePath]['tokenizedContent'] ?? '';
                                    if ($tokenizedContent !== '') {
                                        $data[$sheetKey]['lDEF'][$sheetElementKey]['el'][$valueSectionContainerKey][$valueContainerType]['el'][$containerElement]['vDEF']
                                            = $this->processSoftReferencesSubstTokens($tokenizedContent, $softrefsByPath, $table, (string)$origUid);
                                    }
                                }
                            }
                        }
                    }
                } elseif (isset($data[$sheetKey]['lDEF'][$sheetElementKey]['vDEF'])) {
                    // Simple field element.
                    $structurePath = $sheetKey . '/lDEF/' . $sheetElementKey . '/vDEF/';
                    $softrefsByPath = array_filter($softrefs, static fn(array $softref): bool => $softref['structurePath'] === $structurePath);
                    if (!empty($softrefsByPath)) {
                        $tokenizedContent = $this->dat['records'][$table . ':' . $origUid]['rels'][$field]['flexFormRels']['softrefs'][$structurePath]['tokenizedContent'] ?? '';
                        if ($tokenizedContent !== '') {
                            $data[$sheetKey]['lDEF'][$sheetElementKey]['vDEF']
                                = $this->processSoftReferencesSubstTokens($tokenizedContent, $softrefsByPath, $table, (string)$origUid);
                        }
                    }
                }
            }
        }
        return $data;
    }

    /**
     * Substitution of soft reference tokens
     *
     * @param string $tokenizedContent Content of field with soft reference tokens in.
     * @param array $softrefs Soft references
     * @param string $table Table of record for which the processing occurs
     * @param string $uid UID of record from table
     * @return string The input content with tokens substituted according to entries in $softrefs
     */
    protected function processSoftReferencesSubstTokens(string $tokenizedContent, array $softrefs, string $table, string $uid): string
    {
        foreach ($softrefs as $softref) {
            $tokenID = $softref['subst']['tokenID'];
            $insertValue = $softref['subst']['tokenValue'];
            switch ((string)($this->softrefCfg[$tokenID]['mode'] ?? '')) {
                case self::SOFTREF_IMPORT_MODE_EXCLUDE:
                    // This is the same as handling static relations:
                    // Do not create or update the related file or record and do not change the link in any way,
                    // but use the link as it was when exported.
                    break;
                case self::SOFTREF_IMPORT_MODE_EDITABLE:
                    // This is the same as "exclude" with the option to manually edit the link before importing.
                    $insertValue = $this->softrefInputValues[$tokenID];
                    break;
                default:
                    // This is almost the same as handling relations:
                    // - Adjusting the record reference to link to the already imported record - if any.
                    [$tempTable, $tempUid] = explode(':', (string)($softref['subst']['recordRef'] ?? ':'));
                    if (isset($this->importMapId[$tempTable][$tempUid])) {
                        $insertValue = BackendUtility::wsMapId($tempTable, $this->importMapId[$tempTable][$tempUid]);
                        $tokenValue = (string)$softref['subst']['tokenValue'];
                        if (str_contains($tokenValue, ':')) {
                            [$tokenKey] = explode(':', $tokenValue);
                            $insertValue = $tokenKey . ':' . $insertValue;
                        }
                    }
            }
            // Finally, replace the soft reference token in tokenized content
            $tokenizedContent = str_replace('{softref:' . $tokenID . '}', (string)$insertValue, $tokenizedContent);
        }
        return $tokenizedContent;
    }

    /**
     * @param string $name Name of the hook
     * @param array $params Array with params
     */
    protected function callHook(string $name, array $params): void
    {
        foreach ($GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['ext/impexp/class.tx_impexp.php'][$name] ?? [] as $hook) {
            GeneralUtility::callUserFunction($hook, $params, $this);
        }
    }

    public function setEnableLogging(bool $enableLogging): void
    {
        $this->enableLogging = $enableLogging;
    }
}