This commit is contained in:
2026-09-15 10:35:56 +08:00
commit 782f8ea7aa
4086 changed files with 631288 additions and 0 deletions
@@ -0,0 +1,95 @@
<?php
declare(strict_types=1);
namespace Esun\BlogImport\Console\Command;
use Esun\BlogImport\Model\BlogImporter;
use Magento\Framework\App\Filesystem\DirectoryList;
use Magento\Framework\Console\Cli;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
class ImportBlogCommand extends Command
{
private const COMMAND_NAME = 'esun:blog:import';
public function __construct(
private readonly BlogImporter $blogImporter,
private readonly DirectoryList $directoryList
) {
parent::__construct(self::COMMAND_NAME);
}
protected function configure(): void
{
$this->setDescription('Import eSUN JSON and HTML files into Mageplaza Blog posts.');
$this->addOption('source', null, InputOption::VALUE_REQUIRED, 'Extracted import directory; defaults to var/import.');
$this->addOption('store-id', null, InputOption::VALUE_REQUIRED, 'Magento store ID assigned to posts.', '0');
$this->addOption(
'media-subdir',
null,
InputOption::VALUE_REQUIRED,
'Subdirectory beneath pub/media.',
'mageplaza/blog/post/uploads'
);
$this->addOption('dry-run', null, InputOption::VALUE_NONE, 'Validate and report without writing posts.');
$this->addOption(
'allow-missing-media',
null,
InputOption::VALUE_NONE,
'Continue when HTML references media files that are not present locally.'
);
}
protected function execute(InputInterface $input, OutputInterface $output): int
{
$sourceOption = $input->getOption('source');
$source = is_string($sourceOption) && trim($sourceOption) !== ''
? trim($sourceOption)
: $this->directoryList->getPath(DirectoryList::VAR_DIR) . DIRECTORY_SEPARATOR . 'import';
$storeId = filter_var($input->getOption('store-id'), FILTER_VALIDATE_INT);
if ($storeId === false || $storeId < 0) {
$output->writeln('<error>The --store-id option must be a non-negative integer.</error>');
return Cli::RETURN_FAILURE;
}
$mediaSubdirectory = trim((string)$input->getOption('media-subdir'));
$dryRun = (bool)$input->getOption('dry-run');
$output->writeln(sprintf('<info>%s eSUN Blog import from %s</info>', $dryRun ? 'Validating' : 'Importing', $source));
try {
$stats = $this->blogImporter->import(
$source,
$storeId,
$mediaSubdirectory,
$dryRun,
(bool)$input->getOption('allow-missing-media'),
$output->isVerbose() ? static function (string $message) use ($output): void {
$output->writeln($message);
} : null
);
} catch (\Throwable $exception) {
$output->writeln('<error>' . $exception->getMessage() . '</error>');
return Cli::RETURN_FAILURE;
}
$output->writeln(sprintf('JSON records: %d', $stats['json_records']));
$output->writeln(sprintf('Unique posts: %d', $stats['unique_posts']));
$output->writeln(sprintf('Duplicate records: %d', $stats['duplicate_records']));
$output->writeln(sprintf('Missing media: %d', $stats['missing_media']));
$output->writeln(sprintf(
'%-20s%d',
$dryRun ? 'Media to normalize:' : 'Media normalized:',
$stats['normalized_media']
));
if (!$dryRun) {
$output->writeln(sprintf('Created: %d', $stats['created']));
$output->writeln(sprintf('Updated: %d', $stats['updated']));
$output->writeln(sprintf('Unchanged: %d', $stats['unchanged']));
}
$output->writeln($dryRun ? '<info>Validation completed; no data was written.</info>' : '<info>Blog import completed.</info>');
return Cli::RETURN_SUCCESS;
}
}
@@ -0,0 +1,307 @@
<?php
declare(strict_types=1);
namespace Esun\BlogImport\Model;
use Magento\Framework\App\Cache\TypeListInterface;
use Magento\Framework\App\Filesystem\DirectoryList;
use Magento\Framework\App\ResourceConnection;
use Magento\Framework\Exception\LocalizedException;
use Magento\Framework\Filter\TranslitUrl;
use Mageplaza\Blog\Model\Post;
use Mageplaza\Blog\Model\PostFactory;
use Mageplaza\Blog\Model\ResourceModel\Category\CollectionFactory as CategoryCollectionFactory;
use Mageplaza\Blog\Model\ResourceModel\Post\CollectionFactory as PostCollectionFactory;
use Mageplaza\Blog\Model\ResourceModel\Post as PostResource;
class BlogImporter
{
private const MEDIUMTEXT_MAX_BYTES = 16777215;
private const MAGEPLAZA_POST_MEDIA_PREFIX = 'mageplaza/blog/post/';
public function __construct(
private readonly SourceReader $sourceReader,
private readonly HtmlProcessor $htmlProcessor,
private readonly MediaNormalizer $mediaNormalizer,
private readonly DirectoryList $directoryList,
private readonly TranslitUrl $translitUrl,
private readonly CategoryCollectionFactory $categoryCollectionFactory,
private readonly PostCollectionFactory $postCollectionFactory,
private readonly PostFactory $postFactory,
private readonly PostResource $postResource,
private readonly ResourceConnection $resourceConnection,
private readonly TypeListInterface $cacheTypeList
) {
}
/**
* @param callable(string): void|null $progress
* @return array<string, int>
* @throws LocalizedException
*/
public function import(
string $sourceDirectory,
int $storeId,
string $mediaSubdirectory,
bool $dryRun,
bool $allowMissingMedia,
?callable $progress = null
): array {
$mediaSubdirectory = trim($mediaSubdirectory, '/');
if ($mediaSubdirectory === '' || str_contains($mediaSubdirectory, '..')) {
throw new LocalizedException(__('The media subdirectory is invalid.'));
}
$source = $this->sourceReader->read($sourceDirectory);
$categoryIds = $this->resolveCategoryIds();
$prepared = [];
$missingMedia = [];
$mediaToNormalize = [];
$mediaRoot = $this->directoryList->getPath(DirectoryList::MEDIA)
. DIRECTORY_SEPARATOR . str_replace('/', DIRECTORY_SEPARATOR, $mediaSubdirectory);
foreach ($source['records'] as $record) {
$urlKey = $this->translitUrl->filter(rawurldecode((string)$record['source_url_key']));
if ($urlKey === '') {
throw new LocalizedException(__('URL key is empty after normalization: %1', $record['source_url']));
}
if (isset($prepared[$urlKey]) && $prepared[$urlKey]['source_url'] !== $record['source_url']) {
throw new LocalizedException(__('Two source URLs normalize to the same URL key: %1', $urlKey));
}
$processed = $this->htmlProcessor->process($record['html_path'], $mediaSubdirectory);
$contentBytes = strlen($processed['html']);
if ($contentBytes > self::MEDIUMTEXT_MAX_BYTES) {
throw new LocalizedException(
__(
'Processed HTML for %1 is %2 bytes and exceeds the MEDIUMTEXT limit of %3 bytes.',
$record['source_url'],
$contentBytes,
self::MEDIUMTEXT_MAX_BYTES
)
);
}
foreach ($processed['media'] as $relativePath) {
$mediaPath = $mediaRoot . DIRECTORY_SEPARATOR
. str_replace('/', DIRECTORY_SEPARATOR, $relativePath);
if (!is_file($mediaPath) || !is_readable($mediaPath)) {
$missingMedia[$relativePath] = $relativePath;
} elseif ($this->mediaNormalizer->needsNormalization($mediaPath)) {
$mediaToNormalize[$mediaPath] = $mediaPath;
}
}
if ($record['image_filename'] !== '') {
$coverPath = $mediaRoot . DIRECTORY_SEPARATOR . $record['image_filename'];
if (!is_file($coverPath) || !is_readable($coverPath)) {
$missingMedia[$record['image_filename']] = $record['image_filename'];
} elseif ($this->mediaNormalizer->needsNormalization($coverPath)) {
$mediaToNormalize[$coverPath] = $coverPath;
}
}
$record['url_key'] = $urlKey;
$record['post_content'] = $processed['html'];
$record['image'] = $this->buildPostImagePath($mediaSubdirectory, $record['image_filename']);
if (strlen($record['image']) > 255) {
throw new LocalizedException(__('Post image path exceeds 255 bytes for %1.', $record['source_url']));
}
$record['category_ids'] = array_values(array_map(
static fn(string $category): int => $categoryIds[$category],
array_values($record['categories'])
));
$prepared[$urlKey] = $record;
}
if ($missingMedia !== [] && !$allowMissingMedia) {
$sample = implode(', ', array_slice(array_values($missingMedia), 0, 10));
throw new LocalizedException(
__('Found %1 missing media file(s). First entries: %2', count($missingMedia), $sample)
);
}
$stats = [
'json_records' => (int)$source['json_records'],
'unique_posts' => count($prepared),
'duplicate_records' => (int)$source['duplicate_records'],
'missing_media' => count($missingMedia),
'normalized_media' => count($mediaToNormalize),
'created' => 0,
'updated' => 0,
'unchanged' => 0,
'failed' => 0,
];
if (!$dryRun) {
foreach ($mediaToNormalize as $mediaPath) {
$this->mediaNormalizer->normalize($mediaPath);
}
}
$resolvedPosts = [];
foreach ($prepared as $urlKey => $record) {
$resolvedPosts[$urlKey] = $this->loadPost($record);
}
if ($dryRun) {
return $stats;
}
foreach ($prepared as $urlKey => $record) {
$connection = $this->resourceConnection->getConnection();
$connection->beginTransaction();
try {
[$post, $state] = $resolvedPosts[$urlKey];
$changed = $this->populatePost($post, $record, $storeId);
if ($changed || $state === 'created') {
$this->postResource->save($post);
if ((string)$post->getUrlKey() !== $record['url_key']) {
throw new LocalizedException(
__('Mageplaza changed URL key %1 to %2.', $record['url_key'], $post->getUrlKey())
);
}
$stats[$state]++;
if ($progress !== null) {
$progress(sprintf('%s %s', ucfirst($state), $record['url_key']));
}
} else {
$stats['unchanged']++;
}
$connection->commit();
} catch (\Throwable $exception) {
$connection->rollBack();
$stats['failed']++;
throw new LocalizedException(
__('Failed importing %1: %2', $record['source_url'], $exception->getMessage()),
$exception
);
}
}
$this->cacheTypeList->invalidate('block_html');
$this->cacheTypeList->invalidate('full_page');
return $stats;
}
/** @return array<string, int> */
private function resolveCategoryIds(): array
{
$result = [];
foreach (['industry', 'activities', 'exhibition', 'news'] as $category) {
$collection = $this->categoryCollectionFactory->create()
->addFieldToFilter('url_key', $category)
->setPageSize(2);
if ((int)$collection->getSize() === 0) {
$collection = $this->categoryCollectionFactory->create()
->addFieldToFilter('name', $category)
->setPageSize(2);
}
if ((int)$collection->getSize() !== 1) {
throw new LocalizedException(
__('Expected exactly one Mageplaza Blog category for "%1"; found %2.', $category, $collection->getSize())
);
}
$result[$category] = (int)$collection->getFirstItem()->getId();
}
return $result;
}
/** @return array{0: Post, 1: 'created'|'updated'} */
private function loadPost(array $record): array
{
/** @var Post $post */
$post = $this->postCollectionFactory->create()
->addFieldToFilter('import_source', $record['source_url'])
->setPageSize(1)
->getFirstItem();
if ($post->getId()) {
$collision = $this->postCollectionFactory->create()
->addFieldToFilter('url_key', $record['url_key'])
->setPageSize(1)
->getFirstItem();
if ($collision->getId() && (int)$collision->getId() !== (int)$post->getId()) {
throw new LocalizedException(
__('URL key "%1" is already used by post ID %2.', $record['url_key'], $collision->getId())
);
}
return [$post, 'updated'];
}
$collision = $this->postCollectionFactory->create()
->addFieldToFilter('url_key', $record['url_key'])
->setPageSize(1)
->getFirstItem();
if ($collision->getId()) {
if (!$collision->getData('import_source')
&& trim((string)$collision->getName()) === trim((string)$record['title'])
) {
return [$collision, 'updated'];
}
throw new LocalizedException(
__('URL key "%1" is already used by post ID %2 with a different import source.', $record['url_key'], $collision->getId())
);
}
return [$this->postFactory->create(), 'created'];
}
private function populatePost(Post $post, array $record, int $storeId): bool
{
$data = [
'name' => $record['title'],
'short_description' => $record['short_description'],
'post_content' => $record['post_content'],
'image' => $record['image'],
'store_ids' => [$storeId],
'enabled' => 1,
'url_key' => $record['url_key'],
'in_rss' => 0,
'layout' => '2columns-left',
'allow_comment' => 0,
'meta_title' => $record['title'],
'publish_date' => $record['publish_date'] . ' 00:00:00',
'import_source' => $record['source_url'],
'categories_ids' => $record['category_ids'],
];
if (!$post->getId()) {
$data['created_at'] = $record['publish_date'] . ' 00:00:00';
}
$data['updated_at'] = gmdate('Y-m-d H:i:s');
$changed = !$post->getId();
foreach ($data as $key => $value) {
$oldValue = $key === 'categories_ids' ? $post->getCategoryIds() : $post->getData($key);
if ($key === 'store_ids' && is_string($oldValue)) {
$oldValue = array_filter(explode(',', $oldValue), 'strlen');
}
if ($key === 'updated_at') {
continue;
}
if (is_array($value)) {
$old = array_map('strval', (array)$oldValue);
$new = array_map('strval', $value);
sort($old);
sort($new);
$changed = $changed || $old !== $new;
} else {
$changed = $changed || (string)$oldValue !== (string)$value;
}
}
if ($changed) {
$post->addData($data);
}
return $changed;
}
private function buildPostImagePath(string $mediaSubdirectory, string $imageFilename): string
{
if ($imageFilename === '') {
return '';
}
if (str_starts_with($mediaSubdirectory . '/', self::MAGEPLAZA_POST_MEDIA_PREFIX)) {
$mediaSubdirectory = substr($mediaSubdirectory, strlen(self::MAGEPLAZA_POST_MEDIA_PREFIX));
}
return trim($mediaSubdirectory, '/') . '/' . $imageFilename;
}
}
@@ -0,0 +1,136 @@
<?php
declare(strict_types=1);
namespace Esun\BlogImport\Model;
use Magento\Framework\Exception\LocalizedException;
class HtmlProcessor
{
private const ESUN_MEDIA_HOSTS = ['esun3d.com', 'www.esun3d.com'];
private const MEDIA_EXTENSIONS = [
'avif', 'bmp', 'gif', 'jpeg', 'jpg', 'm4v', 'mov', 'mp4', 'ogg', 'png', 'svg', 'tif', 'tiff', 'webm', 'webp',
];
/**
* @return array{html: string, media: string[]}
* @throws LocalizedException
*/
public function process(string $htmlPath, string $mediaSubdirectory): array
{
$html = file_get_contents($htmlPath);
if ($html === false || trim($html) === '') {
throw new LocalizedException(__('HTML file is empty or unreadable: %1', $htmlPath));
}
$previous = libxml_use_internal_errors(true);
$document = new \DOMDocument('1.0', 'UTF-8');
$loaded = $document->loadHTML(
'<?xml encoding="UTF-8"><div id="esun-import-root">' . $html . '</div>',
LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD
);
libxml_clear_errors();
libxml_use_internal_errors($previous);
if (!$loaded) {
throw new LocalizedException(__('Unable to parse HTML file: %1', $htmlPath));
}
$xpath = new \DOMXPath($document);
$article = $xpath->query(
'//*[contains(concat(" ", normalize-space(@class), " "), " blog-article ")]'
)->item(0);
$contentRoot = $article instanceof \DOMElement
? $article
: $xpath->query('//*[@id="esun-import-root"]')->item(0);
if (!$contentRoot instanceof \DOMElement) {
throw new LocalizedException(__('No importable HTML content found in %1.', $htmlPath));
}
$removeQueries = [
'.//h1[contains(concat(" ", normalize-space(@class), " "), " news-title ")]',
'.//*[contains(concat(" ", normalize-space(@class), " "), " news-time ")]',
'.//script',
'.//*[contains(concat(" ", normalize-space(@class), " "), " picker ")]',
'./*[contains(concat(" ", normalize-space(@class), " "), " clear ")]',
];
foreach ($removeQueries as $query) {
$nodes = iterator_to_array($xpath->query($query, $contentRoot));
foreach ($nodes as $node) {
$node->parentNode?->removeChild($node);
}
}
$media = [];
$mediaReplacements = [];
foreach (iterator_to_array($xpath->query('.//*', $contentRoot)) as $element) {
if (!$element instanceof \DOMElement) {
continue;
}
foreach (['src', 'href', 'poster'] as $attribute) {
if (!$element->hasAttribute($attribute)) {
continue;
}
$relativeMediaPath = $this->getLegacyMediaPath($element->getAttribute($attribute));
if ($relativeMediaPath === null) {
continue;
}
$media[$relativeMediaPath] = $relativeMediaPath;
$placeholder = sprintf('__ESUN_IMPORTED_MEDIA_%d__', count($mediaReplacements));
$mediaReplacements[$placeholder] = sprintf(
"{{media url='%s/%s'}}",
$mediaSubdirectory,
str_replace("'", '&#039;', $relativeMediaPath)
);
$element->setAttribute($attribute, $placeholder);
}
}
$output = '';
foreach ($contentRoot->childNodes as $child) {
$output .= $document->saveHTML($child);
}
$output = trim(strtr($output, $mediaReplacements));
if ($output === '') {
throw new LocalizedException(__('HTML content became empty after processing: %1', $htmlPath));
}
return ['html' => $output, 'media' => array_values($media)];
}
private function getLegacyMediaPath(string $url): ?string
{
$url = html_entity_decode(trim($url), ENT_QUOTES | ENT_HTML5);
$path = '';
if (str_starts_with($url, '/uploads/')) {
$embeddedUrl = rawurldecode(substr($url, strlen('/uploads/')));
if (preg_match('#^(?:https?:)?//#iu', $embeddedUrl)) {
return null;
}
$path = $url;
} elseif (preg_match(
'#^(?:https?:)?//([^/]+)(/.*)$#iu',
$url,
$matches
)) {
if (!in_array(strtolower($matches[1]), self::ESUN_MEDIA_HOSTS, true)) {
return null;
}
$path = $matches[2];
}
if ($path === '') {
return null;
}
$path = (string)preg_split('/[?#]/', $path, 2)[0];
$filename = rawurldecode(basename($path));
if ($filename === '' || $filename === '.' || $filename === '..') {
return null;
}
$extension = strtolower(pathinfo($filename, PATHINFO_EXTENSION));
if (!in_array($extension, self::MEDIA_EXTENSIONS, true)) {
return null;
}
return $filename;
}
}
@@ -0,0 +1,236 @@
<?php
declare(strict_types=1);
namespace Esun\BlogImport\Model;
use Magento\Framework\Exception\LocalizedException;
use Symfony\Component\Process\Exception\ProcessFailedException;
use Symfony\Component\Process\Process;
class MediaNormalizer
{
/** @var array<string, int> */
private const EXPECTED_IMAGE_TYPES = [
'gif' => IMAGETYPE_GIF,
'jpeg' => IMAGETYPE_JPEG,
'jpg' => IMAGETYPE_JPEG,
'png' => IMAGETYPE_PNG,
];
public function needsNormalization(string $path): bool
{
$expectedType = $this->getExpectedType($path);
if ($expectedType === null) {
return false;
}
return $this->getActualType($path) !== $expectedType;
}
/**
* Convert image data in place so that its real format matches its filename extension.
*
* @throws LocalizedException
*/
public function normalize(string $path): bool
{
$expectedType = $this->getExpectedType($path);
if ($expectedType === null) {
return false;
}
$actualType = $this->getActualType($path);
if ($actualType === $expectedType) {
return false;
}
$createCallback = match ($actualType) {
IMAGETYPE_GIF => 'imagecreatefromgif',
IMAGETYPE_JPEG => 'imagecreatefromjpeg',
IMAGETYPE_PNG => 'imagecreatefrompng',
IMAGETYPE_WEBP => 'imagecreatefromwebp',
default => null,
};
if ($createCallback === null || !is_callable($createCallback)) {
throw new LocalizedException(
__('Cannot convert unsupported image data in %1.', $path)
);
}
$sourceImage = @$createCallback($path);
$temporaryPath = tempnam(dirname($path), '.esun-media-');
if ($temporaryPath === false) {
if ($sourceImage instanceof \GdImage) {
imagedestroy($sourceImage);
}
throw new LocalizedException(__('Unable to create a temporary image beside %1.', $path));
}
try {
$written = $sourceImage instanceof \GdImage
? $this->writeImage($sourceImage, $temporaryPath, $expectedType)
: $this->writeAnimatedWebp($path, $temporaryPath, $actualType, $expectedType);
if (!$written) {
throw new LocalizedException(__('Unable to encode image %1.', $path));
}
$permissions = fileperms($path);
if ($permissions !== false) {
@chmod($temporaryPath, $permissions & 0777);
}
if (!@rename($temporaryPath, $path)) {
throw new LocalizedException(__('Unable to replace image %1 after conversion.', $path));
}
} finally {
if ($sourceImage instanceof \GdImage) {
imagedestroy($sourceImage);
}
if (is_file($temporaryPath)) {
@unlink($temporaryPath);
}
}
return true;
}
private function getExpectedType(string $path): ?int
{
$extension = strtolower(pathinfo($path, PATHINFO_EXTENSION));
return self::EXPECTED_IMAGE_TYPES[$extension] ?? null;
}
/** @throws LocalizedException */
private function getActualType(string $path): int
{
$imageInfo = @getimagesize($path);
if ($imageInfo === false || !isset($imageInfo[2])) {
throw new LocalizedException(__('Unable to determine the real image format of %1.', $path));
}
return (int)$imageInfo[2];
}
private function writeImage(\GdImage $sourceImage, string $path, int $expectedType): bool
{
if ($expectedType === IMAGETYPE_JPEG) {
$flattened = imagecreatetruecolor(imagesx($sourceImage), imagesy($sourceImage));
if (!$flattened instanceof \GdImage) {
return false;
}
$white = imagecolorallocate($flattened, 255, 255, 255);
imagefill($flattened, 0, 0, $white);
imagecopy(
$flattened,
$sourceImage,
0,
0,
0,
0,
imagesx($sourceImage),
imagesy($sourceImage)
);
$written = imagejpeg($flattened, $path, 90);
imagedestroy($flattened);
return $written;
}
if ($expectedType === IMAGETYPE_PNG) {
imagealphablending($sourceImage, false);
imagesavealpha($sourceImage, true);
return imagepng($sourceImage, $path, 6);
}
return imagegif($sourceImage, $path);
}
/**
* GD cannot decode animated WebP. Use ffmpeg as a fallback and keep animation when the target is GIF.
*
* @throws LocalizedException
*/
private function writeAnimatedWebp(
string $sourcePath,
string $destinationPath,
int $actualType,
int $expectedType
): bool {
if ($actualType !== IMAGETYPE_WEBP) {
throw new LocalizedException(__('Unable to decode image %1.', $sourcePath));
}
$outputArguments = match ($expectedType) {
IMAGETYPE_GIF => ['-f', 'gif'],
IMAGETYPE_JPEG => ['-frames:v', '1', '-c:v', 'mjpeg', '-f', 'image2'],
IMAGETYPE_PNG => ['-frames:v', '1', '-c:v', 'png', '-f', 'image2'],
default => [],
};
$command = array_merge(
['ffmpeg', '-v', 'error', '-y', '-i', $sourcePath],
$outputArguments,
[$destinationPath]
);
$process = new Process($command);
$process->setTimeout(60);
try {
$process->mustRun();
} catch (ProcessFailedException $ffmpegException) {
@unlink($destinationPath);
try {
return $this->writeFirstWebpFrame($sourcePath, $destinationPath, $expectedType);
} catch (\Throwable $fallbackException) {
throw new LocalizedException(
__(
'Unable to convert animated WebP image %1. Install ffmpeg or the WebP tools (webpmux and dwebp). %2',
$sourcePath,
trim($fallbackException->getMessage())
),
$ffmpegException
);
}
}
return is_file($destinationPath) && filesize($destinationPath) > 0;
}
/**
* Extract a static first frame when ffmpeg cannot decode the animated WebP stream.
*/
private function writeFirstWebpFrame(string $sourcePath, string $destinationPath, int $expectedType): bool
{
$frameWebp = tempnam(dirname($sourcePath), '.esun-webp-frame-');
$framePng = tempnam(dirname($sourcePath), '.esun-png-frame-');
if ($frameWebp === false || $framePng === false) {
if (is_string($frameWebp)) {
@unlink($frameWebp);
}
if (is_string($framePng)) {
@unlink($framePng);
}
throw new LocalizedException(__('Unable to create temporary WebP frame files.'));
}
$frameImage = null;
try {
(new Process(['webpmux', '-get', 'frame', '1', $sourcePath, '-o', $frameWebp]))
->setTimeout(60)
->mustRun();
(new Process(['dwebp', $frameWebp, '-o', $framePng]))
->setTimeout(60)
->mustRun();
$frameImage = @imagecreatefrompng($framePng);
if (!$frameImage instanceof \GdImage) {
throw new LocalizedException(__('Unable to decode the extracted WebP frame.'));
}
return $this->writeImage($frameImage, $destinationPath, $expectedType);
} finally {
if ($frameImage instanceof \GdImage) {
imagedestroy($frameImage);
}
@unlink($frameWebp);
@unlink($framePng);
}
}
}
@@ -0,0 +1,242 @@
<?php
declare(strict_types=1);
namespace Esun\BlogImport\Model;
use Magento\Framework\Exception\LocalizedException;
use Magento\Framework\Serialize\Serializer\Json;
class SourceReader
{
private const SOURCE_FILES = [
'industry' => 'esun_industry_details.json',
'activities' => 'esun_activities_details.json',
'exhibition' => 'esun_exhibition_details.json',
'news' => 'esun_news_details.json',
];
private const LIST_FILES = [
'industry' => 'esun_industry_all.json',
'activities' => 'esun_activities_all.json',
'exhibition' => 'esun_exhibition_all.json',
'news' => 'esun_news_all.json',
];
public function __construct(private readonly Json $json)
{
}
/**
* @return array{records: array<string, array<string, mixed>>, json_records: int, duplicate_records: int}
* @throws LocalizedException
*/
public function read(string $sourceDirectory): array
{
$sourceRoot = realpath($sourceDirectory);
if ($sourceRoot === false || !is_dir($sourceRoot) || !is_readable($sourceRoot)) {
throw new LocalizedException(__('Import source is not a readable directory: %1', $sourceDirectory));
}
$records = [];
$jsonRecords = 0;
foreach (self::SOURCE_FILES as $category => $filename) {
$listItems = $this->readListItems($sourceRoot, $category);
$jsonPath = $sourceRoot . DIRECTORY_SEPARATOR . $filename;
if (!is_file($jsonPath) || !is_readable($jsonPath)) {
throw new LocalizedException(__('Required JSON file is missing or unreadable: %1', $jsonPath));
}
try {
$data = $this->json->unserialize((string)file_get_contents($jsonPath));
} catch (\InvalidArgumentException $exception) {
throw new LocalizedException(__('Invalid JSON in %1: %2', $filename, $exception->getMessage()));
}
if (!is_array($data) || !isset($data['details']) || !is_array($data['details'])) {
throw new LocalizedException(__('The details array is missing from %1.', $filename));
}
if (isset($data['total_count']) && (int)$data['total_count'] !== count($data['details'])) {
throw new LocalizedException(
__('The total_count in %1 does not match its details count.', $filename)
);
}
foreach ($data['details'] as $index => $item) {
$jsonRecords++;
if (!is_array($item)) {
throw new LocalizedException(__('Invalid item %1 in %2.', $index + 1, $filename));
}
$this->validateItem($item, $index, $filename);
$sourceUrl = trim((string)$item['sourceUrl']);
if (!isset($listItems[$sourceUrl])) {
throw new LocalizedException(
__('No matching list item was found for %1 in %2.', $sourceUrl, self::LIST_FILES[$category])
);
}
$listItem = $listItems[$sourceUrl];
$path = (string)parse_url($sourceUrl, PHP_URL_PATH);
$pathSegments = array_values(array_filter(explode('/', trim($path, '/')), 'strlen'));
$sourceUrlKey = $pathSegments ? (string)end($pathSegments) : '';
if ($sourceUrlKey === '') {
throw new LocalizedException(
__('Cannot derive a URL key from item %1 in %2.', $index + 1, $filename)
);
}
$htmlPath = $this->resolveHtmlPath($sourceRoot, (string)$item['localHtmlFile']);
if (isset($records[$sourceUrl])) {
$existing = $records[$sourceUrl];
if ($existing['title'] !== trim((string)$item['title'])
|| $existing['publish_date'] !== (string)$item['publishDate']
) {
throw new LocalizedException(
__('Conflicting duplicate post data for source URL %1.', $sourceUrl)
);
}
if ($records[$sourceUrl]['short_description'] !== ''
&& $listItem['summary'] !== ''
&& $records[$sourceUrl]['short_description'] !== $listItem['summary']
) {
throw new LocalizedException(__('Conflicting summaries for source URL %1.', $sourceUrl));
}
if ($records[$sourceUrl]['image_filename'] !== ''
&& $listItem['image_filename'] !== ''
&& $records[$sourceUrl]['image_filename'] !== $listItem['image_filename']
) {
throw new LocalizedException(__('Conflicting list images for source URL %1.', $sourceUrl));
}
if ($records[$sourceUrl]['short_description'] === '') {
$records[$sourceUrl]['short_description'] = $listItem['summary'];
}
if ($records[$sourceUrl]['image_filename'] === '') {
$records[$sourceUrl]['image_filename'] = $listItem['image_filename'];
}
$records[$sourceUrl]['categories'][$category] = $category;
continue;
}
$records[$sourceUrl] = [
'title' => trim((string)$item['title']),
'source_url' => $sourceUrl,
'source_url_key' => $sourceUrlKey,
'publish_date' => (string)$item['publishDate'],
'html_path' => $htmlPath,
'short_description' => $listItem['summary'],
'image_filename' => $listItem['image_filename'],
'categories' => [$category => $category],
];
}
}
return [
'records' => $records,
'json_records' => $jsonRecords,
'duplicate_records' => $jsonRecords - count($records),
];
}
/**
* @return array<string, array{summary: string, image_filename: string}>
* @throws LocalizedException
*/
private function readListItems(string $sourceRoot, string $category): array
{
$filename = self::LIST_FILES[$category];
$jsonPath = $sourceRoot . DIRECTORY_SEPARATOR . $filename;
if (!is_file($jsonPath) || !is_readable($jsonPath)) {
throw new LocalizedException(__('Required list JSON file is missing or unreadable: %1', $jsonPath));
}
try {
$data = $this->json->unserialize((string)file_get_contents($jsonPath));
} catch (\InvalidArgumentException $exception) {
throw new LocalizedException(__('Invalid JSON in %1: %2', $filename, $exception->getMessage()));
}
if (!is_array($data) || !isset($data['news']) || !is_array($data['news'])) {
throw new LocalizedException(__('The news array is missing from %1.', $filename));
}
if (isset($data['total_count']) && (int)$data['total_count'] !== count($data['news'])) {
throw new LocalizedException(__('The total_count in %1 does not match its news count.', $filename));
}
$result = [];
foreach ($data['news'] as $index => $item) {
if (!is_array($item) || !isset($item['link']) || !is_string($item['link']) || trim($item['link']) === '') {
throw new LocalizedException(__('Missing or invalid link in item %1 of %2.', $index + 1, $filename));
}
$link = trim($item['link']);
if (isset($result[$link])) {
throw new LocalizedException(__('Duplicate link %1 in %2.', $link, $filename));
}
$summary = isset($item['summary']) && is_string($item['summary']) ? trim($item['summary']) : '';
$imageUrl = isset($item['imageUrl']) && is_string($item['imageUrl']) ? trim($item['imageUrl']) : '';
$result[$link] = [
'summary' => $summary,
'image_filename' => $this->extractImageFilename($imageUrl, $index, $filename),
];
}
return $result;
}
/** @throws LocalizedException */
private function extractImageFilename(string $imageUrl, int $index, string $filename): string
{
if ($imageUrl === '') {
return '';
}
$path = (string)preg_split('/[?#]/', $imageUrl, 2)[0];
$imageFilename = rawurldecode(basename($path));
if ($imageFilename === '' || $imageFilename === '.' || $imageFilename === '..') {
throw new LocalizedException(__('Invalid imageUrl in item %1 of %2.', $index + 1, $filename));
}
return $imageFilename;
}
/** @throws LocalizedException */
private function validateItem(array $item, int $index, string $filename): void
{
foreach (['title', 'sourceUrl', 'publishDate', 'localHtmlFile'] as $field) {
if (!isset($item[$field]) || !is_string($item[$field]) || trim($item[$field]) === '') {
throw new LocalizedException(
__('Missing or invalid %1 in item %2 of %3.', $field, $index + 1, $filename)
);
}
}
$sourceUrl = (string)$item['sourceUrl'];
$parts = parse_url($sourceUrl);
if (!is_array($parts)
|| ($parts['scheme'] ?? '') !== 'https'
|| strtolower((string)($parts['host'] ?? '')) !== 'www.esun3d.com'
|| !str_starts_with((string)($parts['path'] ?? ''), '/news/')
) {
throw new LocalizedException(__('Invalid sourceUrl in item %1 of %2.', $index + 1, $filename));
}
$date = \DateTimeImmutable::createFromFormat('!Y-m-d', (string)$item['publishDate']);
if ($date === false || $date->format('Y-m-d') !== $item['publishDate']) {
throw new LocalizedException(__('Invalid publishDate in item %1 of %2.', $index + 1, $filename));
}
}
/** @throws LocalizedException */
private function resolveHtmlPath(string $sourceRoot, string $relativePath): string
{
if ($relativePath === '' || str_contains($relativePath, "\0") || str_starts_with($relativePath, '/')) {
throw new LocalizedException(__('Invalid localHtmlFile path: %1', $relativePath));
}
$htmlPath = realpath($sourceRoot . DIRECTORY_SEPARATOR . $relativePath);
$rootPrefix = rtrim($sourceRoot, DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR;
if ($htmlPath === false
|| !is_file($htmlPath)
|| !is_readable($htmlPath)
|| !str_starts_with($htmlPath, $rootPrefix)
) {
throw new LocalizedException(__('HTML file is missing, unreadable, or outside the source: %1', $relativePath));
}
return $htmlPath;
}
}
+48
View File
@@ -0,0 +1,48 @@
# Esun Blog Import
Imports the four extracted `esun_*_details.json` files and their referenced HTML files into Mageplaza Blog.
## Deployment
Extract `esun.zip` directly into `var/import`, enable the module, and run Magento setup:
```bash
bin/magento module:enable Esun_BlogImport
bin/magento setup:upgrade
```
Validate before importing:
```bash
bin/magento esun:blog:import --dry-run
```
Run the import after validation succeeds:
```bash
bin/magento esun:blog:import
```
The default media location is `pub/media/mageplaza/blog/post/uploads`. Missing media fails validation by default.
The importer also detects JPG, PNG, and GIF files whose binary data has a different format (for example,
WebP data saved with a `.jpg` extension). A real import converts these files in place to the format declared
by the extension so Mageplaza can resize them; `--dry-run` only reports how many require conversion.
Animated WebP data requires `ffmpeg`, or the WebP command-line tools `webpmux` and `dwebp`, on the import host.
`--allow-missing-media` is available for an intentional partial media migration.
Image and video URLs on `esun3d.com` or `www.esun3d.com`, plus source-relative `/uploads/...` URLs, are rewritten to
Magento media directives under the configured media subdirectory. Media URLs on every other domain are left intact.
Each `esun_*_all.json` file supplies list-card data. `summary` is imported into `short_description`, while `imageUrl`
is reduced to its local filename and stored as `uploads/<filename>` in the post `image` field. Mageplaza resolves and
resizes these covers natively from `pub/media/mageplaza/blog/post/uploads`.
Use `--store-id=<id>` to assign posts to a particular store. The default is store ID `0` (all stores).
The importer is idempotent. It identifies imported posts by `import_source`, merges duplicate JSON entries into one
post with multiple category relations, and refuses to overwrite an unrelated post that uses the same URL key.
Imported posts are enabled, excluded from RSS (`in_rss = 0`), use the two-column left-sidebar layout
(`layout = 2columns-left`), and are saved with comments disabled (`allow_comment = 0`).
The module also changes `mageplaza_blog_post.post_content` from `TEXT` to `MEDIUMTEXT`. Processed HTML is validated
against the 16,777,215-byte `MEDIUMTEXT` limit before any post is written.
@@ -0,0 +1,73 @@
<?php
declare(strict_types=1);
namespace Esun\BlogImport\Test\Unit\Model;
use Esun\BlogImport\Model\HtmlProcessor;
use PHPUnit\Framework\TestCase;
class HtmlProcessorTest extends TestCase
{
private string $htmlPath;
protected function setUp(): void
{
$this->htmlPath = tempnam(sys_get_temp_dir(), 'esun-blog-html-');
if ($this->htmlPath === false) {
self::fail('Unable to create a temporary HTML file.');
}
}
protected function tearDown(): void
{
if (isset($this->htmlPath) && is_file($this->htmlPath)) {
unlink($this->htmlPath);
}
}
public function testItExtractsArticleBodyAndRewritesLegacyMedia(): void
{
file_put_contents($this->htmlPath, <<<'HTML'
<section class="main">
<article class="entry blog-article">
<h1 class="news-title">Duplicated title</h1>
<div class="news-time">Post time<script src="//example.test/widget.js"></script></div>
<p><a href="https://www.esun3d.com/uploads/图片1.jpg"><img src="/uploads/图片1.jpg"></a></p>
<img src="https://www.esun3d.com/wp-includes/images/smilies/icon_smile.gif">
<img src="https://www.esunchina.net/uploads/external.jpg">
<img src="/uploads/http://img.xiumi.us/external-wrapped.jpg">
<video src="https://media.example.com/external.mp4"></video>
<a href="https://www.esun3d.com/news/not-media">Regular link</a>
<div class="picker"></div>
</article>
</section>
HTML);
$result = (new HtmlProcessor())->process($this->htmlPath, 'uploads');
self::assertSame(['图片1.jpg', 'icon_smile.gif'], $result['media']);
self::assertStringContainsString("{{media url='uploads/图片1.jpg'}}", $result['html']);
self::assertStringContainsString("{{media url='uploads/icon_smile.gif'}}", $result['html']);
self::assertStringContainsString('https://www.esunchina.net/uploads/external.jpg', $result['html']);
self::assertStringContainsString('/uploads/http://img.xiumi.us/external-wrapped.jpg', $result['html']);
self::assertStringContainsString('https://media.example.com/external.mp4', $result['html']);
self::assertStringContainsString('https://www.esun3d.com/news/not-media', $result['html']);
self::assertStringNotContainsString('Duplicated title', $result['html']);
self::assertStringNotContainsString('news-time', $result['html']);
self::assertStringNotContainsString('<script', $result['html']);
self::assertStringNotContainsString('picker', $result['html']);
}
public function testItLeavesExternalLinksUntouched(): void
{
file_put_contents(
$this->htmlPath,
'<article class="blog-article"><a href="https://example.com/file.pdf">Download</a></article>'
);
$result = (new HtmlProcessor())->process($this->htmlPath, 'upload');
self::assertSame([], $result['media']);
self::assertStringContainsString('https://example.com/file.pdf', $result['html']);
}
}
@@ -0,0 +1,63 @@
<?php
declare(strict_types=1);
namespace Esun\BlogImport\Test\Unit\Model;
use Esun\BlogImport\Model\MediaNormalizer;
use PHPUnit\Framework\TestCase;
class MediaNormalizerTest extends TestCase
{
/** @var string[] */
private array $temporaryFiles = [];
protected function tearDown(): void
{
foreach ($this->temporaryFiles as $file) {
if (is_file($file)) {
unlink($file);
}
}
}
public function testItConvertsWebpDataStoredWithAJpegExtension(): void
{
if (!function_exists('imagewebp')) {
self::markTestSkipped('GD WebP support is required.');
}
$path = $this->temporaryPath('.jpg');
$image = imagecreatetruecolor(2, 2);
imagewebp($image, $path);
imagedestroy($image);
$normalizer = new MediaNormalizer();
self::assertTrue($normalizer->needsNormalization($path));
self::assertTrue($normalizer->normalize($path));
self::assertSame(IMAGETYPE_JPEG, getimagesize($path)[2]);
self::assertFalse($normalizer->needsNormalization($path));
self::assertFalse($normalizer->normalize($path));
}
public function testItIgnoresMediaWithoutAResizableImageExtension(): void
{
$path = $this->temporaryPath('.mp4');
file_put_contents($path, 'not inspected');
$normalizer = new MediaNormalizer();
self::assertFalse($normalizer->needsNormalization($path));
self::assertFalse($normalizer->normalize($path));
}
private function temporaryPath(string $extension): string
{
$base = tempnam(sys_get_temp_dir(), 'esun-media-test-');
if ($base === false) {
self::fail('Unable to create a temporary media file.');
}
unlink($base);
$path = $base . $extension;
$this->temporaryFiles[] = $path;
return $path;
}
}
@@ -0,0 +1,7 @@
<?xml version="1.0"?>
<schema xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:noNamespaceSchemaLocation="urn:magento:framework:Setup/Declaration/Schema/etc/schema.xsd">
<table name="mageplaza_blog_post" resource="default" engine="innodb" comment="Post Table">
<column xsi:type="mediumtext" name="post_content" nullable="true" comment="Post Content"/>
</table>
</schema>
@@ -0,0 +1,7 @@
{
"mageplaza_blog_post": {
"column": {
"post_content": true
}
}
}
+11
View File
@@ -0,0 +1,11 @@
<?xml version="1.0"?>
<config xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:noNamespaceSchemaLocation="urn:magento:framework:ObjectManager/etc/config.xsd">
<type name="Magento\Framework\Console\CommandListInterface">
<arguments>
<argument name="commands" xsi:type="array">
<item name="esunBlogImport" xsi:type="object">Esun\BlogImport\Console\Command\ImportBlogCommand</item>
</argument>
</arguments>
</type>
</config>
+9
View File
@@ -0,0 +1,9 @@
<?xml version="1.0"?>
<config xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:noNamespaceSchemaLocation="urn:magento:framework:Module/etc/module.xsd">
<module name="Esun_BlogImport">
<sequence>
<module name="Mageplaza_Blog"/>
</sequence>
</module>
</config>
+10
View File
@@ -0,0 +1,10 @@
<?php
declare(strict_types=1);
use Magento\Framework\Component\ComponentRegistrar;
ComponentRegistrar::register(
ComponentRegistrar::MODULE,
'Esun_BlogImport',
__DIR__
);