feat: M3 import & indexing engine delivery layer with durable collision handling, space checking and console

This commit is contained in:
Cline committed 2026-10-12 11:29:17 -05:00
1 parent 93d991703c
commit 9d277d9e72
48 files changed
+5147 -4

No files matched your search

+230
View File
@@ -0,0 +1,230 @@
<?php
declare(strict_types=1);
namespace MeshVault\Console;
use Closure;
use MeshVault\Exception\PathSafetyException;
use MeshVault\Indexer\Contracts\FileEnumeratorInterface;
use MeshVault\Indexer\ImportException;
use Throwable;
/**
* Argument parsing and reporting for the meshvault CLI.
*
* Every dependency is injected, so the whole surface is testable without a
* database, a filesystem or a container. Exit codes are a contract:
* 0 success, 1 usage error, 2 runtime failure.
*/
final class Console
{
private const USAGE = 'Usage: meshvault <import|worker|worker-once|help> [arguments]';
private readonly Closure $sleep;
private readonly Closure $stopRequested;
/**
* @param resource $out
* @param resource $err
* @param callable(int): void $sleep
* @param callable(): bool $stopRequested
*/
public function __construct(
private readonly ConsoleServices $services,
private readonly mixed $out,
private readonly mixed $err,
callable $sleep,
callable $stopRequested,
) {
$this->sleep = Closure::fromCallable($sleep);
$this->stopRequested = Closure::fromCallable($stopRequested);
}
/** @param list<string> $argv Arguments after the program name. */
public function run(array $argv): int
{
$command = array_shift($argv);
return match ($command) {
null => $this->usage(self::USAGE),
'help', '--help', '-h' => $this->help(),
'import' => $this->import($argv),
'worker' => $this->worker($argv),
'worker-once' => $this->workerOnce($argv),
default => $this->usage('Unknown command: ' . $command . "\n" . self::USAGE),
};
}
private function help(): int
{
$this->write($this->out, implode("\n", [
self::USAGE,
'',
'Commands:',
' import <source-path> [--ext=stl,3mf] Scan a source tree and queue an import job.',
' worker [--sleep=N] [--max-iterations=N] Drain import jobs until stopped.',
' worker-once Claim and drain at most one job, then exit.',
' help Show this message.',
'',
'Options:',
' --ext=LIST Comma-separated extensions, with or without dots. Defaults to: '
. implode(',', FileEnumeratorInterface::DEFAULT_EXTENSIONS),
' --sleep=N Seconds to wait after an idle poll (default 5).',
' --max-iterations=N Stop after N polls instead of running until signalled.',
]));
return 0;
}
/** @param list<string> $argv */
private function import(array $argv): int
{
$source = null;
$extensions = FileEnumeratorInterface::DEFAULT_EXTENSIONS;
foreach ($argv as $argument) {
if (str_starts_with($argument, '--ext=')) {
$parsed = $this->extensions(substr($argument, 6));
if ($parsed === []) {
return $this->usage('--ext requires at least one extension, e.g. --ext=stl,3mf');
}
$extensions = $parsed;
continue;
}
if (str_starts_with($argument, '-')) {
return $this->usage('Unknown option: ' . $argument);
}
if ($source !== null) {
return $this->usage('import accepts exactly one source path.');
}
$source = $argument;
}
if ($source === null || trim($source) === '') {
return $this->usage('Usage: meshvault import <source-path> [--ext=stl,3mf]');
}
try {
$job = $this->services->import($source, $extensions);
} catch (Throwable $error) {
return $this->fail($error);
}
$this->write($this->out, 'Queued import job ' . $job . ' for ' . $source);
return 0;
}
/** @return list<string> */
private function extensions(string $raw): array
{
$extensions = [];
foreach (explode(',', $raw) as $candidate) {
$extension = strtolower(ltrim(trim($candidate), '.'));
if ($extension !== '' && !in_array($extension, $extensions, true)) {
$extensions[] = $extension;
}
}
return $extensions;
}
/** @param list<string> $argv */
private function workerOnce(array $argv): int
{
if ($argv !== []) {
return $this->usage('worker-once takes no arguments.');
}
try {
$job = $this->services->runWorkerOnce($this->stopRequested);
} catch (Throwable $error) {
return $this->fail($error);
}
$this->write(
$this->out,
$job === null ? 'No pending import jobs.' : 'Processed import job ' . $job . '.'
);
return 0;
}
/** @param list<string> $argv */
private function worker(array $argv): int
{
$sleep = 5;
$maxIterations = null;
foreach ($argv as $argument) {
if (str_starts_with($argument, '--sleep=')) {
$value = substr($argument, 8);
if (!$this->isNonNegativeInteger($value)) {
return $this->usage('--sleep must be a non-negative whole number of seconds.');
}
$sleep = (int) $value;
continue;
}
if (str_starts_with($argument, '--max-iterations=')) {
$value = substr($argument, 17);
if (!$this->isNonNegativeInteger($value) || (int) $value < 1) {
return $this->usage('--max-iterations must be a whole number of at least 1.');
}
$maxIterations = (int) $value;
continue;
}
return $this->usage(
str_starts_with($argument, '-')
? 'Unknown option: ' . $argument
: 'worker takes no positional arguments, got: ' . $argument
);
}
return $this->loop($sleep, $maxIterations);
}
private function loop(int $sleep, ?int $maxIterations): int
{
$iterations = 0;
while ($maxIterations === null || $iterations < $maxIterations) {
try {
$job = $this->services->runWorkerOnce($this->stopRequested);
} catch (Throwable $error) {
return $this->fail($error);
}
$iterations++;
if ($job !== null) {
$this->write($this->out, 'Processed import job ' . $job . '.');
}
// A stop requested mid-job has already been honoured by the worker.
if (($this->stopRequested)()) {
break;
}
if ($job === null) {
($this->sleep)($sleep);
}
}
$this->write($this->out, 'Import worker stopped after ' . $iterations . ' poll(s).');
return 0;
}
private function isNonNegativeInteger(string $value): bool
{
return preg_match('/^[0-9]{1,9}$/D', $value) === 1;
}
private function usage(string $message): int
{
$this->write($this->err, $message);
return 1;
}
private function fail(Throwable $error): int
{
$reason = match (true) {
$error instanceof ImportException => $error->reason,
$error instanceof PathSafetyException => $error->reason,
default => null,
};
// Unexpected errors may carry DSNs or credentials; report only the type.
$this->write($this->err, $reason === null
? 'Error: unexpected ' . $error::class . '. See the application log for details.'
: 'Error [' . $reason . ']: ' . $error->getMessage());
return 2;
}
/** @param resource $stream */
private function write(mixed $stream, string $message): void
{
fwrite($stream, $message . PHP_EOL);
}
}
+30
View File
@@ -0,0 +1,30 @@
<?php
declare(strict_types=1);
namespace MeshVault\Console;
/**
* The only seam between argument parsing and the import engine.
*
* Keeping it this narrow lets the console be tested without a database,
* a filesystem, or a container.
*/
interface ConsoleServices
{
/**
* Queue an import job for a source tree.
*
* @param list<string> $extensions Lowercase extensions without leading dots.
* @return int The created job id.
*/
public function import(string $sourcePath, array $extensions): int;
/**
* Claim and drain at most one job.
*
* @param callable(): bool $shouldStop Consulted by the worker between items.
* @return int|null The processed job id, or null when the queue is idle.
*/
public function runWorkerOnce(callable $shouldStop): ?int;
}
+28
View File
@@ -0,0 +1,28 @@
<?php
declare(strict_types=1);
namespace MeshVault\Console;
use MeshVault\Indexer\ImportService;
/** Binds the console to the real import engine. */
final readonly class ContainerConsoleServices implements ConsoleServices
{
public function __construct(
private ImportService $importService,
private ImportWorkerFactory $workerFactory,
) {
}
public function import(string $sourcePath, array $extensions): int
{
return $this->importService->import($sourcePath, $extensions);
}
public function runWorkerOnce(callable $shouldStop): ?int
{
// The stop callback is per-run, so the worker is built per invocation.
return ($this->workerFactory)($shouldStop)->runOnce();
}
}
+14
View File
@@ -0,0 +1,14 @@
<?php
declare(strict_types=1);
namespace MeshVault\Console;
use MeshVault\Indexer\ImportWorker;
/** Builds an ImportWorker bound to a specific graceful-stop callback. */
interface ImportWorkerFactory
{
/** @param callable(): bool $shouldStop */
public function __invoke(callable $shouldStop): ImportWorker;
}
+39
View File
@@ -0,0 +1,39 @@
<?php
declare(strict_types=1);
namespace MeshVault\Console;
use Closure;
/**
* Defers engine construction until a command actually needs it, so usage
* errors and help never require a configured database or library.
*/
final class LazyConsoleServices implements ConsoleServices
{
private readonly Closure $factory;
private ?ConsoleServices $delegate = null;
/** @param callable(): ConsoleServices $factory */
public function __construct(callable $factory)
{
$this->factory = Closure::fromCallable($factory);
}
public function import(string $sourcePath, array $extensions): int
{
return $this->delegate()->import($sourcePath, $extensions);
}
public function runWorkerOnce(callable $shouldStop): ?int
{
return $this->delegate()->runWorkerOnce($shouldStop);
}
private function delegate(): ConsoleServices
{
return $this->delegate ??= ($this->factory)();
}
}
+23
View File
@@ -0,0 +1,23 @@
<?php
declare(strict_types=1);
namespace MeshVault\Exception;
use RuntimeException;
final class PathSafetyException extends RuntimeException
{
public const INVALID_PATH = 'INVALID_PATH';
public const ABSOLUTE_PATH = 'ABSOLUTE_PATH';
public const PATH_TRAVERSAL = 'PATH_TRAVERSAL';
public const SYMLINK_ESCAPE = 'SYMLINK_ESCAPE';
public const PATH_NOT_FOUND = 'PATH_NOT_FOUND';
public const NOT_DIRECTORY = 'NOT_DIRECTORY';
public const OUTSIDE_LIBRARY_ROOT = 'OUTSIDE_LIBRARY_ROOT';
public function __construct(public readonly string $reason)
{
parent::__construct('Path safety check failed: ' . $reason);
}
}
@@ -0,0 +1,15 @@
<?php
declare(strict_types=1);
namespace MeshVault\Indexer\Contracts;
use MeshVault\Indexer\ScanResult;
interface FileEnumeratorInterface
{
public const DEFAULT_EXTENSIONS = ['stl', 'jpg', 'jpeg', 'png', 'gif', 'pdf', 'txt', 'md'];
/** @param list<string> $extensions Case-insensitive extensions without leading dots. */
public function enumerate(string $sourceRoot, array $extensions = self::DEFAULT_EXTENSIONS): ScanResult;
}
@@ -0,0 +1,31 @@
<?php
declare(strict_types=1);
namespace MeshVault\Indexer\Contracts;
/** Native-style results: failed I/O returns false; callers own and close streams. */
interface FilesystemInterface
{
public function realpath(string $path): string|false;
public function isReadable(string $path): bool;
public function isFile(string $path): bool;
public function isDir(string $path): bool;
public function isSymlink(string $path): bool;
public function getSize(string $path): int|false;
public function getMimeType(string $path): string|false;
/** @return list<string>|false Entry names (not paths), excluding dot entries; false on read failure. */
public function readDirectory(string $path): array|false;
/** @return resource|false Binary stream opened for reading. */
public function readStream(string $path);
/** @return resource|false Binary stream opened for writing, truncating existing contents. */
public function writeStream(string $path);
public function rename(string $from, string $to): bool;
public function unlink(string $path): bool;
public function mkdir(string $path, int $permissions = 0777, bool $recursive = false): bool;
public function freeSpace(string $path): float|false;
}
+134
View File
@@ -0,0 +1,134 @@
<?php
declare(strict_types=1);
namespace MeshVault\Indexer;
use MeshVault\Exception\PathSafetyException;
use MeshVault\Indexer\Contracts\FileEnumeratorInterface;
use MeshVault\Indexer\Contracts\FilesystemInterface;
use MeshVault\Indexer\Filesystem\PathSafety;
use RuntimeException;
/** Point-in-time manifest only: callers must prevent concurrent source-tree mutation. */
final readonly class FileEnumerator implements FileEnumeratorInterface
{
public function __construct(
private FilesystemInterface $filesystem,
private PathSafety $pathSafety,
private int $maxDepth = 128,
) {
}
public function enumerate(string $sourceRoot, array $extensions = self::DEFAULT_EXTENSIONS): ScanResult
{
$files = [];
$errors = [];
$totalBytes = 0;
$allowed = array_fill_keys(array_map('strtolower', $extensions), true);
try {
$root = $this->canonicalSource($sourceRoot);
if (!$this->filesystem->isDir($root) || !$this->filesystem->isReadable($root)) {
throw new RuntimeException('Choose an existing, readable directory.');
}
} catch (RuntimeException $error) {
return new ScanResult([], 0, [$this->scanError($sourceRoot, $error, 'Invalid source root: ')]);
}
// Explicit stack avoids PHP call-stack recursion. Root is depth zero.
$pending = [[$root, '', 0]];
while ($pending !== []) {
[$directory, $prefix, $depth] = array_pop($pending);
try {
$directory = $this->canonicalSource($directory);
if ($depth > $this->maxDepth) {
throw new RuntimeException('Maximum directory depth exceeded; use a shallower source root.');
}
if (!$this->filesystem->isReadable($directory)) {
throw new RuntimeException('Directory is not readable; check permissions.');
}
$entries = $this->filesystem->readDirectory($directory);
if ($entries === false) {
throw new RuntimeException('Cannot read directory; check permissions and availability.');
}
} catch (RuntimeException $error) {
$errors[] = $this->scanError($directory, $error);
continue;
}
foreach ($entries as $entry) {
$path = rtrim($directory, '/\\') . DIRECTORY_SEPARATOR . $entry;
try {
// Directory entries must remain a single portable path component.
if ($entry === '' || $entry === '.' || strpbrk($entry, '/\\') !== false) {
throw new PathSafetyException(PathSafetyException::INVALID_PATH);
}
$relative = $prefix . $entry;
$this->pathSafety->ensureRelative($relative);
$canonical = $this->canonicalSource($path);
if ($this->filesystem->isDir($canonical)) {
$pending[] = [$canonical, $relative . '/', $depth + 1];
continue;
}
if (!isset($allowed[strtolower(pathinfo($entry, PATHINFO_EXTENSION))])) {
continue;
}
if (!$this->filesystem->isFile($canonical) || !$this->filesystem->isReadable($canonical)) {
throw new RuntimeException('Not a readable regular file; check permissions and availability.');
}
$size = $this->filesystem->getSize($canonical);
if ($size === false || $size < 0) {
throw new RuntimeException('Cannot stat file size; check permissions and availability.');
}
$mime = $this->filesystem->getMimeType($canonical);
if ($mime === false || $mime === '') {
throw new RuntimeException('Cannot detect MIME type; check permissions and availability.');
}
if ($size > PHP_INT_MAX - $totalBytes) {
throw new RuntimeException('Total byte count exceeds the supported integer range.');
}
$files[] = new ScannedFile($canonical, $relative, $size, $mime);
$totalBytes += $size;
} catch (RuntimeException $error) {
$errors[] = $this->scanError($path, $error);
}
}
}
return new ScanResult($files, $totalBytes, $errors);
}
private function canonicalSource(string $path): string
{
// Safety must inspect lexical ancestors before canonicalization can hide links.
$path = $this->pathSafety->ensureSourceSafe($path);
if ($this->filesystem->isSymlink($path)) {
throw new PathSafetyException(PathSafetyException::SYMLINK_ESCAPE);
}
$canonical = $this->filesystem->realpath($path);
if ($canonical === false) {
throw new RuntimeException('Cannot resolve source; check existence and permissions.');
}
return $canonical;
}
private function scanError(string $path, RuntimeException $error, string $prefix = ''): ScanError
{
$safetyCodes = [PathSafetyException::SYMLINK_ESCAPE, PathSafetyException::PATH_TRAVERSAL];
$code = $error instanceof PathSafetyException && in_array($error->reason, $safetyCodes, true)
? $error->reason : ScanError::SCAN_READ_ERROR;
$message = $error->getMessage();
if ($error instanceof PathSafetyException) {
$guidance = match ($error->reason) {
PathSafetyException::PATH_NOT_FOUND, PathSafetyException::NOT_DIRECTORY =>
'Choose an existing, readable directory; check the path and permissions.',
PathSafetyException::INVALID_PATH =>
'Choose a non-empty directory path without NUL bytes or URL schemes; '
. 'use forward slashes on POSIX and an absolute path for Windows drives.',
PathSafetyException::PATH_TRAVERSAL => 'Choose a path without parent (..) components.',
PathSafetyException::SYMLINK_ESCAPE => 'Choose a path with no symlinks, including its ancestors.',
default => 'Choose a directory path permitted by the path safety policy.',
};
$message .= '. ' . $guidance;
}
return new ScanError($path, $code, $prefix . $message);
}
}
@@ -0,0 +1,91 @@
<?php
declare(strict_types=1);
namespace MeshVault\Indexer\Filesystem;
use finfo;
use MeshVault\Indexer\Contracts\FilesystemInterface;
/** Thin local I/O adapter; path authorization is the caller's responsibility. */
final readonly class LocalFilesystem implements FilesystemInterface
{
public function realpath(string $path): string|false
{
clearstatcache(true, $path);
return realpath($path);
}
public function isReadable(string $path): bool
{
clearstatcache(true, $path);
return is_readable($path);
}
public function isFile(string $path): bool
{
clearstatcache(true, $path);
return is_file($path);
}
public function isDir(string $path): bool
{
clearstatcache(true, $path);
return is_dir($path);
}
public function isSymlink(string $path): bool
{
clearstatcache(true, $path);
return is_link($path);
}
public function getSize(string $path): int|false
{
clearstatcache(true, $path);
return @filesize($path);
}
public function getMimeType(string $path): string|false
{
return @(new finfo(FILEINFO_MIME_TYPE))->file($path);
}
public function readDirectory(string $path): array|false
{
$entries = @scandir($path);
return $entries === false ? false : array_values(array_diff($entries, ['.', '..']));
}
/** @return resource|false */
public function readStream(string $path)
{
return @fopen($path, 'rb');
}
/** @return resource|false */
public function writeStream(string $path)
{
return @fopen($path, 'wb');
}
public function rename(string $from, string $to): bool
{
return @rename($from, $to);
}
public function unlink(string $path): bool
{
return @unlink($path);
}
public function mkdir(string $path, int $permissions = 0777, bool $recursive = false): bool
{
return @mkdir($path, $permissions, $recursive);
}
public function freeSpace(string $path): float|false
{
return @disk_free_space($path);
}
}
+146
View File
@@ -0,0 +1,146 @@
<?php
declare(strict_types=1);
namespace MeshVault\Indexer\Filesystem;
use MeshVault\Exception\PathSafetyException as Error;
/**
* Point-in-time checks, not race-proof authorization. Callers must control directory
* mutation and recheck before I/O; no check here makes a later open/rename atomic.
*/
final readonly class PathSafety
{
private string $libraryRoot;
public function __construct(string $libraryRoot)
{
$root = $this->ensureSourceSafe($libraryRoot);
if (!is_dir($root)) {
throw new Error(Error::NOT_DIRECTORY);
}
$this->libraryRoot = $root;
}
/** The canonical, realpath-resolved library root. */
public function libraryRoot(): string
{
return $this->libraryRoot;
}
/** Sources may be outside the managed library, but must already exist. */
public function ensureSourceSafe(string $path): string
{
// On POSIX a backslash is a filename character, not a separator. Reject it
// before normalization so lexical checks cannot inspect a different tree.
if (PHP_OS_FAMILY !== 'Windows' && str_contains($path, '\\')) {
throw new Error(Error::INVALID_PATH);
}
$path = $this->validate($path);
// Drive-relative paths can resolve differently from the lexical link-check path.
if (preg_match('~^[A-Za-z]:(?!/)~', $path) === 1) {
throw new Error(Error::INVALID_PATH);
}
$this->rejectLinks($path);
$canonical = realpath($path);
if ($canonical === false) {
throw new Error(Error::PATH_NOT_FOUND);
}
return $canonical;
}
/** Relative destinations are resolved under the library; missing suffixes are allowed. */
public function ensureDestSafe(string $path): string
{
$path = $this->validate($path);
if (!$this->isAbsolute($path)) {
$this->ensureRelative($path);
$path = $this->libraryRoot . DIRECTORY_SEPARATOR . $path;
}
$this->rejectLinks($path);
$ancestor = $path;
$suffix = [];
while (($canonical = realpath($ancestor)) === false) {
$parent = dirname($ancestor);
if ($parent === $ancestor) {
throw new Error(Error::PATH_NOT_FOUND);
}
array_unshift($suffix, basename($ancestor));
$ancestor = $parent;
}
if ($suffix !== [] && !is_dir($canonical)) {
throw new Error(Error::NOT_DIRECTORY);
}
$root = rtrim($this->libraryRoot, '/\\');
$comparison = $canonical;
if (PHP_OS_FAMILY === 'Windows') {
$root = strtolower($root);
$comparison = strtolower($comparison);
}
if ($comparison !== $root && !str_starts_with($comparison, $root . DIRECTORY_SEPARATOR)) {
throw new Error(Error::OUTSIDE_LIBRARY_ROOT);
}
return $suffix === [] ? $canonical
: rtrim($canonical, '/\\') . DIRECTORY_SEPARATOR . implode(DIRECTORY_SEPARATOR, $suffix);
}
/** Validate without rewriting the supplied relative name. */
public function ensureRelative(string $path): string
{
$normalized = $this->validate($path);
if (str_starts_with($normalized, '/') || preg_match('/^[A-Za-z]:/', $normalized)) {
throw new Error(Error::ABSOLUTE_PATH);
}
return $path;
}
private function validate(string $path): string
{
if ($path === '' || str_contains($path, "\0") || str_contains($path, '://')) {
throw new Error(Error::INVALID_PATH);
}
$path = str_replace('\\', '/', $path);
foreach (explode('/', $path) as $index => $part) {
if ($part === '..') {
throw new Error(Error::PATH_TRAVERSAL);
}
// Win32 trims trailing dots/spaces and interprets colons as stream names.
$drive = $index === 0 && preg_match('/^[A-Za-z]:/', $part) === 1;
if (
PHP_OS_FAMILY === 'Windows'
&& (($part !== '.' && preg_match('/[. ]$/', $part))
|| (str_contains($part, ':') && !$drive))
) {
throw new Error(Error::INVALID_PATH);
}
}
return $path;
}
private function isAbsolute(string $path): bool
{
return str_starts_with($path, '/')
|| (PHP_OS_FAMILY === 'Windows' && preg_match('~^[A-Za-z]:/~', $path) === 1);
}
/** Inspect the lexical path before realpath can hide a link, including dangling links. */
private function rejectLinks(string $path): void
{
if (!$this->isAbsolute($path)) {
$path = getcwd() . '/' . $path;
}
$path = rtrim($path, '/');
while (true) {
clearstatcache(true, $path);
if (is_link($path)) {
throw new Error(Error::SYMLINK_ESCAPE);
}
$parent = dirname($path);
if ($parent === $path || $path === '') {
break;
}
$path = $parent;
}
}
}
+17
View File
@@ -0,0 +1,17 @@
<?php
declare(strict_types=1);
namespace MeshVault\Indexer;
use RuntimeException;
use Throwable;
/** Machine-readable reason independent of the human-readable message. */
final class ImportException extends RuntimeException
{
public function __construct(public readonly string $reason, string $message = '', ?Throwable $previous = null)
{
parent::__construct($message === '' ? $reason : $message, 0, $previous);
}
}
+94
View File
@@ -0,0 +1,94 @@
<?php
declare(strict_types=1);
namespace MeshVault\Indexer;
use MeshVault\Indexer\Contracts\FileEnumeratorInterface;
use MeshVault\Repository\CatalogRepositoryInterface;
use MeshVault\Repository\ImportJobRepositoryInterface;
use Throwable;
/**
* Turns a source tree into a durable import job manifest.
*
* Grouping policy (explicit, never inferred from file contents): each file is
* assigned to the model named after its first directory component below the
* source root; files sitting directly in the source root join a model named
* after the source directory itself. Every model lives in a folder of the same
* name directly under the library root, so scans never silently merge trees.
* No filesystem mutation happens here; the worker owns all copying.
*/
final readonly class ImportService
{
public function __construct(
private FileEnumeratorInterface $enumerator,
private ImportJobRepositoryInterface $jobs,
private CatalogRepositoryInterface $catalog,
private SpaceChecker $space,
private string $libraryRoot,
) {
}
/**
* Scan, preflight free space, then persist the manifest atomically.
*
* @param list<string> $extensions Case-insensitive extensions without dots.
* @throws ImportException SCAN_EMPTY, DISK_FULL, SPACE_CHECK_FAILED, DB_ERROR.
*/
public function import(string $sourcePath, array $extensions = FileEnumeratorInterface::DEFAULT_EXTENSIONS): int
{
$scan = $this->enumerator->enumerate($sourcePath, $extensions);
if ($scan->files === []) {
$detail = $scan->errors === [] ? 'No importable files were found.' : $scan->errors[0]->message;
throw new ImportException('SCAN_EMPTY', $detail);
}
// Preflight before writing any row so a full disk never leaves a job behind.
$this->space->check($this->libraryRoot, $scan->totalBytes);
try {
return $this->jobs->atomic(
fn (): int => $this->jobs->create($sourcePath, $this->manifest($sourcePath, $scan))
);
} catch (ImportException $error) {
throw $error;
} catch (Throwable $error) {
throw new ImportException('DB_ERROR', 'Could not persist the import job.', $error);
}
}
/** @return list<array<string, mixed>> */
private function manifest(string $sourcePath, ScanResult $scan): array
{
$models = [];
$items = [];
foreach ($scan->files as $file) {
$group = $this->group($sourcePath, $file);
$models[$group] ??= $this->catalog->ensureModel('/' . $group, $group);
$items[] = [
'source_path' => $file->sourcePath,
'size' => $file->size,
'model_id' => $models[$group],
];
}
// Scan failures become terminal items so the job reports them durably.
foreach ($scan->errors as $error) {
$items[] = [
'source_path' => $error->sourcePath,
'size' => 0,
'model_id' => null,
'status' => 'Failed',
'failure_reason' => $error->code . ': ' . $error->message,
];
}
return $items;
}
private function group(string $sourcePath, ScannedFile $file): string
{
$parent = trim(str_replace('\\', '/', dirname($file->relativePath)), '/');
if ($parent === '' || $parent === '.') {
return basename(rtrim(str_replace('\\', '/', $sourcePath), '/'));
}
return explode('/', $parent)[0];
}
}
+237
View File
@@ -0,0 +1,237 @@
<?php
declare(strict_types=1);
namespace MeshVault\Indexer;
use MeshVault\Exception\PathSafetyException;
use MeshVault\Indexer\Filesystem\PathSafety;
use MeshVault\Indexer\SpaceChecker;
use MeshVault\Repository\CatalogRepositoryInterface;
use MeshVault\Repository\ImportJobRepositoryInterface;
use Psr\Log\LoggerInterface;
use Throwable;
/**
* Serializes every managed-library mutation behind one advisory lock file and
* drives a claimed job item by item.
*
* Durability contract: the planned managed path plus the source digest are
* committed BEFORE any byte is published. After a crash the recovered worker
* re-reads that plan, verifies the expected destination, and completes the
* catalog instead of copying again, so restarts never orphan or duplicate
* files. Items fail independently; one failure never abandons the remaining
* work, and the job reports Failed only once every item is terminal.
* A graceful-stop callback is consulted before each item and leaves the job
* Pending (resumable) rather than terminal.
*/
final readonly class ImportWorker
{
public const LOCK_FILE = '.meshvault-import.lock';
/** @param null|callable(): bool $shouldStop Consulted before each item. */
public function __construct(
private ImportJobRepositoryInterface $jobs,
private CatalogRepositoryInterface $catalog,
private StreamingHasher $hasher,
private VerifiedCopier $copier,
private StlValidator $validator,
private PathSafety $pathSafety,
private SpaceChecker $spaceChecker,
private string $libraryRoot,
private LoggerInterface $logger,
private mixed $shouldStop = null,
) {
}
/**
* Claim and drain at most one job. Returns the job id, or null when idle.
*
* @throws ImportException LOCK_UNAVAILABLE when another worker holds the library.
*/
public function runOnce(): ?int
{
$lock = $this->lock();
try {
// Recovery runs under the lock, so no live worker can be reset.
$this->jobs->resumeInterrupted();
$job = $this->jobs->claimNext();
if ($job === null) {
return null;
}
$jobId = (int) $job['id'];
$this->drain($jobId);
return $jobId;
} finally {
flock($lock, LOCK_UN);
fclose($lock);
}
}
/** @return resource */
private function lock()
{
$path = $this->pathSafety->ensureDestSafe($this->libraryRoot . DIRECTORY_SEPARATOR . self::LOCK_FILE);
$handle = @fopen($path, 'c');
if ($handle === false) {
throw new ImportException('LOCK_UNAVAILABLE', 'Cannot open the library lock file.');
}
if (!flock($handle, LOCK_EX | LOCK_NB)) {
fclose($handle);
throw new ImportException('LOCK_UNAVAILABLE', 'Another import worker holds the library lock.');
}
return $handle;
}
private function drain(int $jobId): void
{
foreach ($this->jobs->items($jobId) as $item) {
if (!in_array($item['status'], ['Pending', 'Copying'], true)) {
continue;
}
if ($this->shouldStop !== null && ($this->shouldStop)()) {
// Graceful stop: return the job and its unfinished items to Pending.
$this->jobs->release($jobId);
$this->logger->info('Import worker stopped gracefully.', ['job_id' => $jobId]);
return;
}
$this->process($jobId, $item);
}
}
/** @param array<string, mixed> $item */
private function process(int $jobId, array $item): void
{
$itemId = (int) $item['id'];
try {
$plan = $this->plan($item);
$destination = $this->pathSafety->ensureDestSafe(
$this->libraryRoot . DIRECTORY_SEPARATOR
. str_replace('/', DIRECTORY_SEPARATOR, $plan['managed_path'])
);
$published = $this->publish($plan, $destination);
$relative = $this->relative($published);
// Catalog and progress commit together, so a crash here simply replays.
$this->jobs->atomic(function () use ($item, $itemId, $plan, $published, $relative): void {
// Update managed_path to the actual published path (with collision suffix if any)
// so import_items.managed_path always points at the published file.
$this->jobs->updateManagedPath($itemId, $relative);
if ($item['model_id'] !== null) {
$this->catalog->recordFile(
(int) $item['model_id'],
$relative,
$plan['hash'],
$plan['size'],
null,
'Present'
);
}
$this->jobs->finishItem($itemId, 'Completed');
});
} catch (Throwable $error) {
$this->failItem($jobId, $itemId, $error);
}
}
/**
* Persist the immutable plan (destination plus digest) before publication.
*
* @param array<string, mixed> $item
* @return array{source: string, managed_path: string, hash: string, size: int}
*/
private function plan(array $item): array
{
$source = (string) $item['source_path'];
if (strtolower(pathinfo($source, PATHINFO_EXTENSION)) === 'stl') {
$this->validator->validate($source);
}
if ($item['managed_path'] !== null && $item['hash'] !== null) {
// Recovered plan: trust the committed intent, never choose a new name.
return [
'source' => $source,
'managed_path' => (string) $item['managed_path'],
'hash' => (string) $item['hash'],
'size' => (int) $item['size'],
];
}
$hash = $this->hasher->hash($source);
$size = filesize($source);
if ($size === false) {
throw new ImportException('SCAN_READ_ERROR', 'Cannot determine the source size.');
}
$managed = $this->managedPath($item, $source);
$this->jobs->planItem((int) $item['id'], $managed, $hash, $size);
return ['source' => $source, 'managed_path' => $managed, 'hash' => $hash, 'size' => $size];
}
/** @param array<string, mixed> $item */
private function managedPath(array $item, string $source): string
{
$folder = 'unfiled';
if ($item['model_id'] !== null) {
$aggregate = $this->catalog->aggregate((int) $item['model_id']);
if ($aggregate !== null) {
$folder = trim((string) $aggregate['folder']['path'], '/');
}
}
return ($folder === '' ? '' : $folder . '/') . basename(str_replace('\\', '/', $source));
}
/**
* Publish, or adopt an already-published destination left by a crashed run.
*
* @param array{source: string, managed_path: string, hash: string, size: int} $plan
* @return string The actual published path (may have collision suffix)
*/
private function publish(array $plan, string $destination): string
{
$directory = dirname($destination);
if (!is_dir($directory) && !@mkdir($directory, 0775, true) && !is_dir($directory)) {
throw new ImportException('COPY_FAILED', 'Cannot create the managed destination directory.');
}
// Re-validate the destination with PathSafety and reject links before adopting.
$destination = $this->pathSafety->ensureDestSafe($destination);
clearstatcache(true, $destination);
if (is_file($destination)) {
// Idempotent restart: the planned destination already holds our bytes.
if (hash_file('sha256', $destination) === $plan['hash']) {
return $destination;
}
// Different content at the planned destination: let VerifiedCopier resolve the collision
// by appending a _001 suffix. This keeps restart idempotent (we don't re-copy the same
// file) while still allowing two same-basename sources with different content to coexist.
} else {
// New file being copied: check free space before copying
$this->spaceChecker->check($directory, $plan['size']);
}
return $this->copier->copy($plan['source'], $destination, $plan['size'], $plan['hash']);
}
private function relative(string $published): string
{
// Use the canonical (realpath-resolved) root from PathSafety so prefix comparison
// is reliable even when the configured library root contains symlinks or case differences.
$prefix = rtrim(str_replace('\\', '/', $this->pathSafety->libraryRoot()), '/') . '/';
$path = str_replace('\\', '/', $published);
if (!str_starts_with($path, $prefix)) {
throw new ImportException('PATH_TRAVERSAL', 'The published file escaped the managed library.');
}
return substr($path, strlen($prefix));
}
private function failItem(int $jobId, int $itemId, Throwable $error): void
{
$reason = match (true) {
$error instanceof ImportException => $error->reason,
$error instanceof PathSafetyException => $error->reason,
default => 'COPY_FAILED',
};
$this->logger->error('Import item failed.', [
'job_id' => $jobId,
'item_id' => $itemId,
'reason' => $reason,
]);
// Failures are per item: the loop continues with the remaining work.
$this->jobs->finishItem($itemId, 'Failed', $reason . ': ' . $error->getMessage());
}
}
+17
View File
@@ -0,0 +1,17 @@
<?php
declare(strict_types=1);
namespace MeshVault\Indexer;
final readonly class ScanError
{
public const SCAN_READ_ERROR = 'SCAN_READ_ERROR';
public function __construct(
public string $sourcePath,
public string $code,
public string $message,
) {
}
}
+16
View File
@@ -0,0 +1,16 @@
<?php
declare(strict_types=1);
namespace MeshVault\Indexer;
final readonly class ScanResult
{
/** @param list<ScannedFile> $files @param list<ScanError> $errors */
public function __construct(
public array $files,
public int $totalBytes,
public array $errors,
) {
}
}
+16
View File
@@ -0,0 +1,16 @@
<?php
declare(strict_types=1);
namespace MeshVault\Indexer;
final readonly class ScannedFile
{
public function __construct(
public string $sourcePath,
public string $relativePath,
public int $size,
public string $mimeType,
) {
}
}
+46
View File
@@ -0,0 +1,46 @@
<?php
declare(strict_types=1);
namespace MeshVault\Indexer;
use Closure;
use InvalidArgumentException;
use Throwable;
/** Advisory preflight only: free space may change immediately after checking. */
final readonly class SpaceChecker
{
private Closure $freeSpace;
/** Margin is a fraction of payload bytes, not a fraction of disk capacity. */
public function __construct(private float $margin = 0.05, ?callable $freeSpace = null)
{
if (!is_finite($margin) || $margin < 0) {
throw new InvalidArgumentException('Space margin must be finite and nonnegative.');
}
$this->freeSpace = $freeSpace === null
? static fn (string $path): float|false => @disk_free_space($path)
: Closure::fromCallable($freeSpace);
}
/** Require ceil(bytes * (1 + margin)) available bytes on an existing destination directory. */
public function check(string $directory, int $bytes): void
{
if ($bytes < 0) {
throw new InvalidArgumentException('Payload size must be nonnegative.');
}
try {
$free = ($this->freeSpace)($directory);
} catch (Throwable $error) {
throw new ImportException('SPACE_CHECK_FAILED', 'Cannot query destination free space.', $error);
}
if ((!is_float($free) && !is_int($free)) || !is_finite((float) $free) || $free < 0) {
throw new ImportException('SPACE_CHECK_FAILED', 'Cannot determine destination free space.');
}
$required = ceil($bytes + $bytes * $this->margin);
if (!is_finite($required) || $free < $required) {
throw new ImportException('DISK_FULL', 'Insufficient free space including the safety margin.');
}
}
}
+204
View File
@@ -0,0 +1,204 @@
<?php
declare(strict_types=1);
namespace MeshVault\Indexer;
use Closure;
use MeshVault\Indexer\Filesystem\PathSafety;
/**
* Structural validation, not manifoldness, winding or non-degeneracy validation.
* Empty meshes are rejected in both encodings. Exact binary length wins over a
* "solid" header. Binary attributes are opaque; all 12 floats must be finite.
* ASCII uses lowercase, one statement per LF/CRLF line (4096 bytes excluding LF),
* optional blank lines and printable names (opening/closing names need not match).
* One solid only; no comments, BOM, bare-CR lines or trailing non-whitespace.
* O(1) auxiliary memory, O(file size) time. Callers must prevent source mutation;
* PathSafety checks and final size checks do not provide a race-proof snapshot.
*/
final readonly class StlValidator
{
/** @param null|Closure(resource, int): (string|false) $reader Test seam; like fread. */
public function __construct(private PathSafety $pathSafety, private ?Closure $reader = null)
{
}
public function validate(string $path): void
{
$path = $this->pathSafety->ensureSourceSafe($path);
if (!is_file($path) || ($stream = @fopen($path, 'rb')) === false) {
throw new ImportException('SCAN_READ_ERROR', 'SCAN_READ_ERROR: Cannot open STL file.');
}
try {
$stat = @fstat($stream);
if ($stat === false || !is_int($stat['size']) || $stat['size'] < 0) {
throw new ImportException('SCAN_READ_ERROR', 'Cannot determine STL size.');
}
$size = $stat['size'];
$header = $this->readUpTo($stream, 84);
$count = strlen($header) === 84 ? unpack('Vcount', substr($header, 80, 4))['count'] : 0;
// Division avoids overflowing 84 + 50 * an untrusted uint32 count.
$binary = $size >= 84 && ($size - 84) % 50 === 0 && intdiv($size - 84, 50) === $count;
if ($binary) {
if ($count === 0) {
$this->invalid('Empty mesh.');
}
for ($index = 0; $index < $count; ++$index) {
$record = $this->readUpTo($stream, 50);
if (strlen($record) !== 50) {
$this->invalid('Truncated binary facet.');
}
foreach (unpack('g12', substr($record, 0, 48)) as $number) {
if (!is_finite($number)) {
$this->invalid('Non-finite binary coordinate or normal.');
}
}
}
if ($this->readUpTo($stream, 1) !== '') {
$this->invalid('Trailing binary data.');
}
} else {
if (@fseek($stream, 0) !== 0) {
throw new ImportException('SCAN_READ_ERROR', 'Cannot rewind STL file.');
}
$this->validateAscii($stream);
}
$after = @fstat($stream);
if ($after === false || $after['size'] !== $size) {
throw new ImportException('SCAN_READ_ERROR', 'STL size changed during validation.');
}
} finally {
fclose($stream);
}
}
/** @param resource $stream */
private function readUpTo($stream, int $length): string
{
$result = '';
while (strlen($result) < $length) {
$remaining = $length - strlen($result);
try {
$chunk = $this->reader === null ? @fread($stream, $remaining) : ($this->reader)($stream, $remaining);
} catch (\Throwable $error) {
throw new ImportException('SCAN_READ_ERROR', 'Failed reading STL stream.', $error);
}
if ($chunk === false || ($chunk === '' && !feof($stream)) || strlen($chunk) > $remaining) {
throw new ImportException('SCAN_READ_ERROR', 'Failed reading STL stream.');
}
if ($chunk === '') {
break;
}
$result .= $chunk;
}
return $result;
}
/** @param resource $stream @return \Generator<int, string> */
private function lines($stream): \Generator
{
$pending = '';
while (($chunk = $this->readUpTo($stream, 4096)) !== '') {
$pending .= $chunk;
while (($newline = strpos($pending, "\n")) !== false) {
if ($newline > 4096) {
$this->invalid('ASCII line exceeds 4096 bytes.');
}
yield substr($pending, 0, $newline);
$pending = substr($pending, $newline + 1);
}
if (strlen($pending) > 4096) {
$this->invalid('ASCII line exceeds 4096 bytes.');
}
}
if ($pending !== '') {
yield $pending;
}
}
/** @param resource $stream */
private function validateAscii($stream): void
{
$state = 'solid';
$hasFacet = false;
foreach ($this->lines($stream) as $line) {
$line = str_ends_with($line, "\r") ? substr($line, 0, -1) : $line;
if (preg_match('/[^\x20-\x7e\t]/', $line)) {
$this->invalid('Non-ASCII or control byte.');
}
$line = trim($line, " \t");
if ($line === '') {
continue;
}
switch ($state) {
case 'solid':
if (!preg_match('/^solid(?:[ \t].*)?$/D', $line)) {
$this->invalid('Expected solid.');
}
$state = 'facet';
break;
case 'facet':
if (preg_match('/^endsolid(?:[ \t].*)?$/D', $line)) {
if (!$hasFacet) {
$this->invalid('Empty mesh.');
}
$state = 'done';
break;
}
$this->vector($line, 'facet normal');
$state = 'outer';
break;
case 'outer':
if (!preg_match('/^outer[ \t]+loop$/D', $line)) {
$this->invalid('Expected outer loop.');
}
$state = 'vertex1';
break;
case 'vertex1':
case 'vertex2':
case 'vertex3':
$this->vector($line, 'vertex');
$state = ['vertex1' => 'vertex2', 'vertex2' => 'vertex3', 'vertex3' => 'endloop'][$state];
break;
case 'endloop':
case 'endfacet':
if ($line !== $state) {
$this->invalid('Expected ' . $state . '.');
}
if ($state === 'endfacet') {
$hasFacet = true;
$state = 'facet';
} else {
$state = 'endfacet';
}
break;
default:
$this->invalid('Trailing ASCII data.');
}
}
if ($state !== 'done') {
$this->invalid('Incomplete ASCII STL.');
}
}
private function vector(string $line, string $prefix): void
{
$number = '([+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?)';
$prefix = str_replace(' ', '[ \t]+', $prefix);
$pattern = '/^' . $prefix . '[ \t]+' . $number . '[ \t]+' . $number . '[ \t]+' . $number . '$/D';
if (!preg_match($pattern, $line, $m)) {
$this->invalid('Expected a normal or vertex with three decimal coordinates.');
}
foreach (array_slice($m, 1) as $value) {
if (!is_finite((float) $value)) {
$this->invalid('Non-finite ASCII coordinate or normal.');
}
}
}
private function invalid(string $message): never
{
throw new ImportException('STL_INVALID', $message);
}
}
+134
View File
@@ -0,0 +1,134 @@
<?php
declare(strict_types=1);
namespace MeshVault\Indexer;
use Closure;
use MeshVault\Exception\PathSafetyException;
use MeshVault\Indexer\Filesystem\PathSafety;
use Throwable;
/**
* Bounded SHA-256 with best-effort mutation detection, not an atomic snapshot.
* Stat timestamps may have coarse resolution and some platforms lack useful inode
* identities. Callers must prevent concurrent writers for a strong guarantee.
*/
final readonly class StreamingHasher
{
public const CHUNK_SIZE = 131072;
private const FIELDS = ['dev', 'ino', 'mode', 'size', 'mtime', 'ctime'];
private ?Closure $chunkObserver;
/**
* Observer receives cumulative bytes after each nonempty chunk; exceptions propagate.
* Trusted fault hooks override native operations: open(path): resource|false,
* read(stream, length): string|false, stat(path): array|false,
* fstat(stream): array|false. The hasher owns and closes the opened stream.
*
* @param array<string, callable> $hooks
*/
public function __construct(
private PathSafety $pathSafety,
?callable $chunkObserver = null,
private array $hooks = [],
) {
$this->chunkObserver = $chunkObserver === null ? null : Closure::fromCallable($chunkObserver);
}
/** @throws ImportException Safety reasons are preserved; I/O reasons use HASH_* tokens. */
public function hash(string $path): string
{
$canonical = $this->safePath($path);
$before = $this->metadata('stat', $canonical);
if (($before['mode'] & 0170000) !== 0100000) {
throw new ImportException('HASH_NOT_REGULAR_FILE', 'Hashing requires a regular file.');
}
$stream = $this->io('open', 'HASH_OPEN_FAILED', $canonical);
if (!is_resource($stream)) {
throw new ImportException('HASH_OPEN_FAILED', 'Unable to open source for hashing.');
}
try {
$this->unchanged($before, $this->metadata('fstat', $stream));
$context = hash_init('sha256');
$bytes = 0;
while (!feof($stream)) {
$chunk = $this->io('read', 'HASH_READ_FAILED', $stream, self::CHUNK_SIZE);
if (!is_string($chunk) || ($chunk === '' && !feof($stream))) {
throw new ImportException('HASH_READ_FAILED', 'Source read failed or made no progress.');
}
if ($chunk === '') {
break;
}
$bytes += strlen($chunk);
if ($bytes > $before['size']) {
throw new ImportException('HASH_SOURCE_CHANGED', 'Source grew while hashing.');
}
hash_update($context, $chunk);
if ($this->chunkObserver !== null) {
($this->chunkObserver)($bytes);
}
}
$this->unchanged($before, $this->metadata('fstat', $stream));
if ($this->safePath($path) !== $canonical) {
throw new ImportException('HASH_SOURCE_CHANGED', 'Source path changed while hashing.');
}
$this->unchanged($before, $this->metadata('stat', $canonical));
if ($bytes !== $before['size']) {
throw new ImportException('HASH_SOURCE_CHANGED', 'Source length changed while hashing.');
}
return hash_final($context);
} finally {
fclose($stream);
}
}
private function safePath(string $path): string
{
clearstatcache(true);
try {
return $this->pathSafety->ensureSourceSafe($path);
} catch (PathSafetyException $error) {
throw new ImportException($error->reason, 'Unsafe hashing source: ' . $error->reason, $error);
}
}
private function metadata(string $operation, mixed $target): array
{
clearstatcache(true);
$stat = $this->io($operation, 'HASH_STAT_FAILED', $target);
foreach (self::FIELDS as $field) {
if (!is_array($stat) || !isset($stat[$field]) || !is_int($stat[$field])) {
throw new ImportException('HASH_STAT_FAILED', 'Unable to inspect source metadata.');
}
}
return $stat;
}
private function unchanged(array $before, array $after): void
{
foreach (self::FIELDS as $field) {
if ($before[$field] !== $after[$field]) {
throw new ImportException('HASH_SOURCE_CHANGED', 'Source metadata changed while hashing: ' . $field);
}
}
}
private function io(string $operation, string $reason, mixed ...$arguments): mixed
{
try {
if (isset($this->hooks[$operation])) {
return ($this->hooks[$operation])(...$arguments);
}
return match ($operation) {
'open' => @fopen($arguments[0], 'rb'),
'read' => @fread($arguments[0], $arguments[1]),
'stat' => @stat($arguments[0]),
'fstat' => @fstat($arguments[0]),
};
} catch (Throwable $error) {
throw new ImportException($reason, 'Hashing I/O failed: ' . $operation, $error);
}
}
}
+183
View File
@@ -0,0 +1,183 @@
<?php
declare(strict_types=1);
namespace MeshVault\Indexer;
use MeshVault\Exception\PathSafetyException;
use MeshVault\Indexer\Filesystem\PathSafety;
use Throwable;
/**
* Verified, no-overwrite publication via hard link (never rename fallback).
* PathSafety checks are point-in-time: callers MUST control source/destination
* tree mutation. Native PHP streams lack openat/O_NOFOLLOW; checks cannot prevent
* adversarial path replacement between checking and I/O. Atomic publication does
* not imply power-loss durability (no directory fsync), nor source immutability.
*/
final readonly class VerifiedCopier
{
/**
* Trusted test seams only; replacements follow native return conventions:
* read(resource, int): string|false; write(resource, string): int|false;
* flush(resource): bool; link(string $temporary, string $destination): bool.
* A successful replacement must actually perform the native operation.
*
* @param array<string, callable> $operations
*/
public function __construct(private PathSafety $pathSafety, private array $operations = [])
{
foreach ($operations as $name => $operation) {
if (!in_array($name, ['read', 'write', 'flush', 'link'], true) || !is_callable($operation)) {
throw new \InvalidArgumentException('Unknown or non-callable copy operation: ' . $name);
}
}
}
/**
* Copy an existing regular source to a filename under the managed library.
* Parent must exist. Returns absolute published path; collisions try _001
* through _999 before the last extension. Never deletes/overwrites a winner.
* Throws PathSafetyException unchanged; ImportException reasons are
* COPY_FAILED, HASH_MISMATCH (size or digest), COLLISION, INVALID_ARGUMENT.
*/
public function copy(string $source, string $destination, int $expectedSize, string $expectedSha256): string
{
if ($expectedSize < 0 || preg_match('/\A[0-9a-fA-F]{64}\z/', $expectedSha256) !== 1) {
throw new ImportException('INVALID_ARGUMENT', 'Expected a nonnegative size and SHA-256 hex digest.');
}
$source = $this->pathSafety->ensureSourceSafe($source);
$destination = $this->pathSafety->ensureDestSafe($destination);
if (!is_file($source) || !is_dir(dirname($destination))) {
throw new ImportException('COPY_FAILED', 'Source must be regular and destination parent must exist.');
}
$input = $output = null;
$temporary = null;
try {
$input = @fopen($this->pathSafety->ensureSourceSafe($source), 'rb');
if ($input === false) {
throw new ImportException('COPY_FAILED', 'Cannot open source.');
}
$candidate = dirname($destination) . DIRECTORY_SEPARATOR
. '.meshvault-' . bin2hex(random_bytes(16)) . '.tmp';
$candidate = $this->pathSafety->ensureDestSafe($candidate);
$output = @fopen($candidate, 'x+b');
if ($output === false) {
throw new ImportException('COPY_FAILED', 'Cannot exclusively create temporary file.');
}
$temporary = $candidate; // Only clean up a file we successfully created.
$size = 0;
$hash = hash_init('sha256');
while (!feof($input)) {
$bytes = $this->invoke('read', $input, 131072);
if ($bytes === false || ($bytes === '' && !feof($input))) {
throw new ImportException('COPY_FAILED', 'Source read failed or stalled.');
}
$length = strlen($bytes);
$size += $length;
if ($size > $expectedSize) {
throw new ImportException('HASH_MISMATCH', 'Source exceeds expected size.');
}
hash_update($hash, $bytes);
for ($offset = 0; $offset < $length; $offset += $written) {
$written = $this->invoke('write', $output, substr($bytes, $offset));
if (!is_int($written) || $written <= 0 || $written > $length - $offset) {
throw new ImportException('COPY_FAILED', 'Temporary write failed or stalled.');
}
}
}
if ($size !== $expectedSize || !hash_equals(strtolower($expectedSha256), hash_final($hash))) {
throw new ImportException('HASH_MISMATCH', 'Source size or SHA-256 differs from expected metadata.');
}
if (!$this->invoke('flush', $output)) {
throw new ImportException('COPY_FAILED', 'Temporary flush failed.');
}
$this->verify($output, $expectedSize, strtolower($expectedSha256));
fclose($input);
$input = null;
if (!fclose($output)) {
throw new ImportException('COPY_FAILED', 'Temporary close failed.');
}
$output = null;
return $this->publish($temporary, $destination);
} catch (ImportException | PathSafetyException $error) {
throw $error;
} catch (Throwable $error) {
throw new ImportException('COPY_FAILED', 'Copy operation failed.', $error);
} finally {
if (is_resource($input)) {
fclose($input);
}
if (is_resource($output)) {
fclose($output);
}
if ($temporary !== null) {
// Cleanup must never throw: a thrown exception here would mask the original
// error (HASH_MISMATCH, COPY_FAILED, etc.) that triggered the finally block.
// Publication may already have succeeded; never remove its final name.
@unlink($temporary);
}
}
}
/** @param resource $stream */
private function verify($stream, int $size, string $digest): void
{
if (!rewind($stream)) {
throw new ImportException('COPY_FAILED', 'Cannot rewind temporary file.');
}
$hash = hash_init('sha256');
$actualSize = 0;
while (!feof($stream)) {
$bytes = $this->invoke('read', $stream, 131072);
if ($bytes === false || ($bytes === '' && !feof($stream))) {
throw new ImportException('COPY_FAILED', 'Cannot read back temporary file.');
}
$actualSize += strlen($bytes);
hash_update($hash, $bytes);
}
if ($actualSize !== $size || !hash_equals($digest, hash_final($hash))) {
throw new ImportException('HASH_MISMATCH', 'Written size or SHA-256 differs from expected metadata.');
}
}
private function publish(string $temporary, string $destination): string
{
$extension = pathinfo($destination, PATHINFO_EXTENSION);
$stem = $extension === '' ? $destination : substr($destination, 0, -strlen($extension) - 1);
for ($suffix = 0; $suffix <= 999; ++$suffix) {
$candidate = $suffix === 0 ? $destination
: $stem . sprintf('_%03d', $suffix) . ($extension === '' ? '' : '.' . $extension);
$candidate = $this->pathSafety->ensureDestSafe($candidate);
clearstatcache(true, $candidate);
if (file_exists($candidate)) {
continue;
}
$this->pathSafety->ensureDestSafe($temporary);
if ($this->invoke('link', $temporary, $candidate)) {
return $candidate;
}
// A failed link is a collision only if another entry now occupies it.
$this->pathSafety->ensureDestSafe($candidate);
clearstatcache(true, $candidate);
if (!file_exists($candidate)) {
throw new ImportException('COPY_FAILED', 'Hard-link publication failed.');
}
}
throw new ImportException('COLLISION', 'Destination and all 999 suffixes are occupied.');
}
private function invoke(string $operation, mixed ...$arguments): mixed
{
if (isset($this->operations[$operation])) {
return ($this->operations[$operation])(...$arguments);
}
return match ($operation) {
'read' => @fread(...$arguments),
'write' => @fwrite(...$arguments),
'flush' => @fflush(...$arguments),
'link' => @link(...$arguments),
};
}
}
@@ -0,0 +1,24 @@
<?php
declare(strict_types=1);
namespace MeshVault\Repository;
interface CatalogRepositoryInterface
{
public function ensureModel(string $folderPath, string $modelName): int;
public function recordFile(
int $modelId,
string $managedPath,
string $hash,
int $size,
?string $mime = null,
string $status = 'Present'
): int;
/** @return array{model: array, folder: array, files: array}|null */
public function aggregate(int $modelId): ?array;
public function setFileStatus(int $fileId, string $status): void;
}
@@ -0,0 +1,36 @@
<?php
declare(strict_types=1);
namespace MeshVault\Repository;
interface ImportJobRepositoryInterface
{
public function create(string $sourcePath, array $items): int;
public function get(int $jobId): ?array;
public function items(int $jobId): array;
public function claimNext(): ?array;
/** Caller MUST hold the global library flock; never use age-based recovery. */
public function resumeInterrupted(): void;
public function planItem(int $itemId, string $managedPath, string $hash, int $size): void;
public function updateManagedPath(int $itemId, string $managedPath): void;
public function finishItem(int $itemId, string $status, ?string $failureReason = null): void;
public function release(int $jobId): void;
public function cancel(int $jobId): void;
public function fail(int $jobId, string $failureReason): void;
public function completeEmpty(int $jobId): void;
/** All repositories participating in the callback must share this PDO connection. */
public function atomic(callable $callback): mixed;
}
+122
View File
@@ -0,0 +1,122 @@
<?php
declare(strict_types=1);
namespace MeshVault\Repository;
use DomainException;
use InvalidArgumentException;
use PDO;
final class PdoCatalogRepository extends PdoRepository implements CatalogRepositoryInterface
{
public function ensureModel(string $folderPath, string $modelName): int
{
if ($modelName === '') {
throw new InvalidArgumentException('Model name must not be empty.');
}
return $this->transaction(function () use ($folderPath, $modelName): int {
$folder = $this->ensureFolder($folderPath);
$this->execute(
'INSERT INTO models (folder_id, name) VALUES (?, ?) ON CONFLICT (folder_id, name) DO NOTHING',
[$folder, $modelName]
);
return (int) $this->row(
'SELECT id FROM models WHERE folder_id = ? AND name = ?',
[$folder, $modelName]
)['id'];
});
}
public function recordFile(
int $modelId,
string $managedPath,
string $hash,
int $size,
?string $mime = null,
string $status = 'Present'
): int {
$this->validateStatus($status);
if ($managedPath === '' || $hash === '' || $size < 0) {
throw new InvalidArgumentException('A file requires a path, hash and non-negative size.');
}
return $this->transaction(function () use ($modelId, $managedPath, $hash, $size, $mime, $status): int {
// Serialize catalog writers on the root row, including paths assigned to different models.
$this->ensureFolder('/');
$this->execute("UPDATE folders SET name = name WHERE path = '/'");
$existing = $this->row('SELECT * FROM files WHERE managed_path = ?', [$managedPath]);
if ($existing !== null) {
if (
(int) $existing['model_id'] !== $modelId || $existing['hash'] !== $hash
|| (int) $existing['size'] !== $size
) {
throw new DomainException('Managed path already records a different file.');
}
return (int) $existing['id'];
}
$folder = $this->ensureFolder(dirname(str_replace('\\', '/', $managedPath)));
$file = $this->row(
'INSERT INTO files (model_id, folder_id, name, managed_path, hash, size, mime_type, status)'
. ' VALUES (?, ?, ?, ?, ?, ?, ?, ?) RETURNING id',
[$modelId, $folder, basename($managedPath), $managedPath, $hash, $size, $mime, $status]
);
return (int) $file['id'];
});
}
public function aggregate(int $modelId): ?array
{
return $this->transaction(function () use ($modelId): ?array {
$model = $this->row('SELECT * FROM models WHERE id = ?', [$modelId]);
if ($model === null) {
return null;
}
return [
'model' => $model,
'folder' => $this->row('SELECT * FROM folders WHERE id = ?', [$model['folder_id']]),
'files' => $this->execute('SELECT * FROM files WHERE model_id = ? ORDER BY id', [$modelId])
->fetchAll(PDO::FETCH_ASSOC),
];
});
}
public function setFileStatus(int $fileId, string $status): void
{
$this->validateStatus($status);
if (
$this->execute(
'UPDATE files SET status = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ?',
[$status, $fileId]
)->rowCount() === 0
) {
throw new DomainException('Unknown file.');
}
}
private function validateStatus(string $status): void
{
if (!in_array($status, ['Present', 'Missing', 'Updated', 'Duplicate'], true)) {
throw new InvalidArgumentException('Invalid file status.');
}
}
private function ensureFolder(string $path): int
{
$segments = array_values(array_filter(explode('/', str_replace('\\', '/', $path)), 'strlen'));
$segments = array_values(array_filter($segments, static fn (string $part): bool => $part !== '.'));
if (in_array('..', $segments, true)) {
throw new InvalidArgumentException('Folder traversal is not allowed.');
}
$parent = null;
$current = '';
foreach (array_merge([''], $segments) as $segment) {
$current = $segment === '' ? '/' : rtrim($current, '/') . '/' . $segment;
$this->execute(
'INSERT INTO folders (parent_id, name, path) VALUES (?, ?, ?) ON CONFLICT (path) DO NOTHING',
[$parent, $segment === '' ? '/' : $segment, $current]
);
$parent = (int) $this->row('SELECT id FROM folders WHERE path = ?', [$current])['id'];
}
return $parent;
}
}
+311
View File
@@ -0,0 +1,311 @@
<?php
declare(strict_types=1);
namespace MeshVault\Repository;
use DomainException;
use InvalidArgumentException;
use PDO;
final class PdoImportJobRepository extends PdoRepository implements ImportJobRepositoryInterface
{
public function create(string $sourcePath, array $items): int
{
if ($sourcePath === '') {
throw new InvalidArgumentException('Source path must not be empty.');
}
foreach ($items as $index => $item) {
if (!isset($item['source_path']) || !isset($item['size'])) {
throw new InvalidArgumentException("Item $index requires source_path and size.");
}
if ($item['size'] < 0) {
throw new InvalidArgumentException("Item $index size must be non-negative.");
}
$allowed = ['Pending', 'Copying', 'Completed', 'Failed', 'Skipped'];
if (isset($item['status']) && !in_array($item['status'], $allowed, true)) {
throw new InvalidArgumentException("Item $index has invalid status.");
}
}
return $this->transaction(function () use ($sourcePath, $items): int {
$job = $this->row(
'INSERT INTO import_jobs (source_path, total_files, total_bytes) VALUES (?, ?, ?) RETURNING id',
[$sourcePath, count($items), array_sum(array_column($items, 'size'))]
);
$jobId = (int) $job['id'];
if ($items === []) {
// Empty manifests stay claimable; completion is an explicit caller decision.
return $jobId;
}
$statement = $this->pdo->prepare(
'INSERT INTO import_items (job_id, source_path, size, model_id, status, failure_reason)'
. ' VALUES (?, ?, ?, ?, ?, ?)'
);
foreach ($items as $item) {
$status = $item['status'] ?? 'Pending';
$failureReason = $item['failure_reason'] ?? null;
$statement->execute([
$jobId,
$item['source_path'],
$item['size'],
$item['model_id'] ?? null,
$status,
$failureReason,
]);
}
// Scan-time failures arrive terminal, so publish progress immediately.
$this->refresh($jobId);
return $jobId;
});
}
public function get(int $jobId): ?array
{
return $this->row('SELECT * FROM import_jobs WHERE id = ?', [$jobId]);
}
public function items(int $jobId): array
{
return $this->execute('SELECT * FROM import_items WHERE job_id = ? ORDER BY id', [$jobId])
->fetchAll(PDO::FETCH_ASSOC);
}
public function claimNext(): ?array
{
return $this->transaction(function (): ?array {
if ($this->pdo->getAttribute(PDO::ATTR_DRIVER_NAME) === 'pgsql') {
// SKIP LOCKED lets concurrent workers claim distinct jobs without blocking.
$candidate = $this->row(
"SELECT id FROM import_jobs WHERE status = 'Pending'"
. ' ORDER BY id FOR UPDATE SKIP LOCKED LIMIT 1'
);
if ($candidate === null) {
return null;
}
return $this->row(
"UPDATE import_jobs SET status = 'Copying', started_at = CURRENT_TIMESTAMP"
. ' WHERE id = ? RETURNING *',
[$candidate['id']]
);
}
// SQLite serializes writers, so a single conditional UPDATE is already atomic.
return $this->row(
"UPDATE import_jobs SET status = 'Copying', started_at = CURRENT_TIMESTAMP"
. " WHERE id = (SELECT id FROM import_jobs WHERE status = 'Pending' ORDER BY id LIMIT 1)"
. ' RETURNING *'
);
});
}
public function resumeInterrupted(): void
{
$this->transaction(function (): void {
// Plans (managed_path/hash) are deliberately preserved for idempotent recovery.
$this->execute(
"UPDATE import_items SET status = 'Pending', started_at = NULL WHERE status = 'Copying'"
. " AND job_id IN (SELECT id FROM import_jobs WHERE status IN ('Scanning', 'Copying'))"
);
$this->execute(
"UPDATE import_jobs SET status = 'Pending' WHERE status IN ('Scanning', 'Copying')"
);
});
}
public function planItem(int $itemId, string $managedPath, string $hash, int $size): void
{
if ($managedPath === '' || $hash === '' || $size < 0) {
throw new InvalidArgumentException('A plan requires a path, hash and non-negative size.');
}
$this->transaction(function () use ($itemId, $managedPath, $hash, $size): void {
$item = $this->lockedItem($itemId);
if (in_array($item['status'], ['Completed', 'Failed', 'Skipped'], true)) {
throw new DomainException('Cannot plan a terminal import item.');
}
if ($item['managed_path'] !== null) {
if (
$item['managed_path'] !== $managedPath || $item['hash'] !== $hash
|| (int) $item['size'] !== $size
) {
throw new DomainException('An import plan is immutable once recorded.');
}
return;
}
$this->execute(
"UPDATE import_items SET status = 'Copying', managed_path = ?, hash = ?, size = ?,"
. ' started_at = CURRENT_TIMESTAMP WHERE id = ?',
[$managedPath, $hash, $size, $itemId]
);
});
}
public function updateManagedPath(int $itemId, string $managedPath): void
{
if ($managedPath === '') {
throw new InvalidArgumentException('A managed path must not be empty.');
}
$this->transaction(function () use ($itemId, $managedPath): void {
$item = $this->lockedItem($itemId);
if (in_array($item['status'], ['Completed', 'Failed', 'Skipped'], true)) {
throw new DomainException('Cannot update a terminal import item.');
}
$this->execute(
'UPDATE import_items SET managed_path = ? WHERE id = ?',
[$managedPath, $itemId]
);
});
}
public function finishItem(int $itemId, string $status, ?string $failureReason = null): void
{
if (!in_array($status, ['Completed', 'Failed', 'Skipped'], true)) {
throw new InvalidArgumentException('Only terminal item statuses may finish an item.');
}
$this->transaction(function () use ($itemId, $status, $failureReason): void {
$item = $this->lockedItem($itemId);
if (in_array($item['status'], ['Completed', 'Failed', 'Skipped'], true)) {
if ($item['status'] !== $status) {
throw new DomainException('Import item already reached a different terminal status.');
}
return;
}
$this->execute(
'UPDATE import_items SET status = ?, failure_reason = ?, completed_at = CURRENT_TIMESTAMP'
. ' WHERE id = ?',
[$status, $failureReason, $itemId]
);
$this->refresh((int) $item['job_id']);
});
}
public function release(int $jobId): void
{
$this->transaction(function () use ($jobId): void {
$this->execute(
"UPDATE import_items SET status = 'Pending', started_at = NULL"
. " WHERE job_id = ? AND status = 'Copying'",
[$jobId]
);
$this->execute(
"UPDATE import_jobs SET status = 'Pending' WHERE id = ? AND status IN ('Scanning', 'Copying')",
[$jobId]
);
});
}
public function cancel(int $jobId): void
{
$this->terminate($jobId, 'Cancelled', 'Skipped', null);
}
public function fail(int $jobId, string $failureReason): void
{
if ($failureReason === '') {
throw new InvalidArgumentException('A failure reason is required.');
}
$this->terminate($jobId, 'Failed', 'Failed', $failureReason);
}
public function completeEmpty(int $jobId): void
{
$this->transaction(function () use ($jobId): void {
$job = $this->row('SELECT * FROM import_jobs WHERE id = ?', [$jobId]);
if ($job === null) {
throw new DomainException('Unknown import job.');
}
if ($this->items($jobId) !== []) {
throw new DomainException('Import job still has items.');
}
if (in_array($job['status'], ['Completed', 'Failed', 'Cancelled'], true)) {
return;
}
$this->execute(
"UPDATE import_jobs SET status = 'Completed', processed_files = 0,"
. ' completed_at = CURRENT_TIMESTAMP WHERE id = ?',
[$jobId]
);
});
}
public function atomic(callable $callback): mixed
{
return $this->transaction($callback);
}
/** Terminal job transitions share one shape: finish open items, then close the job. */
private function terminate(int $jobId, string $jobStatus, string $itemStatus, ?string $reason): void
{
$this->transaction(function () use ($jobId, $jobStatus, $itemStatus, $reason): void {
$job = $this->row('SELECT * FROM import_jobs WHERE id = ?', [$jobId]);
if ($job === null) {
throw new DomainException('Unknown import job.');
}
if (in_array($job['status'], ['Completed', 'Failed', 'Cancelled'], true)) {
return;
}
$this->execute(
'UPDATE import_items SET status = ?, failure_reason = ?, completed_at = CURRENT_TIMESTAMP'
. " WHERE job_id = ? AND status IN ('Pending', 'Copying')",
[$itemStatus, $reason, $jobId]
);
$this->execute(
'UPDATE import_jobs SET status = ?, processed_files = ?, copied_bytes = ?,'
. ' completed_at = CURRENT_TIMESTAMP WHERE id = ?',
[$jobStatus, $this->processedFiles($jobId), $this->copiedBytes($jobId), $jobId]
);
});
}
/** Progress is recomputed from items so repeated or crashed updates cannot drift. */
private function refresh(int $jobId): void
{
$job = $this->row('SELECT * FROM import_jobs WHERE id = ?', [$jobId]);
if ($job === null || in_array($job['status'], ['Completed', 'Failed', 'Cancelled'], true)) {
return;
}
$total = (int) $this->row('SELECT COUNT(*) AS total FROM import_items WHERE job_id = ?', [$jobId])['total'];
$processed = $this->processedFiles($jobId);
$failed = (int) $this->row(
"SELECT COUNT(*) AS failed FROM import_items WHERE job_id = ? AND status = 'Failed'",
[$jobId]
)['failed'];
if ($total === 0 || $processed < $total) {
$this->execute(
'UPDATE import_jobs SET processed_files = ?, copied_bytes = ? WHERE id = ?',
[$processed, $this->copiedBytes($jobId), $jobId]
);
return;
}
$this->execute(
'UPDATE import_jobs SET status = ?, processed_files = ?, copied_bytes = ?,'
. ' completed_at = CURRENT_TIMESTAMP WHERE id = ?',
[$failed > 0 ? 'Failed' : 'Completed', $processed, $this->copiedBytes($jobId), $jobId]
);
}
private function processedFiles(int $jobId): int
{
return (int) $this->row(
'SELECT COUNT(*) AS processed FROM import_items WHERE job_id = ?'
. " AND status IN ('Completed', 'Failed', 'Skipped')",
[$jobId]
)['processed'];
}
private function copiedBytes(int $jobId): int
{
return (int) $this->row(
"SELECT COALESCE(SUM(size), 0) AS copied FROM import_items WHERE job_id = ? AND status = 'Completed'",
[$jobId]
)['copied'];
}
private function lockedItem(int $itemId): array
{
$suffix = $this->pdo->getAttribute(PDO::ATTR_DRIVER_NAME) === 'pgsql' ? ' FOR UPDATE' : '';
$item = $this->row('SELECT * FROM import_items WHERE id = ?' . $suffix, [$itemId]);
if ($item === null) {
throw new DomainException('Unknown import item.');
}
return $item;
}
}
+61
View File
@@ -0,0 +1,61 @@
<?php
declare(strict_types=1);
namespace MeshVault\Repository;
use PDO;
use PDOStatement;
use Throwable;
/** @internal Shared SQL and composable transaction support. */
abstract class PdoRepository
{
public function __construct(protected PDO $pdo)
{
$pdo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
}
protected function execute(string $sql, array $parameters = []): PDOStatement
{
$statement = $this->pdo->prepare($sql);
$statement->execute($parameters);
return $statement;
}
protected function row(string $sql, array $parameters = []): ?array
{
$statement = $this->execute($sql, $parameters);
$row = $statement->fetch(PDO::FETCH_ASSOC);
$statement->closeCursor();
return $row === false ? null : $row;
}
protected function transaction(callable $callback): mixed
{
$nested = $this->pdo->inTransaction();
$savepoint = 'repository_' . bin2hex(random_bytes(8));
if ($nested) {
$this->pdo->exec('SAVEPOINT ' . $savepoint);
} else {
$this->pdo->beginTransaction();
}
try {
$result = $callback();
if ($nested) {
$this->pdo->exec('RELEASE SAVEPOINT ' . $savepoint);
} else {
$this->pdo->commit();
}
return $result;
} catch (Throwable $error) {
if ($nested) {
$this->pdo->exec('ROLLBACK TO SAVEPOINT ' . $savepoint);
$this->pdo->exec('RELEASE SAVEPOINT ' . $savepoint);
} elseif ($this->pdo->inTransaction()) {
$this->pdo->rollBack();
}
throw $error;
}
}
}