feat: M3 import & indexing engine delivery layer with durable collision handling, space checking and console

This commit is contained in:
Cline committed 2026-10-12 11:29:17 -05:00
1 parent 93d991703c
commit 9d277d9e72
48 files changed
+5147 -4

No files matched your search

@@ -0,0 +1,24 @@
<?php
declare(strict_types=1);
namespace MeshVault\Repository;
interface CatalogRepositoryInterface
{
public function ensureModel(string $folderPath, string $modelName): int;
public function recordFile(
int $modelId,
string $managedPath,
string $hash,
int $size,
?string $mime = null,
string $status = 'Present'
): int;
/** @return array{model: array, folder: array, files: array}|null */
public function aggregate(int $modelId): ?array;
public function setFileStatus(int $fileId, string $status): void;
}
@@ -0,0 +1,36 @@
<?php
declare(strict_types=1);
namespace MeshVault\Repository;
interface ImportJobRepositoryInterface
{
public function create(string $sourcePath, array $items): int;
public function get(int $jobId): ?array;
public function items(int $jobId): array;
public function claimNext(): ?array;
/** Caller MUST hold the global library flock; never use age-based recovery. */
public function resumeInterrupted(): void;
public function planItem(int $itemId, string $managedPath, string $hash, int $size): void;
public function updateManagedPath(int $itemId, string $managedPath): void;
public function finishItem(int $itemId, string $status, ?string $failureReason = null): void;
public function release(int $jobId): void;
public function cancel(int $jobId): void;
public function fail(int $jobId, string $failureReason): void;
public function completeEmpty(int $jobId): void;
/** All repositories participating in the callback must share this PDO connection. */
public function atomic(callable $callback): mixed;
}
+122
View File
@@ -0,0 +1,122 @@
<?php
declare(strict_types=1);
namespace MeshVault\Repository;
use DomainException;
use InvalidArgumentException;
use PDO;
final class PdoCatalogRepository extends PdoRepository implements CatalogRepositoryInterface
{
public function ensureModel(string $folderPath, string $modelName): int
{
if ($modelName === '') {
throw new InvalidArgumentException('Model name must not be empty.');
}
return $this->transaction(function () use ($folderPath, $modelName): int {
$folder = $this->ensureFolder($folderPath);
$this->execute(
'INSERT INTO models (folder_id, name) VALUES (?, ?) ON CONFLICT (folder_id, name) DO NOTHING',
[$folder, $modelName]
);
return (int) $this->row(
'SELECT id FROM models WHERE folder_id = ? AND name = ?',
[$folder, $modelName]
)['id'];
});
}
public function recordFile(
int $modelId,
string $managedPath,
string $hash,
int $size,
?string $mime = null,
string $status = 'Present'
): int {
$this->validateStatus($status);
if ($managedPath === '' || $hash === '' || $size < 0) {
throw new InvalidArgumentException('A file requires a path, hash and non-negative size.');
}
return $this->transaction(function () use ($modelId, $managedPath, $hash, $size, $mime, $status): int {
// Serialize catalog writers on the root row, including paths assigned to different models.
$this->ensureFolder('/');
$this->execute("UPDATE folders SET name = name WHERE path = '/'");
$existing = $this->row('SELECT * FROM files WHERE managed_path = ?', [$managedPath]);
if ($existing !== null) {
if (
(int) $existing['model_id'] !== $modelId || $existing['hash'] !== $hash
|| (int) $existing['size'] !== $size
) {
throw new DomainException('Managed path already records a different file.');
}
return (int) $existing['id'];
}
$folder = $this->ensureFolder(dirname(str_replace('\\', '/', $managedPath)));
$file = $this->row(
'INSERT INTO files (model_id, folder_id, name, managed_path, hash, size, mime_type, status)'
. ' VALUES (?, ?, ?, ?, ?, ?, ?, ?) RETURNING id',
[$modelId, $folder, basename($managedPath), $managedPath, $hash, $size, $mime, $status]
);
return (int) $file['id'];
});
}
public function aggregate(int $modelId): ?array
{
return $this->transaction(function () use ($modelId): ?array {
$model = $this->row('SELECT * FROM models WHERE id = ?', [$modelId]);
if ($model === null) {
return null;
}
return [
'model' => $model,
'folder' => $this->row('SELECT * FROM folders WHERE id = ?', [$model['folder_id']]),
'files' => $this->execute('SELECT * FROM files WHERE model_id = ? ORDER BY id', [$modelId])
->fetchAll(PDO::FETCH_ASSOC),
];
});
}
public function setFileStatus(int $fileId, string $status): void
{
$this->validateStatus($status);
if (
$this->execute(
'UPDATE files SET status = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ?',
[$status, $fileId]
)->rowCount() === 0
) {
throw new DomainException('Unknown file.');
}
}
private function validateStatus(string $status): void
{
if (!in_array($status, ['Present', 'Missing', 'Updated', 'Duplicate'], true)) {
throw new InvalidArgumentException('Invalid file status.');
}
}
private function ensureFolder(string $path): int
{
$segments = array_values(array_filter(explode('/', str_replace('\\', '/', $path)), 'strlen'));
$segments = array_values(array_filter($segments, static fn (string $part): bool => $part !== '.'));
if (in_array('..', $segments, true)) {
throw new InvalidArgumentException('Folder traversal is not allowed.');
}
$parent = null;
$current = '';
foreach (array_merge([''], $segments) as $segment) {
$current = $segment === '' ? '/' : rtrim($current, '/') . '/' . $segment;
$this->execute(
'INSERT INTO folders (parent_id, name, path) VALUES (?, ?, ?) ON CONFLICT (path) DO NOTHING',
[$parent, $segment === '' ? '/' : $segment, $current]
);
$parent = (int) $this->row('SELECT id FROM folders WHERE path = ?', [$current])['id'];
}
return $parent;
}
}
+311
View File
@@ -0,0 +1,311 @@
<?php
declare(strict_types=1);
namespace MeshVault\Repository;
use DomainException;
use InvalidArgumentException;
use PDO;
final class PdoImportJobRepository extends PdoRepository implements ImportJobRepositoryInterface
{
public function create(string $sourcePath, array $items): int
{
if ($sourcePath === '') {
throw new InvalidArgumentException('Source path must not be empty.');
}
foreach ($items as $index => $item) {
if (!isset($item['source_path']) || !isset($item['size'])) {
throw new InvalidArgumentException("Item $index requires source_path and size.");
}
if ($item['size'] < 0) {
throw new InvalidArgumentException("Item $index size must be non-negative.");
}
$allowed = ['Pending', 'Copying', 'Completed', 'Failed', 'Skipped'];
if (isset($item['status']) && !in_array($item['status'], $allowed, true)) {
throw new InvalidArgumentException("Item $index has invalid status.");
}
}
return $this->transaction(function () use ($sourcePath, $items): int {
$job = $this->row(
'INSERT INTO import_jobs (source_path, total_files, total_bytes) VALUES (?, ?, ?) RETURNING id',
[$sourcePath, count($items), array_sum(array_column($items, 'size'))]
);
$jobId = (int) $job['id'];
if ($items === []) {
// Empty manifests stay claimable; completion is an explicit caller decision.
return $jobId;
}
$statement = $this->pdo->prepare(
'INSERT INTO import_items (job_id, source_path, size, model_id, status, failure_reason)'
. ' VALUES (?, ?, ?, ?, ?, ?)'
);
foreach ($items as $item) {
$status = $item['status'] ?? 'Pending';
$failureReason = $item['failure_reason'] ?? null;
$statement->execute([
$jobId,
$item['source_path'],
$item['size'],
$item['model_id'] ?? null,
$status,
$failureReason,
]);
}
// Scan-time failures arrive terminal, so publish progress immediately.
$this->refresh($jobId);
return $jobId;
});
}
public function get(int $jobId): ?array
{
return $this->row('SELECT * FROM import_jobs WHERE id = ?', [$jobId]);
}
public function items(int $jobId): array
{
return $this->execute('SELECT * FROM import_items WHERE job_id = ? ORDER BY id', [$jobId])
->fetchAll(PDO::FETCH_ASSOC);
}
public function claimNext(): ?array
{
return $this->transaction(function (): ?array {
if ($this->pdo->getAttribute(PDO::ATTR_DRIVER_NAME) === 'pgsql') {
// SKIP LOCKED lets concurrent workers claim distinct jobs without blocking.
$candidate = $this->row(
"SELECT id FROM import_jobs WHERE status = 'Pending'"
. ' ORDER BY id FOR UPDATE SKIP LOCKED LIMIT 1'
);
if ($candidate === null) {
return null;
}
return $this->row(
"UPDATE import_jobs SET status = 'Copying', started_at = CURRENT_TIMESTAMP"
. ' WHERE id = ? RETURNING *',
[$candidate['id']]
);
}
// SQLite serializes writers, so a single conditional UPDATE is already atomic.
return $this->row(
"UPDATE import_jobs SET status = 'Copying', started_at = CURRENT_TIMESTAMP"
. " WHERE id = (SELECT id FROM import_jobs WHERE status = 'Pending' ORDER BY id LIMIT 1)"
. ' RETURNING *'
);
});
}
public function resumeInterrupted(): void
{
$this->transaction(function (): void {
// Plans (managed_path/hash) are deliberately preserved for idempotent recovery.
$this->execute(
"UPDATE import_items SET status = 'Pending', started_at = NULL WHERE status = 'Copying'"
. " AND job_id IN (SELECT id FROM import_jobs WHERE status IN ('Scanning', 'Copying'))"
);
$this->execute(
"UPDATE import_jobs SET status = 'Pending' WHERE status IN ('Scanning', 'Copying')"
);
});
}
public function planItem(int $itemId, string $managedPath, string $hash, int $size): void
{
if ($managedPath === '' || $hash === '' || $size < 0) {
throw new InvalidArgumentException('A plan requires a path, hash and non-negative size.');
}
$this->transaction(function () use ($itemId, $managedPath, $hash, $size): void {
$item = $this->lockedItem($itemId);
if (in_array($item['status'], ['Completed', 'Failed', 'Skipped'], true)) {
throw new DomainException('Cannot plan a terminal import item.');
}
if ($item['managed_path'] !== null) {
if (
$item['managed_path'] !== $managedPath || $item['hash'] !== $hash
|| (int) $item['size'] !== $size
) {
throw new DomainException('An import plan is immutable once recorded.');
}
return;
}
$this->execute(
"UPDATE import_items SET status = 'Copying', managed_path = ?, hash = ?, size = ?,"
. ' started_at = CURRENT_TIMESTAMP WHERE id = ?',
[$managedPath, $hash, $size, $itemId]
);
});
}
public function updateManagedPath(int $itemId, string $managedPath): void
{
if ($managedPath === '') {
throw new InvalidArgumentException('A managed path must not be empty.');
}
$this->transaction(function () use ($itemId, $managedPath): void {
$item = $this->lockedItem($itemId);
if (in_array($item['status'], ['Completed', 'Failed', 'Skipped'], true)) {
throw new DomainException('Cannot update a terminal import item.');
}
$this->execute(
'UPDATE import_items SET managed_path = ? WHERE id = ?',
[$managedPath, $itemId]
);
});
}
public function finishItem(int $itemId, string $status, ?string $failureReason = null): void
{
if (!in_array($status, ['Completed', 'Failed', 'Skipped'], true)) {
throw new InvalidArgumentException('Only terminal item statuses may finish an item.');
}
$this->transaction(function () use ($itemId, $status, $failureReason): void {
$item = $this->lockedItem($itemId);
if (in_array($item['status'], ['Completed', 'Failed', 'Skipped'], true)) {
if ($item['status'] !== $status) {
throw new DomainException('Import item already reached a different terminal status.');
}
return;
}
$this->execute(
'UPDATE import_items SET status = ?, failure_reason = ?, completed_at = CURRENT_TIMESTAMP'
. ' WHERE id = ?',
[$status, $failureReason, $itemId]
);
$this->refresh((int) $item['job_id']);
});
}
public function release(int $jobId): void
{
$this->transaction(function () use ($jobId): void {
$this->execute(
"UPDATE import_items SET status = 'Pending', started_at = NULL"
. " WHERE job_id = ? AND status = 'Copying'",
[$jobId]
);
$this->execute(
"UPDATE import_jobs SET status = 'Pending' WHERE id = ? AND status IN ('Scanning', 'Copying')",
[$jobId]
);
});
}
public function cancel(int $jobId): void
{
$this->terminate($jobId, 'Cancelled', 'Skipped', null);
}
public function fail(int $jobId, string $failureReason): void
{
if ($failureReason === '') {
throw new InvalidArgumentException('A failure reason is required.');
}
$this->terminate($jobId, 'Failed', 'Failed', $failureReason);
}
public function completeEmpty(int $jobId): void
{
$this->transaction(function () use ($jobId): void {
$job = $this->row('SELECT * FROM import_jobs WHERE id = ?', [$jobId]);
if ($job === null) {
throw new DomainException('Unknown import job.');
}
if ($this->items($jobId) !== []) {
throw new DomainException('Import job still has items.');
}
if (in_array($job['status'], ['Completed', 'Failed', 'Cancelled'], true)) {
return;
}
$this->execute(
"UPDATE import_jobs SET status = 'Completed', processed_files = 0,"
. ' completed_at = CURRENT_TIMESTAMP WHERE id = ?',
[$jobId]
);
});
}
public function atomic(callable $callback): mixed
{
return $this->transaction($callback);
}
/** Terminal job transitions share one shape: finish open items, then close the job. */
private function terminate(int $jobId, string $jobStatus, string $itemStatus, ?string $reason): void
{
$this->transaction(function () use ($jobId, $jobStatus, $itemStatus, $reason): void {
$job = $this->row('SELECT * FROM import_jobs WHERE id = ?', [$jobId]);
if ($job === null) {
throw new DomainException('Unknown import job.');
}
if (in_array($job['status'], ['Completed', 'Failed', 'Cancelled'], true)) {
return;
}
$this->execute(
'UPDATE import_items SET status = ?, failure_reason = ?, completed_at = CURRENT_TIMESTAMP'
. " WHERE job_id = ? AND status IN ('Pending', 'Copying')",
[$itemStatus, $reason, $jobId]
);
$this->execute(
'UPDATE import_jobs SET status = ?, processed_files = ?, copied_bytes = ?,'
. ' completed_at = CURRENT_TIMESTAMP WHERE id = ?',
[$jobStatus, $this->processedFiles($jobId), $this->copiedBytes($jobId), $jobId]
);
});
}
/** Progress is recomputed from items so repeated or crashed updates cannot drift. */
private function refresh(int $jobId): void
{
$job = $this->row('SELECT * FROM import_jobs WHERE id = ?', [$jobId]);
if ($job === null || in_array($job['status'], ['Completed', 'Failed', 'Cancelled'], true)) {
return;
}
$total = (int) $this->row('SELECT COUNT(*) AS total FROM import_items WHERE job_id = ?', [$jobId])['total'];
$processed = $this->processedFiles($jobId);
$failed = (int) $this->row(
"SELECT COUNT(*) AS failed FROM import_items WHERE job_id = ? AND status = 'Failed'",
[$jobId]
)['failed'];
if ($total === 0 || $processed < $total) {
$this->execute(
'UPDATE import_jobs SET processed_files = ?, copied_bytes = ? WHERE id = ?',
[$processed, $this->copiedBytes($jobId), $jobId]
);
return;
}
$this->execute(
'UPDATE import_jobs SET status = ?, processed_files = ?, copied_bytes = ?,'
. ' completed_at = CURRENT_TIMESTAMP WHERE id = ?',
[$failed > 0 ? 'Failed' : 'Completed', $processed, $this->copiedBytes($jobId), $jobId]
);
}
private function processedFiles(int $jobId): int
{
return (int) $this->row(
'SELECT COUNT(*) AS processed FROM import_items WHERE job_id = ?'
. " AND status IN ('Completed', 'Failed', 'Skipped')",
[$jobId]
)['processed'];
}
private function copiedBytes(int $jobId): int
{
return (int) $this->row(
"SELECT COALESCE(SUM(size), 0) AS copied FROM import_items WHERE job_id = ? AND status = 'Completed'",
[$jobId]
)['copied'];
}
private function lockedItem(int $itemId): array
{
$suffix = $this->pdo->getAttribute(PDO::ATTR_DRIVER_NAME) === 'pgsql' ? ' FOR UPDATE' : '';
$item = $this->row('SELECT * FROM import_items WHERE id = ?' . $suffix, [$itemId]);
if ($item === null) {
throw new DomainException('Unknown import item.');
}
return $item;
}
}
+61
View File
@@ -0,0 +1,61 @@
<?php
declare(strict_types=1);
namespace MeshVault\Repository;
use PDO;
use PDOStatement;
use Throwable;
/** @internal Shared SQL and composable transaction support. */
abstract class PdoRepository
{
public function __construct(protected PDO $pdo)
{
$pdo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
}
protected function execute(string $sql, array $parameters = []): PDOStatement
{
$statement = $this->pdo->prepare($sql);
$statement->execute($parameters);
return $statement;
}
protected function row(string $sql, array $parameters = []): ?array
{
$statement = $this->execute($sql, $parameters);
$row = $statement->fetch(PDO::FETCH_ASSOC);
$statement->closeCursor();
return $row === false ? null : $row;
}
protected function transaction(callable $callback): mixed
{
$nested = $this->pdo->inTransaction();
$savepoint = 'repository_' . bin2hex(random_bytes(8));
if ($nested) {
$this->pdo->exec('SAVEPOINT ' . $savepoint);
} else {
$this->pdo->beginTransaction();
}
try {
$result = $callback();
if ($nested) {
$this->pdo->exec('RELEASE SAVEPOINT ' . $savepoint);
} else {
$this->pdo->commit();
}
return $result;
} catch (Throwable $error) {
if ($nested) {
$this->pdo->exec('ROLLBACK TO SAVEPOINT ' . $savepoint);
$this->pdo->exec('RELEASE SAVEPOINT ' . $savepoint);
} elseif ($this->pdo->inTransaction()) {
$this->pdo->rollBack();
}
throw $error;
}
}
}