Files
MeshVault/tests/Integration/ImportPipelineTest.php
T
Cline a829d6e8c6 feat: catalog shared contracts (Tasks 1-4)
Task 1: Establish catalog identity and atomic file ownership
- CatalogService, CatalogServiceInterface, ModelMetadata DTO
- PdoCatalogRepository with atomic operations, savepoint support
- FOR UPDATE file locks in ascending order, FOR NO KEY UPDATE for metadata
- FK-compatible lock composition documented (spec §6)

Task 2: M3 import & indexing engine delivery layer
- ImportService, ImportWorker with durable collision handling
- SpaceChecker, StreamingHasher, VerifiedCopier, StlValidator
- FileEnumerator, LocalFilesystem, PathSafety
- Console command wiring via ContainerWiringTest

Task 3: Importer integration and repair
- LegacyGuessedImportGroupingAdapter isolates first-directory guessing
- ImportService uses adapter; ImportWorker uses CatalogServiceInterface
- Null owners remain null; DomainException mapped to CATALOG_CONFLICT
- SQL failure after registration/path update with caller sentinel rollback

Task 4: PostgreSQL contention evidence and bounded acceptance
- 6 new PostgresCatalogContractsTest cases with independent sessions
- Generated schemas, 250ms lock timeouts, 5s statement timeouts
- Ascending file locks, FOR SHARE contention, stale-owner rejection
- Rollback/retry, nested transaction survival, root creation contention
- Negative controls: replay FOR UPDATE removal (11 failures), metadata FOR UPDATE (4 FK KEY SHARE timeouts)
- Fresh Docker PostgreSQL 17 / PHP 8.4.26: 473 tests, 2718 assertions, 10 skips, exit 0

All local: 474 tests, 2347 assertions, 54 skips, foundation/lint clean.
2026-10-13 19:41:03 -05:00

519 lines
23 KiB
PHP

<?php
declare(strict_types=1);
namespace MeshVault\Tests\Integration;
use MeshVault\Indexer\FileEnumerator;
use MeshVault\Indexer\Filesystem\LocalFilesystem;
use MeshVault\Indexer\Filesystem\PathSafety;
use MeshVault\Indexer\ImportException;
use MeshVault\Indexer\ImportService;
use MeshVault\Indexer\ImportWorker;
use MeshVault\Indexer\SpaceChecker;
use MeshVault\Indexer\StlValidator;
use MeshVault\Indexer\StreamingHasher;
use MeshVault\Indexer\VerifiedCopier;
use MeshVault\Repository\PdoCatalogRepository;
use MeshVault\Repository\PdoImportJobRepository;
use PDO;
use PHPUnit\Framework\TestCase;
use Psr\Log\NullLogger;
final class ImportPipelineTest extends TestCase
{
private PDO $pdo;
private string $database;
private string $library;
private string $source;
private PdoImportJobRepository $jobs;
private PdoCatalogRepository $catalog;
protected function setUp(): void
{
$this->database = tempnam(sys_get_temp_dir(), 'meshvault-pipeline-');
$this->pdo = RepositoryDatabase::migrate($this->database);
$this->jobs = new PdoImportJobRepository($this->pdo);
$this->catalog = new PdoCatalogRepository($this->pdo);
$this->library = RepositoryDatabase::directory('library');
$this->source = RepositoryDatabase::directory('source');
}
protected function tearDown(): void
{
unset($this->pdo, $this->jobs, $this->catalog);
RepositoryDatabase::cleanup($this->database);
if (isset($this->library)) {
RepositoryDatabase::remove($this->library);
}
if (isset($this->source)) {
RepositoryDatabase::remove($this->source);
}
}
private function service(?SpaceChecker $space = null): ImportService
{
$safety = new PathSafety($this->source);
return new ImportService(
new FileEnumerator(new LocalFilesystem(), $safety),
$this->jobs,
new \MeshVault\Indexer\LegacyGuessedImportGroupingAdapter($this->catalog),
$space ?? new SpaceChecker(),
$this->library,
);
}
private function worker(?callable $shouldStop = null): ImportWorker
{
$safety = new PathSafety($this->library);
$sourceSafety = new PathSafety($this->source);
return new ImportWorker(
$this->jobs,
new \MeshVault\Catalog\CatalogService($this->catalog),
new StreamingHasher($sourceSafety),
new VerifiedCopier($safety),
new StlValidator($sourceSafety),
$safety,
new SpaceChecker(),
$this->library,
new NullLogger(),
$shouldStop,
);
}
private function writeStl(string $relative, string $variant = 'default'): string
{
$path = $this->source . DIRECTORY_SEPARATOR . str_replace('/', DIRECTORY_SEPARATOR, $relative);
if (!is_dir(dirname($path))) {
mkdir(dirname($path), 0777, true);
}
$facet = "facet normal 0 0 1\nouter loop\nvertex 0 0 0\nvertex 1 0 0\nvertex 0 1 0\nendloop\nendfacet\n";
$content = "solid test_$variant\n" . $facet . "endsolid test_$variant\n";
file_put_contents($path, $content);
return $path;
}
public function testNullOwnerIsRegisteredAndScanFailuresAreNot(): void
{
$source = $this->writeStl('loose.stl');
$job = $this->jobs->create($this->source, [
['source_path' => $source, 'size' => filesize($source), 'model_id' => null],
['source_path' => 'unreadable', 'size' => 0, 'model_id' => null,
'status' => 'Failed', 'failure_reason' => 'SCAN_READ_ERROR'],
]);
$this->worker()->runOnce();
$files = $this->pdo->query('SELECT * FROM files')->fetchAll(PDO::FETCH_ASSOC);
self::assertCount(1, $files);
self::assertNull($files[0]['model_id']);
self::assertSame('Completed', $this->jobs->items($job)[0]['status']);
}
public function testRegistrationConflictReportsCatalogConflictAndPreservesBytes(): void
{
$source = $this->writeStl('planes/wing.stl');
$job = $this->service()->import($this->source);
$other = $this->catalog->ensureModel('/other', 'other');
$id = $this->catalog->recordFile($other, 'planes/wing.stl', hash_file('sha256', $source), filesize($source));
$sourceBefore = PhysicalFileSnapshot::capture($this->source);
$this->worker()->runOnce();
$item = $this->jobs->items($job)[0];
self::assertSame('Failed', $item['status']);
self::assertStringStartsWith('CATALOG_CONFLICT:', $item['failure_reason']);
self::assertSame($other, (int) $this->catalog->file($id)['model_id']);
self::assertSame($sourceBefore, PhysicalFileSnapshot::capture($this->source));
self::assertSame(
$sourceBefore['planes/wing.stl']['bytes'],
file_get_contents($this->library . '/planes/wing.stl')
);
}
public function testCompletionRegistersBeforeUpdatingItemPath(): void
{
if ($this->pdo->getAttribute(PDO::ATTR_DRIVER_NAME) !== 'sqlite') {
self::markTestSkipped('SQLite ordering assertion trigger.');
}
$this->writeStl('planes/wing.stl');
$job = $this->service()->import($this->source);
$this->pdo->exec("CREATE TRIGGER catalog_first BEFORE UPDATE OF managed_path ON import_items"
. " WHEN OLD.hash IS NOT NULL AND NOT EXISTS (SELECT 1 FROM files WHERE managed_path = NEW.managed_path)"
. " BEGIN SELECT RAISE(ABORT, 'catalog must precede progress'); END");
$this->worker()->runOnce();
self::assertSame('Completed', $this->jobs->items($job)[0]['status']);
}
public function testMissingNonNullOwnerFailsRatherThanBecomingUnassigned(): void
{
if ($this->pdo->getAttribute(PDO::ATTR_DRIVER_NAME) !== 'sqlite') {
self::markTestSkipped('SQLite fixture deliberately simulates a stale manifest owner.');
}
$source = $this->writeStl('loose.stl');
$this->pdo->exec('PRAGMA foreign_keys = OFF');
$job = $this->jobs->create($this->source, [
['source_path' => $source, 'size' => filesize($source), 'model_id' => 999999],
]);
$this->pdo->exec('PRAGMA foreign_keys = ON');
$sourceBefore = PhysicalFileSnapshot::capture($this->source);
$this->worker()->runOnce();
$item = $this->jobs->items($job)[0];
self::assertSame('Failed', $item['status']);
self::assertStringStartsWith('CATALOG_CONFLICT:', $item['failure_reason']);
self::assertSame(0, (int) $this->pdo->query('SELECT COUNT(*) FROM files')->fetchColumn());
self::assertSame($sourceBefore, PhysicalFileSnapshot::capture($this->source));
self::assertSame($sourceBefore['loose.stl']['bytes'], file_get_contents($this->library . '/unfiled/loose.stl'));
}
public function testCompletionSqlFailureRollsBackCatalogPathAndProgressInsideCallerTransaction(): void
{
if ($this->pdo->getAttribute(PDO::ATTR_DRIVER_NAME) !== 'sqlite') {
self::markTestSkipped('SQLite failure-injection trigger.');
}
$source = $this->writeStl('planes/wing.stl');
$job = $this->service()->import($this->source);
mkdir($this->library . '/planes');
file_put_contents($this->library . '/planes/wing.stl', 'different bytes');
$sourceBefore = PhysicalFileSnapshot::capture($this->source);
$libraryBefore = PhysicalFileSnapshot::capture($this->library);
// Fail after registration and item-path update, during progress refresh.
$this->pdo->exec("CREATE TRIGGER reject_completion BEFORE UPDATE ON import_jobs"
. " WHEN NEW.status = 'Completed' BEGIN SELECT RAISE(ABORT, 'injected completion failure'); END");
$this->pdo->beginTransaction();
$sentinel = $this->catalog->createModel('/sentinel', 'sentinel');
$this->worker()->runOnce();
self::assertTrue($this->pdo->inTransaction());
self::assertNotNull($this->catalog->aggregate($sentinel));
self::assertSame(0, (int) $this->pdo->query('SELECT COUNT(*) FROM files')->fetchColumn());
$item = $this->jobs->items($job)[0];
self::assertSame('planes/wing.stl', $item['managed_path']);
self::assertSame('Failed', $item['status']);
self::assertStringStartsWith('COPY_FAILED:', $item['failure_reason']);
self::assertSame('Failed', $this->jobs->get($job)['status']);
self::assertSame($sourceBefore, PhysicalFileSnapshot::capture($this->source));
$published = PhysicalFileSnapshot::capture($this->library);
self::assertSame($libraryBefore['planes/wing.stl'], $published['planes/wing.stl']);
self::assertSame($sourceBefore['planes/wing.stl']['bytes'], $published['planes/wing_001.stl']['bytes']);
self::assertSame($sourceBefore['planes/wing.stl']['size'], $published['planes/wing_001.stl']['size']);
$this->pdo->rollBack();
self::assertSame($sourceBefore, PhysicalFileSnapshot::capture($this->source));
self::assertSame($published, PhysicalFileSnapshot::capture($this->library));
self::assertNull($this->catalog->aggregate($sentinel));
self::assertSame('Pending', $this->jobs->items($job)[0]['status']);
}
public function testImportCreatesJobWithLegacyGuessedSubfolderGrouping(): void
{
$this->writeStl('planes/wing.stl');
$this->writeStl('planes/body.stl');
$this->writeStl('loose.stl');
$job = $this->service()->import($this->source);
$items = $this->jobs->items($job);
self::assertCount(3, $items);
self::assertSame('Pending', $this->jobs->get($job)['status']);
$models = [];
foreach ($items as $item) {
self::assertNotNull($item['model_id']);
$aggregate = $this->catalog->aggregate((int) $item['model_id']);
$models[$aggregate['model']['name']] = $aggregate['folder']['path'];
}
// Legacy guessed grouping: one model per immediate source subfolder; root files join
// a model named after the source directory itself.
self::assertSame('/planes', $models['planes']);
self::assertSame('/' . basename($this->source), $models[basename($this->source)]);
}
public function testImportRejectsEmptySourceAndInsufficientSpace(): void
{
try {
$this->service()->import($this->source);
self::fail('Empty source accepted');
} catch (ImportException $error) {
self::assertSame('SCAN_EMPTY', $error->reason);
}
$this->writeStl('planes/wing.stl');
$full = new SpaceChecker(0.05, static fn (): float => 1.0);
try {
$this->service($full)->import($this->source);
self::fail('Full disk accepted');
} catch (ImportException $error) {
self::assertSame('DISK_FULL', $error->reason);
}
self::assertSame(0, (int) $this->pdo->query('SELECT COUNT(*) FROM import_jobs')->fetchColumn());
}
public function testWorkerCopiesPlansAndCompletesCatalog(): void
{
$this->writeStl('planes/wing.stl');
$job = $this->service()->import($this->source);
self::assertSame($job, $this->worker()->runOnce());
$item = $this->jobs->items($job)[0];
self::assertSame('Completed', $item['status']);
self::assertSame('Completed', $this->jobs->get($job)['status']);
self::assertNotNull($item['managed_path']);
$published = $this->library . DIRECTORY_SEPARATOR
. str_replace('/', DIRECTORY_SEPARATOR, $item['managed_path']);
self::assertFileExists($published);
self::assertSame(hash_file('sha256', $published), $item['hash']);
$files = $this->catalog->aggregate((int) $item['model_id'])['files'];
self::assertCount(1, $files);
self::assertSame('Present', $files[0]['status']);
self::assertSame($item['managed_path'], $files[0]['managed_path']);
self::assertNull($this->worker()->runOnce());
}
public function testFailedItemsDoNotStopRemainingWork(): void
{
$this->writeStl('planes/wing.stl');
$broken = $this->writeStl('planes/broken.stl');
file_put_contents($broken, 'not an stl at all');
$job = $this->service()->import($this->source);
$this->worker()->runOnce();
$statuses = array_column($this->jobs->items($job), 'status');
self::assertContains('Failed', $statuses);
self::assertContains('Completed', $statuses);
self::assertSame('Failed', $this->jobs->get($job)['status']);
self::assertSame(2, (int) $this->jobs->get($job)['processed_files']);
$failure = array_values(array_filter(
$this->jobs->items($job),
static fn (array $item): bool => $item['status'] === 'Failed'
))[0];
self::assertStringContainsString('STL_INVALID', (string) $failure['failure_reason']);
}
public function testCrashRecoveryCompletesPlannedItemWithoutDuplicateCopy(): void
{
$this->writeStl('planes/wing.stl');
$job = $this->service()->import($this->source);
$this->worker()->runOnce();
$item = $this->jobs->items($job)[0];
// Simulate a crash after the copy but before the catalog/progress commit.
$this->pdo->exec(
"UPDATE import_items SET status = 'Copying', completed_at = NULL WHERE id = " . (int) $item['id']
);
$this->pdo->exec("UPDATE import_jobs SET status = 'Copying', processed_files = 0 WHERE id = $job");
$this->pdo->exec('DELETE FROM files');
$this->jobs->resumeInterrupted();
self::assertSame($job, $this->worker()->runOnce());
self::assertSame('Completed', $this->jobs->items($job)[0]['status']);
self::assertSame($item['managed_path'], $this->jobs->items($job)[0]['managed_path']);
$directory = dirname($this->library . DIRECTORY_SEPARATOR
. str_replace('/', DIRECTORY_SEPARATOR, (string) $item['managed_path']));
self::assertCount(1, array_diff(scandir($directory), ['.', '..']));
self::assertCount(1, $this->catalog->aggregate((int) $item['model_id'])['files']);
}
public function testGracefulStopLeavesJobResumable(): void
{
$this->writeStl('planes/wing.stl');
$this->writeStl('planes/body.stl');
$job = $this->service()->import($this->source);
$calls = 0;
$this->worker(static function () use (&$calls): bool {
// Stop only after the first item has been processed.
return $calls++ > 0;
})->runOnce();
self::assertSame('Pending', $this->jobs->get($job)['status']);
$statuses = array_column($this->jobs->items($job), 'status');
self::assertContains('Pending', $statuses);
self::assertContains('Completed', $statuses);
self::assertSame($job, $this->worker()->runOnce());
self::assertSame('Completed', $this->jobs->get($job)['status']);
self::assertSame(2, (int) $this->jobs->get($job)['processed_files']);
}
public function testConcurrentWorkerCannotHoldTheLibraryLock(): void
{
$this->writeStl('planes/wing.stl');
$job = $this->service()->import($this->source);
$lock = fopen($this->library . DIRECTORY_SEPARATOR . '.meshvault-import.lock', 'c');
self::assertNotFalse($lock);
self::assertTrue(flock($lock, LOCK_EX | LOCK_NB));
try {
$this->worker()->runOnce();
self::fail('Worker ignored the global library lock');
} catch (ImportException $error) {
self::assertSame('LOCK_UNAVAILABLE', $error->reason);
} finally {
flock($lock, LOCK_UN);
fclose($lock);
}
self::assertSame('Pending', $this->jobs->get($job)['status']);
}
public function testTwoSameBasenameSourcesPublishWithCollisionSuffixAndBothRowsPointToRealFiles(): void
{
// Two sources with same basename but DIFFERENT content in different subdirectories
$this->writeStl('planes/wing.stl', 'wing1');
$this->writeStl('planes/sub/wing.stl', 'wing2');
$job = $this->service()->import($this->source);
$this->worker()->runOnce();
$items = $this->jobs->items($job);
self::assertCount(2, $items);
// Both should complete successfully
$statuses = array_column($items, 'status');
self::assertSame(['Completed', 'Completed'], $statuses);
// Both managed_path entries should point to actual published files (with suffixes)
foreach ($items as $item) {
$published = $this->library . DIRECTORY_SEPARATOR
. str_replace('/', DIRECTORY_SEPARATOR, $item['managed_path']);
self::assertFileExists($published, "Published file missing for managed_path: " . $item['managed_path']);
self::assertSame(hash_file('sha256', $published), $item['hash']);
}
// Filenames should differ (one gets _001 suffix)
$basenames = array_map('basename', array_column($items, 'managed_path'));
self::assertNotSame($basenames[0], $basenames[1], 'Collision suffix not applied - both have same basename');
self::assertMatchesRegularExpression('/^wing(_001)?\\.stl$/', $basenames[0]);
self::assertMatchesRegularExpression('/^wing(_001)?\\.stl$/', $basenames[1]);
}
public function testCrashAfterPublishBeforeCompletionResumesWithoutRecopyOrCollision(): void
{
$this->writeStl('planes/wing.stl');
$job = $this->service()->import($this->source);
$this->worker()->runOnce();
$item = $this->jobs->items($job)[0];
// Simulate crash AFTER copy but BEFORE catalog/progress commit
// Set item to Copying (plan already committed, file already copied)
$this->pdo->exec(
"UPDATE import_items SET status = 'Copying', completed_at = NULL WHERE id = " . (int) $item['id']
);
$this->pdo->exec("UPDATE import_jobs SET status = 'Copying', processed_files = 0 WHERE id = $job");
$this->pdo->exec('DELETE FROM files'); // Remove catalog entry but keep file on disk
// Resume and run again
$this->jobs->resumeInterrupted();
$resumedJob = $this->worker()->runOnce();
self::assertSame($job, $resumedJob);
// Item should complete without re-copying
$resumedItem = $this->jobs->items($job)[0];
self::assertSame('Completed', $resumedItem['status']);
self::assertSame($item['managed_path'], $resumedItem['managed_path'], 'managed_path changed on resume');
// Only ONE file should exist in the destination directory (no duplicate copy)
$directory = dirname($this->library . DIRECTORY_SEPARATOR
. str_replace('/', DIRECTORY_SEPARATOR, (string) $item['managed_path']));
self::assertCount(1, array_diff(scandir($directory), ['.', '..']), 'Duplicate file created on resume');
// Catalog should have exactly one entry
self::assertCount(1, $this->catalog->aggregate((int) $item['model_id'])['files']);
}
public function testAdoptExistingRevalidatesWithPathSafetyAndRejectsLinks(): void
{
// Create a legitimate file first
$this->writeStl('planes/wing.stl');
$job = $this->service()->import($this->source);
$this->worker()->runOnce();
$item = $this->jobs->items($job)[0];
$managedPath = $item['managed_path'];
$publishedPath = $this->library . DIRECTORY_SEPARATOR . str_replace('/', DIRECTORY_SEPARATOR, $managedPath);
// Verify it exists and is a regular file
self::assertFileExists($publishedPath);
self::assertFalse(is_link($publishedPath));
// Now simulate a crash and replace the file with a symlink
$this->pdo->exec(
"UPDATE import_items SET status = 'Copying', completed_at = NULL WHERE id = " . (int) $item['id']
);
$this->pdo->exec("UPDATE import_jobs SET status = 'Copying', processed_files = 0 WHERE id = $job");
$this->pdo->exec('DELETE FROM files');
// Replace the file with a symlink pointing outside the library
$outside = $this->source . DIRECTORY_SEPARATOR . 'outside.stl';
file_put_contents($outside, 'malicious');
unlink($publishedPath);
if (!@symlink($outside, $publishedPath)) {
self::markTestSkipped('Host cannot create symlinks.');
}
// Resume - should reject the symlink and fail the item
$this->jobs->resumeInterrupted();
$this->worker()->runOnce();
$resumedItem = $this->jobs->items($job)[0];
self::assertSame('Failed', $resumedItem['status']);
self::assertStringContainsString('SYMLINK_ESCAPE', $resumedItem['failure_reason']);
}
public function testSpaceCheckerPreflightBeforeEachCopy(): void
{
$this->writeStl('planes/wing.stl');
$this->writeStl('planes/body.stl');
$job = $this->service()->import($this->source);
// Simulate disk full on second item
$spaceChecks = 0;
$spaceChecker = new SpaceChecker(0.05, static function (string $path) use (&$spaceChecks): float {
$spaceChecks++;
// First check passes, second fails
return $spaceChecks === 1 ? 1000000.0 : 100.0;
});
// Create worker with custom SpaceChecker
$safety = new PathSafety($this->library);
$sourceSafety = new PathSafety($this->source);
$worker = new ImportWorker(
$this->jobs,
new \MeshVault\Catalog\CatalogService($this->catalog),
new StreamingHasher($sourceSafety),
new VerifiedCopier($safety),
new StlValidator($sourceSafety),
$safety,
$spaceChecker,
$this->library,
new NullLogger(),
);
$worker->runOnce();
$items = $this->jobs->items($job);
$statuses = array_column($items, 'status');
// First item should complete, second should fail with DISK_FULL
self::assertSame('Completed', $statuses[0]);
self::assertSame('Failed', $statuses[1]);
self::assertStringContainsString('DISK_FULL', $items[1]['failure_reason']);
// Job should be Failed overall but first item completed
self::assertSame('Failed', $this->jobs->get($job)['status']);
}
public function testWorkerUsesCanonicalLibraryRootForRelativePaths(): void
{
// A trailing separator is safe but differs from the canonical root.
// Symlink roots are deliberately rejected by PathSafety.
$rawLibrary = $this->library . DIRECTORY_SEPARATOR;
$this->writeStl('planes/wing.stl');
$job = $this->service()->import($this->source);
$safety = new PathSafety($rawLibrary);
self::assertNotSame($rawLibrary, $safety->libraryRoot());
$sourceSafety = new PathSafety($this->source);
$worker = new ImportWorker(
$this->jobs,
new \MeshVault\Catalog\CatalogService($this->catalog),
new StreamingHasher($sourceSafety),
new VerifiedCopier($safety),
new StlValidator($sourceSafety),
$safety,
new SpaceChecker(),
$rawLibrary,
new NullLogger(),
);
$worker->runOnce();
$item = $this->jobs->items($job)[0];
self::assertSame('Completed', $item['status']);
self::assertStringStartsWith('planes/', $item['managed_path']);
$published = $this->library . DIRECTORY_SEPARATOR
. str_replace('/', DIRECTORY_SEPARATOR, $item['managed_path']);
self::assertFileExists($published);
}
}