Cline
c1ef2ee97a
Add isolated PostgreSQL fixtures, two-session locking coverage, and a disposable remote Docker test runner. Correct the canonical-root fixture and document verified SQLite and PostgreSQL results.
413 lines
17 KiB
PHP
413 lines
17 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
namespace MeshVault\Tests\Integration;
|
|
|
|
use MeshVault\Indexer\FileEnumerator;
|
|
use MeshVault\Indexer\Filesystem\LocalFilesystem;
|
|
use MeshVault\Indexer\Filesystem\PathSafety;
|
|
use MeshVault\Indexer\ImportException;
|
|
use MeshVault\Indexer\ImportService;
|
|
use MeshVault\Indexer\ImportWorker;
|
|
use MeshVault\Indexer\SpaceChecker;
|
|
use MeshVault\Indexer\StlValidator;
|
|
use MeshVault\Indexer\StreamingHasher;
|
|
use MeshVault\Indexer\VerifiedCopier;
|
|
use MeshVault\Repository\PdoCatalogRepository;
|
|
use MeshVault\Repository\PdoImportJobRepository;
|
|
use PDO;
|
|
use PHPUnit\Framework\TestCase;
|
|
use Psr\Log\NullLogger;
|
|
|
|
final class ImportPipelineTest extends TestCase
|
|
{
|
|
private PDO $pdo;
|
|
private string $database;
|
|
private string $library;
|
|
private string $source;
|
|
private PdoImportJobRepository $jobs;
|
|
private PdoCatalogRepository $catalog;
|
|
|
|
protected function setUp(): void
|
|
{
|
|
$this->database = tempnam(sys_get_temp_dir(), 'meshvault-pipeline-');
|
|
$this->pdo = RepositoryDatabase::migrate($this->database);
|
|
$this->jobs = new PdoImportJobRepository($this->pdo);
|
|
$this->catalog = new PdoCatalogRepository($this->pdo);
|
|
$this->library = RepositoryDatabase::directory('library');
|
|
$this->source = RepositoryDatabase::directory('source');
|
|
}
|
|
|
|
protected function tearDown(): void
|
|
{
|
|
unset($this->pdo, $this->jobs, $this->catalog);
|
|
RepositoryDatabase::cleanup($this->database);
|
|
if (isset($this->library)) {
|
|
RepositoryDatabase::remove($this->library);
|
|
}
|
|
if (isset($this->source)) {
|
|
RepositoryDatabase::remove($this->source);
|
|
}
|
|
}
|
|
|
|
private function service(?SpaceChecker $space = null): ImportService
|
|
{
|
|
$safety = new PathSafety($this->source);
|
|
return new ImportService(
|
|
new FileEnumerator(new LocalFilesystem(), $safety),
|
|
$this->jobs,
|
|
$this->catalog,
|
|
$space ?? new SpaceChecker(),
|
|
$this->library,
|
|
);
|
|
}
|
|
|
|
private function worker(?callable $shouldStop = null): ImportWorker
|
|
{
|
|
$safety = new PathSafety($this->library);
|
|
$sourceSafety = new PathSafety($this->source);
|
|
return new ImportWorker(
|
|
$this->jobs,
|
|
$this->catalog,
|
|
new StreamingHasher($sourceSafety),
|
|
new VerifiedCopier($safety),
|
|
new StlValidator($sourceSafety),
|
|
$safety,
|
|
new SpaceChecker(),
|
|
$this->library,
|
|
new NullLogger(),
|
|
$shouldStop,
|
|
);
|
|
}
|
|
|
|
private function writeStl(string $relative, string $variant = 'default'): string
|
|
{
|
|
$path = $this->source . DIRECTORY_SEPARATOR . str_replace('/', DIRECTORY_SEPARATOR, $relative);
|
|
if (!is_dir(dirname($path))) {
|
|
mkdir(dirname($path), 0777, true);
|
|
}
|
|
$facet = "facet normal 0 0 1\nouter loop\nvertex 0 0 0\nvertex 1 0 0\nvertex 0 1 0\nendloop\nendfacet\n";
|
|
$content = "solid test_$variant\n" . $facet . "endsolid test_$variant\n";
|
|
file_put_contents($path, $content);
|
|
return $path;
|
|
}
|
|
|
|
public function testImportCreatesJobWithExplicitSubfolderGrouping(): void
|
|
{
|
|
$this->writeStl('planes/wing.stl');
|
|
$this->writeStl('planes/body.stl');
|
|
$this->writeStl('loose.stl');
|
|
$job = $this->service()->import($this->source);
|
|
$items = $this->jobs->items($job);
|
|
self::assertCount(3, $items);
|
|
self::assertSame('Pending', $this->jobs->get($job)['status']);
|
|
$models = [];
|
|
foreach ($items as $item) {
|
|
self::assertNotNull($item['model_id']);
|
|
$aggregate = $this->catalog->aggregate((int) $item['model_id']);
|
|
$models[$aggregate['model']['name']] = $aggregate['folder']['path'];
|
|
}
|
|
// Explicit policy: one model per immediate source subfolder; root files join
|
|
// a model named after the source directory itself.
|
|
self::assertSame('/planes', $models['planes']);
|
|
self::assertSame('/' . basename($this->source), $models[basename($this->source)]);
|
|
}
|
|
|
|
public function testImportRejectsEmptySourceAndInsufficientSpace(): void
|
|
{
|
|
try {
|
|
$this->service()->import($this->source);
|
|
self::fail('Empty source accepted');
|
|
} catch (ImportException $error) {
|
|
self::assertSame('SCAN_EMPTY', $error->reason);
|
|
}
|
|
$this->writeStl('planes/wing.stl');
|
|
$full = new SpaceChecker(0.05, static fn (): float => 1.0);
|
|
try {
|
|
$this->service($full)->import($this->source);
|
|
self::fail('Full disk accepted');
|
|
} catch (ImportException $error) {
|
|
self::assertSame('DISK_FULL', $error->reason);
|
|
}
|
|
self::assertSame(0, (int) $this->pdo->query('SELECT COUNT(*) FROM import_jobs')->fetchColumn());
|
|
}
|
|
|
|
public function testWorkerCopiesPlansAndCompletesCatalog(): void
|
|
{
|
|
$this->writeStl('planes/wing.stl');
|
|
$job = $this->service()->import($this->source);
|
|
self::assertSame($job, $this->worker()->runOnce());
|
|
$item = $this->jobs->items($job)[0];
|
|
self::assertSame('Completed', $item['status']);
|
|
self::assertSame('Completed', $this->jobs->get($job)['status']);
|
|
self::assertNotNull($item['managed_path']);
|
|
$published = $this->library . DIRECTORY_SEPARATOR
|
|
. str_replace('/', DIRECTORY_SEPARATOR, $item['managed_path']);
|
|
self::assertFileExists($published);
|
|
self::assertSame(hash_file('sha256', $published), $item['hash']);
|
|
$files = $this->catalog->aggregate((int) $item['model_id'])['files'];
|
|
self::assertCount(1, $files);
|
|
self::assertSame('Present', $files[0]['status']);
|
|
self::assertSame($item['managed_path'], $files[0]['managed_path']);
|
|
self::assertNull($this->worker()->runOnce());
|
|
}
|
|
|
|
public function testFailedItemsDoNotStopRemainingWork(): void
|
|
{
|
|
$this->writeStl('planes/wing.stl');
|
|
$broken = $this->writeStl('planes/broken.stl');
|
|
file_put_contents($broken, 'not an stl at all');
|
|
$job = $this->service()->import($this->source);
|
|
$this->worker()->runOnce();
|
|
$statuses = array_column($this->jobs->items($job), 'status');
|
|
self::assertContains('Failed', $statuses);
|
|
self::assertContains('Completed', $statuses);
|
|
self::assertSame('Failed', $this->jobs->get($job)['status']);
|
|
self::assertSame(2, (int) $this->jobs->get($job)['processed_files']);
|
|
$failure = array_values(array_filter(
|
|
$this->jobs->items($job),
|
|
static fn (array $item): bool => $item['status'] === 'Failed'
|
|
))[0];
|
|
self::assertStringContainsString('STL_INVALID', (string) $failure['failure_reason']);
|
|
}
|
|
|
|
public function testCrashRecoveryCompletesPlannedItemWithoutDuplicateCopy(): void
|
|
{
|
|
$this->writeStl('planes/wing.stl');
|
|
$job = $this->service()->import($this->source);
|
|
$this->worker()->runOnce();
|
|
$item = $this->jobs->items($job)[0];
|
|
// Simulate a crash after the copy but before the catalog/progress commit.
|
|
$this->pdo->exec(
|
|
"UPDATE import_items SET status = 'Copying', completed_at = NULL WHERE id = " . (int) $item['id']
|
|
);
|
|
$this->pdo->exec("UPDATE import_jobs SET status = 'Copying', processed_files = 0 WHERE id = $job");
|
|
$this->pdo->exec('DELETE FROM files');
|
|
$this->jobs->resumeInterrupted();
|
|
self::assertSame($job, $this->worker()->runOnce());
|
|
self::assertSame('Completed', $this->jobs->items($job)[0]['status']);
|
|
self::assertSame($item['managed_path'], $this->jobs->items($job)[0]['managed_path']);
|
|
$directory = dirname($this->library . DIRECTORY_SEPARATOR
|
|
. str_replace('/', DIRECTORY_SEPARATOR, (string) $item['managed_path']));
|
|
self::assertCount(1, array_diff(scandir($directory), ['.', '..']));
|
|
self::assertCount(1, $this->catalog->aggregate((int) $item['model_id'])['files']);
|
|
}
|
|
|
|
public function testGracefulStopLeavesJobResumable(): void
|
|
{
|
|
$this->writeStl('planes/wing.stl');
|
|
$this->writeStl('planes/body.stl');
|
|
$job = $this->service()->import($this->source);
|
|
$calls = 0;
|
|
$this->worker(static function () use (&$calls): bool {
|
|
// Stop only after the first item has been processed.
|
|
return $calls++ > 0;
|
|
})->runOnce();
|
|
self::assertSame('Pending', $this->jobs->get($job)['status']);
|
|
$statuses = array_column($this->jobs->items($job), 'status');
|
|
self::assertContains('Pending', $statuses);
|
|
self::assertContains('Completed', $statuses);
|
|
self::assertSame($job, $this->worker()->runOnce());
|
|
self::assertSame('Completed', $this->jobs->get($job)['status']);
|
|
self::assertSame(2, (int) $this->jobs->get($job)['processed_files']);
|
|
}
|
|
|
|
public function testConcurrentWorkerCannotHoldTheLibraryLock(): void
|
|
{
|
|
$this->writeStl('planes/wing.stl');
|
|
$job = $this->service()->import($this->source);
|
|
$lock = fopen($this->library . DIRECTORY_SEPARATOR . '.meshvault-import.lock', 'c');
|
|
self::assertNotFalse($lock);
|
|
self::assertTrue(flock($lock, LOCK_EX | LOCK_NB));
|
|
try {
|
|
$this->worker()->runOnce();
|
|
self::fail('Worker ignored the global library lock');
|
|
} catch (ImportException $error) {
|
|
self::assertSame('LOCK_UNAVAILABLE', $error->reason);
|
|
} finally {
|
|
flock($lock, LOCK_UN);
|
|
fclose($lock);
|
|
}
|
|
self::assertSame('Pending', $this->jobs->get($job)['status']);
|
|
}
|
|
|
|
public function testTwoSameBasenameSourcesPublishWithCollisionSuffixAndBothRowsPointToRealFiles(): void
|
|
{
|
|
// Two sources with same basename but DIFFERENT content in different subdirectories
|
|
$this->writeStl('planes/wing.stl', 'wing1');
|
|
$this->writeStl('planes/sub/wing.stl', 'wing2');
|
|
$job = $this->service()->import($this->source);
|
|
$this->worker()->runOnce();
|
|
|
|
$items = $this->jobs->items($job);
|
|
self::assertCount(2, $items);
|
|
|
|
// Both should complete successfully
|
|
$statuses = array_column($items, 'status');
|
|
self::assertSame(['Completed', 'Completed'], $statuses);
|
|
|
|
// Both managed_path entries should point to actual published files (with suffixes)
|
|
foreach ($items as $item) {
|
|
$published = $this->library . DIRECTORY_SEPARATOR
|
|
. str_replace('/', DIRECTORY_SEPARATOR, $item['managed_path']);
|
|
self::assertFileExists($published, "Published file missing for managed_path: " . $item['managed_path']);
|
|
self::assertSame(hash_file('sha256', $published), $item['hash']);
|
|
}
|
|
|
|
// Filenames should differ (one gets _001 suffix)
|
|
$basenames = array_map('basename', array_column($items, 'managed_path'));
|
|
self::assertNotSame($basenames[0], $basenames[1], 'Collision suffix not applied - both have same basename');
|
|
self::assertMatchesRegularExpression('/^wing(_001)?\\.stl$/', $basenames[0]);
|
|
self::assertMatchesRegularExpression('/^wing(_001)?\\.stl$/', $basenames[1]);
|
|
}
|
|
|
|
public function testCrashAfterPublishBeforeCompletionResumesWithoutRecopyOrCollision(): void
|
|
{
|
|
$this->writeStl('planes/wing.stl');
|
|
$job = $this->service()->import($this->source);
|
|
$this->worker()->runOnce();
|
|
$item = $this->jobs->items($job)[0];
|
|
|
|
// Simulate crash AFTER copy but BEFORE catalog/progress commit
|
|
// Set item to Copying (plan already committed, file already copied)
|
|
$this->pdo->exec(
|
|
"UPDATE import_items SET status = 'Copying', completed_at = NULL WHERE id = " . (int) $item['id']
|
|
);
|
|
$this->pdo->exec("UPDATE import_jobs SET status = 'Copying', processed_files = 0 WHERE id = $job");
|
|
$this->pdo->exec('DELETE FROM files'); // Remove catalog entry but keep file on disk
|
|
|
|
// Resume and run again
|
|
$this->jobs->resumeInterrupted();
|
|
$resumedJob = $this->worker()->runOnce();
|
|
self::assertSame($job, $resumedJob);
|
|
|
|
// Item should complete without re-copying
|
|
$resumedItem = $this->jobs->items($job)[0];
|
|
self::assertSame('Completed', $resumedItem['status']);
|
|
self::assertSame($item['managed_path'], $resumedItem['managed_path'], 'managed_path changed on resume');
|
|
|
|
// Only ONE file should exist in the destination directory (no duplicate copy)
|
|
$directory = dirname($this->library . DIRECTORY_SEPARATOR
|
|
. str_replace('/', DIRECTORY_SEPARATOR, (string) $item['managed_path']));
|
|
self::assertCount(1, array_diff(scandir($directory), ['.', '..']), 'Duplicate file created on resume');
|
|
|
|
// Catalog should have exactly one entry
|
|
self::assertCount(1, $this->catalog->aggregate((int) $item['model_id'])['files']);
|
|
}
|
|
|
|
public function testAdoptExistingRevalidatesWithPathSafetyAndRejectsLinks(): void
|
|
{
|
|
// Create a legitimate file first
|
|
$this->writeStl('planes/wing.stl');
|
|
$job = $this->service()->import($this->source);
|
|
$this->worker()->runOnce();
|
|
$item = $this->jobs->items($job)[0];
|
|
$managedPath = $item['managed_path'];
|
|
$publishedPath = $this->library . DIRECTORY_SEPARATOR . str_replace('/', DIRECTORY_SEPARATOR, $managedPath);
|
|
|
|
// Verify it exists and is a regular file
|
|
self::assertFileExists($publishedPath);
|
|
self::assertFalse(is_link($publishedPath));
|
|
|
|
// Now simulate a crash and replace the file with a symlink
|
|
$this->pdo->exec(
|
|
"UPDATE import_items SET status = 'Copying', completed_at = NULL WHERE id = " . (int) $item['id']
|
|
);
|
|
$this->pdo->exec("UPDATE import_jobs SET status = 'Copying', processed_files = 0 WHERE id = $job");
|
|
$this->pdo->exec('DELETE FROM files');
|
|
|
|
// Replace the file with a symlink pointing outside the library
|
|
$outside = $this->source . DIRECTORY_SEPARATOR . 'outside.stl';
|
|
file_put_contents($outside, 'malicious');
|
|
unlink($publishedPath);
|
|
if (!@symlink($outside, $publishedPath)) {
|
|
self::markTestSkipped('Host cannot create symlinks.');
|
|
}
|
|
|
|
// Resume - should reject the symlink and fail the item
|
|
$this->jobs->resumeInterrupted();
|
|
$this->worker()->runOnce();
|
|
|
|
$resumedItem = $this->jobs->items($job)[0];
|
|
self::assertSame('Failed', $resumedItem['status']);
|
|
self::assertStringContainsString('SYMLINK_ESCAPE', $resumedItem['failure_reason']);
|
|
}
|
|
|
|
public function testSpaceCheckerPreflightBeforeEachCopy(): void
|
|
{
|
|
$this->writeStl('planes/wing.stl');
|
|
$this->writeStl('planes/body.stl');
|
|
$job = $this->service()->import($this->source);
|
|
|
|
// Simulate disk full on second item
|
|
$spaceChecks = 0;
|
|
$spaceChecker = new SpaceChecker(0.05, static function (string $path) use (&$spaceChecks): float {
|
|
$spaceChecks++;
|
|
// First check passes, second fails
|
|
return $spaceChecks === 1 ? 1000000.0 : 100.0;
|
|
});
|
|
|
|
// Create worker with custom SpaceChecker
|
|
$safety = new PathSafety($this->library);
|
|
$sourceSafety = new PathSafety($this->source);
|
|
$worker = new ImportWorker(
|
|
$this->jobs,
|
|
$this->catalog,
|
|
new StreamingHasher($sourceSafety),
|
|
new VerifiedCopier($safety),
|
|
new StlValidator($sourceSafety),
|
|
$safety,
|
|
$spaceChecker,
|
|
$this->library,
|
|
new NullLogger(),
|
|
);
|
|
$worker->runOnce();
|
|
|
|
$items = $this->jobs->items($job);
|
|
$statuses = array_column($items, 'status');
|
|
|
|
// First item should complete, second should fail with DISK_FULL
|
|
self::assertSame('Completed', $statuses[0]);
|
|
self::assertSame('Failed', $statuses[1]);
|
|
self::assertStringContainsString('DISK_FULL', $items[1]['failure_reason']);
|
|
|
|
// Job should be Failed overall but first item completed
|
|
self::assertSame('Failed', $this->jobs->get($job)['status']);
|
|
}
|
|
|
|
public function testWorkerUsesCanonicalLibraryRootForRelativePaths(): void
|
|
{
|
|
// A trailing separator is safe but differs from the canonical root.
|
|
// Symlink roots are deliberately rejected by PathSafety.
|
|
$rawLibrary = $this->library . DIRECTORY_SEPARATOR;
|
|
|
|
$this->writeStl('planes/wing.stl');
|
|
$job = $this->service()->import($this->source);
|
|
|
|
$safety = new PathSafety($rawLibrary);
|
|
self::assertNotSame($rawLibrary, $safety->libraryRoot());
|
|
$sourceSafety = new PathSafety($this->source);
|
|
$worker = new ImportWorker(
|
|
$this->jobs,
|
|
$this->catalog,
|
|
new StreamingHasher($sourceSafety),
|
|
new VerifiedCopier($safety),
|
|
new StlValidator($sourceSafety),
|
|
$safety,
|
|
new SpaceChecker(),
|
|
$rawLibrary,
|
|
new NullLogger(),
|
|
);
|
|
|
|
$worker->runOnce();
|
|
$item = $this->jobs->items($job)[0];
|
|
|
|
self::assertSame('Completed', $item['status']);
|
|
self::assertStringStartsWith('planes/', $item['managed_path']);
|
|
$published = $this->library . DIRECTORY_SEPARATOR
|
|
. str_replace('/', DIRECTORY_SEPARATOR, $item['managed_path']);
|
|
self::assertFileExists($published);
|
|
}
|
|
}
|