Concrete TypeScript signatures for every public surface this plan introduces. These are the contracts; block plans implement them.
Conventions:
- All shape types are
readonlywhere structurally immutable. - All string IDs use branded types where existing libs already use them (e.g.
PipelineIdfromoperation-orchestrator). - All async APIs return
Promise<T>(no callbacks). - Error types extend
Errorwith acodediscriminator for stable consumer detection.
@isis/scene-from-image-composer#
Public entry points#
ts
// libs/isis/scene-from-image-composer/src/composer.ts
/**
* Convenience API. Runs the full composer pipeline in-process with auto-
* approving checkpoints. For production use the advanced API.
*/
export async function composeSceneFromImage(
input: SceneFromImageRequest
): Promise<SceneFromImageResult>;
/**
* Advanced API. Returns a PipelineDefinition consumable by a configured
* PipelineOrchestrator. Caller controls persistence, observability,
* checkpoint handling.
*/
export function createSceneFromImagePipeline(
input: SceneFromImageRequest
): PipelineDefinition;
SceneFromImageRequest#
ts
export interface SceneFromImageRequest {
// Identity
readonly assetId: string;
readonly tenantId?: string;
readonly userId?: string;
// Input
readonly sourceImage: Buffer;
readonly mimeType?: string; // 'image/png' default
// Provider injection (composer doesn't construct providers itself)
readonly providers: {
readonly marble: MarbleProvider;
readonly mesh?: I3DProvider; // Hunyuan/Tripo/Meshy/etc.
readonly sfx?: ISFXProvider;
};
// Optional grounding (block B+)
readonly groundingAdapter?: ThreeDQualityGroundingVlmAdapter;
// Object sourcing (block B explicit-only; block C unlocks all four)
readonly objects?: readonly ObjectSpec[];
readonly detectionPolicy?: DetectionPolicy; // for vlm-detected mode
// Audio (block B onwards)
readonly ambient?: { readonly prompt: string; readonly durationSec?: number };
// Per-phase opt-out
readonly phases?: {
readonly skipGrounding?: boolean;
readonly skipObjects?: boolean;
readonly skipAudio?: boolean;
readonly skipBake?: boolean; // block E onwards
};
// Output mode (block B = transform-only; block E adds 'baked')
readonly outputMode?: 'transform-only' | 'baked'; // default 'baked' once E lands
readonly bakeOptions?: BakeOptions; // block E
// Checkpoint policy per phase (block A)
readonly checkpoints?: Partial<Record<PhaseId, CheckpointPolicy>>;
// Dispatch (block G)
readonly dispatchMode?: 'in-process' | 'job-envelope';
readonly dispatchConfig?: {
readonly redisUrl: string;
readonly channelPrefix?: string;
};
// Persistence (block H)
readonly repositoryConfig?: PipelineRepositoryConfig;
// Policy + observability (block I)
readonly policyContext?: PolicyContext;
readonly checkpointHandler?: CheckpointHandlerSpec;
}
SceneFromImageResult#
ts
export interface SceneFromImageResult {
readonly manifest: OutputManifest;
readonly phaseOutputs: {
readonly environment: EnvironmentPhaseOutput;
readonly grounding?: GroundingPhaseOutput;
readonly detection?: DetectionPhaseOutput; // block C
readonly objects?: ObjectsPhaseOutput;
readonly align?: AlignPhaseOutput;
readonly bake?: BakePhaseOutput; // block E
readonly audio?: AudioPhaseOutput;
};
readonly checkpointHistory: readonly CheckpointDecision[];
readonly metrics: {
readonly totalDurationMs: number;
readonly perPhaseMs: Partial<Record<PhaseId, number>>;
readonly costCentsTotal?: number; // block I
readonly costCentsByPhase?: Partial<Record<PhaseId, number>>;
};
readonly lineage?: LineageGraph; // block I
readonly galleryRecordId?: string; // block I
}
ObjectSpec (block C)#
ts
export type BBox = {
readonly x: number;
readonly y: number; // 0..1 normalised
readonly w: number;
readonly h: number;
};
export type ObjectSpec =
| {
readonly source: 'explicit';
readonly id: string;
readonly prompt: string;
readonly anchor: AnchorSpec;
readonly providerHint?: string;
}
| {
readonly source: 'cropped';
readonly id: string;
readonly bbox: BBox;
readonly prompt?: string;
readonly anchor: AnchorSpec;
readonly removeBackground?: boolean;
readonly providerHint?: string;
}
| {
readonly source: 'vlm-detected';
readonly id: string;
readonly bbox: BBox;
readonly label: string;
readonly confidence: number;
readonly anchor: AnchorSpec;
readonly providerHint?: string;
};
AnchorSpec (block C)#
ts
export type AnchorSpec =
| {
readonly type: 'absolute';
readonly position: readonly [number, number, number];
}
| { readonly type: 'centre-ground' }
| { readonly type: 'on-floor-front-centre' }
| { readonly type: 'bbox-floor'; readonly bbox: BBox }
| {
readonly type: 'forward-of';
readonly otherId: string;
readonly distanceMeters: number;
}
| {
readonly type: 'right-of';
readonly otherId: string;
readonly distanceMeters: number;
}
| {
readonly type: 'surface-snapped';
readonly hint: readonly [number, number, number];
};
PhaseId#
ts
export type PhaseId =
| 'environment'
| 'grounding'
| 'detection' // block C
| 'objects'
| 'align'
| 'bake' // block E
| 'audio'
| 'manifest';
Errors#
ts
export class ComposerError extends Error {
readonly code: ComposerErrorCode;
readonly phase?: PhaseId;
readonly cause?: Error;
constructor(args: {
code: ComposerErrorCode;
message: string;
phase?: PhaseId;
cause?: Error;
});
}
export type ComposerErrorCode =
| 'invalid-input'
| 'phase-failed'
| 'checkpoint-rejected'
| 'checkpoint-timed-out'
| 'budget-exceeded'
| 'license-incompatible'
| 'dispatch-error'
| 'persistence-error';
@isis/operation-orchestrator checkpoint extensions (block A)#
Types#
ts
// libs/isis/operation-orchestrator/src/checkpoint/checkpoint-types.ts
export interface CheckpointArtifact {
readonly schemaVersion: '1.0.0';
readonly key: string;
readonly mimeType: string; // 'application/json' | 'image/png' | etc.
readonly inlineData?: unknown;
readonly artifactUri?: string;
}
export interface CheckpointDecision {
readonly status: 'approved' | 'rejected' | 'modified' | 'timed-out';
readonly reviewer: string; // 'human:elikem', 'auto:timeout-5m', 'consensus:abc'
readonly reviewedAt: string; // ISO
readonly notes?: string;
readonly modifications?: Readonly<Record<string, unknown>>;
}
export type CheckpointTimeoutAction = 'approve' | 'reject' | 'escalate';
export interface CheckpointPolicy {
readonly timeoutMs: number;
readonly onTimeout: CheckpointTimeoutAction;
readonly escalationHandler?: (
artifact: CheckpointArtifact
) => Promise<CheckpointDecision>;
}
export interface AwaitingCheckpoint {
readonly pipelineId: PipelineId;
readonly chainId: ChainId;
readonly phaseId: PhaseId;
readonly artifact: CheckpointArtifact;
readonly awaitedAt: Date;
readonly deadlineAt: Date;
}
Coordinator#
ts
// libs/isis/operation-orchestrator/src/checkpoint/checkpoint-coordinator.ts
export class CheckpointCoordinator {
awaitDecision(args: {
pipelineId: PipelineId;
chainId: ChainId;
phaseId: PhaseId;
artifact: CheckpointArtifact;
policy: CheckpointPolicy;
}): Promise<CheckpointDecision>;
recordDecision(args: {
pipelineId: PipelineId;
phaseId: PhaseId;
decision: CheckpointDecision;
}): void;
getAwaiting(pipelineId: PipelineId): readonly AwaitingCheckpoint[];
}
export class CheckpointAlreadyDecidedError extends Error {
readonly code: 'checkpoint-already-decided';
readonly phaseId: PhaseId;
}
Pipeline-orchestrator additions#
ts
class PipelineOrchestrator {
// existing surface...
approveCheckpoint(
pipelineId: PipelineId,
phaseId: PhaseId,
decision: CheckpointDecision
): Promise<void>;
getAwaitingCheckpoints(
pipelineId?: PipelineId
): readonly AwaitingCheckpoint[];
// block H additions
resumePipeline(options: {
readonly pipelineId: PipelineId;
readonly fromPhaseId?: PhaseId;
readonly modifications?: Readonly<Record<string, unknown>>;
}): Promise<void>;
}
// New ChainStatus value
export const ChainStatus: {
// existing...
readonly AWAITING_APPROVAL: 'AWAITING_APPROVAL';
};
@isis/gaussian-splatting .spz codec (block D)#
ts
// libs/isis/gaussian-splatting/src/exporters/spz-exporter.ts
export interface SpzExportOptions {
readonly fractionalBits?: number; // default 12
readonly antialiased?: boolean; // default false
readonly shDegree?: 0 | 1 | 2 | 3; // inferred from input if omitted
}
export class SpzExporter {
encode(scene: LoadedScene, options?: SpzExportOptions): Promise<Buffer>;
}
// libs/isis/gaussian-splatting/src/exporters/spz-decoder.ts
export interface SpzDecodeResult {
readonly scene: LoadedScene;
readonly metadata: {
readonly version: number;
readonly fractionalBits: number;
readonly antialiased: boolean;
readonly shDegree: number;
};
}
export class SpzDecoder {
decode(buffer: Buffer): Promise<SpzDecodeResult>;
}
export function createSpzExporter(): SpzExporter;
export function createSpzDecoder(): SpzDecoder;
@isis/gaussian-splatting bake module (block E)#
ts
// libs/isis/gaussian-splatting/src/bake/types.ts
export type BakeMethod = 'surface-sampling' | 'hybrid';
export interface BakeRequest {
readonly mesh: Buffer; // GLB bytes
readonly transform: Float32Array; // 16-element column-major 4×4
readonly method: BakeMethod;
readonly targetGaussianCount?: number; // default 5_000
readonly refinementSteps?: number; // default 200 (hybrid only)
readonly cameraCount?: number; // default 16 (hybrid only)
readonly lightingContext?: { readonly irradianceMap: IrradianceMap }; // block F
}
export interface BakedGaussians {
readonly positions: Float32Array;
readonly scales: Float32Array;
readonly rotations: Float32Array;
readonly opacities: Float32Array;
readonly colors: Float32Array;
readonly sphericalHarmonics?: Float32Array;
readonly count: number;
}
export interface BakeResult {
readonly baked: BakedGaussians;
readonly metrics: { readonly durationMs: number; readonly steps?: number };
}
export async function bakeMesh(request: BakeRequest): Promise<BakeResult>;
// libs/isis/gaussian-splatting/src/bake/merge/merge-splats.ts
export interface MergeOptions {
readonly enableDepthSort?: boolean;
readonly enableOcclusionCull?: boolean;
readonly enableDedupe?: boolean;
readonly dedupeEpsilon?: number;
readonly dedupeColorEpsilon?: number;
readonly canonicalView?: {
readonly position: readonly [number, number, number];
readonly target: readonly [number, number, number];
};
readonly occlusionVoxelSize?: number;
}
export function mergeSplats(
scenes: readonly LoadedScene[],
options?: MergeOptions
): LoadedScene;
@isis/gaussian-splatting lighting module (block F)#
ts
// libs/isis/gaussian-splatting/src/lighting/types.ts
export type IrradianceMap =
| { readonly encoding: 'sh9'; readonly coefficients: Float32Array /* 27 */ }
| {
readonly encoding: 'env-map';
readonly width: number;
readonly height: number;
readonly data: Float32Array;
};
export interface EnvironmentLightField {
readonly origin: readonly [number, number, number];
readonly map: IrradianceMap;
readonly metadata: {
readonly sourceGaussianCount: number;
readonly sampledRadiusMeters: number;
readonly estimationMethod: 'gaussian-weighted-sh' | 'voxel-aggregate';
};
}
export interface LightingEstimateOptions {
readonly encoding?: 'sh9' | 'env-map';
readonly samplingRadiusMeters?: number;
readonly maxSamples?: number;
}
export function estimateEnvironmentLightField(
scene: LoadedScene,
placementPoint: readonly [number, number, number],
options?: LightingEstimateOptions
): EnvironmentLightField;
export interface PbrSurfaceSample {
readonly position: readonly [number, number, number];
readonly normal: readonly [number, number, number];
readonly albedo: readonly [number, number, number];
readonly roughness?: number;
readonly metallic?: number;
}
export function applyPbrShading(
sample: PbrSurfaceSample,
irradiance: IrradianceMap
): readonly [number, number, number];
@isis/3d-quality-gates composition coherence gate (block F)#
ts
// libs/isis/3d-quality-gates/src/delivery/composition-coherence-gate.ts
export interface ThreeDQualityCoherenceFeatureAdapter {
readonly id: string;
extractFeatures(input: {
image: Buffer;
mask: Uint8Array;
}): Promise<Float32Array>;
}
export interface CompositionCoherenceEvaluation {
readonly assetId: string;
readonly evaluatedAt: string;
readonly adapterId: string;
readonly perRenderScores: readonly number[]; // 0..1 per render
readonly meanScore: number;
readonly decision: 'pass' | 'soft-fail' | 'hard-fail';
readonly thresholds: {
readonly passMin: number; // default 0.85
readonly softFailMin: number; // default 0.7
};
}
export async function evaluateCompositionCoherence(input: {
readonly assetId: string;
readonly composedRenders: readonly { image: Buffer; cameraIndex: number }[];
readonly placementMasks: readonly Uint8Array[]; // per render
readonly adapter: ThreeDQualityCoherenceFeatureAdapter;
readonly thresholds?: Partial<{ passMin: number; softFailMin: number }>;
}): Promise<CompositionCoherenceEvaluation>;
@isis/job-envelope extensions (block G)#
ts
// libs/isis/job-envelope/src/job-envelope.ts (extended)
export const ISIS_GENERATION_TYPES = [
// existing...
'scene-from-image-bake',
] as const;
// libs/isis/job-envelope/src/payloads.ts (new)
export type IsisJobPayload<T extends IsisGenerationType> =
T extends 'image-to-3d'
? ImageTo3DPayload
: T extends 'gaussian-splatting'
? GaussianSplattingPayload
: T extends 'mesh-processing'
? MeshProcessingPayload
: T extends 'scene-from-image-bake'
? SceneFromImageBakePayload
: never;
@isis/operation-orchestrator persistence (block H)#
ts
// libs/isis/operation-orchestrator/src/persistence/pipeline-repository.ts
export interface PersistedPipeline {
readonly id: PipelineId;
readonly name: string;
readonly status: ChainStatus;
readonly startedAt: Date;
readonly completedAt?: Date;
readonly metadata?: Readonly<Record<string, unknown>>;
readonly chains: readonly PersistedChain[];
}
export interface PersistedChain {
readonly id: ChainId;
readonly pipelineId: PipelineId;
readonly name: string;
readonly status: ChainStatus;
readonly phases: readonly PersistedPhase[];
}
export interface PersistedPhase {
readonly id: PhaseId;
readonly chainId: ChainId;
readonly name: string;
readonly phaseType: PhaseType;
readonly status: ChainStatus;
readonly startedAt?: Date;
readonly completedAt?: Date;
readonly artifacts: readonly PersistedArtifact[];
}
export interface PersistedArtifact {
readonly key: string;
readonly mimeType: string;
readonly inlineData?: unknown;
readonly artifactUri?: string;
}
export interface PipelineRepository {
createPipeline(
input: Omit<PersistedPipeline, 'id'>
): Promise<PersistedPipeline>;
loadPipeline(id: PipelineId): Promise<PersistedPipeline | null>;
updatePipelineStatus(id: PipelineId, status: ChainStatus): Promise<void>;
storeArtifact(phaseId: PhaseId, artifact: PersistedArtifact): Promise<void>;
loadArtifact(
phaseId: PhaseId,
key: string
): Promise<PersistedArtifact | null>;
recordAwaitingCheckpoint(args: {
phaseId: PhaseId;
artifact: CheckpointArtifact;
deadlineAt: Date;
}): Promise<void>;
recordCheckpointDecision(args: {
phaseId: PhaseId;
decision: CheckpointDecision;
}): Promise<void>;
loadAwaitingCheckpoints(
pipelineId?: PipelineId
): Promise<readonly AwaitingCheckpoint[]>;
loadPipelineSnapshot(id: PipelineId): Promise<PipelineSnapshot | null>;
}
export type PipelineRepositoryConfig =
| { readonly type: 'in-memory' }
| { readonly type: 'postgres'; readonly connectionString: string };
OutputManifest.metadata.sceneFromImage extension (block B+, evolved through F, I)#
ts
// libs/isis/scene-from-image-composer/src/manifest-extension.ts
export interface SceneComposerManifestMetadata {
readonly schemaVersion: '1.0.0';
readonly assetId: string;
readonly sceneBoundsMeters: {
readonly min: readonly [number, number, number];
readonly max: readonly [number, number, number];
};
readonly placements: readonly {
readonly fileId: string; // OutputFileId
readonly transform: readonly number[]; // 16-element column-major
readonly anchor: AnchorSpec;
}[];
readonly bakeMetadata?: {
// block E
readonly method: BakeMethod;
readonly gaussianCount: number;
readonly mergeOptions: MergeOptions;
};
readonly lightingMetadata?: {
// block F
readonly shCoefficients?: readonly number[]; // 27 floats
readonly estimationMethod: string;
readonly perMeshLightFields?: readonly {
meshIndex: number;
encoding: 'sh9' | 'env-map';
}[];
};
readonly checkpointHistory: readonly CheckpointDecision[];
readonly groundingEvaluation?: ThreeDQualityInputGroundingEvaluation;
readonly coherenceEvaluation?: CompositionCoherenceEvaluation;
readonly policyContext?: {
// block I
readonly licensePolicyId: string;
readonly budgetCents: number;
readonly spentCents: number;
};
}
declare module '@isis/outputs' {
interface OutputManifestMetadata {
sceneFromImage?: SceneComposerManifestMetadata;
}
}
Stable invariants across the surface#
assetIdis the same across the request, the manifest, the gallery record, and every persistence row. It's the join key for cross-system audit.Float32Array(16)for 4×4 transforms, column-major. Three.js, Unreal, glTF, andgsplat.jsall consume this layout (with a per-engine upload-time transpose for row-major engines, which we don't do here).- All quaternion serialisation:
[x, y, z, w]ordering. Confirmed against the existingMarbleProviderand the splat-renderer's loaded scene. - All durations:
numberof milliseconds. - All sizes: bytes.
- All angles: degrees in user-facing API; radians internally.