Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 77 additions & 0 deletions packages/db/src/etl/required-power-publication.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,42 @@ function multinodeFixture() {
});
return dir;
}
function centralSingleNodeFixture() {
const dir = fixture();
const files = {
'power_audit_golden/LOGS/power/samples.csv':
'timestamp_unix,hostname,gpu_uuid,power_w\n1700000000,golden-node,GPU-golden,500\n',
'power_audit_golden/LOGS/power/manifest.json': JSON.stringify({
status: 'complete',
publication_valid: true,
}),
'power_audit_golden/LOGS/power/windows/point.json': JSON.stringify({
benchmark_start_time_unix: 1700000000,
benchmark_end_time_unix: 1700000002,
}),
};
for (const [file, contents] of Object.entries(files)) {
fs.mkdirSync(path.dirname(path.join(dir, file)), { recursive: true });
fs.writeFileSync(path.join(dir, file), contents);
}
changeArtifact(dir, auditPath, (audit) => {
audit.selected_window = {
benchmark_type: 'agentx',
concurrency: 1,
window_file: 'windows/point.json',
};
audit.per_gpu_energy_j = { 'golden-node/GPU-golden': 1000 };
audit.per_gpu_role = { 'golden-node/GPU-golden': 'agg' };
});
changeManifest(dir, (manifest) => {
manifest.points[0].artifacts = [auditPath, benchmarkPath, ...Object.keys(files)].map(
(file) => ({ path: file, sha256: sha256(dir, file), validation_state: 'valid' }),
);
});
for (const file of ['gpu_metrics.csv', 'gpu_metrics_identity.csv', 'power_node.txt'])
fs.rmSync(path.join(dir, 'agentic_golden', file));
return dir;
}
const nativeNodes = { 'node-a': 'GPU-a', 'node-b': 'GPU-b' };
/** Non-disagg two-node bundle whose telemetry is per-node nvidia-smi with signed receipts. */
function nativeFixture() {
Expand Down Expand Up @@ -257,6 +293,47 @@ describe('required power publication contract', () => {
});
expect(() => verifyRequiredPowerArtifacts(dir, source)).toThrow('unsupported telemetry_kind');
});
it('accepts native central telemetry for a single-node point without legacy identity files', () => {
const dir = centralSingleNodeFixture();
const scope = verifyRequiredPowerArtifacts(dir, source);
expect(scope?.points).toHaveLength(1);
expect(scope?.points[0].config.isMultinode).toBe(false);
expect(scope?.points[0].metrics.total_gpu_energy_j).toBe(1000);
changeArtifact(dir, auditPath, (audit) => {
audit.per_gpu_energy_j = { 'another-node/GPU-golden': 1000 };
audit.per_gpu_role = { 'another-node/GPU-golden': 'agg' };
});
expect(() => verifyRequiredPowerArtifacts(dir, source)).toThrow(
'physical GPU evidence differs from audit',
);
});
it('requires the native single-node measurement window to be hash-bound', () => {
const dir = centralSingleNodeFixture();
changeManifest(dir, (manifest) => {
manifest.points[0].artifacts = manifest.points[0].artifacts.filter(
(file: { path: string }) => !file.path.includes('/windows/'),
);
});
expect(() => verifyRequiredPowerArtifacts(dir, source)).toThrow(
'missing central measurement window artifact',
);
});
it('excludes service-only nodes while requiring every compute node in the power evidence', () => {
const dir = multinodeFixture();
changeManifest(dir, (manifest) => {
Object.assign(manifest.matrix.multi_node.agentic[0], {
'node-count': 3,
'compute-node-count': 2,
});
});
expect(verifyRequiredPowerArtifacts(dir, source)?.points).toHaveLength(1);
for (const count of [0, 1, 3, 4, 1.5, null]) {
changeManifest(dir, (manifest) => {
manifest.matrix.multi_node.agentic[0]['compute-node-count'] = count;
});
expect(() => verifyRequiredPowerArtifacts(dir, source)).toThrow('Required power:');
}
});
it('accepts a native multinode bundle and rejects a receipt whose telemetry hash differs', () => {
const dir = nativeFixture();
expect(verifyRequiredPowerArtifacts(dir, source)?.points).toHaveLength(1);
Expand Down
30 changes: 15 additions & 15 deletions packages/db/src/etl/required-power-publication.ts
Original file line number Diff line number Diff line change
Expand Up @@ -631,16 +631,12 @@ function verifiedAudit(
return audit;
}

/**
* Which collector produced the evidence. Only native per-node SMI audits name
* their kind; central DCGM (multinode) and per-job SMI (single node) audits
* carry none. Any other kind is a collector this verifier does not know.
*/
type TelemetryKind = 'native_multinode' | 'central_multinode' | 'single_node';
type TelemetryKind = 'native_multinode' | 'central' | 'single_node';

function telemetryKind(audit: JsonRow, isMultinode: boolean): TelemetryKind {
const kind = audit.telemetry_kind;
if (kind === undefined || kind === null) return isMultinode ? 'central_multinode' : 'single_node';
if (kind === undefined || kind === null)
return isMultinode || audit.selected_window !== undefined ? 'central' : 'single_node';
if (kind === 'native_multinode_smi' && isMultinode) return 'native_multinode';
throw new Error(
`Required power: unsupported telemetry_kind ${JSON.stringify(kind)} for a ${isMultinode ? 'multinode' : 'single-node'} point`,
Expand All @@ -658,7 +654,7 @@ function verifyTelemetryEvidence(
oneEvidence(evidence, 'gpu_metrics.csv');
break;
}
case 'central_multinode': {
case 'central': {
oneEvidence(evidence, 'samples.csv');
oneEvidence(evidence, 'manifest.json');
if (
Expand Down Expand Up @@ -707,12 +703,6 @@ function verifyNativeNodeReceipts(evidence: ReadonlyMap<string, Evidence>, audit
}
}

/**
* The audit's per-GPU energy as `{ node, gpu_uuid, role, energy_j }`. Single-node
* audits key energy by GPU index and name the node and UUIDs in sibling files.
* Multinode audits key it by `node/uuid` with `per_gpu_role`, or by UUID alone for
* native collection, whose receipts name each UUID's node.
*/
function auditDevices(
kind: TelemetryKind,
audit: JsonRow,
Expand Down Expand Up @@ -854,7 +844,17 @@ function verifyDeviceEnergy(
if (!audited.some((actual) => isDeepStrictEqual(actual, device)))
throw new Error(`Required power: physical GPU evidence differs from audit for ${key}`);
}
if (nodes.size !== (matrix['node-count'] ?? 1))
const scheduledNodes = positive(matrix['node-count'] ?? 1, 'scheduled node count');
const computeNodes =
matrix['compute-node-count'] === undefined
? scheduledNodes
: positive(matrix['compute-node-count'], 'compute node count');
if (
!Number.isSafeInteger(scheduledNodes) ||
!Number.isSafeInteger(computeNodes) ||
computeNodes > scheduledNodes ||
nodes.size !== computeNodes
)
throw new Error(
`Required power: participating node count differs from required matrix for ${key}`,
);
Expand Down
Loading