/* BioReveal — Unified BioScan (one capture, three signals + voice transcript).
*
* 25 seconds. Camera + mic on simultaneously.
* - Per-frame face RGB → rPPG vitals (HR, breathing, autonomic activation)
* - Final face frame → /api/skin (hydration, redness, texture, spots)
* - Mic captures voice → DSP features (pitch/jitter/energy)
* - Web Speech API transcribes voice in real-time (the user describes
* what they want to improve — that text becomes a primary AI input)
*
* Captured numbers are NOT shown to user (they're estimates, can be wrong).
* Just "✓ Captured" — the AI synthesizer turns the raw signals into useful
* analysis on the Reveal page.
*/
const StepBioscan = ({ value, onChange }) => {
const videoRef = React.useRef(null);
const canvasRef = React.useRef(null);
const streamRef = React.useRef(null);
const audioCtxRef = React.useRef(null);
const sourceRef = React.useRef(null);
const processorRef = React.useRef(null);
const audioSamplesRef = React.useRef([]);
const sampleRateRef = React.useRef(48000);
const landmarkerRef = React.useRef(null);
const captureBufRef = React.useRef({ r: [], g: [], b: [] });
const startTsRef = React.useRef(0);
const rafRef = React.useRef(0);
const recognitionRef = React.useRef(null);
const transcriptRef = React.useRef("");
const [status, setStatus] = React.useState("idle"); // idle | loading | requesting | capturing | analyzing | done | error | denied
const [progress, setProgress] = React.useState(0);
const [usingMP, setUsingMP] = React.useState(false);
const [errMsg, setErrMsg] = React.useState("");
const [interimTranscript, setInterimTranscript] = React.useState("");
const [captured, setCaptured] = React.useState(value?.captured || false);
const [hadVitals, setHadVitals] = React.useState(false);
const [hadSkin, setHadSkin] = React.useState(false);
const [hadVoice, setHadVoice] = React.useState(false);
const [hadTranscript, setHadTranscript] = React.useState(false);
const CAPTURE_SECONDS = 25;
const FS = 30;
const cleanup = () => {
cancelAnimationFrame(rafRef.current);
try { processorRef.current?.disconnect(); } catch(_){}
try { sourceRef.current?.disconnect(); } catch(_){}
try { audioCtxRef.current?.close(); } catch(_){}
try { recognitionRef.current?.stop(); } catch(_){}
if (streamRef.current) {
streamRef.current.getTracks().forEach(t => t.stop());
streamRef.current = null;
}
};
React.useEffect(() => () => cleanup(), []);
const tryLoadFaceLM = async () => {
if (landmarkerRef.current) return landmarkerRef.current;
try {
const mod = await import("https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.18/+esm");
const { FilesetResolver, FaceLandmarker } = mod;
const fileset = await FilesetResolver.forVisionTasks("https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.18/wasm");
const lm = await FaceLandmarker.createFromOptions(fileset, {
baseOptions: { modelAssetPath: "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task" },
runningMode: "VIDEO", numFaces: 1,
});
landmarkerRef.current = lm;
return lm;
} catch(_) { return null; }
};
const startTranscription = () => {
const SR = window.SpeechRecognition || window.webkitSpeechRecognition;
if (!SR) { console.warn("Web Speech API not supported"); return; }
const rec = new SR();
rec.continuous = true;
rec.interimResults = true;
rec.lang = "en-US";
transcriptRef.current = "";
rec.onresult = (e) => {
let final = "", interim = "";
for (let i = e.resultIndex; i < e.results.length; i++) {
const t = e.results[i][0].transcript;
if (e.results[i].isFinal) final += t + " "; else interim += t;
}
if (final) transcriptRef.current += final;
setInterimTranscript((transcriptRef.current + " " + interim).trim());
};
rec.onerror = (e) => { console.warn("speech rec err", e.error); };
try { rec.start(); recognitionRef.current = rec; } catch(e){ console.warn(e); }
};
const start = async () => {
setErrMsg("");
setInterimTranscript("");
setCaptured(false);
setStatus("loading");
const lm = await tryLoadFaceLM();
setUsingMP(!!lm);
setStatus("requesting");
let stream;
try {
stream = await navigator.mediaDevices.getUserMedia({
video: { facingMode: "user", width: { ideal: 720 }, height: { ideal: 720 }, frameRate: { ideal: 30 } },
audio: { echoCancellation: true, noiseSuppression: true },
});
} catch (err) {
setErrMsg(err?.name === "NotAllowedError"
? "Camera + mic permission needed. Click the lock icon in the address bar to allow both, then retry."
: err.message);
setStatus(err?.name === "NotAllowedError" ? "denied" : "error");
return;
}
streamRef.current = stream;
const v = videoRef.current;
v.srcObject = stream;
try { await v.play(); } catch(_){}
// Audio pipeline (DSP features)
const audioStream = new MediaStream(stream.getAudioTracks());
const ctx = new (window.AudioContext || window.webkitAudioContext)();
audioCtxRef.current = ctx;
sampleRateRef.current = ctx.sampleRate;
const src = ctx.createMediaStreamSource(audioStream);
sourceRef.current = src;
const proc = ctx.createScriptProcessor(2048, 1, 1);
processorRef.current = proc;
audioSamplesRef.current = [];
proc.onaudioprocess = (e) => {
audioSamplesRef.current.push(new Float32Array(e.inputBuffer.getChannelData(0)));
};
src.connect(proc); proc.connect(ctx.destination);
// Speech-to-text (free, browser-native)
startTranscription();
setStatus("capturing");
startTsRef.current = performance.now();
captureBufRef.current = { r: [], g: [], b: [] };
const canvas = canvasRef.current;
const cctx = canvas.getContext("2d", { willReadFrequently: true });
canvas.width = v.videoWidth || 720;
canvas.height = v.videoHeight || 720;
const tick = async (ts) => {
if (!streamRef.current) return;
const elapsed = (ts - startTsRef.current) / 1000;
if (elapsed >= CAPTURE_SECONDS) { await analyzeAll(canvas); return; }
try {
cctx.drawImage(v, 0, 0, canvas.width, canvas.height);
const W = canvas.width, H = canvas.height;
let rsum = 0, gsum = 0, bsum = 0, count = 0;
if (lm) {
let result;
try { result = lm.detectForVideo(v, ts); } catch(_){}
const lms = result?.faceLandmarks?.[0];
if (lms && lms.length > 200) {
const idxs = [50, 280, 10, 117, 346];
const half = 14;
for (const i of idxs) {
const p = lms[i];
const cx = Math.round(p.x * W), cy = Math.round(p.y * H);
const x0 = Math.max(0, cx - half), y0 = Math.max(0, cy - half);
const w = Math.min(W - x0, half * 2), h = Math.min(H - y0, half * 2);
const data = cctx.getImageData(x0, y0, w, h).data;
for (let j = 0; j < data.length; j += 4) {
rsum += data[j]; gsum += data[j+1]; bsum += data[j+2]; count++;
}
}
}
}
if (count === 0) {
const w = Math.round(W * 0.4), h = Math.round(H * 0.4);
const x0 = Math.max(0, Math.round(W/2 - w/2)), y0 = Math.max(0, Math.round(H * 0.45 - h/2));
const data = cctx.getImageData(x0, y0, w, h).data;
for (let j = 0; j < data.length; j += 64) {
rsum += data[j]; gsum += data[j+1]; bsum += data[j+2]; count++;
}
}
if (count > 0) {
captureBufRef.current.r.push(rsum / count);
captureBufRef.current.g.push(gsum / count);
captureBufRef.current.b.push(bsum / count);
}
setProgress(Math.min(100, Math.round((elapsed / CAPTURE_SECONDS) * 100)));
} catch(e){ console.warn(e); }
rafRef.current = requestAnimationFrame(tick);
};
rafRef.current = requestAnimationFrame(tick);
};
const analyzeAll = async (canvas) => {
setStatus("analyzing");
cleanup();
const buf = captureBufRef.current;
const audio = audioSamplesRef.current;
const transcript = transcriptRef.current.trim();
let skinResult = null;
try {
const blob = await new Promise(res => canvas.toBlob(res, "image/jpeg", 0.85));
if (blob) {
const r = await fetch("/api/skin", { method: "POST", headers: { "Content-Type": "image/jpeg" }, body: blob });
const d = await r.json();
if (d.ok) skinResult = d;
}
} catch(e){ console.warn("skin failed", e); }
let vitalsResult = null;
if (buf.r.length >= 60) {
try {
const r = await fetch("/api/vitals", {
method: "POST", headers: { "Content-Type": "application/json" },
body: JSON.stringify({ r: buf.r, g: buf.g, b: buf.b, fs: FS }),
});
const d = await r.json();
if (d.ok) vitalsResult = d;
} catch(e){ console.warn("vitals failed", e); }
}
let voiceResult = null;
if (audio.length > 0) {
try { voiceResult = computeVoiceFeatures(audio, sampleRateRef.current); }
catch(e){ console.warn("voice failed", e); }
}
if (voiceResult && transcript) voiceResult.transcript = transcript;
if (!vitalsResult && !skinResult && !voiceResult) {
setStatus("error");
setErrMsg("Couldn't extract any signals — try again with better light + a quieter room.");
return;
}
setHadVitals(!!vitalsResult);
setHadSkin(!!skinResult);
setHadVoice(!!voiceResult);
setHadTranscript(!!transcript);
setCaptured(true);
onChange({ captured: true, vitals: vitalsResult, skin: skinResult, voice: voiceResult });
setStatus("done");
};
const skip = () => { cleanup(); onChange({ skipped: true }); setStatus("done"); setCaptured(false); };
const retry = () => { setStatus("idle"); setProgress(0); setErrMsg(""); setCaptured(false); setInterimTranscript(""); transcriptRef.current = ""; };
return (
<>
02 / 06 · The bioscan · Optional
25 seconds. Tell us what you want fixed.
Camera and mic come on. While the AI reads your face and voice, you talk about what you want to upgrade. That's the whole thing. The AI takes it from there.
{status === "idle" && (
When the scan starts, say out loud:
"I want to fix my ___ and feel ___" — just talk. Pick from:
{["hair", "skin", "muscle", "weight", "energy", "sleep", "focus", "drive", "look younger", "joint pain"].map(g => (
{g}
))}
Be honest. Tired? Say tired. Frustrated with your skin? Say it. The AI uses every word.
)}
{status === "idle" && (
Tap "Begin BioScan"
)}
{status === "loading" && (
Loading face model~3MB · first time only
)}
{status === "requesting" && (
Allow camera + mic
)}
{status === "capturing" && (
<>
● BIOSCAN · {progress}%
{usingMP ? "FACE-LOCKED" : "CENTER-ROI"}
TELL ME WHAT YOU WANT FIXED
{interimTranscript || (start speaking…) }
HOLD STILL · KEEP TALKING
)}
{status === "analyzing" && (
Processing all 3 signal channels…
)}
{status === "done" && captured && (
✓
BIOSCAN COMPLETE
The AI puts it all together on the next page. Real read. No raw numbers.
)}
{(status === "error" || status === "denied") && (
BIOSCAN FAILED
{errMsg || "Unknown error"}
)}
{status === "idle" && (
<>
● Begin BioScan
Skip
)}
{(status === "loading" || status === "requesting" || status === "capturing" || status === "analyzing") && (
Working…
)}
{status === "done" && (
<>
{captured ? "✓ BioScan saved" : "✓ Skipped"}
{captured && Re-scan }
)}
{(status === "error" || status === "denied") && (
<>
Try again
Skip
)}
ⓘ WHAT THE AI READS · Your heart rate (from skin color), your skin, your voice, and what you say out loud. Raw numbers stay private. You get the read.
);
};
const Pill = ({ on, label }) => (
{label}
);
// ── Voice DSP (client-side feature extraction) ──
function computeVoiceFeatures(samplesArrays, sr) {
const total = samplesArrays.reduce((s,c)=>s+c.length, 0);
const samples = new Float32Array(total);
let off = 0;
for (const c of samplesArrays) { samples.set(c, off); off += c.length; }
if (samples.length < sr * 5) return null;
const frameSize = Math.round(sr * 0.05);
const hop = Math.round(sr * 0.025);
const detectPitch = (frame) => {
const minHz = 70, maxHz = 400;
const minLag = Math.floor(sr / maxHz), maxLag = Math.floor(sr / minHz);
let bestLag = 0, bestCorr = 0, sumE = 0;
for (let i = 0; i < frame.length; i++) sumE += frame[i] * frame[i];
if (sumE < 0.001) return 0;
for (let lag = minLag; lag <= maxLag; lag++) {
let corr = 0;
for (let i = 0; i < frame.length - lag; i++) corr += frame[i] * frame[i+lag];
if (corr > bestCorr) { bestCorr = corr; bestLag = lag; }
}
if (bestCorr / sumE < 0.3) return 0;
return sr / bestLag;
};
const rms = (frame) => { let s=0; for (let i=0;i {
const N = 256;
const sub = new Float32Array(N);
const stride = Math.max(1, Math.floor(frame.length / N));
for (let i = 0; i < N; i++) sub[i] = frame[i*stride] || 0;
let num = 0, den = 0;
for (let k = 1; k < N/2; k++) {
let re = 0, im = 0;
for (let n = 0; n < N; n++) {
const a = -2 * Math.PI * k * n / N;
re += sub[n] * Math.cos(a); im += sub[n] * Math.sin(a);
}
const mag = Math.sqrt(re*re + im*im);
const f = (k * sr / stride) / N;
num += f * mag; den += mag;
}
return den > 0 ? num / den : 0;
};
const pitches = [], energies = [], centroids = [];
let voicedFrames = 0, totalFrames = 0;
for (let i = 0; i + frameSize < samples.length; i += hop) {
const frame = samples.subarray(i, i + frameSize);
const e = rms(frame);
energies.push(e); totalFrames++;
if (e < 0.005) continue;
const p = detectPitch(frame);
if (p > 0) {
pitches.push(p); voicedFrames++;
if (totalFrames % 4 === 0) centroids.push(spectralCentroid(frame));
}
}
if (pitches.length < 20) return null;
const mean = (a) => a.reduce((s,x)=>s+x,0)/a.length;
const std = (a) => { const m = mean(a); return Math.sqrt(mean(a.map(x=>(x-m)**2))); };
const meanPitch = mean(pitches);
const stdPitch = std(pitches);
const meanEnergy = mean(energies);
const meanCentroid = centroids.length ? mean(centroids) : 0;
let jitter = 0;
for (let i = 1; i < pitches.length; i++) jitter += Math.abs(pitches[i] - pitches[i-1]);
jitter = (jitter / (pitches.length - 1)) / meanPitch;
const stress_score = Math.round(Math.min(100, Math.max(0, jitter*1500 + (stdPitch < 8 ? 25 : 0) + (meanCentroid < 800 ? 15 : 0))));
const fatigue_score = Math.round(Math.min(100, Math.max(0, (meanEnergy < 0.02 ? 40 : 20) + (meanCentroid < 700 ? 25 : 0) + (stdPitch < 10 ? 20 : 0))));
const energy_score = Math.round(Math.min(100, Math.max(0, meanEnergy*2000 + stdPitch*0.8 + (meanCentroid > 1000 ? 15 : 0))));
const mood_class = stdPitch > 15 && jitter < 0.04 ? "upbeat" : jitter > 0.06 ? "tense" : meanEnergy < 0.02 ? "flat" : "neutral";
return {
ok: true,
voiced_ratio: Math.round((voicedFrames / totalFrames) * 100) / 100,
mean_pitch_hz: Math.round(meanPitch),
pitch_std_hz: Math.round(stdPitch * 10) / 10,
mean_energy: Math.round(meanEnergy * 1000) / 1000,
spectral_centroid_hz: Math.round(meanCentroid),
jitter: Math.round(jitter * 1000) / 1000,
stress_score, fatigue_score, energy_score, mood_class,
};
}
window.BR_STEPS = window.BR_STEPS || {};
window.BR_STEPS.StepBioscan = StepBioscan;