const gentleCheckInResponse = `
I want to make sure I understand how you're feeling.
It sounds like you're going through a difficult time.
Would you like to:
- Talk more about what's on your mind?
- Try a grounding exercise together?
- Look at some coping strategies?
I'm here to listen.
`;
Level 3: Safety Resources (Medium Risk)
const safetyResourcesResponse = `
I hear that you're struggling, and I want you to know that support is available.
Here are some resources that might help:
📞 988 Suicide & Crisis Lifeline: Call or text 988
💬 Crisis Text Line: Text HOME to 741741
🌐 SAMHSA Helpline: 1-800-662-4357
Would you like to talk about what's going on, or would connecting with one of these resources feel right?
`;
Level 4: Crisis Protocol (High Risk)
const crisisProtocolResponse = `
I'm concerned about what you've shared, and I want to make sure you're safe.
Right now, I'd like you to consider reaching out to someone who can help:
🆘 If you're in immediate danger: Call 911
📞 988 Suicide & Crisis Lifeline: Call or text 988 (24/7)
💬 Crisis Text Line: Text HOME to 741741
These are trained counselors who understand what you're going through.
Is there someone you trust - a friend, family member, or sponsor - who you could reach out to right now?
I'm still here with you.
`;
Level 5: Emergency Escalation (Critical)
const emergencyResponse = `
I'm very concerned about your safety right now.
🆘 Please call 911 or go to your nearest emergency room immediately.
If you can't do that, please call 988 right now - they can help.
Your life matters. Please reach out for help right now.
[EMERGENCY CONTACTS DISPLAYED]
`;
// System action: Flag for human review, notify emergency contact if configured
Implementation in Chat Handler
// src/lib/ai/chat-handler.ts
export async function handleChatMessage(
message: string,
conversationId: string,
userId: string
): Promise<ChatResponse> {
// 1. Assess crisis level BEFORE generating AI response
const crisisAssessment = await assessCrisisLevel(
message,
await getConversationHistory(conversationId),
await getUserCheckIns(userId, 7)
);
// 2. Log assessment (for safety review)
await logCrisisAssessment(userId, conversationId, crisisAssessment);
// 3. Handle based on level
if (crisisAssessment.level === 'critical') {
// Don't use AI - provide immediate crisis response
await notifyEmergencyContact(userId);
await flagForHumanReview(conversationId, 'critical_crisis');
return {
message: emergencyResponse,
showEmergencyContacts: true,
disableChat: true, // Prevent further AI interaction
crisisLevel: 'critical',
};
}
if (crisisAssessment.level === 'high') {
// Provide crisis resources, continue with caution
await flagForHumanReview(conversationId, 'high_risk');
return {
message: crisisProtocolResponse,
showCrisisResources: true,
crisisLevel: 'high',
};
}
// 4. For lower levels, proceed with AI but inject safety context
const systemPrompt = getSystemPromptWithSafetyContext(crisisAssessment);
const aiResponse = await generateAIResponse(message, systemPrompt);
// 5. Post-process AI response for safety
const safeResponse = await validateResponseSafety(aiResponse);
return {
message: safeResponse,
crisisLevel: crisisAssessment.level,
};
}
// src/lib/crisis/emergency-contacts.ts
export async function notifyEmergencyContact(userId: string): Promise<void> {
const contacts = await getEmergencyContacts(userId);
if (contacts.length === 0) {
// Log that no emergency contact was available
await logCrisisEvent(userId, 'no_emergency_contact');
return;
}
const primaryContact = contacts[0];
// Send notification (SMS, email, or push)
await sendEmergencyNotification(primaryContact, {
type: 'crisis_alert',
message: `${userName} may be in crisis and could use your support. ` +
`Please reach out to them if you can.`,
// Never include conversation content - privacy
});
// Audit log
await logCrisisEvent(userId, 'emergency_contact_notified', {
contactId: primaryContact.id,
});
}
Safety Guardrails for AI
// Prompt injection for safety context
const safetySystemPrompt = `
CRITICAL SAFETY INSTRUCTIONS:
1. You are NOT a therapist or crisis counselor
2. For ANY mention of self-harm, suicide, or harming others:
- Express genuine concern
- Provide crisis resources (988, Crisis Text Line)
- Encourage professional help
- Do NOT attempt to counsel through crisis
3. Never minimize feelings or use toxic positivity
4. Never promise confidentiality about safety concerns
5. Always validate emotions while encouraging professional support
6. If user mentions relapse, acknowledge and provide SAMHSA helpline
`;
// Response validation
async function validateResponseSafety(response: string): Promise<string> {
const unsafePatterns = [
/don't (call|reach out|get help)/i,
/you don't need (help|therapy|a professional)/i,
/just (think positive|be happy|get over it)/i,
/it's not that (bad|serious)/i,
];
for (const pattern of unsafePatterns) {
if (pattern.test(response)) {
// Flag for review and return safe fallback
await flagForReview('unsafe_response_pattern');
return getSafeFallbackResponse();
}
}
return response;
}
Audit & Compliance
// All crisis events must be logged for review
export async function logCrisisEvent(
userId: string,
eventType: CrisisEventType,
details?: Record<string, unknown>
): Promise<void> {
await db.insert(crisisEvents).values({
id: generateId(),
userId,
eventType,
details: JSON.stringify(sanitizeDetails(details)),
createdAt: new Date(),
reviewed: false, // Requires human review
});
// Critical events trigger immediate notification
if (isCriticalEvent(eventType)) {
await notifyOnCallStaff(userId, eventType);
}
}
Testing Crisis Features
// NEVER use real crisis content in tests
describe('Crisis Detection', () => {
it('detects high-risk indicators', () => {
// Use clearly artificial test phrases
const result = assessCrisisLevel(
'[TEST_HIGH_RISK_INDICATOR]',
[],
[]
);
expect(result.level).toBe('high');
});
it('provides appropriate resources', () => {
const response = getCrisisResponse('high');
expect(response).toContain('988');
expect(response).toContain('Crisis Text Line');
});
});