Bright Data Local Dev Loop

Overview

Set up a fast, reproducible local development workflow for Bright Data scraping projects with mocked proxy responses, cached results, and vitest integration.

Prerequisites

• Completed brightdata-install-auth setup
• Node.js 18+ with npm/pnpm
• brd-ca.crt SSL certificate downloaded

Instructions

Step 1: Create Project Structure

my-scraper/
├── src/
│   ├── brightdata/
│   │   ├── proxy.ts         # Proxy configuration helper
│   │   ├── scraper.ts       # Scraping functions
│   │   └── cache.ts         # Response caching for dev
│   └── index.ts
├── tests/
│   ├── fixtures/            # Cached HTML responses
│   │   └── example.html
│   └── scraper.test.ts
├── .env.local               # Local credentials (git-ignored)
├── .env.example             # Template for team
├── brd-ca.crt               # Bright Data SSL cert (git-ignored)
└── package.json

Step 2: Build Proxy Configuration Module

// src/brightdata/proxy.ts
import 'dotenv/config';

export interface BrightDataProxy {
  host: string;
  port: number;
  auth: { username: string; password: string };
}

export function getProxy(options?: {
  country?: string;
  city?: string;
  session?: string;
}): BrightDataProxy {
  const { BRIGHTDATA_CUSTOMER_ID, BRIGHTDATA_ZONE, BRIGHTDATA_ZONE_PASSWORD } = process.env;
  if (!BRIGHTDATA_CUSTOMER_ID || !BRIGHTDATA_ZONE || !BRIGHTDATA_ZONE_PASSWORD) {
    throw new Error('Missing BRIGHTDATA_* environment variables');
  }

  let username = `brd-customer-${BRIGHTDATA_CUSTOMER_ID}-zone-${BRIGHTDATA_ZONE}`;
  if (options?.country) username += `-country-${options.country}`;
  if (options?.city) username += `-city-${options.city}`;
  if (options?.session) username += `-session-${options.session}`;

  return {
    host: 'brd.superproxy.io',
    port: 33335,
    auth: { username, password: BRIGHTDATA_ZONE_PASSWORD },
  };
}

Step 3: Add Response Cache for Development

// src/brightdata/cache.ts — cache scraped pages to avoid burning proxy credits
import { createHash } from 'crypto';
import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs';
import { join } from 'path';

const CACHE_DIR = join(process.cwd(), '.scrape-cache');

export function getCachedResponse(url: string): string | null {
  const key = createHash('md5').update(url).digest('hex');
  const path = join(CACHE_DIR, `${key}.html`);
  return existsSync(path) ? readFileSync(path, 'utf-8') : null;
}

export function setCachedResponse(url: string, html: string): void {
  mkdirSync(CACHE_DIR, { recursive: true });
  const key = createHash('md5').update(url).digest('hex');
  writeFileSync(join(CACHE_DIR, `${key}.html`), html);
}