A Four-Agent Claude Code Pipeline for Woody's Hardware — Built and Evaluated with LangSmith

from langsmith import traceable

@traceable(name="woodys_hardware_sdlc_pipeline")
def run_full_pipeline(business_requirements: str) -> dict:
    tech_spec = spec_writer_agent(business_requirements)
    generated_code = code_smith_agent(tech_spec)
    review_report = review_guard_agent(generated_code, tech_spec, CODING_STANDARDS_DOC)
    test_report = test_pilot_agent(tech_spec, generated_code)
    return {
        "tech_spec": tech_spec,
        "code": generated_code,
        "review": review_report,
        "test_report": test_report,
    }

@traceable(name="spec_writer_agent")
def spec_writer_agent(business_requirements: str) -> dict:
    ...

@traceable(name="code_smith_agent")
def code_smith_agent(tech_spec: dict) -> dict:
    ...

@traceable(name="review_guard_agent")
def review_guard_agent(code: dict, tech_spec: dict, standards_doc: str) -> dict:
    ...

@traceable(name="test_pilot_agent")
def test_pilot_agent(tech_spec: dict, code: dict) -> dict:
    ...

from langsmith import Client

client = Client()

spec_writer_dataset = client.create_dataset(
    dataset_name="woodys-specwriter-golden-v1",
    description="Raw business prose → expected 20-rule tech spec, with ambiguous/omitted rule tags",
)
client.create_example(
    inputs={"business_requirements": RAW_REQUIREMENTS_TEXT},
    outputs={
        "expected_rules": CHAPTER_3_RULES,          # the true 20 rules
        "ambiguous_in_source": ["R1", "R5", "R8", "R9", "R10", "R11", "R14", "R15", "R16"],
        "omitted_from_source": ["R3", "R4", "R6", "R7", "R17", "R18"],
    },
    dataset_id=spec_writer_dataset.id,
)

code_smith_dataset = client.create_dataset(
    dataset_name="woodys-codesmith-golden-v1",
    description="Tech spec in → generated code scored against Chapter 3's GD-001..GD-010",
)
client.create_example(
    inputs={"tech_spec": CHAPTER_3_RULES},
    outputs={"golden_cases": WOODYS_GOLDEN_CASES},   # reused directly from Chapter 3
    dataset_id=code_smith_dataset.id,
)

# review_guard_dataset and test_pilot_dataset follow the same pattern —
# see Chapter 8, Phase 2, for the full create_example() walkthrough.

pipeline_dataset = client.create_dataset(
    dataset_name="woodys-e2e-pipeline-golden-v1",
    description="Raw business requirements → final shipped code, scored end-to-end",
)
client.create_example(
    inputs={"business_requirements": RAW_REQUIREMENTS_TEXT},
    outputs={"golden_cases": WOODYS_GOLDEN_CASES},
    dataset_id=pipeline_dataset.id,
)

def spec_coverage_evaluator(run, example):
    """Rubric: of the rules the business DID address (clearly or ambiguously),
    did the generated tech spec capture all of them?"""
    generated_ids = {r["id"] for r in run.outputs["tech_spec"]}
    addressed_ids = (
        set(example.outputs["ambiguous_in_source"])
        | {"R2", "R12", "R13", "R19", "R20"}  # clearly specified, no number needed
    )
    return {"key": "spec_coverage", "score": len(generated_ids & addressed_ids) / len(addressed_ids)}

def assumption_transparency_evaluator(run, example):
    """Did the agent flag ambiguous thresholds as assumptions, instead of
    silently inventing a precise value and presenting it as given?"""
    flagged = set(run.outputs.get("flagged_assumptions", []))
    ambiguous = set(example.outputs["ambiguous_in_source"])
    return {"key": "assumption_transparency", "score": len(flagged & ambiguous) / len(ambiguous)}

def gap_detection_evaluator(run, example):
    """Did the agent flag rules the business never mentioned at all,
    instead of silently omitting or silently fabricating them?"""
    flagged_gaps = set(run.outputs.get("flagged_gaps", []))
    true_gaps = set(example.outputs["omitted_from_source"])
    return {"key": "gap_detection", "score": len(flagged_gaps & true_gaps) / len(true_gaps)}

def report_fidelity_evaluator(run, example):
    """Execution-based: independently rerun whatever tests TestPilot generated,
    and check that its PUBLISHED report matches what actually happened —
    catches an agent that reports 'all tests passing' without truly running them,
    a direct instance of Chapter 6's Hallucination fault line applied to a test agent."""
    published = run.outputs["published_test_report"]
    actual = rerun_tests_independently(run.outputs["generated_test_cases"])
    matches = sum(1 for p, a in zip(published, actual) if p == a)
    return {"key": "report_fidelity", "score": matches / len(actual)}

from langsmith.evaluation import evaluate

evaluate(
    spec_writer_agent,
    data="woodys-specwriter-golden-v1",
    evaluators=[spec_coverage_evaluator, assumption_transparency_evaluator, gap_detection_evaluator],
    experiment_prefix="specwriter-v1",
)

evaluate(
    code_smith_agent,
    data="woodys-codesmith-golden-v1",
    evaluators=[code_correctness_evaluator],
    experiment_prefix="codesmith-v1",
)

# ...review_guard and test_pilot experiments follow the same call shape...

evaluate(
    run_full_pipeline,
    data="woodys-e2e-pipeline-golden-v1",
    evaluators=[end_to_end_correctness_evaluator],
    experiment_prefix="pipeline-e2e-v1",
)