Data Codebook Example

data-codebook/
├── hank.json                    # Main configuration (below)
├── prompts/
│   ├── observe.md               # Observation codon prompt
│   ├── generate.md              # Schema generation prompt
│   ├── validate.md              # Validation loop prompt
│   └── document.md              # Documentation prompt
├── sentinels/
│   ├── narrator.sentinel.json   # Progress narrator
│   └── cost-tracker.sentinel.json # Cost monitoring
├── templates/
│   └── typescript/              # TypeScript project template
│       ├── package.json
│       ├── tsconfig.json
│       └── src/
└── data/                        # Your CSV files
    ├── users.csv
    └── orders.csv

{
  "meta": {
    "name": "Data Codebook Generator",
    "version": "1.0.0",
    "description": "Generate documented Zod schemas from CSV files"
  },
  "overrides": {
    "model": "sonnet",
    "dataHashTimeLimit": 10000
  },
  "hank": [
    {
      "id": "observe",
      "name": "Observe Data Structure",
      "model": "haiku",
      "continuationMode": "fresh",
      "promptFile": "./prompts/observe.md",
      "rigSetup": [
        {
          "type": "command",
          "command": {
            "run": "mkdir -p notes",
            "workingDirectory": "project"
          }
        }
      ],
      "checkpointedFiles": ["notes/**/*"]
    },
    {
      "id": "generate",
      "name": "Generate Zod Schemas",
      "model": "sonnet",
      "continuationMode": "fresh",
      "promptFile": "./prompts/generate.md",
      "rigSetup": [
        {
          "type": "copy",
          "copy": {
            "from": "./templates/typescript",
            "to": "src"
          }
        },
        {
          "type": "command",
          "command": {
            "run": "bun install",
            "workingDirectory": "lastCopied"
          }
        }
      ],
      "checkpointedFiles": ["src/schemas/**/*.ts", "src/package.json"],
      "sentinels": [
        { "sentinelConfig": "./sentinels/narrator.sentinel.json" },
        { "sentinelConfig": "./sentinels/cost-tracker.sentinel.json" }
      ]
    },
    {
      "type": "loop",
      "id": "validate",
      "name": "Schema Validation Loop",
      "description": "Iteratively fix schema issues until typecheck passes",
      "terminateOn": {
        "type": "iterationLimit",
        "limit": 3
      },
      "codons": [
        {
          "id": "fix-schemas",
          "name": "Validate and Fix Schemas",
          "model": "sonnet",
          "continuationMode": "continue-previous",
          "promptFile": "./prompts/validate.md",
          "checkpointedFiles": ["src/schemas/**/*.ts"],
          "rigSetup": [
            {
              "type": "command",
              "command": {
                "run": "bun run typecheck || true",
                "workingDirectory": "project"
              },
              "allowFailure": true
            }
          ],
          "sentinels": [
            { "sentinelConfig": "./sentinels/narrator.sentinel.json" }
          ]
        }
      ]
    },
    {
      "id": "document",
      "name": "Generate Documentation",
      "model": "sonnet",
      "continuationMode": "fresh",
      "promptFile": "./prompts/document.md",
      "rigSetup": [
        {
          "type": "command",
          "command": {
            "run": "mkdir -p docs",
            "workingDirectory": "project"
          }
        }
      ],
      "checkpointedFiles": ["docs/**/*"],
      "outputFiles": [
        {
          "copy": ["src/schemas/**/*.ts", "docs/**/*"],
          "beforeCopy": [
            {
              "type": "command",
              "command": {
                "run": "bun run typecheck",
                "workingDirectory": "project"
              }
            }
          ]
        }
      ],
      "sentinels": [
        { "sentinelConfig": "./sentinels/narrator.sentinel.json" },
        { "sentinelConfig": "./sentinels/cost-tracker.sentinel.json" }
      ]
    }
  ]
}

"meta": {
  "name": "Data Codebook Generator",
  "version": "1.0.0",
  "description": "Generate documented Zod schemas from CSV files"
}

"overrides": {
  "model": "sonnet",
  "dataHashTimeLimit": 10000
}

{
  "id": "observe",
  "name": "Observe Data Structure",
  "model": "haiku",
  "continuationMode": "fresh",
  "promptFile": "./prompts/observe.md",
  "rigSetup": [
    {
      "type": "command",
      "command": {
        "run": "mkdir -p notes",
        "workingDirectory": "project"
      }
    }
  ],
  "checkpointedFiles": ["notes/**/*"]
}

{
  "id": "generate",
  "name": "Generate Zod Schemas",
  "model": "sonnet",
  "continuationMode": "fresh",
  "promptFile": "./prompts/generate.md",
  "rigSetup": [
    {
      "type": "copy",
      "copy": { "from": "./templates/typescript", "to": "src" }
    },
    {
      "type": "command",
      "command": { "run": "bun install", "workingDirectory": "lastCopied" }
    }
  ],
  "checkpointedFiles": ["src/schemas/**/*.ts", "src/package.json"],
  "sentinels": [
    { "sentinelConfig": "./sentinels/narrator.sentinel.json" },
    { "sentinelConfig": "./sentinels/cost-tracker.sentinel.json" }
  ]
}

{
  "type": "loop",
  "id": "validate",
  "name": "Schema Validation Loop",
  "terminateOn": {
    "type": "iterationLimit",
    "limit": 3
  },
  "codons": [
    {
      "id": "fix-schemas",
      "name": "Validate and Fix Schemas",
      "model": "sonnet",
      "continuationMode": "continue-previous",
      "promptFile": "./prompts/validate.md",
      "rigSetup": [
        {
          "type": "command",
          "command": {
            "run": "bun run typecheck || true",
            "workingDirectory": "project"
          },
          "allowFailure": true
        }
      ],
      "sentinels": [{ "sentinelConfig": "./sentinels/narrator.sentinel.json" }]
    }
  ]
}

{
  "id": "document",
  "name": "Generate Documentation",
  "model": "sonnet",
  "continuationMode": "fresh",
  "promptFile": "./prompts/document.md",
  "rigSetup": [
    {
      "type": "command",
      "command": { "run": "mkdir -p docs", "workingDirectory": "project" }
    }
  ],
  "checkpointedFiles": ["docs/**/*"],
  "outputFiles": [
    {
      "copy": ["src/schemas/**/*.ts", "docs/**/*"],
      "beforeCopy": [
        {
          "type": "command",
          "command": {
            "run": "bun run typecheck",
            "workingDirectory": "project"
          }
        }
      ]
    }
  ],
  "sentinels": [
    { "sentinelConfig": "./sentinels/narrator.sentinel.json" },
    { "sentinelConfig": "./sentinels/cost-tracker.sentinel.json" }
  ]
}

{
  "id": "narrator",
  "name": "Progress Narrator",
  "model": "anthropic/claude-haiku-4-5",
  "trigger": {
    "type": "event",
    "on": ["assistant.action", "tool.result"]
  },
  "execution": { "strategy": "debounce", "milliseconds": 10000 },
  "systemPromptText": "You are a technical writer summarizing AI agent progress. Be concise and factual.",
  "userPromptText": "Based on these events, write a brief paragraph about what the agent just accomplished:\n\n<%= JSON.stringify(it.events, null, 2) %>",
  "joinString": "\n\n"
}

{
  "id": "cost-tracker",
  "name": "Cost Tracker",
  "model": "anthropic/claude-haiku-4-5",
  "trigger": { "type": "event", "on": ["token.usage"] },
  "execution": { "strategy": "timeWindow", "milliseconds": 30000 },
  "userPromptText": "Summarize the token usage so far. Calculate total input tokens, output tokens, and estimated cost. List the most expensive operations.\n\nEvents:\n<%= JSON.stringify(it.events, null, 2) %>",
  "joinString": "\n\n---\n\n"
}

# Data Observation Task
 
Examine the CSV files in the `read_only_data_source/data/` directory.
 
For each CSV file:
 
1. List all columns with inferred data types.
2. Note any patterns (IDs, dates, enums, foreign keys).
3. Identify relationships between files (e.g., user_id references).
4. Record sample values and constraints.
 
Create a file called `notes/observations.md` with your findings.
Structure it clearly with headers for each CSV file.
 
Be thorough but concise. Focus on what a schema author would need to know.

# Schema Generation Task
 
Read the observations in `notes/observations.md`.
 
Based on those observations, create Zod schemas for each CSV file in `src/schemas/`.
 
Requirements:
 
- One schema file per CSV (e.g., `src/schemas/users.ts`, `src/schemas/orders.ts`).
- Include JSDoc comments explaining each field.
- Add appropriate validations (email format, date strings, enums).
- Create a barrel export in `src/schemas/index.ts`.
 
Example schema structure:
 
```typescript
import { z } from "zod";
 
/**
 * User record from users.csv
 */
export const UserSchema = z.object({
  id: z.number().int().positive(),
  name: z.string().min(1),
  email: z.string().email(),
  created_at: z.string().datetime(),
  status: z.enum(["active", "inactive"]),
});
 
export type User = z.infer<typeof UserSchema>;
```

### Validation Prompt

```markdown filename="prompts/validate.md"
# Schema Validation Task

Run the TypeScript type checker:

```bash
cd src && bun run typecheck

cd src && bun test

### Documentation Prompt

```markdown filename="prompts/document.md"
# Documentation Generation Task

Create comprehensive documentation for the schemas in `src/schemas/`.

Generate `docs/CODEBOOK.md` with:

1. **Overview**: What datasets this codebook covers.
2. **Schemas**: For each schema:
   - Table name and purpose
   - Field-by-field documentation
   - Example valid/invalid values
   - Relationships to other tables
3. **Usage**: How to import and use the schemas.

Make it readable by non-developers. Explain what the data means, not just the types.

Also create `docs/CHANGELOG.md` documenting what was generated.

{
  "name": "data-schemas",
  "version": "1.0.0",
  "type": "module",
  "scripts": {
    "typecheck": "tsc --noEmit",
    "test": "bun test"
  },
  "dependencies": {
    "zod": "^3.22.0"
  },
  "devDependencies": {
    "typescript": "^5.0.0",
    "@types/bun": "latest"
  }
}

{
  "compilerOptions": {
    "target": "ES2022",
    "module": "ESNext",
    "moduleResolution": "bundler",
    "strict": true,
    "esModuleInterop": true,
    "skipLibCheck": true,
    "outDir": "./dist"
  },
  "include": ["src/**/*"]
}

# Run with your data directory
hankweave --data ./data
 
# Validate configuration without running
hankweave --validate
 
# Force a new execution (ignore any checkpoints)
hankweave --data ./data --start-new

hankweave-results/
├── src/
│   └── schemas/
│       ├── users.ts
│       ├── orders.ts
│       └── index.ts
└── docs/
    ├── CODEBOOK.md
    └── CHANGELOG.md

.hankweave/sentinels/outputs/
├── narrator/
│   └── narrator-document-{timestamp}.md
└── cost-tracker/
    └── cost-tracker-document-{timestamp}.md

"terminateOn": {
  "type": "contextExceeded"
}

{
  "id": "qa-review",
  "name": "Quality Reviewer",
  "model": "anthropic/claude-haiku-4-5",
  "trigger": {
    "type": "event",
    "on": ["file.updated"],
    "conditions": [
      { "operator": "matches", "path": "path", "value": ".*\\.ts$" }
    ]
  },
  "execution": { "strategy": "debounce", "milliseconds": 5000 },
  "userPromptText": "Review this TypeScript file for quality issues (missing types, poor naming, security issues):\n\n<%= JSON.stringify(it.events, null, 2) %>"
}

{
  "type": "loop",
  "id": "validate",
  "terminateOn": { "type": "iterationLimit", "limit": 5 },
  "codons": [
    {
      "id": "quick-fix",
      "model": "haiku",
      "continuationMode": "fresh",
      "promptText": "Run typecheck. If there are simple errors, fix them."
    },
    {
      "id": "deep-fix",
      "model": "sonnet",
      "continuationMode": "fresh",
      "promptText": "Run typecheck. If there are complex logical errors, analyze and fix them."
    }
  ]
}